Make Autotest compatible with the new version of dpdispatcher. (#667)

* Change mv to cp when downloading softlinks.

* Add explanations relevant to POTCAR's examples.

* Fix CH4's vasp-md INCAR example.

* Fix bug in make_submission of Auto-test

* Add ase, custodian, GromacsWrapper in conda packages.

* Fix bug of fp_task_min

* Add gromacswrapper to conda recipe

* Revert "Fix bug of fp_task_min"

This reverts commit 8d7f8a7.

* Add numkit recipe

* Add `convert_mdata` definition to fix simplify bug.

* Debug info

* Debug info

* Fix bug in autotest

* Check autotest confdir

* Debug autotest command

* Change run_task to [run_task]

* Combining tasks together in common_equi

* Combining dispatcher in common_equi

* Delete debugging info.

* Delete irrelevant conda file

* Fix bug in common_equi

* Add error log in common_prop

Author: AnguseZhang

dpgen/auto_test/common_equi.py

@@ -114,46 +114,6 @@ def make_equi(confs,
         inter.make_input_file(ii, 'relaxation', relax_param)
 
 
-def worker(work_path,
-           run_task,
-           forward_common_files,
-           forward_files,
-           backward_files,
-           mdata,
-           inter_type):
-    machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
-    disp = make_dispatcher(machine, resources, work_path, [run_task], group_size)
-    print("%s --> Runing... " % (work_path))
-
-    api_version = mdata.get('api_version', '0.9')
-    if LooseVersion(api_version) < LooseVersion('1.0'):
-        warnings.warn(f"the dpdispatcher will be updated to new version."
-                      f"And the interface may be changed. Please check the documents for more details")
-        disp.run_jobs(resources,
-                      command,
-                      work_path,
-                      [run_task],
-                      group_size,
-                      forward_common_files,
-                      forward_files,
-                      backward_files,
-                      outlog='outlog',
-                      errlog='errlog')
-    elif LooseVersion(api_version) >= LooseVersion('1.0'):
-        submission = make_submission(
-            mdata_machine=machine,
-            mdata_resources=resources,
-            commands=[command],
-            work_path=work_path,
-            run_tasks=run_task,
-            group_size=group_size,
-            forward_common_files=forward_common_files,
-            forward_files=forward_files,
-            backward_files=backward_files,
-            outlog='outlog',
-            errlog='errlog'
-        )
-        submission.run_submission()
 
 def run_equi(confs,
              inter_param,
@@ -166,18 +126,16 @@ def run_equi(confs,
     conf_dirs.sort()
 
     processes = len(conf_dirs)
-    pool = Pool(processes=processes)
-    print("Submit job via %d processes" % processes)
 
     # generate a list of task names like mp-xxx/relaxation/relax_task
     # ...
     work_path_list = []
     for ii in conf_dirs:
-        work_path_list.append(os.path.abspath(os.path.join(ii, 'relaxation')))
+        work_path_list.append(os.path.join(ii, 'relaxation'))
     all_task = []
     for ii in work_path_list:
         all_task.append(os.path.join(ii, 'relax_task'))
-
+    run_tasks = all_task
     inter_type = inter_param['type']
     # vasp
     if inter_type == "vasp":
@@ -186,41 +144,51 @@ def run_equi(confs,
         mdata = convert_mdata(mdata, ["model_devi"])
     else:
         raise RuntimeError("unknown task %s, something wrong" % inter_type)
-
+    
     # dispatch the tasks
     # POSCAR here is useless
     virtual_calculator = make_calculator(inter_param, "POSCAR")
     forward_files = virtual_calculator.forward_files()
     forward_common_files = virtual_calculator.forward_common_files()
     backward_files = virtual_calculator.backward_files()
     #    backward_files += logs
-    # ...
-    run_tasks = util.collect_task(all_task, inter_type)
-    if len(run_tasks) == 0:
-        return
-    else:
-        run_tasks = [os.path.basename(ii) for ii in all_task]
-        machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
-        print('%d tasks will be submited '%len(run_tasks))
-        multiple_ret = []
-        for ii in range(len(work_path_list)):
-            work_path = work_path_list[ii]
-
-            ret = pool.apply_async(worker, (work_path,
-                                            run_tasks[ii],
-                                            forward_common_files,
-                                            forward_files,
-                                            backward_files,
-                                            mdata,
-                                            inter_type,
-                                            ))
-            multiple_ret.append(ret)
-        pool.close()
-        pool.join()
-        for ii in range(len(multiple_ret)):
-            if not multiple_ret[ii].successful():
-                raise RuntimeError("Task %d is not successful! work_path: %s " % (ii, work_path_list[ii]))
-        print('finished')
+    machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
+    work_path = os.getcwd()
+    print("%s --> Runing... " % (work_path))
+
+    api_version = mdata.get('api_version', '0.9')
+    if LooseVersion(api_version) < LooseVersion('1.0'):
+        warnings.warn(f"the dpdispatcher will be updated to new version."
+                      f"And the interface may be changed. Please check the documents for more details")
+        disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
+        disp.run_jobs(resources,
+                      command,
+                      work_path,
+                      run_tasks,
+                      group_size,
+                      forward_common_files,
+                      forward_files,
+                      backward_files,
+                      outlog='outlog',
+                      errlog='errlog')
+    elif LooseVersion(api_version) >= LooseVersion('1.0'):
+    
+        submission = make_submission(
+            mdata_machine=machine,
+            mdata_resources=resources,
+            commands=[command],
+            work_path=work_path,
+            run_tasks=run_tasks,
+            group_size=group_size,
+            forward_common_files=forward_common_files,
+            forward_files=forward_files,
+            backward_files=backward_files,
+            outlog='outlog',
+            errlog='errlog'
+        )
+        submission.run_submission()
+    
+    
 
 def post_equi(confs, inter_param):
     # find all POSCARs and their name like mp-xxx

dpgen/auto_test/common_prop.py

@@ -173,6 +173,7 @@ def run_property(confs,
     pool.join()
     for ii in range(len(multiple_ret)):
         if not multiple_ret[ii].successful():
+            print("ERROR:", multiple_ret[ii].get())
             raise RuntimeError("Job %d is not successful!" % ii)
     print('%d jobs are finished' % len(multiple_ret))
 
@@ -186,11 +187,11 @@ def worker(work_path,
            inter_type):
     run_tasks = [os.path.basename(ii) for ii in all_task]
     machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
-    disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
     api_version = mdata.get('api_version', '0.9')
     if LooseVersion(api_version) < LooseVersion('1.0'):
         warnings.warn(f"the dpdispatcher will be updated to new version."
             f"And the interface may be changed. Please check the documents for more details")
+        disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
         disp.run_jobs(resources,
                   command,
                   work_path,