Merge remote-tracking branch 'upstream/devel' into devel

Author: Han Wang

doc/conf.py

@@ -38,6 +38,7 @@
 # ]
 
 extensions = [
+    'deepmodeling_sphinx',
     "sphinx_rtd_theme",
     'myst_parser',
     'sphinx.ext.autosummary',

doc/requirements.txt

@@ -3,3 +3,4 @@ recommonmark==0.7.1
 sphinx_rtd_theme==0.5.2
 sphinx_markdown_tables==0.0.15
 myst-parser==0.13.7
+deepmodeling_sphinx

dpgen/auto_test/common_equi.py

@@ -114,46 +114,6 @@ def make_equi(confs,
         inter.make_input_file(ii, 'relaxation', relax_param)
 
 
-def worker(work_path,
-           run_task,
-           forward_common_files,
-           forward_files,
-           backward_files,
-           mdata,
-           inter_type):
-    machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
-    disp = make_dispatcher(machine, resources, work_path, [run_task], group_size)
-    print("%s --> Runing... " % (work_path))
-
-    api_version = mdata.get('api_version', '0.9')
-    if LooseVersion(api_version) < LooseVersion('1.0'):
-        warnings.warn(f"the dpdispatcher will be updated to new version."
-                      f"And the interface may be changed. Please check the documents for more details")
-        disp.run_jobs(resources,
-                      command,
-                      work_path,
-                      [run_task],
-                      group_size,
-                      forward_common_files,
-                      forward_files,
-                      backward_files,
-                      outlog='outlog',
-                      errlog='errlog')
-    elif LooseVersion(api_version) >= LooseVersion('1.0'):
-        submission = make_submission(
-            mdata_machine=machine,
-            mdata_resources=resources,
-            commands=[command],
-            work_path=work_path,
-            run_tasks=run_task,
-            group_size=group_size,
-            forward_common_files=forward_common_files,
-            forward_files=forward_files,
-            backward_files=backward_files,
-            outlog='outlog',
-            errlog='errlog'
-        )
-        submission.run_submission()
 
 def run_equi(confs,
              inter_param,
@@ -166,18 +126,16 @@ def run_equi(confs,
     conf_dirs.sort()
 
     processes = len(conf_dirs)
-    pool = Pool(processes=processes)
-    print("Submit job via %d processes" % processes)
 
     # generate a list of task names like mp-xxx/relaxation/relax_task
     # ...
     work_path_list = []
     for ii in conf_dirs:
-        work_path_list.append(os.path.abspath(os.path.join(ii, 'relaxation')))
+        work_path_list.append(os.path.join(ii, 'relaxation'))
     all_task = []
     for ii in work_path_list:
         all_task.append(os.path.join(ii, 'relax_task'))
-
+    run_tasks = all_task
     inter_type = inter_param['type']
     # vasp
     if inter_type == "vasp":
@@ -186,41 +144,51 @@ def run_equi(confs,
         mdata = convert_mdata(mdata, ["model_devi"])
     else:
         raise RuntimeError("unknown task %s, something wrong" % inter_type)
-
+    
     # dispatch the tasks
     # POSCAR here is useless
     virtual_calculator = make_calculator(inter_param, "POSCAR")
     forward_files = virtual_calculator.forward_files()
     forward_common_files = virtual_calculator.forward_common_files()
     backward_files = virtual_calculator.backward_files()
     #    backward_files += logs
-    # ...
-    run_tasks = util.collect_task(all_task, inter_type)
-    if len(run_tasks) == 0:
-        return
-    else:
-        run_tasks = [os.path.basename(ii) for ii in all_task]
-        machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
-        print('%d tasks will be submited '%len(run_tasks))
-        multiple_ret = []
-        for ii in range(len(work_path_list)):
-            work_path = work_path_list[ii]
-
-            ret = pool.apply_async(worker, (work_path,
-                                            run_tasks[ii],
-                                            forward_common_files,
-                                            forward_files,
-                                            backward_files,
-                                            mdata,
-                                            inter_type,
-                                            ))
-            multiple_ret.append(ret)
-        pool.close()
-        pool.join()
-        for ii in range(len(multiple_ret)):
-            if not multiple_ret[ii].successful():
-                raise RuntimeError("Task %d is not successful! work_path: %s " % (ii, work_path_list[ii]))
-        print('finished')
+    machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
+    work_path = os.getcwd()
+    print("%s --> Runing... " % (work_path))
+
+    api_version = mdata.get('api_version', '0.9')
+    if LooseVersion(api_version) < LooseVersion('1.0'):
+        warnings.warn(f"the dpdispatcher will be updated to new version."
+                      f"And the interface may be changed. Please check the documents for more details")
+        disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
+        disp.run_jobs(resources,
+                      command,
+                      work_path,
+                      run_tasks,
+                      group_size,
+                      forward_common_files,
+                      forward_files,
+                      backward_files,
+                      outlog='outlog',
+                      errlog='errlog')
+    elif LooseVersion(api_version) >= LooseVersion('1.0'):
+    
+        submission = make_submission(
+            mdata_machine=machine,
+            mdata_resources=resources,
+            commands=[command],
+            work_path=work_path,
+            run_tasks=run_tasks,
+            group_size=group_size,
+            forward_common_files=forward_common_files,
+            forward_files=forward_files,
+            backward_files=backward_files,
+            outlog='outlog',
+            errlog='errlog'
+        )
+        submission.run_submission()
+    
+    
 
 def post_equi(confs, inter_param):
     # find all POSCARs and their name like mp-xxx

dpgen/auto_test/common_prop.py

@@ -173,6 +173,7 @@ def run_property(confs,
     pool.join()
     for ii in range(len(multiple_ret)):
         if not multiple_ret[ii].successful():
+            print("ERROR:", multiple_ret[ii].get())
             raise RuntimeError("Job %d is not successful!" % ii)
     print('%d jobs are finished' % len(multiple_ret))
 
@@ -186,11 +187,11 @@ def worker(work_path,
            inter_type):
     run_tasks = [os.path.basename(ii) for ii in all_task]
     machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
-    disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
     api_version = mdata.get('api_version', '0.9')
     if LooseVersion(api_version) < LooseVersion('1.0'):
         warnings.warn(f"the dpdispatcher will be updated to new version."
             f"And the interface may be changed. Please check the documents for more details")
+        disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
         disp.run_jobs(resources,
                   command,
                   work_path,

dpgen/generator/run.py

@@ -394,6 +394,8 @@ def make_train (iter_index,
             else:
                 jinput['model']['descriptor']['seed'] = random.randrange(sys.maxsize) % (2**32)
             jinput['model']['fitting_net']['seed'] = random.randrange(sys.maxsize) % (2**32)
+            if 'type_embedding' in jinput['model']:
+                jinput['model']['type_embedding']['seed'] = random.randrange(sys.maxsize) % (2**32)
             jinput['training']['seed'] = random.randrange(sys.maxsize) % (2**32)
         else:
             raise RuntimeError("DP-GEN currently only supports for DeePMD-kit 1.x or 2.x version!" )

dpgen/simplify/simplify.py

@@ -22,7 +22,8 @@
 from dpgen import dlog
 from dpgen import SHORT_CMD
 from dpgen.util import sepline
-from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher
+from distutils.version import LooseVersion
+from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher, make_submission
 from dpgen.generator.run import make_train, run_train, post_train, run_fp, post_fp, fp_name, model_devi_name, train_name, train_task_fmt, sys_link_fp_vasp_pp, make_fp_vasp_incar, make_fp_vasp_kp, make_fp_vasp_cp_cvasp, data_system_fmt, model_devi_task_fmt, fp_task_fmt
 # TODO: maybe the following functions can be moved to dpgen.util
 from dpgen.generator.lib.utils import log_iter, make_iter_name, create_path, record_iter
@@ -245,7 +246,7 @@ def run_model_devi(iter_index, jdata, mdata):
     commands = []
     detail_file_names = []
     for ii, mm in enumerate(task_model_list):
-        detail_file_name = "{prefix}.{ii}".format(
+        detail_file_name = "{prefix}-{ii}".format(
             prefix=detail_file_name_prefix,
             ii=ii,
         )
@@ -268,17 +269,36 @@ def run_model_devi(iter_index, jdata, mdata):
     forward_files = [rest_data_name]
     backward_files = sum([[pf+".e.out", pf+".f.out", pf+".v.out"] for pf in detail_file_names], [])
 
-    dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size)
-    dispatcher.run_jobs(mdata['model_devi_resources'],
+    api_version = mdata.get('api_version', '0.9')
+    if LooseVersion(api_version) < LooseVersion('1.0'):
+        warnings.warn(f"the dpdispatcher will be updated to new version."
+            f"And the interface may be changed. Please check the documents for more details")
+        dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size)
+        dispatcher.run_jobs(mdata['model_devi_resources'],
                         commands,
                         work_path,
                         run_tasks,
                         model_devi_group_size,
                         model_names,
                         forward_files,
                         backward_files,
-                        outlog='model_devi.log',
-                        errlog='model_devi.log')
+                        outlog = 'model_devi.log',
+                        errlog = 'model_devi.log')
+
+    elif LooseVersion(api_version) >= LooseVersion('1.0'):
+        submission = make_submission(
+            mdata['model_devi_machine'],
+            mdata['model_devi_resources'],
+            commands=commands,
+            work_path=work_path,
+            run_tasks=run_tasks,
+            group_size=model_devi_group_size,
+            forward_common_files=model_names,
+            forward_files=forward_files,
+            backward_files=backward_files,
+            outlog = 'model_devi.log',
+            errlog = 'model_devi.log')
+        submission.run_submission()
 
 
 def post_model_devi(iter_index, jdata, mdata):
@@ -309,13 +329,13 @@ def post_model_devi(iter_index, jdata, mdata):
             sys_name = os.path.basename(task).split('.')[1]
             all_names.add(sys_name)
         # e.out
-        details_e = glob.glob(os.path.join(task, "{}.*.e.out".format(detail_file_name_prefix)))
+        details_e = glob.glob(os.path.join(task, "{}-*.e.out".format(detail_file_name_prefix)))
         e_all = np.array([np.loadtxt(detail_e, ndmin=2)[:, 1] for detail_e in details_e])
         e_std = np.std(e_all, axis=0)
         n_frame = e_std.size
 
         # f.out
-        details_f = glob.glob(os.path.join(task, "{}.*.f.out".format(detail_file_name_prefix)))
+        details_f = glob.glob(os.path.join(task, "{}-*.f.out".format(detail_file_name_prefix)))
         f_all = np.array([np.loadtxt(detail_f, ndmin=2)[:, 3:6].reshape((n_frame, -1, 3)) for detail_f in details_f])
         # (n_model, n_frame, n_atom, 3)
         f_std = np.std(f_all, axis=0)