Enable Ruff E, W, F, RUF, NPY, and DTZ rules with comprehensive error fixes

.pre-commit-config.yaml

@@ -17,8 +17,10 @@ repos:
 # Python
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.1.3
+    rev: v0.12.12
     hooks:
+    - id: ruff
+      args: [--fix]
     - id: ruff-format
 -   repo: https://github.com/PyCQA/isort
     rev: 5.12.0

dpgen2/__init__.py

@@ -2,5 +2,5 @@
     from ._version import version as __version__
 except ImportError:
     from .__about__ import (
-        __version__,
+        __version__ as __version__,
     )

dpgen2/conf/__init__.py

@@ -2,7 +2,7 @@
     AlloyConfGenerator,
 )
 from .conf_generator import (
-    ConfGenerator,
+    ConfGenerator as ConfGenerator,
 )
 from .file_conf import (
     FileConfGenerator,

dpgen2/conf/alloy_conf.py

@@ -5,7 +5,6 @@
 )
 from typing import (
     List,
-    Optional,
     Tuple,
     Union,
 )
@@ -14,7 +13,6 @@
 import numpy as np
 from dargs import (
     Argument,
-    Variant,
 )
 
 from .conf_generator import (

dpgen2/conf/file_conf.py

@@ -1,19 +1,16 @@
 import glob
-import os
 from pathlib import (
     Path,
 )
 from typing import (
     List,
     Optional,
-    Tuple,
     Union,
 )
 
 import dpdata
 from dargs import (
     Argument,
-    Variant,
 )
 
 from .conf_generator import (

dpgen2/entrypoint/args.py

@@ -9,13 +9,9 @@
     Variant,
 )
 
-import dpgen2
 from dpgen2.conf import (
     conf_styles,
 )
-from dpgen2.constants import (
-    default_image,
-)
 from dpgen2.exploration.report import (
     conv_styles,
 )
@@ -83,7 +79,6 @@ def dp_dist_train_args():
 
 def dp_train_args():
     doc_numb_models = "Number of models trained for evaluating the model deviation"
-    doc_config = "Configuration of training"
     doc_template_script = "File names of the template training script. It can be a `List[str]`, the length of which is the same as `numb_models`. Each template script in the list is used to train a model. Can be a `str`, the models share the same template training script. "
     doc_init_models_paths = "the paths to initial models"
     doc_init_models_uri = "The URI of initial models"

dpgen2/entrypoint/common.py

@@ -3,24 +3,16 @@
     Path,
 )
 from typing import (
-    Dict,
     List,
-    Optional,
     Union,
 )
 
 import dflow
 
 from dpgen2.utils import (
     bohrium_config_from_dict,
-    dump_object_to_file,
-    load_object_from_file,
-    matched_step_key,
-    print_keys_in_nice_format,
-    sort_slice_ops,
     workflow_config_from_dict,
 )
-from dpgen2.utils.step_config import normalize as normalize_step_dict
 
 
 def global_config_workflow(

dpgen2/entrypoint/download.py

@@ -3,7 +3,6 @@
     Dict,
     List,
     Optional,
-    Union,
 )
 
 from dflow import (
@@ -14,9 +13,6 @@
 from dpgen2.entrypoint.common import (
     global_config_workflow,
 )
-from dpgen2.utils.dflow_query import (
-    matched_step_key,
-)
 from dpgen2.utils.download_dpgen2_artifacts import (
     download_dpgen2_artifacts,
     download_dpgen2_artifacts_by_def,

dpgen2/entrypoint/main.py

@@ -1,21 +1,12 @@
 import argparse
 import json
 import logging
-import os
 import textwrap
 from typing import (
     List,
     Optional,
 )
 
-import dflow
-from dflow import (
-    Step,
-    Steps,
-    Workflow,
-    download_artifact,
-    upload_artifact,
-)
 
 from dpgen2 import (
     __version__,
@@ -41,11 +32,8 @@
     status,
 )
 from .submit import (
-    make_concurrent_learning_op,
-    make_naive_exploration_scheduler,
     resubmit_concurrent_learning,
     submit_concurrent_learning,
-    workflow_concurrent_learning,
 )
 from .watch import (
     default_watching_keys,
@@ -327,7 +315,7 @@ def main():
     logging.basicConfig(level=logging.INFO)
 
     args = parse_args()
-    dict_args = vars(args)
+    vars(args)
 
     if args.command == "submit":
         with open(args.CONFIG) as fp:

dpgen2/entrypoint/showkey.py

@@ -1,11 +1,3 @@
-import glob
-import os
-import pickle
-from pathlib import (
-    Path,
-)
-
-import dpdata
 from dflow import (
     Workflow,
 )
@@ -20,6 +12,8 @@
 from dpgen2.utils import (
     print_keys_in_nice_format,
 )
+import functools
+import operator
 
 
 def showkey(
@@ -32,7 +26,7 @@ def showkey(
 
     wf = Workflow(id=wf_id)
     folded_keys = get_resubmit_keys(wf)
-    all_step_keys = sum(folded_keys.values(), [])
+    all_step_keys = functools.reduce(operator.iadd, folded_keys.values(), [])
     prt_str = print_keys_in_nice_format(
         all_step_keys,
         ["run-train", "run-lmp", "run-fp", "diffcsp-gen", "run-relax"],

dpgen2/entrypoint/status.py

@@ -1,9 +1,7 @@
 import logging
 from typing import (
     Dict,
-    List,
     Optional,
-    Union,
 )
 
 from dflow import (
@@ -15,7 +13,6 @@
     global_config_workflow,
 )
 from dpgen2.utils.dflow_query import (
-    get_all_schedulers,
     get_last_scheduler,
 )
 

dpgen2/entrypoint/submit.py

@@ -1,10 +1,7 @@
 import copy
-import glob
 import json
 import logging
 import os
-import pickle
-import re
 from copy import (
     deepcopy,
 )
@@ -15,44 +12,20 @@
     Dict,
     List,
     Optional,
-    Tuple,
-    Type,
-    Union,
 )
 
-import dpdata
 from dflow import (
     ArgoStep,
-    InputArtifact,
-    InputParameter,
-    Inputs,
-    OutputArtifact,
-    OutputParameter,
-    Outputs,
     S3Artifact,
     Step,
-    Steps,
     Workflow,
-    argo_range,
-    download_artifact,
     upload_artifact,
 )
-from dflow.python import (
-    OP,
-    OPIO,
-    Artifact,
-    FatalError,
-    OPIOSign,
-    PythonOPTemplate,
-    TransientError,
-    upload_packages,
-)
 
 from dpgen2.conf import (
     conf_styles,
 )
 from dpgen2.constants import (
-    default_host,
     default_image,
 )
 from dpgen2.entrypoint.args import normalize as normalize_args
@@ -65,7 +38,6 @@
     TrajRenderLammps,
 )
 from dpgen2.exploration.report import (
-    ExplorationReportTrustLevelsRandom,
     conv_styles,
 )
 from dpgen2.exploration.scheduler import (
@@ -78,11 +50,7 @@
     conf_filter_styles,
 )
 from dpgen2.exploration.task import (
-    CustomizedLmpTemplateTaskGroup,
     ExplorationStage,
-    ExplorationTask,
-    LmpTemplateTaskGroup,
-    NPTTaskGroup,
     caly_normalize,
     diffcsp_normalize,
     make_calypso_task_group_from_config,
@@ -131,18 +99,16 @@
 )
 from dpgen2.utils import (
     BinaryFileInput,
-    bohrium_config_from_dict,
-    dump_object_to_file,
     get_artifact_from_uri,
     get_subkey,
-    load_object_from_file,
     matched_step_key,
     print_keys_in_nice_format,
     sort_slice_ops,
     upload_artifact_and_print_uri,
-    workflow_config_from_dict,
 )
 from dpgen2.utils.step_config import normalize as normalize_step_dict
+import functools
+import operator
 
 default_config = normalize_step_dict(
     {
@@ -465,14 +431,14 @@ def get_systems_from_data(data, data_prefix=None):
     assert isinstance(data, list)
     if data_prefix is not None:
         data = [os.path.join(data_prefix, ii) for ii in data]
-    data = sum([expand_sys_str(ii) for ii in data], [])
+    data = functools.reduce(operator.iadd, [expand_sys_str(ii) for ii in data], [])
     return data
 
 
 def workflow_concurrent_learning(
     config: Dict,
 ) -> Step:
-    default_config = config["default_step_config"]
+    config["default_step_config"]
 
     train_config = config["train"]["config"]
     explore_config = config["explore"]["config"]
@@ -571,9 +537,9 @@ def workflow_concurrent_learning(
         "teacher_model_path" in explore_config
         and explore_config["teacher_model_path"] is not None
     ):
-        assert os.path.exists(
-            explore_config["teacher_model_path"]
-        ), f"No such file: {explore_config['teacher_model_path']}"
+        assert os.path.exists(explore_config["teacher_model_path"]), (
+            f"No such file: {explore_config['teacher_model_path']}"
+        )
         explore_config["teacher_model_path"] = BinaryFileInput(
             explore_config["teacher_model_path"]
         )
@@ -586,12 +552,12 @@ def workflow_concurrent_learning(
     fp_config["run"] = config["fp"]["run_config"]
     fp_config["extra_output_files"] = config["fp"]["extra_output_files"]
     if fp_style == "deepmd":
-        assert (
-            "teacher_model_path" in fp_config["run"]
-        ), f"Cannot find 'teacher_model_path' in config['fp']['run_config'] when fp_style == 'deepmd'"
-        assert os.path.exists(
-            fp_config["run"]["teacher_model_path"]
-        ), f"No such file: {fp_config['run']['teacher_model_path']}"
+        assert "teacher_model_path" in fp_config["run"], (
+            "Cannot find 'teacher_model_path' in config['fp']['run_config'] when fp_style == 'deepmd'"
+        )
+        assert os.path.exists(fp_config["run"]["teacher_model_path"]), (
+            f"No such file: {fp_config['run']['teacher_model_path']}"
+        )
         fp_config["run"]["teacher_model_path"] = BinaryFileInput(
             fp_config["run"]["teacher_model_path"]
         )
@@ -677,13 +643,13 @@ def get_scheduler_ids(
         if get_subkey(ii.key, 1) == "scheduler":
             scheduler_ids.append(idx)
     scheduler_keys = [reuse_step[ii].key for ii in scheduler_ids]
-    assert (
-        sorted(scheduler_keys) == scheduler_keys
-    ), "The scheduler keys are not properly sorted"
+    assert sorted(scheduler_keys) == scheduler_keys, (
+        "The scheduler keys are not properly sorted"
+    )
 
     if len(scheduler_ids) == 0:
         logging.warning(
-            "No scheduler found in the workflow, " "does not do any replacement."
+            "No scheduler found in the workflow, does not do any replacement."
         )
     return scheduler_ids
 
@@ -891,7 +857,7 @@ def resubmit_concurrent_learning(
             # reuse the super OP iif all steps within it are reused
             if set(v) == set(folded_keys[k]):
                 reused_folded_keys[k] = [k]
-        reused_keys = sum(reused_folded_keys.values(), [])
+        reused_keys = functools.reduce(operator.iadd, reused_folded_keys.values(), [])
     reuse_step = old_wf.query_step(key=reused_keys, sort_by_generation=True)
 
     wf = submit_concurrent_learning(