Streamline plot making

Author: staadecker

benchmarks/.gitignore

@@ -1,3 +1,2 @@
 */data/**
-*/results/*.tsv
-*.csv
+*/results/*

benchmarks/Snakefile

@@ -1,50 +1,58 @@
+import importlib
+import os
+from snakemake.utils import available_cpu_count
+
 configfile: "config.yaml"
 
-def generate_all_runs():
+def generate_all_runs(problem):
     import itertools
 
-    problem_size_pairs = []
-    for problem, problem_data in config["problems"].items():
-        if problem_data is None or "size" not in problem_data:
-            problem_size_pairs.append((problem, "0"))
-        else:
-            for size in problem_data["size"]:
-                problem_size_pairs.append((problem, size))
+    problem_data = config["problems"][problem]
+    if problem_data is None or "size" not in problem_data:
+        sizes = ["0"]
+    else:
+        sizes = problem_data["size"]
 
     runs = [
-        (problem, size, library, solver)
-        for (problem, size), library, solver in itertools.product(
-            problem_size_pairs,
+        (size, library, solver)
+        for size, library, solver in itertools.product(
+            sizes,
             config["libraries"],
             config["solvers"]
         )
     ]
+
+    benchmarks = [
+        importlib.import_module(f"{problem}.bm_{library.lower()}").Bench
+        for size, library, solver in runs
+    ]
+
+    runs = [
+        r for r, benchmark in zip(
+            runs, benchmarks
+        ) if benchmark.MAX_SIZE == None or r[0] <= benchmark.MAX_SIZE
+    ]
+
     return runs
 
 rule all:
     input:
-        expand("{problem}/results/{solver}.png", problem=config["problems"], solver=config["solvers"]),
-        "results.csv"
+        expand("{problem}/results/benchmark_results.png", problem=config["problems"])
 
 rule plot_results:
     input:
-        "results.csv"
+        lambda wildcards: [
+            f"{wildcards.problem}/results/{library}_{solver}_{size}.tsv"
+            for size, library, solver in generate_all_runs(wildcards.problem)
+        ]
     output:
-        mem_plot="{problem}/results/memory_{solver}.png",
-        time_plot="{problem}/results/time_{solver}.png"
+        "{problem}/results/benchmark_results.csv",
+        "{problem}/results/benchmark_results.png"
     script:
         "plot_results.py"
 
-rule collect_benchmarks:
-    input:
-        [f"{problem}/results/{library}_{solver}_{size}.tsv"
-         for problem, size, library, solver in generate_all_runs()]
-    output:
-        "results.csv"
-    script:
-        "collect_benchmarks.py"
-
 rule run_benchmark:
+    threads: available_cpu_count()  # Force to use all cores to avoid parallel execution
     benchmark:
         repeat("{problem}/results/{library}_{solver}_{size}.tsv", config["repeat"])
     shell:

benchmarks/collect_benchmarks.py

@@ -1,11 +1,12 @@
-repeat: 1
+repeat: 1  # TODO repeat
 problems:
-  facility_problem:
+  facility_location:
     size:
       - 25
       - 50
       - 75
       - 100
+      - 150
   # energy_problem:
 
 solvers:
@@ -15,7 +16,7 @@ solvers:
 
 libraries:
   - pyoframe
-  # - pyomo
+  - pyomo
   # - cvxpy
   - gurobipy
   # - jump

benchmarks/config.yaml

@@ -2,13 +2,14 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "id": "6211c84b",
    "metadata": {},
    "outputs": [],
    "source": [
     "import polars as pl\n",
-    "from util import mock_snakemake\n",
+    "\n",
+    "from benchmarks.util import mock_snakemake\n",
     "\n",
     "if \"snakemake\" not in globals() or hasattr(snakemake, \"mock\"):  # noqa: F821\n",
     "    snakemake = mock_snakemake(\"compute_capacity_factors\")\n",

benchmarks/energy_planning/scripts/compute_capacity_factors.py.ipynb

@@ -8,7 +8,8 @@
    "outputs": [],
    "source": [
     "import polars as pl\n",
-    "from util import mock_snakemake\n",
+    "\n",
+    "from benchmarks.util import mock_snakemake\n",
     "\n",
     "if \"snakemake\" not in globals() or hasattr(snakemake, \"mock\"):  # noqa: F821\n",
     "    snakemake = mock_snakemake(\"process_generator_data\")"

benchmarks/energy_planning/scripts/process_generator_data.py.ipynb

@@ -11,7 +11,8 @@
     "\n",
     "import pandas as pd\n",
     "import polars as pl\n",
-    "from util import mock_snakemake\n",
+    "\n",
+    "from benchmarks.util import mock_snakemake\n",
     "\n",
     "if \"snakemake\" not in globals() or hasattr(snakemake, \"mock\"):  # noqa: F821\n",
     "    snakemake = mock_snakemake(\"process_line_data\")"

benchmarks/energy_planning/scripts/process_lines_json.py.ipynb

@@ -2,13 +2,14 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "372bc740",
    "metadata": {},
    "outputs": [],
    "source": [
     "import polars as pl\n",
-    "from util import mock_snakemake\n",
+    "\n",
+    "from benchmarks.util import mock_snakemake\n",
     "\n",
     "DEBUG = False\n",
     "\n",

benchmarks/energy_planning/scripts/process_load_data.py.ipynb

@@ -10,7 +10,8 @@
 """
 
 import polars as pl
-from util import mock_snakemake
+
+from benchmarks.util import mock_snakemake
 
 f_bus = pl.col("from_bus")
 t_bus = pl.col("to_bus")

benchmarks/energy_planning/scripts/simplify_network.py

@@ -1,99 +1,6 @@
-"""
-`mock_snakemake` is inspired from PyPSA-Eur (MIT license, see https://github.com/PyPSA/pypsa-eur/blob/master/scripts/_helpers.py#L476)
-"""
-
 from __future__ import annotations
 
-from pathlib import Path
-
-
-def mock_snakemake(rulename, **wildcards):
-    """
-    This function is expected to be executed from the 'scripts'-directory of '
-    the snakemake project. It returns a snakemake.script.Snakemake object,
-    based on the Snakefile.
-
-    If a rule has wildcards, you have to specify them in **wildcards.
-
-    **wildcards:
-        keyword arguments fixing the wildcards. Only necessary if wildcards are
-        needed.
-
-    Parameters
-    ----------
-    rulename: str
-        name of the rule for which the snakemake object should be generated
-    """
-    import os
-
-    import snakemake as sm
-    from snakemake.api import Workflow
-    from snakemake.common import SNAKEFILE_CHOICES
-    from snakemake.script import Snakemake
-    from snakemake.settings.types import (
-        ConfigSettings,
-        DAGSettings,
-        ResourceSettings,
-        StorageSettings,
-        WorkflowSettings,
-    )
-
-    script_dir = Path(__file__).parent.resolve()
-    root_dir = script_dir.parent.parent
-    current_dir = os.getcwd()
-    os.chdir(root_dir)
-
-    try:
-        for p in SNAKEFILE_CHOICES:
-            p = root_dir / p
-            if os.path.exists(p):
-                snakefile = p
-                break
-        else:
-            raise FileNotFoundError(
-                f"Could not find a Snakefile in {root_dir} or its subdirectories."
-            )
-        workflow = Workflow(
-            ConfigSettings(),
-            ResourceSettings(),
-            WorkflowSettings(),
-            StorageSettings(),
-            DAGSettings(rerun_triggers=[]),
-            storage_provider_settings=dict(),
-        )
-        workflow.include(snakefile)
-        workflow.global_resources = {}
-        rule = workflow.get_rule(rulename)
-        dag = sm.dag.DAG(workflow, rules=[rule])
-        job = sm.jobs.Job(rule, dag, wildcards)
-
-        def make_accessable(*ios):
-            for io in ios:
-                for i, _ in enumerate(io):
-                    io[i] = os.path.abspath(io[i])
-
-        make_accessable(job.input, job.output, job.log)
-        snakemake = Snakemake(
-            job.input,
-            job.output,
-            job.params,
-            job.wildcards,
-            job.threads,
-            job.resources,
-            job.log,
-            job.dag.workflow.config,
-            job.rule.name,
-            None,
-        )
-        # create log and output dir if not existent
-        for path in list(snakemake.log) + list(snakemake.output):
-            Path(path).parent.mkdir(parents=True, exist_ok=True)
-    finally:
-        os.chdir(current_dir)
-
-    snakemake.mock = True
-    return snakemake
-
+from benchmarks.util import mock_snakemake
 
 if __name__ == "__main__":
     # Example usage