* python/benchmarks/bench.py: Add support for setup= and check= arguments.

Author: MLopez-Ibanez

python/benchmarks/bench.py

@@ -78,22 +78,35 @@ def get_package_version(package):
         case "DEAP_er":
             # Requires version >= 0.2.0
             from deap_er import __version__ as version
-        case "desdeo":
+        case "desdeo" | "seqme":
             # It does not provide __version__ !
             from importlib.metadata import version as get_version
 
-            version = get_version("desdeo")
+            version = get_version(package)
         case _:
             raise ValueError(f"unknown package {package}")
 
     return version
 
 
+def check_float_values(a, b, what, n, name):
+    assert np.isclose(a, b), (
+        f"In {name}, maxrow={n}, {what}={b}  not equal to moocore={a}"
+    )
+
+
 class Bench:
     cpu_model = cpuinfo.get_cpu_info()["brand_raw"]
 
     def __init__(
-        self, name, n, bench, report_values=None, return_all_values=False
+        self,
+        name,
+        n,
+        bench,
+        setup=None,
+        check=None,
+        report_values=None,
+        return_all_values=False,
     ):
         self.name = name
         self.n = n
@@ -112,14 +125,20 @@ def __init__(
         else:
             self.values = None
             self.value_label = None
+        self.setup = setup
+        self.check = check
 
     def keys(self):
         return self.bench.keys()
 
-    def __call__(self, what, n, *args, **kwargs):
-        # FIXME: Ideally, bench() would call fun for each value in self.n
-        assert n in self.n
-        # FIXME: Allow passing a setup() function.
+    def bench1(self, what, n, *args, **kwargs):
+        if self.setup:
+            if isinstance(self.setup, dict):
+                setup = self.setup.get(what)
+                if setup:
+                    args = (setup(*args, **kwargs),)
+                    kwargs = {}
+
         fun = self.bench[what]
         duration, value = timeit.Timer(lambda: fun(*args, **kwargs)).timeit(
             number=3
@@ -130,6 +149,22 @@ def __call__(self, what, n, *args, **kwargs):
         print(f"{self.name}:{n}:{what}:{duration}")
         return value
 
+    def __call__(self, n, *args, **kwargs):
+        # FIXME: Ideally, bench() would call fun for each value in self.n
+        assert n in self.n
+        values = {
+            what: self.bench1(what, n, *args, **kwargs) for what in self.keys()
+        }
+        if self.check:
+            a = values["moocore"]
+            for what in self.keys():
+                if what == "moocore":
+                    continue
+                b = values[what]
+                self.check(a, b, what=what, n=n, name=self.name)
+
+        return values
+
     def plots(self, title, file_prefix, log="y", relative=False):
         for what in self.keys():
             self.times[what] = np.asarray(self.times[what])

python/benchmarks/bench_epsilon.py

@@ -5,7 +5,7 @@
 
 """
 
-from bench import Bench, read_data
+from bench import Bench, read_data, check_float_values
 
 import numpy as np
 import moocore
@@ -43,25 +43,12 @@
             "moocore": lambda z, ref=ref: moocore.epsilon_additive(z, ref=ref),
             "jMetalPy": lambda z, eps=jmetal_EPS(ref): eps.compute(z),
         },
+        check=check_float_values,
     )
 
-    values = {}
     for maxrow in n:
-        z = x[:maxrow, :]
-        for what in bench.keys():
-            values[what] = bench(what, maxrow, z)
+        values = bench(maxrow, x[:maxrow, :])
 
-        # Check values
-        for what in bench.keys():
-            if what == "moocore":
-                continue
-            a = values["moocore"]
-            b = values[what]
-            assert np.isclose(a, b), (
-                f"In {name}, maxrow={maxrow}, {what}={b}  not equal to moocore={a}"
-            )
-
-    del values
     bench.plots(file_prefix=file_prefix, title=title)
 
 if "__file__" not in globals():  # Running interactively.

python/benchmarks/bench_hv.py

@@ -47,6 +47,12 @@
 }
 
 
+def check_values(a, b, what, n, name):
+    assert np.isclose(a, b), (
+        f"In {name}, maxrow={n}, {what}={b}  not equal to moocore={a}"
+    )
+
+
 title = "HV computation"
 file_prefix = "hv"
 names = files.keys()
@@ -72,30 +78,20 @@
             ref_point=torch.from_numpy(-ref)
         ): hv.compute(z)
 
-    bench = Bench(name=name, n=n, bench=benchmarks)
-    values = {}
+    bench = Bench(
+        name=name,
+        n=n,
+        # Exclude the conversion to torch from the timing.
+        setup={"botorch": lambda z: torch.from_numpy(-z)},
+        # elif what == "trieste":
+        #     zz = tf.convert_to_tensor(z)
+        bench=benchmarks,
+        check=check_values,
+    )
+
     for maxrow in n:
-        z = x[:maxrow, :]
-        for what in bench.keys():
-            if what == "botorch":
-                zz = torch.from_numpy(-z)
-            # elif what == "trieste":
-            #     zz = tf.convert_to_tensor(z)
-            else:
-                zz = z
-            values[what] = bench(what, maxrow, zz)
-
-        # Check values
-        for what in bench.keys():
-            if what == "moocore":
-                continue
-            a = values["moocore"]
-            b = values[what]
-            assert np.isclose(a, b), (
-                f"In {name}, maxrow={maxrow}, {what}={b}  not equal to moocore={a}"
-            )
-
-    del values
+        values = bench(maxrow, x[:maxrow, :])
+
     bench.plots(file_prefix=file_prefix, title=title)
 
 if "__file__" not in globals():  # Running interactively.

python/benchmarks/bench_hvapprox.py

@@ -107,14 +107,11 @@ def time_hv_exact(name, maxrow, z, ref):
     bench = Bench(
         name=name, n=n, bench=benchmarks, report_values="HV Relative Error"
     )
-    res = {}
     for maxrow in n:
         z = x[:maxrow, :]
         exact = time_hv_exact(name, maxrow, z, ref)
-        for what in bench.keys():
-            res[what] = bench(what, maxrow, z, exact=exact)
+        res = bench(maxrow, z, exact=exact)
 
-    del res
     bench.plots(file_prefix=file_prefix, title=title)
 
 if "__file__" not in globals():  # Running interactively.

python/benchmarks/bench_igdplus.py

@@ -5,7 +5,7 @@
 
 """
 
-from bench import Bench, read_data
+from bench import Bench, read_data, check_float_values
 
 import numpy as np
 import moocore
@@ -47,25 +47,12 @@
             # FIXME: Currently DESDEO is a thousand times slower than moocore, so it is not worth running it.
             # "desdeo": lambda z, ref=ref: desdeo_igd_plus(z, reference_set=ref).igd_plus,
         },
+        check=check_float_values,
     )
 
-    values = {}
     for maxrow in n:
-        z = x[:maxrow, :]
-        for what in bench.keys():
-            values[what] = bench(what, maxrow, z)
+        values = bench(maxrow, x[:maxrow, :])
 
-        # Check values
-        for what in bench.keys():
-            if what == "moocore":
-                continue
-            a = values["moocore"]
-            b = values[what]
-            assert np.isclose(a, b), (
-                f"In {name}, maxrow={maxrow}, {what}={b}  not equal to moocore={a}"
-            )
-
-    del values
     bench.plots(file_prefix=file_prefix, title=title)
 
 if "__file__" not in globals():  # Running interactively.

python/benchmarks/bench_ndom.py

@@ -25,6 +25,8 @@
 
 from paretoset import paretoset
 
+from seqme.core.rank import is_pareto_front as seqme_is_pareto_front
+
 # See https://github.com/multi-objective/testsuite/tree/main/data
 files = {
     # range are the (start, stop, num) parameters for np.geomspace()
@@ -50,6 +52,18 @@ def get_dataset(name):
     return moocore.get_dataset(files[name]["file"])[:, :-1]
 
 
+# Exclude the conversion to torch from the timing.
+setup = {"botorch": lambda z: torch.from_numpy(z)}
+
+
+def check_values(a, b, what, n, name):
+    np.testing.assert_allclose(
+        a,
+        b,
+        err_msg=f"In {name}, maxrow={n}, {what}={b}  not equal to moocore={a}",
+    )
+
+
 title = "is_non_dominated(keep_weakly=False)"
 file_prefix = "ndom"
 names = files.keys()
@@ -60,6 +74,7 @@ def get_dataset(name):
     bench = Bench(
         name=name,
         n=n,
+        setup=setup,
         bench={
             "moocore": lambda z: moocore.is_nondominated(
                 z, maximise=True, keep_weakly=False
@@ -69,38 +84,17 @@ def get_dataset(name):
                 z, sense=z.shape[1] * ["max"], distinct=True, use_numba=True
             ),
         },
+        check=check_values,
     )
 
-    values = {}
     for maxrow in n:
-        z = x[:maxrow, :]
-        for what in bench.keys():
-            if what == "botorch":
-                # Exclude the conversion to torch from the timing.
-                zz = torch.from_numpy(z)
-            else:
-                zz = z
-            values[what] = bench(what, maxrow, zz)
-
-        # Check values
-        for what in bench.keys():
-            if what == "moocore":
-                continue
-            a = values["moocore"]
-            b = values[what]
-            np.testing.assert_array_equal(
-                a,
-                b,
-                err_msg=f"In {name}, maxrow={maxrow}, {what}={b}  not equal to moocore={a}",
-            )
-
-    del values
+        values = bench(maxrow, x[:maxrow, :])
+
     bench.plots(file_prefix=file_prefix, title=title, log="xy")
 
 
 title = "is_non_dominated(keep_weakly=True)"
 file_prefix = "wndom"
-
 names = files.keys()
 for name in names:
     x = get_dataset(name)
@@ -109,6 +103,7 @@ def get_dataset(name):
     bench = Bench(
         name=name,
         n=n,
+        setup=setup,
         bench={
             "moocore": lambda z: bool2pos(
                 moocore.is_nondominated(z, maximise=True, keep_weakly=True)
@@ -128,33 +123,16 @@ def get_dataset(name):
                 -z, only_non_dominated_front=True
             ),
             "desdeo": lambda z: bool2pos(desdeo_is_nondominated(-z)),
+            "seqme": lambda z: bool2pos(
+                seqme_is_pareto_front(-z, assume_unique_lexsorted=True)
+            ),
         },
+        check=check_values,
     )
 
-    values = {}
     for maxrow in n:
-        z = x[:maxrow, :]
-        for what in bench.keys():
-            if what == "botorch":
-                # Exclude the conversion to torch from the timing.
-                zz = torch.from_numpy(z)
-            else:
-                zz = z
-            values[what] = bench(what, maxrow, zz)
-
-        # Check values
-        for what in bench.keys():
-            if what == "moocore":
-                continue
-            a = values["moocore"]
-            b = values[what]
-            np.testing.assert_allclose(
-                a,
-                b,
-                err_msg=f"In {name}, maxrow={maxrow}, {what}={b}  not equal to moocore={a}",
-            )
-
-    del values
+        values = bench(maxrow, x[:maxrow, :])
+
     bench.plots(file_prefix=file_prefix, title=title, log="xy")
 
 

python/benchmarks/bench_ndsort.py

@@ -57,27 +57,16 @@ def get_dataset(name):
             "pymoo": lambda z, nds=pymoo_NDS(): nds.do(z, return_rank=True)[1],
             "desdeo": lambda z: desdeo_nds(z).argmax(axis=0),
         },
+        check=lambda a, b, what, n, name: np.testing.assert_array_equal(
+            a,
+            b,
+            err_msg=f"In {name}, maxrow={n}, {what}={b}  not equal to moocore={a}",
+        ),
     )
 
-    values = {}
     for maxrow in n:
-        z = x[:maxrow, :]
-        for what in bench.keys():
-            values[what] = bench(what, maxrow, z)
-
-        # Check values
-        for what in bench.keys():
-            if what == "moocore":
-                continue
-            a = values["moocore"]
-            b = values[what]
-            np.testing.assert_array_equal(
-                a,
-                b,
-                err_msg=f"In {name}, maxrow={maxrow}, {what}={b}  not equal to moocore={a}",
-            )
-
-    del values
+        values = bench(maxrow, x[:maxrow, :])
+
     bench.plots(file_prefix=file_prefix, title=title, log="xy")
 
 if "__file__" not in globals():  # Running interactively.