Add faster list solver to benchmarks

- Better results vs pyvsc.
- Still include the old one for reference as it's slower now.

Author: will-keen

benchmarks/__main__.py

@@ -16,7 +16,8 @@
 from benchmarks.pyvsc.in_keyword import vsc_in, cr_in, cr_in_order
 from benchmarks.pyvsc.ldinstr import vsc_ldinstr
 from benchmarks.pyvsc.randlist import vscRandListSumZero, \
-    crRandListSumZero, vscRandListUnique, crRandListUnique
+    crRandListSumZero, vscRandListUnique, crRandListUnique, \
+    crRandListSumZeroFaster, crRandListUniqueFaster
 from examples.ldinstr import ldInstr
 
 
@@ -122,13 +123,17 @@ def test_randlist_sumzero(self):
         randobjs = {
             'vsc': vscRandListSumZero(),
             'cr': crRandListSumZero(Random(0)),
+            'cr_faster': crRandListSumZeroFaster(Random(0)),
         }
         def check(results):
             self.assertGreater(results['cr'][1], results['vsc'][1])
+            self.assertGreater(results['cr_faster'][1], results['vsc'][1])
             # This testcase is typically 20x faster, which may vary depending
             # on machine. Ensure it doesn't fall below 15x.
             speedup = results['cr'][1] / results['vsc'][1]
             self.assertGreater(speedup, 15, "Performance has degraded!")
+            speedup = results['cr_faster'][1] / results['vsc'][1]
+            self.assertGreater(speedup, 15, "Performance has degraded!")
         self.run_benchmark(randobjs, 100, check)
 
     def test_randlist_unique(self):
@@ -138,12 +143,19 @@ def test_randlist_unique(self):
         randobjs = {
             'vsc': vscRandListUnique(),
             'cr': crRandListUnique(Random(0)),
+            'cr_faster': crRandListUniqueFaster(Random(0)),
         }
         def check(results):
+            self.assertGreater(results['cr_faster'][1], results['vsc'][1])
             self.assertGreater(results['cr'][1], results['vsc'][1])
+            # With the naive solver, this testcase is typically 3-4x faster,
+            # which may vary depending on machine. Ensure it doesn't fall
+            # below 2x.
+            speedup = results['cr'][1] / results['vsc'][1]
+            self.assertGreater(speedup, 2, "Performance has degraded!")
             # This testcase is typically 10-13x faster, which may vary depending
             # on machine. Ensure it doesn't fall below 10x.
-            speedup = results['cr'][1] / results['vsc'][1]
+            speedup = results['cr_faster'][1] / results['vsc'][1]
             self.assertGreater(speedup, 10, "Performance has degraded!")
         self.run_benchmark(randobjs, 100, check)
 

benchmarks/pyvsc/randlist.py

@@ -37,6 +37,16 @@ def sum_0(listvar):
         self.add_constraint(sum_0, ('listvar',))
 
 
+class crRandListSumZeroFaster(RandObj):
+
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.add_rand_var('listvar', domain=range(-10, 11), length=10, disable_naive_list_solver=True)
+        def sum_0(listvar):
+            return sum(listvar) == 0
+        self.add_constraint(sum_0, ('listvar',))
+
+
 @vsc.randobj
 class vscRandListUnique(object):
 
@@ -59,3 +69,15 @@ class crRandListUnique(RandObj):
     def __init__(self, *args):
         super().__init__(*args)
         self.add_rand_var('listvar', domain=range(10), length=10, list_constraints=[unique])
+
+
+class crRandListUniqueFaster(RandObj):
+
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.add_rand_var('listvar',
+            domain=range(10),
+            length=10,
+            list_constraints=[unique],
+            disable_naive_list_solver=True,
+        )