txgraph: check that DoWork finds optimal if given high budget (tests)

Author: sipa

src/test/fuzz/cluster_linearize.cpp

@@ -1167,24 +1167,9 @@ FUZZ_TARGET(clusterlin_linearize)
     }
 
     // If the iteration count is sufficiently high, an optimal linearization must be found.
-    // Each linearization step can use up to 2^(k-1) iterations, with steps k=1..n. That sum is
-    // 2^n - 1.
-    const uint64_t n = depgraph.TxCount();
-    if (n <= 19 && iter_count > (uint64_t{1} << n)) {
+    if (iter_count >= MaxOptimalLinearizationIters(depgraph.TxCount())) {
         assert(optimal);
     }
-    // Additionally, if the assumption of sqrt(2^k)+1 iterations per step holds, plus ceil(k/4)
-    // start-up cost per step, plus ceil(n^2/64) start-up cost overall, we can compute the upper
-    // bound for a whole linearization (summing for k=1..n) using the Python expression
-    // [sum((k+3)//4 + int(math.sqrt(2**k)) + 1 for k in range(1, n + 1)) + (n**2 + 63) // 64 for n in range(0, 35)]:
-    static constexpr uint64_t MAX_OPTIMAL_ITERS[] = {
-        0, 4, 8, 12, 18, 26, 37, 51, 70, 97, 133, 182, 251, 346, 480, 666, 927, 1296, 1815, 2545,
-        3576, 5031, 7087, 9991, 14094, 19895, 28096, 39690, 56083, 79263, 112041, 158391, 223936,
-        316629, 447712
-    };
-    if (n < std::size(MAX_OPTIMAL_ITERS) && iter_count >= MAX_OPTIMAL_ITERS[n]) {
-        Assume(optimal);
-    }
 
     // If Linearize claims optimal result, run quality tests.
     if (optimal) {

src/test/fuzz/txgraph.cpp

@@ -5,6 +5,7 @@
 #include <cluster_linearize.h>
 #include <test/fuzz/FuzzedDataProvider.h>
 #include <test/fuzz/fuzz.h>
+#include <test/util/cluster_linearize.h>
 #include <test/util/random.h>
 #include <txgraph.h>
 #include <util/bitset.h>
@@ -730,16 +731,38 @@ FUZZ_TARGET(txgraph)
             } else if (command-- == 0) {
                 // DoWork.
                 uint64_t iters = provider.ConsumeIntegralInRange<uint64_t>(0, alt ? 10000 : 255);
-                if (real->DoWork(iters)) {
-                    for (unsigned level = 0; level < sims.size(); ++level) {
-                        // DoWork() will not optimize oversized levels.
-                        if (sims[level].IsOversized()) continue;
-                        // DoWork() will not touch the main level if a builder is present.
-                        if (level == 0 && !block_builders.empty()) continue;
-                        // If neither of the two above conditions holds, and DoWork() returned
-                        // then the level is optimal.
+                bool ret = real->DoWork(iters);
+                uint64_t iters_for_optimal{0};
+                for (unsigned level = 0; level < sims.size(); ++level) {
+                    // DoWork() will not optimize oversized levels, or the main level if a builder
+                    // is present. Note that this impacts the DoWork() return value, as true means
+                    // that non-optimal clusters may remain within such oversized or builder-having
+                    // levels.
+                    if (sims[level].IsOversized()) continue;
+                    if (level == 0 && !block_builders.empty()) continue;
+                    // If neither of the two above conditions holds, and DoWork() returned true,
+                    // then the level is optimal.
+                    if (ret) {
                         sims[level].real_is_optimal = true;
                     }
+                    // Compute how many iterations would be needed to make everything optimal.
+                    for (auto component : sims[level].GetComponents()) {
+                        auto iters_opt_this_cluster = MaxOptimalLinearizationIters(component.Count());
+                        if (iters_opt_this_cluster > acceptable_iters) {
+                            // If the number of iterations required to linearize this cluster
+                            // optimally exceeds acceptable_iters, DoWork() may process it in two
+                            // stages: once to acceptable, and once to optimal.
+                            iters_for_optimal += iters_opt_this_cluster + acceptable_iters;
+                        } else {
+                            iters_for_optimal += iters_opt_this_cluster;
+                        }
+                    }
+                }
+                if (!ret) {
+                    // DoWork can only have more work left if the requested number of iterations
+                    // was insufficient to linearize everything optimally within the levels it is
+                    // allowed to touch.
+                    assert(iters <= iters_for_optimal);
                 }
                 break;
             } else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) {

src/test/util/cluster_linearize.h

@@ -394,6 +394,29 @@ void SanityCheck(const DepGraph<SetType>& depgraph, std::span<const DepGraphInde
     }
 }
 
+inline uint64_t MaxOptimalLinearizationIters(DepGraphIndex cluster_count)
+{
+    // We assume sqrt(2^k)+1 candidate-finding iterations per candidate to be found, plus ceil(k/4)
+    // startup cost when up to k unlinearization transactions remain, plus ceil(n^2/64) overall
+    // startup cost in Linearize. Thus, we can compute the upper bound for a whole linearization
+    // (summing for k=1..n) using the Python expression:
+    //
+    //   [sum((k+3)//4 + math.isqrt(2**k) + 1 for k in range(1, n + 1)) + (n**2 + 63) // 64 for n in range(0, 65)]
+    //
+    // Note that these are just assumptions, as the proven upper bound grows with 2^k, not
+    // sqrt(2^k).
+    static constexpr uint64_t MAX_OPTIMAL_ITERS[65] = {
+        0, 4, 8, 12, 18, 26, 37, 51, 70, 97, 133, 182, 251, 346, 480, 666, 927, 1296, 1815, 2545,
+        3576, 5031, 7087, 9991, 14094, 19895, 28096, 39690, 56083, 79263, 112041, 158391, 223936,
+        316629, 447712, 633086, 895241, 1265980, 1790280, 2531747, 3580335, 5063259, 7160424,
+        10126257, 14320575, 20252230, 28640853, 40504150, 57281380, 81007962, 114562410, 162015557,
+        229124437, 324030718, 458248463, 648061011, 916496483, 1296121563, 1832992493, 2592242635,
+        3665984477, 5184484745, 7331968412, 10368968930, 14663936244
+    };
+    assert(cluster_count < sizeof(MAX_OPTIMAL_ITERS) / sizeof(MAX_OPTIMAL_ITERS[0]));
+    return MAX_OPTIMAL_ITERS[cluster_count];
+}
+
 } // namespace
 
 #endif // BITCOIN_TEST_UTIL_CLUSTER_LINEARIZE_H