Merge bitcoin/bitcoin#32540: rest: fetch spent transaction outputs by blockhash

c48846e doc: add release notes for #32540 (Roman Zeyde)
d4e212e rest: fetch spent transaction outputs by blockhash (Roman Zeyde)

Pull request description:

  Today, it is possible to fetch a block's spent prevouts in order to build an external index by using the `/rest/block/BLOCKHASH.json` endpoint. However, its performance is low due to JSON serialization overhead.

  We can significantly optimize it by adding a new [REST API](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md) endpoint, using a binary response format (returning a collection of spent txout lists, one per each block transaction):

  ```
  $ BLOCKHASH=00000000000000000002a7c4c1e48d76c5a37902165a270156b7a8d72728a054

  $ ab -k -c 1 -n 100 http://localhost:8332/rest/block/$BLOCKHASH.json
  Document Length:        13278152 bytes
  Requests per second:    3.53 [#/sec] (mean)
  Time per request:       283.569 [ms] (mean)

  $ ab -k -c 1 -n 10000 http://localhost:8332/rest/spenttxouts/$BLOCKHASH.bin
  Document Length:        195591 bytes
  Requests per second:    254.47 [#/sec] (mean)
  Time per request:       3.930 [ms] (mean)
  ```

  Currently, this PR is being used and tested by Bindex[^1].

  This PR would allow to improve the performance of external indexers such as electrs[^2], ElectrumX[^3], Fulcrum[^4] and Blockbook[^5].

  [^1]: https://github.com/romanz/bindex-rs
  [^2]: https://github.com/romanz/electrs (also [blockstream.info](https://github.com/Blockstream/electrs) and [mempool.space](https://github.com/mempool/electrs) forks)
  [^3]: https://github.com/spesmilo/electrumx
  [^4]: https://github.com/cculianu/Fulcrum
  [^5]: https://github.com/trezor/blockbook

ACKs for top commit:
  maflcko:
    re-ACK c48846e 📶
  TheCharlatan:
    Re-ACK c48846e
  achow101:
    ACK c48846e

Tree-SHA512: cf423541be90d6615289760494ae849b7239b69427036db6cc528ac81df10900f514471d81a460125522c5ffa31e9747ddfca187a1f93151e4ae77fe773c6b7b

Author: achow101

doc/release-notes-32540.md

@@ -0,0 +1,5 @@
+New RPCs
+--------
+
+- A new REST API endpoint (`/rest/spenttxouts/BLOCKHASH`) has been introduced for
+  efficiently fetching spent transaction outputs using the block's undo data (#32540).

src/rest.cpp

@@ -27,6 +27,7 @@
 #include <streams.h>
 #include <sync.h>
 #include <txmempool.h>
+#include <undo.h>
 #include <util/any.h>
 #include <util/check.h>
 #include <util/strencodings.h>
@@ -281,6 +282,113 @@ static bool rest_headers(const std::any& context,
     }
 }
 
+/**
+ * Serialize spent outputs as a list of per-transaction CTxOut lists using binary format.
+ */
+static void SerializeBlockUndo(DataStream& stream, const CBlockUndo& block_undo)
+{
+    WriteCompactSize(stream, block_undo.vtxundo.size() + 1);
+    WriteCompactSize(stream, 0); // block_undo.vtxundo doesn't contain coinbase tx
+    for (const CTxUndo& tx_undo : block_undo.vtxundo) {
+        WriteCompactSize(stream, tx_undo.vprevout.size());
+        for (const Coin& coin : tx_undo.vprevout) {
+            coin.out.Serialize(stream);
+        }
+    }
+}
+
+/**
+ * Serialize spent outputs as a list of per-transaction CTxOut lists using JSON format.
+ */
+static void BlockUndoToJSON(const CBlockUndo& block_undo, UniValue& result)
+{
+    result.push_back({UniValue::VARR}); // block_undo.vtxundo doesn't contain coinbase tx
+    for (const CTxUndo& tx_undo : block_undo.vtxundo) {
+        UniValue tx_prevouts(UniValue::VARR);
+        for (const Coin& coin : tx_undo.vprevout) {
+            UniValue prevout(UniValue::VOBJ);
+            prevout.pushKV("value", ValueFromAmount(coin.out.nValue));
+
+            UniValue script_pub_key(UniValue::VOBJ);
+            ScriptToUniv(coin.out.scriptPubKey, /*out=*/script_pub_key, /*include_hex=*/true, /*include_address=*/true);
+            prevout.pushKV("scriptPubKey", std::move(script_pub_key));
+
+            tx_prevouts.push_back(std::move(prevout));
+        }
+        result.push_back(std::move(tx_prevouts));
+    }
+}
+
+static bool rest_spent_txouts(const std::any& context, HTTPRequest* req, const std::string& strURIPart)
+{
+    if (!CheckWarmup(req)) {
+        return false;
+    }
+    std::string param;
+    const RESTResponseFormat rf = ParseDataFormat(param, strURIPart);
+    std::vector<std::string> path = SplitString(param, '/');
+
+    std::string hashStr;
+    if (path.size() == 1) {
+        // path with query parameter: /rest/spenttxouts/<hash>
+        hashStr = path[0];
+    } else {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/spenttxouts/<hash>.<ext>");
+    }
+
+    auto hash{uint256::FromHex(hashStr)};
+    if (!hash) {
+        return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr);
+    }
+
+    ChainstateManager* chainman = GetChainman(context, req);
+    if (!chainman) {
+        return false;
+    }
+
+    const CBlockIndex* pblockindex = WITH_LOCK(cs_main, return chainman->m_blockman.LookupBlockIndex(*hash));
+    if (!pblockindex) {
+        return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found");
+    }
+
+    CBlockUndo block_undo;
+    if (pblockindex->nHeight > 0 && !chainman->m_blockman.ReadBlockUndo(block_undo, *pblockindex)) {
+        return RESTERR(req, HTTP_NOT_FOUND, hashStr + " undo not available");
+    }
+
+    switch (rf) {
+    case RESTResponseFormat::BINARY: {
+        DataStream ssSpentResponse{};
+        SerializeBlockUndo(ssSpentResponse, block_undo);
+        req->WriteHeader("Content-Type", "application/octet-stream");
+        req->WriteReply(HTTP_OK, ssSpentResponse);
+        return true;
+    }
+
+    case RESTResponseFormat::HEX: {
+        DataStream ssSpentResponse{};
+        SerializeBlockUndo(ssSpentResponse, block_undo);
+        const std::string strHex{HexStr(ssSpentResponse) + "\n"};
+        req->WriteHeader("Content-Type", "text/plain");
+        req->WriteReply(HTTP_OK, strHex);
+        return true;
+    }
+
+    case RESTResponseFormat::JSON: {
+        UniValue result(UniValue::VARR);
+        BlockUndoToJSON(block_undo, result);
+        std::string strJSON = result.write() + "\n";
+        req->WriteHeader("Content-Type", "application/json");
+        req->WriteReply(HTTP_OK, strJSON);
+        return true;
+    }
+
+    default: {
+        return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
+    }
+    }
+}
+
 static bool rest_block(const std::any& context,
                        HTTPRequest* req,
                        const std::string& strURIPart,
@@ -1021,6 +1129,7 @@ static const struct {
       {"/rest/deploymentinfo/", rest_deploymentinfo},
       {"/rest/deploymentinfo", rest_deploymentinfo},
       {"/rest/blockhashbyheight/", rest_blockhash_by_height},
+      {"/rest/spenttxouts/", rest_spent_txouts},
 };
 
 void StartREST(const std::any& context)

test/functional/interface_rest.py

@@ -6,6 +6,7 @@
 
 from decimal import Decimal
 from enum import Enum
+from io import BytesIO
 import http.client
 import json
 import typing
@@ -15,6 +16,7 @@
 from test_framework.messages import (
     BLOCK_HEADER_SIZE,
     COIN,
+    deser_block_spent_outputs,
 )
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
@@ -426,6 +428,34 @@ def run_test(self):
         assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}"))
         assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}"))
 
+        self.log.info("Test the /spenttxouts URI")
+
+        block_count = self.nodes[0].getblockcount()
+        for height in range(0, block_count + 1):
+            blockhash = self.nodes[0].getblockhash(height)
+            spent_bin = self.test_rest_request(f"/spenttxouts/{blockhash}", req_type=ReqType.BIN, ret_type=RetType.BYTES)
+            spent_hex = self.test_rest_request(f"/spenttxouts/{blockhash}", req_type=ReqType.HEX, ret_type=RetType.BYTES)
+            spent_json = self.test_rest_request(f"/spenttxouts/{blockhash}", req_type=ReqType.JSON, ret_type=RetType.JSON)
+
+            assert_equal(bytes.fromhex(spent_hex.decode()), spent_bin)
+
+            spent = deser_block_spent_outputs(BytesIO(spent_bin))
+            block = self.nodes[0].getblock(blockhash, 3)  # return prevout for each input
+            assert_equal(len(spent), len(block["tx"]))
+            assert_equal(len(spent_json), len(block["tx"]))
+
+            for i, tx in enumerate(block["tx"]):
+                prevouts = [txin["prevout"] for txin in tx["vin"] if "coinbase" not in txin]
+                # compare with `getblock` JSON output (coinbase tx has no prevouts)
+                actual = [(txout.scriptPubKey.hex(), Decimal(txout.nValue) / COIN) for txout in spent[i]]
+                expected = [(p["scriptPubKey"]["hex"], p["value"]) for p in prevouts]
+                assert_equal(expected, actual)
+                # also compare JSON format
+                actual = [(prevout["scriptPubKey"], prevout["value"]) for prevout in spent_json[i]]
+                expected = [(p["scriptPubKey"], p["value"]) for p in prevouts]
+                assert_equal(expected, actual)
+
+
         self.log.info("Test the /deploymentinfo URI")
 
         deployment_info = self.nodes[0].getdeploymentinfo()

test/functional/test_framework/messages.py

@@ -230,6 +230,11 @@ def ser_string_vector(l):
     return r
 
 
+def deser_block_spent_outputs(f):
+    nit = deser_compact_size(f)
+    return [deser_vector(f, CTxOut) for _ in range(nit)]
+
+
 def from_hex(obj, hex_string):
     """Deserialize from a hex string representation (e.g. from RPC)