From 7311de8896cd7771f05031fb604711e39420bda6 Mon Sep 17 00:00:00 2001 From: Federico Mastellone Date: Tue, 24 Jun 2025 19:33:23 +0000 Subject: [PATCH 1/6] wb | extract utxo utils from voting workload --- nix/workbench/workload/utils/utxo.nix | 517 ++++++++++++++++++++++++++ nix/workbench/workload/voting.nix | 496 +----------------------- 2 files changed, 525 insertions(+), 488 deletions(-) create mode 100644 nix/workbench/workload/utils/utxo.nix diff --git a/nix/workbench/workload/utils/utxo.nix b/nix/workbench/workload/utils/utxo.nix new file mode 100644 index 00000000000..b31183193a9 --- /dev/null +++ b/nix/workbench/workload/utils/utxo.nix @@ -0,0 +1,517 @@ +{ coreutils +, cardano-cli +, jq +, testnet_magic +# Max number of '--tx-out' when splitting funds. +, outs_per_split_transaction ? 100 +# Submit the transaction up to this times on timeout. +, funds_submit_tries ? 3 +# Sleeps. +# Used when splitting funds to wait for funds to arrive, as this initial funds +# are sent from a different process (not genesis) this works as a semaphore! +, wait_any_utxo_tries ? 30 +, wait_any_utxo_sleep ? 10 # 5 minutes in 10s steps. +# Used when splitting funds, it waits for the expected UTxO to arrive to the +# change-address and re-submits the transaction if necessary! +, wait_utxo_id_tries ? 18 +, wait_utxo_id_sleep ? 10 # 3 minutes in 10s steps. +# Used when waiting for the recently created proposal. +, wait_proposal_id_tries ? 30 +, wait_proposal_id_sleep ? 10 # 5 minutes +# Use to wait for all proposals to be available before we start voting. +# As nodes will end their splitting phases at different times, this parameters +# work as a formation lap before race start =). +# No tries, waits forever! +, wait_proposals_count_sleep ? 10 +}: +'' +################################################################################ +# Give a node name ("node-0", "explorer", etc) returns the node's socket path. +################################################################################ +function get_socket_path { + + # Function arguments. + local node_str=$1 # node name / folder to find the socket. + + local socket_path="../../''${node_str}/node.socket" + ${coreutils}/bin/echo "''${socket_path}" +} + +################################################################################ +# Given a "tx.signed" returns a JSON object that has as "tx_id" and "tx_ix" +# properties the TxHash#TxIx of the FIRST occurrence of the provided address in +# its "outputs" and in the "value" property the lovelace it will contain. +# For example: {"tx_id":"0000000000", "tx_ix": 0, "value":123456}. +# If NO ADDRESS IS SUPPLIED as argument to this function we use/assume the last +# output is the change address (--change-address) and you want to use that one +# to calculate a future expected UTxO. +################################################################################ +function calculate_next_utxo { + + # Function arguments. + local tx_signed=$1 + local addr=''${2:-null} + + local tx_id + # Prints a transaction identifier. + tx_id="$( \ + ${cardano-cli}/bin/cardano-cli conway transaction txid \ + --tx-file "''${tx_signed}" \ + )" + # View transaction as JSON and get index of FIRST output containing "$addr". + ${cardano-cli}/bin/cardano-cli debug transaction view \ + --output-json \ + --tx-file "''${tx_signed}" \ + | ${jq}/bin/jq --raw-output \ + --argjson tx_id "\"''${tx_id}\"" \ + --argjson addr "\"''${addr}\"" \ + ' + ( + if $addr == null or $addr == "null" + then + (.outputs | length - 1) + else + ( + .outputs + | map(.address == $addr) + | index(true) + ) + end + ) as $tx_ix + | { "tx_id": $tx_id + , "tx_ix": $tx_ix + , "value": ( .outputs[$tx_ix].amount.lovelace ) + } + ' +} + +################################################################################ +# Store the pre-calculated "cached" future UTxO of this address. +# (Only useful if an address is always used from the same node/socket/path). +################################################################################ +function store_address_utxo_expected { + + # Function arguments. + local tx_signed=$1 + local addr=$2 + + local utxo_file=./addr."''${addr}".json # Store in workload's directory! + calculate_next_utxo \ + "''${tx_signed}" \ + "''${addr}" \ + > "''${utxo_file}" +} + +################################################################################ +# Get pre-calculated "cached" future UTxO TxHash#TxIx suitable to use as a part +# of a "--tx-in" argument. Returns an "empty" string if not available. +# (Only useful if an address is always used from the same node/socket/path). +################################################################################ +function get_address_utxo_expected_id { + + # Function arguments. + local addr=$1 + + local utxo_file=./addr."''${addr}".json # Store in workload's directory! + if test -f "''${utxo_file}" + then + ${jq}/bin/jq --raw-output \ + '( .tx_id + "#" + (.tx_ix | tostring) )' \ + "''${utxo_file}" + fi +} + +################################################################################ +# Get pre-calculated "cached" future UTxO lovelace amount suitable to use as +# part of a "--tx-in" argument. Returns an "empty" string if not available. +# (This only works if an address is always used from the same node/socket/path). +################################################################################ +function get_address_utxo_expected_value { + + # Function arguments. + local addr=$1 + + local utxo_file=./addr."''${addr}".json # Store in workload's directory! + if test -f "''${utxo_file}" + then + ${jq}/bin/jq --raw-output '.value' "''${utxo_file}" + fi +} + +################################################################################ +# Give a "tx.signed" filepath returns "true" or "false". +# Not to be run during the benchmarking phase: lots of queries! +################################################################################ +function is_tx_in_mempool { + + # Function arguments. + local node_str=$1 # node name / folder where to store the files. + local tx_signed=$2 + + # Only defined in functions that use it. + local socket_path + socket_path="$(get_socket_path "''${node_str}")" + + local tx_id + tx_id="$( \ + ${cardano-cli}/bin/cardano-cli conway transaction txid \ + --tx-file "''${tx_signed}" \ + )" + ${cardano-cli}/bin/cardano-cli conway query tx-mempool \ + tx-exists "''${tx_id}" \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + | ${jq}/bin/jq --raw-output \ + .exists +} + +################################################################################ +# Function to submit the funds-splitting tx and retry if needed. +# Not to be run during the benchmarking phase: lots of queries! +################################################################################ +function funds_submit_retry { + + # Function arguments. + local node_str=$1 # node name / folder to find the socket to use. + local tx_signed=$2 # tx to send and maybe re-send. + local addr=$3 # Address to wait for (UTxO id must be cached). + + # Only defined in functions that use it. + local socket_path + socket_path="$(get_socket_path "''${node_str}")" + + local utxo_id + utxo_id="$(get_address_utxo_expected_id "''${addr}")" + + local contains_addr="false" + local submit_tries=${toString funds_submit_tries} + while test ! "''${contains_addr}" = "true" + do + if test "''${submit_tries}" -le 0 + then + # Time's up! + ${coreutils}/bin/echo "funds_submit_retry: Timeout waiting for: ''${addr} - ''${utxo_id}" + exit 1 + else + + # Some debugging. + ${coreutils}/bin/echo "funds_submit_retry: submit: ''${tx_signed} (''${submit_tries})" + + # (Re)Submit transaction ignoring errors. + ${cardano-cli}/bin/cardano-cli conway transaction submit \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + --tx-file "''${tx_signed}" \ + || true + submit_tries="$((submit_tries - 1))" + + # Wait for the transaction to NOT be in the mempool anymore + local in_mempool="true" + while test ! "''${in_mempool}" = "false" + do + ${coreutils}/bin/sleep 1 + in_mempool="$(is_tx_in_mempool "''${node_str}" "''${tx_signed}")" + done + + # Some loops to see if the expected UTxO of this address appears. + local utxo_tries=${toString wait_utxo_id_tries} + while test ! "''${contains_addr}" = "true" && test "''${utxo_tries}" -gt 0 + do + ${coreutils}/bin/sleep ${toString wait_utxo_id_sleep} + # Some debugging. + ${coreutils}/bin/echo "funds_submit_retry: wait_utxo_id: ''${utxo_id} (''${utxo_tries})" + contains_addr="$( \ + ${cardano-cli}/bin/cardano-cli conway query utxo \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + --address "''${addr}" \ + --output-json \ + | ${jq}/bin/jq --raw-output \ + --argjson utxo_id "\"''${utxo_id}\"" \ + 'keys | any(. == $utxo_id) // false' \ + )" + utxo_tries="$((utxo_tries - 1))" + done + + fi + done + +} + +################################################################################ +# Evenly split the first UTxO of this key to the addresses in the array! +# Does it in batchs so we don't exceed "maxTxSize" of 16384. +# Stores the future UTxOs of all addresses in files for later references. +# Not to be run during the benchmarking phase: waits for funds between batchs! +################################################################################ +function funds_from_to { + + # Function arguments. + local node_str=''${1}; shift # node name / folder to find the socket to use. + local utxo_vkey=''${1}; shift # In + local utxo_skey=''${1}; shift # In + local reminder=''${1}; shift # Funds to keep in the origin address. + local donation=''${1}; shift # To treasury. + local addrs_array=("$@") # Outs + + # Only defined in functions that use it. + local socket_path + socket_path="$(get_socket_path "''${node_str}")" + + # Get the "in" address and its first UTxO only once we have the lock. + local funds_addr + funds_addr="$( \ + ${cardano-cli}/bin/cardano-cli address build \ + --testnet-magic ${toString testnet_magic} \ + --payment-verification-key-file "''${utxo_vkey}" \ + )" + # This three only needed for the first batch and to calculate funds per node. + local funds_json funds_tx funds_lovelace + funds_json="$( \ + ${cardano-cli}/bin/cardano-cli conway query utxo \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + --address "''${funds_addr}" \ + --output-json \ + )" + funds_tx="$( \ + ${coreutils}/bin/echo "''${funds_json}" \ + | ${jq}/bin/jq -r \ + 'keys[0]' \ + )" + funds_lovelace="$( \ + ${coreutils}/bin/echo "''${funds_json}" \ + | ${jq}/bin/jq -r \ + --arg keyName "''${funds_tx}" \ + '.[$keyName].value.lovelace' \ + )" + + # Calculate how much lovelace for each output address. + local outs_count per_out_lovelace + outs_count="''${#addrs_array[@]}" + ### HACK: Fees! Always using 550000!!! + ### With 2 outputs: "Estimated transaction fee: 172233 Lovelace" + ### With 10 outputs: "Estimated transaction fee: 186665 Lovelace" + ### With 53 outputs: "Estimated transaction fee: 264281 Lovelace" + ### With 150 outputs: "Estimated transaction fee: 439357 Lovelace" + ### With 193 outputs: "Estimated transaction fee: 516929 Lovelace" + per_out_lovelace="$( \ + ${jq}/bin/jq -r --null-input \ + --argjson numerator "''${funds_lovelace}" \ + --argjson denominator "''${outs_count}" \ + --argjson reminder "''${reminder}" \ + --argjson donation "''${donation}" \ + '( + ( $numerator + - $reminder + - $donation + - ( 550000 + * ( ($denominator / ${toString outs_per_split_transaction}) | ceil ) + ) + ) + / $denominator + | round + )' \ + )" + + # Split the funds in batchs (donations only happen in the first batch). + local i=0 + local txOuts_args_array=() txOuts_addrs_array=() + local batch=${toString outs_per_split_transaction} + local tx_in tx_filename + local treasury_donation_args_array=() + for addr in "''${addrs_array[@]}" + do + i="$((i + 1))" + # Build the "--tx-out" arguments array of this batch. + txOuts_args_array+=("--tx-out") + txOuts_args_array+=("''${addr}+''${per_out_lovelace}") + txOuts_addrs_array+=("''${addr}") + + # We send if last addr in the for loop or batch max exceeded. + if test "$i" -ge "''${#addrs_array[@]}" || test "$i" -ge "$batch" + then + if test "$batch" -eq ${toString outs_per_split_transaction} + then + # First transaction. + # The input comes from the function arguments. + tx_in="''${funds_tx}" + # Treasury donation happens only once. + if ! test "''${donation}" = "0" + then + treasury_donation_args_array=("--treasury-donation" "''${donation}") + fi + else + # Not the first batch. + # The input comes from the last transaction submitted. + # No need to wait for it because the submission function does this! + tx_in="$(get_address_utxo_expected_id "''${funds_addr}")" + # Treasury donation happens only once. + treasury_donation_args_array=() + fi + + # Some debugging! + ${coreutils}/bin/echo "funds_from_to: ''${utxo_vkey} (''${funds_addr}): --tx-in ''${tx_in}" + + # Send this batch to each node! + # Build transaction. + tx_filename=./funds_from_to."''${funds_addr}"."''${i}" + ${cardano-cli}/bin/cardano-cli conway transaction build \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + --tx-in "''${tx_in}" \ + ''${txOuts_args_array[@]} \ + ''${treasury_donation_args_array[@]} \ + --change-address "''${funds_addr}" \ + --out-file "''${tx_filename}.raw" + # Sign transaction. + ${cardano-cli}/bin/cardano-cli conway transaction sign \ + --testnet-magic ${toString testnet_magic} \ + --signing-key-file "''${utxo_skey}" \ + --tx-body-file "''${tx_filename}.raw" \ + --out-file "''${tx_filename}.signed" + + # Store outs/addresses next UTxO. + for addr_cache in "''${txOuts_addrs_array[@]}" + do + store_address_utxo_expected \ + "''${tx_filename}.signed" \ + "''${addr_cache}" + done + # Without the change address we can't wait for the funds after submission + # or calculate the next input to use if an extra batch is needed! + store_address_utxo_expected \ + "''${tx_filename}.signed" \ + "''${funds_addr}" + + # Submit transaction and wait for settlement. + funds_submit_retry \ + "''${node_str}" \ + "''${tx_filename}.signed" \ + "''${funds_addr}" + + # Reset variables for next batch iteration. + txOuts_args_array=() txOuts_addrs_array=() + batch="$((batch + ${toString outs_per_split_transaction}))" + fi + done +} + +################################################################################ +# Waits until the UTxOs of this address are not empty (errors on timeout). +# Not to be run during the benchmarking phase: lots of queries! +################################################################################ +function wait_any_utxo { + + # Function arguments. + local node_str=$1 # node name / folder to find the socket to use. + local addr=$2 + + # Only defined in functions that use it. + local socket_path + socket_path="$(get_socket_path "''${node_str}")" + + local tries=${toString wait_any_utxo_tries} + local utxos_json="{}" + while test "''${utxos_json}" = "{}" + do + if test "''${tries}" -le 0 + then + # Time's up! + ${coreutils}/bin/echo "wait_any_utxo: Timeout waiting for: ''${addr}" + exit 1 + fi + utxos_json="$( \ + ${cardano-cli}/bin/cardano-cli conway query utxo \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + --address "''${addr}" \ + --output-json \ + )" + if ! test "''${tries}" = ${toString wait_any_utxo_tries} + then + ${coreutils}/bin/sleep ${toString wait_any_utxo_sleep} + fi + tries="$((tries - 1))" + done +} + +################################################################################ +# Waits until an specific proposal appears or fails. +# Not to be run during the benchmarking phase: lots of queries! +################################################################################ +function wait_proposal_id { + + # Function arguments. + local node_str=$1 # node name / folder to find the socket to use. + local tx_signed=$2 + + # Only defined in functions that use it. + local socket_path + socket_path="$(get_socket_path "''${node_str}")" + + # Get proposal's "txId" from the "--tx-file". + local tx_id + tx_id="$( \ + ${cardano-cli}/bin/cardano-cli conway transaction txid \ + --tx-file "''${tx_signed}" \ + )" + + local contains_proposal="false" + local tries=${toString wait_proposal_id_tries} + while test "''${contains_proposal}" = "false" + do + if test "''${tries}" -le 0 + then + # Time's up! + ${coreutils}/bin/echo "wait_proposal_id: Timeout waiting for: ''${tx_id}" + exit 1 + else + # No "--output-json" needed. + contains_proposal="$( \ + ${cardano-cli}/bin/cardano-cli conway query gov-state \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + | ${jq}/bin/jq --raw-output \ + --argjson tx_id "\"''${tx_id}\"" \ + '.proposals | any(.actionId.txId == $tx_id) // false' \ + )" + if ! test "''${tries}" = ${toString wait_proposal_id_tries} + then + ${coreutils}/bin/sleep ${toString wait_proposal_id_sleep} + fi + tries="$((tries - 1))" + fi + done +} + +################################################################################ +# Waits until an specific number of proposals are visible. +# Not to be run during the benchmarking phase: lots of queries! +################################################################################ +function wait_proposals_count { + + # Function arguments. + local node_str=$1 # node name / folder to find the socket to use. + local count=$2 + + # Only defined in functions that use it. + local socket_path + socket_path="$(get_socket_path "''${node_str}")" + + local contains_proposals="false" + while test "''${contains_proposals}" = "false" + do + # No "--output-json" needed. + contains_proposals="$( \ + ${cardano-cli}/bin/cardano-cli conway query gov-state \ + --testnet-magic ${toString testnet_magic} \ + --socket-path "''${socket_path}" \ + | ${jq}/bin/jq --raw-output \ + --argjson count "''${count}" \ + '.proposals | length == $count // false' \ + )" + ${coreutils}/bin/sleep ${toString wait_proposals_count_sleep} + done +} +'' diff --git a/nix/workbench/workload/voting.nix b/nix/workbench/workload/voting.nix index 922ccc7bb2d..b6246b76ac0 100644 --- a/nix/workbench/workload/voting.nix +++ b/nix/workbench/workload/voting.nix @@ -141,494 +141,14 @@ in '' # desired_producer_tps: ${toString desired_producer_tps} # desired_producer_sleep: ${toString desired_producer_sleep} -################################################################################ -# Give a node name ("node-0", "explorer", etc) returns the node's socket path. -################################################################################ -function get_socket_path { - - # Function arguments. - local node_str=$1 # node name / folder to find the socket. - - local socket_path="../../''${node_str}/node.socket" - ${coreutils}/bin/echo "''${socket_path}" -} - -################################################################################ -# Given a "tx.signed" returns a JSON object that has as "tx_id" and "tx_ix" -# properties the TxHash#TxIx of the FIRST occurrence of the provided address in -# its "outputs" and in the "value" property the lovelace it will contain. -# For example: {"tx_id":"0000000000", "tx_ix": 0, "value":123456}. -# If NO ADDRESS IS SUPPLIED as argument to this function we use/assume the last -# output is the change address (--change-address) and you want to use that one -# to calculate a future expected UTxO. -################################################################################ -function calculate_next_utxo { - - # Function arguments. - local tx_signed=$1 - local addr=''${2:-null} - - local tx_id - # Prints a transaction identifier. - tx_id="$( \ - ${cardano-cli}/bin/cardano-cli conway transaction txid \ - --tx-file "''${tx_signed}" \ - )" - # View transaction as JSON and get index of FIRST output containing "$addr". - ${cardano-cli}/bin/cardano-cli debug transaction view \ - --output-json \ - --tx-file "''${tx_signed}" \ - | ${jq}/bin/jq --raw-output \ - --argjson tx_id "\"''${tx_id}\"" \ - --argjson addr "\"''${addr}\"" \ - ' - ( - if $addr == null or $addr == "null" - then - (.outputs | length - 1) - else - ( - .outputs - | map(.address == $addr) - | index(true) - ) - end - ) as $tx_ix - | { "tx_id": $tx_id - , "tx_ix": $tx_ix - , "value": ( .outputs[$tx_ix].amount.lovelace ) - } - ' -} - -################################################################################ -# Store the pre-calculated "cached" future UTxO of this address. -# (Only useful if an address is always used from the same node/socket/path). -################################################################################ -function store_address_utxo_expected { - - # Function arguments. - local tx_signed=$1 - local addr=$2 - - local utxo_file=./addr."''${addr}".json # Store in workload's directory! - calculate_next_utxo \ - "''${tx_signed}" \ - "''${addr}" \ - > "''${utxo_file}" -} - -################################################################################ -# Get pre-calculated "cached" future UTxO TxHash#TxIx suitable to use as a part -# of a "--tx-in" argument. Returns an "empty" string if not available. -# (Only useful if an address is always used from the same node/socket/path). -################################################################################ -function get_address_utxo_expected_id { - - # Function arguments. - local addr=$1 - - local utxo_file=./addr."''${addr}".json # Store in workload's directory! - if test -f "''${utxo_file}" - then - ${jq}/bin/jq --raw-output \ - '( .tx_id + "#" + (.tx_ix | tostring) )' \ - "''${utxo_file}" - fi -} - -################################################################################ -# Get pre-calculated "cached" future UTxO lovelace amount suitable to use as -# part of a "--tx-in" argument. Returns an "empty" string if not available. -# (This only works if an address is always used from the same node/socket/path). -################################################################################ -function get_address_utxo_expected_value { - - # Function arguments. - local addr=$1 - - local utxo_file=./addr."''${addr}".json # Store in workload's directory! - if test -f "''${utxo_file}" - then - ${jq}/bin/jq --raw-output '.value' "''${utxo_file}" - fi -} - -################################################################################ -# Give a "tx.signed" filepath returns "true" or "false". -# Not to be run during the benchmarking phase: lots of queries! -################################################################################ -function is_tx_in_mempool { - - # Function arguments. - local node_str=$1 # node name / folder where to store the files. - local tx_signed=$2 - - # Only defined in functions that use it. - local socket_path - socket_path="$(get_socket_path "''${node_str}")" - - local tx_id - tx_id="$( \ - ${cardano-cli}/bin/cardano-cli conway transaction txid \ - --tx-file "''${tx_signed}" \ - )" - ${cardano-cli}/bin/cardano-cli conway query tx-mempool \ - tx-exists "''${tx_id}" \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - | ${jq}/bin/jq --raw-output \ - .exists -} - -################################################################################ -# Function to submit the funds-splitting tx and retry if needed. -# Not to be run during the benchmarking phase: lots of queries! -################################################################################ -function funds_submit_retry { - - # Function arguments. - local node_str=$1 # node name / folder to find the socket to use. - local tx_signed=$2 # tx to send and maybe re-send. - local addr=$3 # Address to wait for (UTxO id must be cached). - - # Only defined in functions that use it. - local socket_path - socket_path="$(get_socket_path "''${node_str}")" - - local utxo_id - utxo_id="$(get_address_utxo_expected_id "''${addr}")" - - local contains_addr="false" - local submit_tries=${toString funds_submit_tries} - while test ! "''${contains_addr}" = "true" - do - if test "''${submit_tries}" -le 0 - then - # Time's up! - ${coreutils}/bin/echo "funds_submit_retry: Timeout waiting for: ''${addr} - ''${utxo_id}" - exit 1 - else - - # Some debugging. - ${coreutils}/bin/echo "funds_submit_retry: submit: ''${tx_signed} (''${submit_tries})" - - # (Re)Submit transaction ignoring errors. - ${cardano-cli}/bin/cardano-cli conway transaction submit \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - --tx-file "''${tx_signed}" \ - || true - submit_tries="$((submit_tries - 1))" - - # Wait for the transaction to NOT be in the mempool anymore - local in_mempool="true" - while test ! "''${in_mempool}" = "false" - do - ${coreutils}/bin/sleep 1 - in_mempool="$(is_tx_in_mempool "''${node_str}" "''${tx_signed}")" - done - - # Some loops to see if the expected UTxO of this address appears. - local utxo_tries=${toString wait_utxo_id_tries} - while test ! "''${contains_addr}" = "true" && test "''${utxo_tries}" -gt 0 - do - ${coreutils}/bin/sleep ${toString wait_utxo_id_sleep} - # Some debugging. - ${coreutils}/bin/echo "funds_submit_retry: wait_utxo_id: ''${utxo_id} (''${utxo_tries})" - contains_addr="$( \ - ${cardano-cli}/bin/cardano-cli conway query utxo \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - --address "''${addr}" \ - --output-json \ - | ${jq}/bin/jq --raw-output \ - --argjson utxo_id "\"''${utxo_id}\"" \ - 'keys | any(. == $utxo_id) // false' \ - )" - utxo_tries="$((utxo_tries - 1))" - done - - fi - done - -} - -################################################################################ -# Evenly split the first UTxO of this key to the addresses in the array! -# Does it in batchs so we don't exceed "maxTxSize" of 16384. -# Stores the future UTxOs of all addresses in files for later references. -# Not to be run during the benchmarking phase: waits for funds between batchs! -################################################################################ -function funds_from_to { - - # Function arguments. - local node_str=''${1}; shift # node name / folder to find the socket to use. - local utxo_vkey=''${1}; shift # In - local utxo_skey=''${1}; shift # In - local reminder=''${1}; shift # Funds to keep in the origin address. - local donation=''${1}; shift # To treasury. - local addrs_array=("$@") # Outs - - # Only defined in functions that use it. - local socket_path - socket_path="$(get_socket_path "''${node_str}")" - - # Get the "in" address and its first UTxO only once we have the lock. - local funds_addr - funds_addr="$( \ - ${cardano-cli}/bin/cardano-cli address build \ - --testnet-magic ${toString testnet_magic} \ - --payment-verification-key-file "''${utxo_vkey}" \ - )" - # This three only needed for the first batch and to calculate funds per node. - local funds_json funds_tx funds_lovelace - funds_json="$( \ - ${cardano-cli}/bin/cardano-cli conway query utxo \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - --address "''${funds_addr}" \ - --output-json \ - )" - funds_tx="$( \ - ${coreutils}/bin/echo "''${funds_json}" \ - | ${jq}/bin/jq -r \ - 'keys[0]' \ - )" - funds_lovelace="$( \ - ${coreutils}/bin/echo "''${funds_json}" \ - | ${jq}/bin/jq -r \ - --arg keyName "''${funds_tx}" \ - '.[$keyName].value.lovelace' \ - )" - - # Calculate how much lovelace for each output address. - local outs_count per_out_lovelace - outs_count="''${#addrs_array[@]}" - ### HACK: Fees! Always using 550000!!! - ### With 2 outputs: "Estimated transaction fee: 172233 Lovelace" - ### With 10 outputs: "Estimated transaction fee: 186665 Lovelace" - ### With 53 outputs: "Estimated transaction fee: 264281 Lovelace" - ### With 150 outputs: "Estimated transaction fee: 439357 Lovelace" - ### With 193 outputs: "Estimated transaction fee: 516929 Lovelace" - per_out_lovelace="$( \ - ${jq}/bin/jq -r --null-input \ - --argjson numerator "''${funds_lovelace}" \ - --argjson denominator "''${outs_count}" \ - --argjson reminder "''${reminder}" \ - --argjson donation "''${donation}" \ - '( - ( $numerator - - $reminder - - $donation - - ( 550000 - * ( ($denominator / ${toString outs_per_split_transaction}) | ceil ) - ) - ) - / $denominator - | round - )' \ - )" - - # Split the funds in batchs (donations only happen in the first batch). - local i=0 - local txOuts_args_array=() txOuts_addrs_array=() - local batch=${toString outs_per_split_transaction} - local tx_in tx_filename - local treasury_donation_args_array=() - for addr in "''${addrs_array[@]}" - do - i="$((i + 1))" - # Build the "--tx-out" arguments array of this batch. - txOuts_args_array+=("--tx-out") - txOuts_args_array+=("''${addr}+''${per_out_lovelace}") - txOuts_addrs_array+=("''${addr}") - - # We send if last addr in the for loop or batch max exceeded. - if test "$i" -ge "''${#addrs_array[@]}" || test "$i" -ge "$batch" - then - if test "$batch" -eq ${toString outs_per_split_transaction} - then - # First transaction. - # The input comes from the function arguments. - tx_in="''${funds_tx}" - # Treasury donation happens only once. - if ! test "''${donation}" = "0" - then - treasury_donation_args_array=("--treasury-donation" "''${donation}") - fi - else - # Not the first batch. - # The input comes from the last transaction submitted. - # No need to wait for it because the submission function does this! - tx_in="$(get_address_utxo_expected_id "''${funds_addr}")" - # Treasury donation happens only once. - treasury_donation_args_array=() - fi - - # Some debugging! - ${coreutils}/bin/echo "funds_from_to: ''${utxo_vkey} (''${funds_addr}): --tx-in ''${tx_in}" - - # Send this batch to each node! - # Build transaction. - tx_filename=./funds_from_to."''${funds_addr}"."''${i}" - ${cardano-cli}/bin/cardano-cli conway transaction build \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - --tx-in "''${tx_in}" \ - ''${txOuts_args_array[@]} \ - ''${treasury_donation_args_array[@]} \ - --change-address "''${funds_addr}" \ - --out-file "''${tx_filename}.raw" - # Sign transaction. - ${cardano-cli}/bin/cardano-cli conway transaction sign \ - --testnet-magic ${toString testnet_magic} \ - --signing-key-file "''${utxo_skey}" \ - --tx-body-file "''${tx_filename}.raw" \ - --out-file "''${tx_filename}.signed" - - # Store outs/addresses next UTxO. - for addr_cache in "''${txOuts_addrs_array[@]}" - do - store_address_utxo_expected \ - "''${tx_filename}.signed" \ - "''${addr_cache}" - done - # Without the change address we can't wait for the funds after submission - # or calculate the next input to use if an extra batch is needed! - store_address_utxo_expected \ - "''${tx_filename}.signed" \ - "''${funds_addr}" - - # Submit transaction and wait for settlement. - funds_submit_retry \ - "''${node_str}" \ - "''${tx_filename}.signed" \ - "''${funds_addr}" - - # Reset variables for next batch iteration. - txOuts_args_array=() txOuts_addrs_array=() - batch="$((batch + ${toString outs_per_split_transaction}))" - fi - done -} - -################################################################################ -# Waits until the UTxOs of this address are not empty (errors on timeout). -# Not to be run during the benchmarking phase: lots of queries! -################################################################################ -function wait_any_utxo { - - # Function arguments. - local node_str=$1 # node name / folder to find the socket to use. - local addr=$2 - - # Only defined in functions that use it. - local socket_path - socket_path="$(get_socket_path "''${node_str}")" - - local tries=${toString wait_any_utxo_tries} - local utxos_json="{}" - while test "''${utxos_json}" = "{}" - do - if test "''${tries}" -le 0 - then - # Time's up! - ${coreutils}/bin/echo "wait_any_utxo: Timeout waiting for: ''${addr}" - exit 1 - fi - utxos_json="$( \ - ${cardano-cli}/bin/cardano-cli conway query utxo \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - --address "''${addr}" \ - --output-json \ - )" - if ! test "''${tries}" = ${toString wait_any_utxo_tries} - then - ${coreutils}/bin/sleep ${toString wait_any_utxo_sleep} - fi - tries="$((tries - 1))" - done -} - -################################################################################ -# Waits until an specific proposal appears or fails. -# Not to be run during the benchmarking phase: lots of queries! -################################################################################ -function wait_proposal_id { - - # Function arguments. - local node_str=$1 # node name / folder to find the socket to use. - local tx_signed=$2 - - # Only defined in functions that use it. - local socket_path - socket_path="$(get_socket_path "''${node_str}")" - - # Get proposal's "txId" from the "--tx-file". - local tx_id - tx_id="$( \ - ${cardano-cli}/bin/cardano-cli conway transaction txid \ - --tx-file "''${tx_signed}" \ - )" - - local contains_proposal="false" - local tries=${toString wait_proposal_id_tries} - while test "''${contains_proposal}" = "false" - do - if test "''${tries}" -le 0 - then - # Time's up! - ${coreutils}/bin/echo "wait_proposal_id: Timeout waiting for: ''${tx_id}" - exit 1 - else - # No "--output-json" needed. - contains_proposal="$( \ - ${cardano-cli}/bin/cardano-cli conway query gov-state \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - | ${jq}/bin/jq --raw-output \ - --argjson tx_id "\"''${tx_id}\"" \ - '.proposals | any(.actionId.txId == $tx_id) // false' \ - )" - if ! test "''${tries}" = ${toString wait_proposal_id_tries} - then - ${coreutils}/bin/sleep ${toString wait_proposal_id_sleep} - fi - tries="$((tries - 1))" - fi - done -} - -################################################################################ -# Waits until an specific number of proposals are visible. -# Not to be run during the benchmarking phase: lots of queries! -################################################################################ -function wait_proposals_count { - - # Function arguments. - local node_str=$1 # node name / folder to find the socket to use. - local count=$2 - - # Only defined in functions that use it. - local socket_path - socket_path="$(get_socket_path "''${node_str}")" - - local contains_proposals="false" - while test "''${contains_proposals}" = "false" - do - # No "--output-json" needed. - contains_proposals="$( \ - ${cardano-cli}/bin/cardano-cli conway query gov-state \ - --testnet-magic ${toString testnet_magic} \ - --socket-path "''${socket_path}" \ - | ${jq}/bin/jq --raw-output \ - --argjson count "''${count}" \ - '.proposals | length == $count // false' \ - )" - ${coreutils}/bin/sleep ${toString wait_proposals_count_sleep} - done +${import ./utils/utxo.nix + { inherit coreutils cardano-cli jq testnet_magic; + inherit outs_per_split_transaction funds_submit_tries; + inherit wait_any_utxo_tries wait_any_utxo_sleep; + inherit wait_utxo_id_tries wait_utxo_id_sleep; + inherit wait_proposal_id_tries wait_proposal_id_sleep; + inherit wait_proposals_count_sleep; + } } ################################################################################ From 3d6db1c0633e2d8140b2eea470bcffc594a1c5c5 Mon Sep 17 00:00:00 2001 From: Federico Mastellone Date: Thu, 16 Oct 2025 15:42:46 +0000 Subject: [PATCH 2/6] wb | extract keys utils from voting workload --- nix/workbench/workload/utils/keys.nix | 113 ++++++++++++++++++++++ nix/workbench/workload/voting.nix | 131 ++++---------------------- 2 files changed, 130 insertions(+), 114 deletions(-) create mode 100644 nix/workbench/workload/utils/keys.nix diff --git a/nix/workbench/workload/utils/keys.nix b/nix/workbench/workload/utils/keys.nix new file mode 100644 index 00000000000..d48d8d1f475 --- /dev/null +++ b/nix/workbench/workload/utils/keys.nix @@ -0,0 +1,113 @@ +{ coreutils +, jq +, cardano-cli +, testnet_magic +}: +'' +################################################################################ +# Hack: Why pre-create and pre-share keys when you can create them on demand? +# (ONLY BECAUSE THIS IS A TEST ENVIRONMENT AND WE ARE ALL FRIENDS!) +# Given a prefix and an x, y and z numbers creates always the same address keys. +# The x, y and z are to allow to have different key "levels", for example a key +# for "node-2", "drep 5" and "proposal 22". +# Supports x=[0..99], y=[0..9999] and z=[0..999999] by adding the Hex chars. +# Returns the file path without the extensions (the ".skey" or ".vkey" part). +# No key is created if the file already exists. +################################################################################ +function create_x_y_z_key_files { + + # Function arguments. + local prefix=$1 # String for the key file name (not for the socket). + local x_i=$2 + local y_n=$3 + local z_n=$4 + + local filename=./"''${prefix}"-"''${y_n}"-"''${z_n}" + # Now with the extensions. + local skey="''${filename}".skey + local vkey="''${filename}".vkey + + # Only create if not already there! + if ! test -f "''${vkey}" + then + ${jq}/bin/jq --null-input \ + --argjson x_n "''${x_n}" \ + --argjson y_n "''${y_n}" \ + --argjson z_n "''${z_n}" \ + ' + {"type": "PaymentSigningKeyShelley_ed25519", + "description": "Payment Signing Key", + "cborHex": ( + "5820b02868d722df021278c78be3b7363759b37f5852b8747b488bab" + + (if $x_n <= 9 + then ("0" + ($x_n | tostring)) + elif $x_n >= 10 and $x_n <= 99 + then ( $x_n | tostring) + else (error ("Node ID above 99")) + end + ) + + (if $y_n <= 9 + then ( "000" + ($y_n | tostring)) + elif $y_n >= 10 and $y_n <= 99 + then ( "00" + ($y_n | tostring)) + elif $y_n >= 100 and $y_n <= 999 + then ( "0" + ($y_n | tostring)) + elif $y_n >= 1000 and $y_n <= 9999 + then ( ($y_n | tostring)) + else (error ("Proposal ID above 9999")) + end + ) + + (if $z_n <= 9 + then ( "00000" + ($z_n | tostring)) + elif $z_n >= 10 and $z_n <= 99 + then ( "0000" + ($z_n | tostring)) + elif $z_n >= 100 and $z_n <= 999 + then ( "000" + ($z_n | tostring)) + elif $z_n >= 1000 and $z_n <= 9999 + then ( "00" + ($z_n | tostring)) + elif $z_n >= 10000 and $z_n <= 99999 + then ( "0" + ($z_n | tostring)) + elif $z_n >= 100000 and $z_n <= 999999 + then ( ($z_n | tostring)) + else (error ("DRep ID above 999999")) + end + ) + ) + } + ' \ + > "''${skey}" + ${cardano-cli}/bin/cardano-cli conway key verification-key \ + --signing-key-file "''${skey}" \ + --verification-key-file "''${vkey}" + fi + ${coreutils}/bin/echo "''${filename}" +} + +################################################################################ +# Get address of the x-y-z key combination! +# Creates the key if it does not already exist. +################################################################################ +function build_x_y_z_address { + + # Function arguments. + local prefix=$1 + local x_n=$2 + local y_n=$3 + local z_n=$4 + + local filename addr + filename="$(create_x_y_z_key_files "''${prefix}" "''${x_n}" "''${y_n}" "''${z_n}")" + addr="''${filename}.addr" + # Only create if not already there! + if ! test -f "''${addr}" + then + local vkey="''${filename}".vkey + ${cardano-cli}/bin/cardano-cli address build \ + --testnet-magic ${toString testnet_magic} \ + --payment-verification-key-file "''${vkey}" \ + > "''${addr}" + fi + ${coreutils}/bin/cat "''${addr}" +} +'' + diff --git a/nix/workbench/workload/voting.nix b/nix/workbench/workload/voting.nix index b6246b76ac0..83228cb9e1e 100644 --- a/nix/workbench/workload/voting.nix +++ b/nix/workbench/workload/voting.nix @@ -141,6 +141,11 @@ in '' # desired_producer_tps: ${toString desired_producer_tps} # desired_producer_sleep: ${toString desired_producer_sleep} +${import ./utils/keys.nix + { inherit coreutils jq cardano-cli testnet_magic; + } +} + ${import ./utils/utxo.nix { inherit coreutils cardano-cli jq testnet_magic; inherit outs_per_split_transaction funds_submit_tries; @@ -151,108 +156,6 @@ ${import ./utils/utxo.nix } } -################################################################################ -# Hack: Given a node "i" and proposal number and a DRep number create always the -# same address keys. -# Only supports up to 99 nodes, 9999 proposals and 999999 DReps by adding the -# missing Hex chars. -# Returns the file path without the extensions (the ".skey" or ".vkey" part). -################################################################################ -function create_node_prop_drep_key_files { - - # Function arguments. - local node_str=$1 # String for the key file name (not for the socket). - local node_i=$2 # This "i" is part of the node name ("node-i"). - local prop_i=$3 - local drep_i=$4 - - local filename=./"''${node_str}"-prop-"''${prop_i}"-drep-"''${drep_i}" - # Now with the extensions. - local skey="''${filename}".skey - local vkey="''${filename}".vkey - - # Only create if not already there! - if ! test -f "''${vkey}" - then - ${jq}/bin/jq --null-input \ - --argjson node_i "''${node_i}" \ - --argjson prop_i "''${prop_i}" \ - --argjson drep_i "''${drep_i}" \ - ' - {"type": "PaymentSigningKeyShelley_ed25519", - "description": "Payment Signing Key", - "cborHex": ( - "5820b02868d722df021278c78be3b7363759b37f5852b8747b488bab" - + (if $node_i <= 9 - then ("0" + ($node_i | tostring)) - elif $node_i >= 10 and $node_i <= 99 - then ( $node_i | tostring) - else (error ("Node ID above 99")) - end - ) - + (if $prop_i <= 9 - then ( "000" + ($prop_i | tostring)) - elif $prop_i >= 10 and $prop_i <= 99 - then ( "00" + ($prop_i | tostring)) - elif $prop_i >= 100 and $prop_i <= 999 - then ( "0" + ($prop_i | tostring)) - elif $prop_i >= 1000 and $prop_i <= 9999 - then ( ($prop_i | tostring)) - else (error ("Proposal ID above 9999")) - end - ) - + (if $drep_i <= 9 - then ( "00000" + ($drep_i | tostring)) - elif $drep_i >= 10 and $drep_i <= 99 - then ( "0000" + ($drep_i | tostring)) - elif $drep_i >= 100 and $drep_i <= 999 - then ( "000" + ($drep_i | tostring)) - elif $drep_i >= 1000 and $drep_i <= 9999 - then ( "00" + ($drep_i | tostring)) - elif $drep_i >= 10000 and $drep_i <= 99999 - then ( "0" + ($drep_i | tostring)) - elif $drep_i >= 100000 and $drep_i <= 999999 - then ( ($drep_i | tostring)) - else (error ("DRep ID above 999999")) - end - ) - ) - } - ' \ - > "''${skey}" - ${cardano-cli}/bin/cardano-cli conway key verification-key \ - --signing-key-file "''${skey}" \ - --verification-key-file "''${vkey}" - fi - ${coreutils}/bin/echo "''${filename}" -} - -################################################################################ -# Get address of the node-proposal-drep combination! -################################################################################ -function build_node_prop_drep_address { - - # Function arguments. - local node_str=$1 # String for the key file name (not for the socket). - local node_i=$2 # This "i" is part of the node name ("node-i"). - local prop_i=$3 - local drep_i=$4 - - local filename addr - filename="$(create_node_prop_drep_key_files "''${node_str}" "''${node_i}" "''${prop_i}" "''${drep_i}")" - addr="''${filename}.addr" - # Only create if not already there! - if ! test -f "''${addr}" - then - local vkey="''${filename}".vkey - ${cardano-cli}/bin/cardano-cli address build \ - --testnet-magic ${toString testnet_magic} \ - --payment-verification-key-file "''${vkey}" \ - > "''${addr}" - fi - ${coreutils}/bin/cat "''${addr}" -} - ################################################################################ # Evenly distribute the "utxo_*key" genesis funds to all producer nodes. # To be called before `governance_funds_producer`. @@ -289,7 +192,7 @@ function governance_funds_genesis { )" local producer_addr # Drep 0 is No DRep (funds for the node). - producer_addr="$(build_node_prop_drep_address "''${producer_name}" "''${producer_i}" 0 0)" + producer_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" 0 0)" producers_addrs_array+=("''${producer_addr}") ${coreutils}/bin/echo "governance_funds_genesis: Splitting to: ''${producer_name} - ''${producer_i} - 0 - (''${producer_addr})" done @@ -326,9 +229,9 @@ function governance_funds_producer { ../../node-specs.json \ )" local producer_addr producer_vkey producer_skey - producer_addr="$(build_node_prop_drep_address "''${producer_name}" "''${producer_i}" 0 0)" - producer_vkey="$(create_node_prop_drep_key_files "''${producer_name}" "''${producer_i}" 0 0)".vkey - producer_skey="$(create_node_prop_drep_key_files "''${producer_name}" "''${producer_i}" 0 0)".skey + producer_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" 0 0)" + producer_vkey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" 0 0)".vkey + producer_skey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" 0 0)".skey # Wait for initial funds to arrive! ${coreutils}/bin/echo "governance_funds_producer: Wait for funds: $(${coreutils}/bin/date --rfc-3339=seconds)" @@ -349,7 +252,7 @@ function governance_funds_producer { for prop_i in {1..${toString proposals_count}} do local producer_prop_addr - producer_prop_addr="$(build_node_prop_drep_address "''${producer_name}" "''${producer_i}" "''${prop_i}" 0)" + producer_prop_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" "''${prop_i}" 0)" producer_prop_addr_array+=("''${producer_prop_addr}") ${coreutils}/bin/echo "governance_funds_producer: Splitting to: ''${producer_name} - ''${producer_i} - ''${prop_i} - ''${producer_prop_addr}" done @@ -374,8 +277,8 @@ function governance_funds_producer { do local producer_prop_vkey producer_prop_skey - producer_prop_vkey="$(create_node_prop_drep_key_files "''${producer_name}" "''${producer_i}" "''${prop_i}" 0)".vkey - producer_prop_skey="$(create_node_prop_drep_key_files "''${producer_name}" "''${producer_i}" "''${prop_i}" 0)".skey + producer_prop_vkey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" "''${prop_i}" 0)".vkey + producer_prop_skey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" "''${prop_i}" 0)".skey local producer_dreps_addrs_array=() local drep_step=0 @@ -385,7 +288,7 @@ function governance_funds_producer { do local producer_drep_addr actual_drep="$((drep_step + i))" - producer_drep_addr="$(build_node_prop_drep_address "''${producer_name}" "''${producer_i}" "''${prop_i}" "''${actual_drep}")" + producer_drep_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" "''${prop_i}" "''${actual_drep}")" producer_dreps_addrs_array+=("''${producer_drep_addr}") ${coreutils}/bin/echo "governance_funds_producer: Splitting to: ''${producer_name} - ''${producer_i} - ''${prop_i} - ''${actual_drep} - ''${producer_drep_addr}" done @@ -522,8 +425,8 @@ function governance_create_withdrawal { socket_path="$(get_socket_path "''${node_str}")" local node_drep_skey node_drep_addr - node_drep_skey="$(create_node_prop_drep_key_files "''${node_str}" "''${node_i}" 0 "''${drep_i}")".skey - node_drep_addr="$(build_node_prop_drep_address "''${node_str}" "''${node_i}" 0 "''${drep_i}")" + node_drep_skey="$(create_x_y_z_key_files "''${node_str}" "''${node_i}" 0 "''${drep_i}")".skey + node_drep_addr="$(build_x_y_z_address "''${node_str}" "''${node_i}" 0 "''${drep_i}")" # Funds needed for this governance action ? local action_deposit @@ -719,8 +622,8 @@ function governance_vote_proposal { for drep_i in ''${dreps_array[*]} do local node_drep_skey node_drep_addr - node_drep_skey="$(create_node_prop_drep_key_files "''${node_str}" "''${node_i}" "''${prop_i}" "''${drep_i}")".skey - node_drep_addr="$(build_node_prop_drep_address "''${node_str}" "''${node_i}" "''${prop_i}" "''${drep_i}")" + node_drep_skey="$(create_x_y_z_key_files "''${node_str}" "''${node_i}" "''${prop_i}" "''${drep_i}")".skey + node_drep_addr="$(build_x_y_z_address "''${node_str}" "''${node_i}" "''${prop_i}" "''${drep_i}")" # UTxO are created for 1 vote per transaction so all runs have the same # number of UTxOs. We grab the funds from the first address/UTxO. if test -z "''${funds_tx-}" From 07ebb273e97eaf02e404369b9f50a9beebf77c53 Mon Sep 17 00:00:00 2001 From: Federico Mastellone Date: Tue, 24 Jun 2025 22:38:00 +0000 Subject: [PATCH 3/6] wb | make workloads compatible with new cardano-cli version --- nix/workbench/workload/utils/utxo.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/workbench/workload/utils/utxo.nix b/nix/workbench/workload/utils/utxo.nix index b31183193a9..9aaa0519cd4 100644 --- a/nix/workbench/workload/utils/utxo.nix +++ b/nix/workbench/workload/utils/utxo.nix @@ -57,6 +57,7 @@ function calculate_next_utxo { tx_id="$( \ ${cardano-cli}/bin/cardano-cli conway transaction txid \ --tx-file "''${tx_signed}" \ + --output-text \ )" # View transaction as JSON and get index of FIRST output containing "$addr". ${cardano-cli}/bin/cardano-cli debug transaction view \ @@ -156,6 +157,7 @@ function is_tx_in_mempool { tx_id="$( \ ${cardano-cli}/bin/cardano-cli conway transaction txid \ --tx-file "''${tx_signed}" \ + --output-text \ )" ${cardano-cli}/bin/cardano-cli conway query tx-mempool \ tx-exists "''${tx_id}" \ @@ -453,8 +455,9 @@ function wait_proposal_id { # Get proposal's "txId" from the "--tx-file". local tx_id tx_id="$( \ - ${cardano-cli}/bin/cardano-cli conway transaction txid \ - --tx-file "''${tx_signed}" \ + ${cardano-cli}/bin/cardano-cli conway transaction txid \ + --tx-file "''${tx_signed}" \ + --output-text \ )" local contains_proposal="false" From 5498f856e7699bab4e81e22df0e8ae64351308d1 Mon Sep 17 00:00:00 2001 From: Federico Mastellone Date: Thu, 16 Oct 2025 16:13:58 +0000 Subject: [PATCH 4/6] SQUASH --- nix/workbench/workload/voting.nix | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/nix/workbench/workload/voting.nix b/nix/workbench/workload/voting.nix index 83228cb9e1e..8faf587fa2b 100644 --- a/nix/workbench/workload/voting.nix +++ b/nix/workbench/workload/voting.nix @@ -11,6 +11,7 @@ let bashInteractive = pkgs.bashInteractive; coreutils = pkgs.coreutils; + wget = pkgs.wget; jq = pkgs.jq; # Avoid rebuilding on every commit because of `set-git-rev`. cardano-cli = pkgs.cardanoNodePackages.cardano-cli.passthru.noGitRev; @@ -347,6 +348,14 @@ function governance_create_constitution { | ${jq}/bin/jq -r \ '.nextRatifyState.nextEnactState.prevGovActionIds' + # Copy base-line anchor. + ${wget}/bin/wget \ + --output-document ./constitution.anchor.json \ + "https://raw.githubusercontent.com/cardano-foundation/CIPs/master/CIP-0100/cip-0100.common.schema.json" + # Calculate anchor hash. + ${cardano-cli}/bin/cardano-cli hash anchor-data \ + --file-text ./constitution.anchor.json \ + --out-file ./constitution.anchor.hash # Create dummy constitution. ${coreutils}/bin/echo "My Constitution: free mate and asado" \ > ./constitution.txt @@ -368,7 +377,7 @@ function governance_create_constitution { ${cardano-cli}/bin/cardano-cli conway governance action create-constitution \ --testnet \ --anchor-url "https://raw.githubusercontent.com/cardano-foundation/CIPs/master/CIP-0100/cip-0100.common.schema.json" \ - --anchor-data-hash "9d99fbca260b2d77e6d3012204e1a8658f872637ae94cdb1d8a53f4369400aa9" \ + --anchor-data-hash "$(${coreutils}/bin/cat ./constitution.anchor.hash)" \ --constitution-url "https://ipfs.io/ipfs/Qmdo2J5vkGKVu2ur43PuTrM7FdaeyfeFav8fhovT6C2tto" \ --constitution-hash "$(${coreutils}/bin/cat ./constitution.hash)" \ --constitution-script-hash "$(${coreutils}/bin/cat ./guardrails-script.hash)" \ @@ -438,12 +447,21 @@ function governance_create_withdrawal { local funds_tx funds_tx="$(get_address_utxo_expected_id "''${node_drep_addr}")" + # Copy base-line anchor. + ${wget}/bin/wget \ + --output-document ./treasury-withdrawal.json \ + "https://raw.githubusercontent.com/cardano-foundation/CIPs/master/CIP-0108/examples/treasury-withdrawal.jsonld" + # Calculate anchor hash. + ${cardano-cli}/bin/cardano-cli hash anchor-data \ + --file-text ./treasury-withdrawal.json \ + --out-file ./treasury-withdrawal.hash + local tx_filename=./create-withdrawal."''${node_str}"."''${drep_i}" # Create action. ${cardano-cli}/bin/cardano-cli conway governance action create-treasury-withdrawal \ --testnet \ --anchor-url "https://raw.githubusercontent.com/cardano-foundation/CIPs/master/CIP-0108/examples/treasury-withdrawal.jsonld" \ - --anchor-data-hash "311b148ca792007a3b1fee75a8698165911e306c3bc2afef6cf0145ecc7d03d4" \ + --anchor-data-hash "$(${coreutils}/bin/cat ./treasury-withdrawal.hash)" \ --governance-action-deposit "''${action_deposit}" \ --transfer 50 \ --deposit-return-stake-verification-key-file ../../genesis/cache-entry/stake-delegators/"delegator''${node_i}"/staking.vkey \ From 61c74847b1a5167e9779f6b167b9b076d07fe9f9 Mon Sep 17 00:00:00 2001 From: Federico Mastellone Date: Wed, 15 Oct 2025 19:02:28 +0000 Subject: [PATCH 5/6] wb | allow workloads in "idle" scenarios (ignores wait_pools flag) --- nix/workbench/scenario.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nix/workbench/scenario.sh b/nix/workbench/scenario.sh index 15e07642989..81589e98c40 100644 --- a/nix/workbench/scenario.sh +++ b/nix/workbench/scenario.sh @@ -35,7 +35,15 @@ case "$op" in scenario_setup_exit_trap "$dir" # Trap start ############ + for workload in $(jq -r '.workloads[] | select(.before_nodes == true) | .name' "$dir"/profile.json) + do + backend start-workload-by-name "$dir" "$workload" + done backend start-nodes "$dir" + for workload in $(jq -r '.workloads[] | select(.before_nodes == false) | .name' "$dir"/profile.json) + do + backend start-workload-by-name "$dir" "$workload" + done backend wait-pools-stopped "$dir" # Trap end ########## From dec0dcf382e6b39793df0f6c344fe930900d5745 Mon Sep 17 00:00:00 2001 From: Federico Mastellone Date: Wed, 15 Oct 2025 18:54:54 +0000 Subject: [PATCH 6/6] WIP --- bench/cardano-profile/cardano-profile.cabal | 1 + .../Benchmarking/Profile/Playground.hs | 20 +- .../Benchmarking/Profile/Workload/Hydra.hs | 36 +++ nix/workbench/workload/hydra.nix | 256 ++++++++++++++++++ 4 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 bench/cardano-profile/src/Cardano/Benchmarking/Profile/Workload/Hydra.hs create mode 100644 nix/workbench/workload/hydra.nix diff --git a/bench/cardano-profile/cardano-profile.cabal b/bench/cardano-profile/cardano-profile.cabal index a9550d9a2af..5e2a64d97c1 100644 --- a/bench/cardano-profile/cardano-profile.cabal +++ b/bench/cardano-profile/cardano-profile.cabal @@ -71,6 +71,7 @@ library , Cardano.Benchmarking.Profile.Vocabulary , Cardano.Benchmarking.Profile.Types , Cardano.Benchmarking.Profile.Workload.CGroupMemory + , Cardano.Benchmarking.Profile.Workload.Hydra , Cardano.Benchmarking.Profile.Workload.Latency , Cardano.Benchmarking.Profile.Workload.Voting build-depends: base >=4.12 && <5 diff --git a/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Playground.hs b/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Playground.hs index 7fd8a68f46f..1c000f1c321 100644 --- a/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Playground.hs +++ b/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Playground.hs @@ -21,7 +21,8 @@ import qualified Cardano.Benchmarking.Profile.Builtin.Miniature as M import qualified Cardano.Benchmarking.Profile.Primitives as P import qualified Cardano.Benchmarking.Profile.Types as Types import qualified Cardano.Benchmarking.Profile.Vocabulary as V -import qualified Cardano.Benchmarking.Profile.Workload.Voting as W +import qualified Cardano.Benchmarking.Profile.Workload.Hydra as WH +import qualified Cardano.Benchmarking.Profile.Workload.Voting as WV -------------------------------------------------------------------------------- @@ -107,6 +108,21 @@ profilesNoEraPlayground = -- Voting profiles. , voting & P.name "development-voting" . P.dreps 1000 - . P.workloadAppend W.votingWorkloadx2 + . P.workloadAppend WV.votingWorkloadx2 . P.traceForwardingOn . P.newTracing + -- Hydra profiles. + , P.empty & P.name "development-hydra" + . P.idle -- No `tx-generator`. + {-- For tx-generators use with a duration: + . P.fixedLoaded . V.valueLocal + --} + . P.workloadAppend WH.hydraWorkload + . P.uniCircle . V.hosts 2 . P.loopback + . V.genesisVariantLatest . V.timescaleCompressed + . V.datasetEmpty -- No UTxO. + -- One for a future tx-generator plus one for each node. + . P.poolBalance 1000000000000000 . P.funds 40000000000000 . P.utxoKeys 3 + . P.traceForwardingOn . P.newTracing + . P.analysisOff -- Nothing to analyze. ] + diff --git a/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Workload/Hydra.hs b/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Workload/Hydra.hs new file mode 100644 index 00000000000..4d302260570 --- /dev/null +++ b/bench/cardano-profile/src/Cardano/Benchmarking/Profile/Workload/Hydra.hs @@ -0,0 +1,36 @@ +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE OverloadedStrings #-} + +-------------------------------------------------------------------------------- + +module Cardano.Benchmarking.Profile.Workload.Hydra ( + hydraWorkload +) where + +-------------------------------------------------------------------------------- + +import Prelude +-- Package: aeson. +import qualified Data.Aeson as Aeson +import qualified Data.Aeson.KeyMap as KeyMap +-- Package: self. +import qualified Cardano.Benchmarking.Profile.Types as Types + +-------------------------------------------------------------------------------- + +hydraWorkload :: Types.Workload +hydraWorkload = Types.Workload { + Types.workloadName = "hydra" + , Types.parameters = KeyMap.fromList [ + ("baseport", Aeson.Number 31000) + -- Up to (maximun) 9 hydra heads per deployed Cardano node. + , ("heads_per_cardano_node", Aeson.Number 2) + ] + , Types.entrypoints = Types.Entrypoints { + Types.pre_generator = Nothing + , Types.producers = "hydra" + } + , Types.before_nodes = False + , Types.wait_pools = True +} + diff --git a/nix/workbench/workload/hydra.nix b/nix/workbench/workload/hydra.nix new file mode 100644 index 00000000000..c1c54e12d38 --- /dev/null +++ b/nix/workbench/workload/hydra.nix @@ -0,0 +1,256 @@ +{ pkgs +, profile +, nodeSpecs +, workload +}: + +let + + # supervisor: make playground-development-hydra + # nomadexec: nix-shell -A 'workbench-shell' --max-jobs 8 --cores 0 --show-trace --argstr profileName development-hydra-coay --argstr backendName nomadexec + + # Packages + ########## + + bashInteractive = pkgs.bashInteractive; + coreutils = pkgs.coreutils; + jq = pkgs.jq; + # Avoid rebuilding on every commit because of `set-git-rev`. + cardano-cli = pkgs.cardanoNodePackages.cardano-cli.passthru.noGitRev; + # Hyra (Release 1.0.0). + commit = "b5e33b55e9fba442c562f82cec6c36b1716d9847"; + flake = (__getFlake "github:cardano-scaling/hydra/${commit}"); + hydra = flake.packages.${builtins.currentSystem}.hydra-node; + + # Parameters + ############ + + testnet_magic = 42; + baseport = workload.parameters.baseport or 31000; + heads_per_cardano_node = workload.parameters.heads_per_cardano_node or 1; + # Filter producers from "node-specs.json". + producers = + builtins.filter + (nodeSpec: nodeSpec.isProducer) + (builtins.attrValues nodeSpecs) + ; + # Construct an "array" with node producers to use in BASH `for` loops. + producers_bash_array = + "(" + + (builtins.concatStringsSep + " " + (builtins.map + (x: "\"" + x.name + "\"") + producers + ) + ) + + ")" + ; + +in '' +${import ./utils/keys.nix + { inherit coreutils jq cardano-cli testnet_magic; + } +} + +${import ./utils/utxo.nix + { inherit coreutils jq cardano-cli testnet_magic; + } +} + +# Waits for all jobs to finish independent of their exit status! +# Returns the first error code obtained if any one fails. +wait_all () { + wait_internal 0 "false" "$@" +} + +# Waits for any job to fail or all to be OK! +# All processes are killed as soon as one fails! +# Returns the first error code obtained if any one fails. +wait_kill_em_all () { + # We are scanning the scene in the city tonite ... searching, seek and destroy + wait_internal 0 "true" "$@" +} + +# Returns 0/success if no process fails, else returns the first error code +# obtained that is not zero. +wait_internal () { + # The initial status for recursion, on first call it should always be zero! + local initial_exit_status=''${1}; shift + # Should all processes be killed as soon as one fails? Else waits for all + # processes to finish independent of their exit status. + local kill_em_all=''${1}; shift + # Array of processes IDs or a jobs specifications. + # If ID is a job specification, waits for all processes in that job's pipeline + local processes_ids=("$@") + # Are there any processes left to wait for ? + if test -n "''${processes_ids[*]:-}" + then + local wait_exit_status + local exited_process_id + # Wait for a single job from the list of processes and returns its exit + # status and the process or job identifier of the job for which the exit + # status is returned is assigned to the variable provided by `-p VAR`. + wait -n -p exited_process_id "''${processes_ids[@]}" + wait_exit_status=$? + # Only if the exit status to return is still zero we care about the + # new exit status. + if test "''${initial_exit_status}" -eq 0 + then + initial_exit_status="''${wait_exit_status}" + fi + # Create a wew array without the newly exited process. + local processes_ids_p=() + for p in "''${processes_ids[@]}" + do + if test "''${p}" != "''${exited_process_id}" + then + processes_ids_p+=("''${p}") + fi + done + # Are there still any processes left to wait for ? + if test -n "''${processes_ids_p[*]:-}" + then + # Keep waiting or kill 'em all ?' + if ! test "''${wait_exit_status}" -eq 0 && test "''${kill_em_all}" = "true" + then + kill "''${processes_p[@]}" 2>/dev/null || true + return "''${wait_exit_status}" + else + # Recursion, wiiiiiiiii! + wait_internal \ + "''${initial_exit_status}" "''${kill_em_all}" "''${processes_ids_p[@]}" + fi + else + return "''${initial_exit_status}" + fi + else + return 0 + fi +} + +############################################################################### +# Workload entrypoint ######################################################### +############################################################################### + +function hydra { + # Run the workflow for each deployed producer node. + local producers=${toString producers_bash_array} + local producers_jobs_array=() + for producer_name in ''${producers[*]} + do + # Checks if the producer node is deployed in this machine. + if test -d "../../''${producer_name}" + then + hydra_producer "''${producer_name}" & + producers_jobs_array+=("$!") + fi + done + wait_all "''${producers_jobs_array[@]}" +} + +############################################################################### +# Producer node entrypoint #################################################### +############################################################################### + +function hydra_producer { + # Function arguments. + local producer_name=$1 # node name / folder to find the socket to use. + + local producer_i + # Lookup producer numeric index by name. + producer_i="$( \ + ${jq}/bin/jq --raw-output \ + --arg keyName "''${producer_name}" \ + '.[$keyName].i' \ + ../../node-specs.json \ + )" + + msg "Starting producer \"''${producer_name}\" (''${producer_i})" + + # Parameters for this producer node: + # - Where to obtain the genesis funds for this producer. + genesis_funds_vkey="../../genesis/cache-entry/utxo-keys/utxo$((producer_i + 1)).vkey" + genesis_funds_skey="../../genesis/cache-entry/utxo-keys/utxo$((producer_i + 1)).skey" + # - IP address and port. + producer_ip="127.0.0.1" + producer_port="$((${toString baseport} + producer_i * 10))" + + msg "Producer params: ''${genesis_funds_vkey} - ''${genesis_funds_skey} - ''${producer_port}" + + # Split funds to each of this producer's head. + producer_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" 0 0)" + producer_vkey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" 0 0)".vkey + producer_skey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" 0 0)".skey + local producer_head_addr_array=() + for head_i in {1..${toString heads_per_cardano_node}} + do + local producer_head_addr + producer_head_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" "''${head_i}" 0)" + producer_head_addr_array+=("''${producer_head_addr}") + ${coreutils}/bin/echo "hydra_producer: Splitting to: ''${producer_name} - ''${producer_i} - ''${head_i} - ''${producer_head_addr}" + done + # Split (no need to wait for the funds or re-submit, function takes care)! + funds_from_to \ + "''${producer_name}" \ + "''${genesis_funds_vkey}" \ + "''${genesis_funds_skey}" \ + 0 \ + 0 \ + "''${producer_head_addr_array[@]}" + + for head_i in $(seq 1 ${toString heads_per_cardano_node}); do + hydra_producer_head \ + "''${producer_name}" \ + "''${producer_i}" \ + "''${producer_port}" \ + "''${head_i}" + done +} + +############################################################################### +# Producer-head entrypoint #################################################### +############################################################################### + +function hydra_producer_head { + # Function arguments. + local producer_name=$1 # node name / folder to find the socket to use. + local producer_i=$2 + local producer_port=$3 + local head_i=$4 + + msg "Starting head: Producer \"''${producer_name}\" (''${producer_i}) head ''${head_i}" + + # Head parameters. + head_vkey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" "''${head_i}" 0)".vkey + head_skey="$(create_x_y_z_key_files "''${producer_name}" "''${producer_i}" "''${head_i}" 0)".skey + head_addr="$(build_x_y_z_address "''${producer_name}" "''${producer_i}" "''${head_i}" 0)" + # - IP address and port. + head_ip="127.0.0.1" + head_port="$((producer_port + head_i))" + + msg "Head parameters: ''${head_port}" + + wait_any_utxo "''${producer_name}" "''${head_addr}" + + ${hydra}/bin/hydra-node \ + --node-id "''${producer_i}" \ + --listen "''${head_ip}:''${head_port}" \ + --advertise "''${head_ip}:''${head_port}" \ + --cardano-verification-key "''${genesis_funds_vkey}" \ + --cardano-signing-key "''${genesis_funds_skey}" \ + --testnet-magic ${toString testnet_magic} \ + --node-socket "../../''${producer_name}/node.socket" +} + +############################################################################### +# Utils ####################################################################### +############################################################################### + +function msg { + # Outputs to stdout, unbuffered if not the message may be lost! + ${coreutils}/bin/stdbuf -o0 \ + ${bashInteractive}/bin/sh -c \ + "${coreutils}/bin/echo -e \"$(${coreutils}/bin/date --rfc-3339=seconds): $1\"" +} +''