Skip to content

Commit 39bc313

Browse files
committed
Add verbose mode with dependency tracking for test debugging
Add -v and -vv flags to print complete request chains that can be piped directly to a handler for manual debugging and reproduction. - Introduces DependencyTracker to build transitive dependency chains from ref usage - Switches to pflag for better CLI flag support - Adds unit tests for dependency tracking logic
1 parent b6775e5 commit 39bc313

File tree

9 files changed

+642
-41
lines changed

9 files changed

+642
-41
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ test:
2222
@echo "Running runner unit tests..."
2323
go test -v ./runner/...
2424
@echo "Running conformance tests with mock handler..."
25-
$(RUNNER_BIN) -handler $(MOCK_HANDLER_BIN)
25+
$(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv
2626

2727
clean:
2828
@echo "Cleaning build artifacts..."

README.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,38 @@ make runner
6666

6767
The runner automatically detects and recovers from crashed/unresponsive handlers, allowing remaining tests to continue.
6868

69+
#### Verbose Flags
70+
71+
- **`-v, --verbose`**: Shows request chains and responses for **failed tests only**
72+
- **`-vv`**: Shows request chains and responses for **all tests** (passed and failed)
73+
74+
The request chains printed by verbose mode can be directly piped to the handler binary for manual debugging:
75+
76+
```bash
77+
# Example output from -vv mode:
78+
# ✓ chain#4 (Get active chain reference from chainstate manager)
79+
#
80+
# Request chain
81+
# ────────────────────────────────────────
82+
# {"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
83+
# {"id":"chain#2","method":"btck_chainstate_manager_create","params":{"context":"$context_ref"},"ref":"$chainstate_manager_ref"}
84+
# {"id":"chain#4","method":"btck_chainstate_manager_get_active_chain","params":{"chainstate_manager":"$chainstate_manager_ref"},"ref":"$chain_ref"}
85+
#
86+
# Response:
87+
# ────────────────────────────────────────
88+
# {"result":"$chain_ref"}
89+
90+
# Copy the request chain and pipe it to your handler for debugging:
91+
echo '{"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
92+
{"id":"chain#2","method":"btck_chainstate_manager_create","params":{"context":"$context_ref"},"ref":"$chainstate_manager_ref"}
93+
{"id":"chain#4","method":"btck_chainstate_manager_get_active_chain","params":{"chainstate_manager":"$chainstate_manager_ref"},"ref":"$chain_ref"}' | ./path/to/your/handler
94+
```
95+
96+
This is particularly useful for:
97+
- Reproducing test failures independently
98+
- Debugging handler behavior without running the full test suite
99+
- Testing individual operations in isolation
100+
69101
### Testing the Runner
70102

71103
Build and test the runner:

cmd/runner/main.go

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,36 @@ package main
22

33
import (
44
"context"
5-
"flag"
65
"fmt"
76
"io/fs"
87
"os"
98
"sort"
109
"strings"
1110
"time"
1211

12+
"github.com/spf13/pflag"
1313
"github.com/stringintech/kernel-bindings-tests/runner"
1414
"github.com/stringintech/kernel-bindings-tests/testdata"
1515
)
1616

1717
func main() {
18-
handlerPath := flag.String("handler", "", "Path to handler binary")
19-
handlerTimeout := flag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
20-
timeout := flag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)")
21-
flag.Parse()
18+
handlerPath := pflag.String("handler", "", "Path to handler binary")
19+
handlerTimeout := pflag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
20+
timeout := pflag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)")
21+
verboseCount := pflag.CountP("verbose", "v", "Verbose mode: -v shows all requests needed to reproduce failed tests, plus received/expected responses; -vv shows this for all tests (passed and failed)")
22+
pflag.Parse()
23+
24+
// Convert verbose count to verbosity level
25+
verbosity := runner.VerbosityQuiet
26+
if *verboseCount >= 2 {
27+
verbosity = runner.VerbosityAlways
28+
} else if *verboseCount == 1 {
29+
verbosity = runner.VerbosityOnFailure
30+
}
2231

2332
if *handlerPath == "" {
24-
fmt.Fprintf(os.Stderr, "Error: -handler flag is required\n")
25-
flag.Usage()
33+
fmt.Fprintf(os.Stderr, "Error: --handler flag is required\n")
34+
pflag.Usage()
2635
os.Exit(1)
2736
}
2837

@@ -69,7 +78,7 @@ func main() {
6978
}
7079

7180
// Run suite
72-
result := testRunner.RunTestSuite(ctx, *suite)
81+
result := testRunner.RunTestSuite(ctx, *suite, verbosity)
7382
printResults(suite, result)
7483

7584
totalPassed += result.PassedTests

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
module github.com/stringintech/kernel-bindings-tests
22

33
go 1.23
4+
5+
require github.com/spf13/pflag v1.0.10

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
2+
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=

runner/dependency_tracker.go

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package runner
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
)
7+
8+
// stateMutatingMethods contains methods that mutate internal state.
9+
// Tests using these methods are assumed to affect all subsequent tests
10+
// and are included in dependency chains printed in verbose mode.
11+
var stateMutatingMethods = map[string]bool{
12+
"btck_chainstate_manager_process_block": true,
13+
}
14+
15+
// DependencyTracker manages test dependencies and builds request chains for verbose output.
16+
// It tracks both explicit ref dependencies and implicit state dependencies.
17+
type DependencyTracker struct {
18+
// refCreators maps reference names to the test index that created them
19+
refCreators map[string]int
20+
21+
// statefulRefs tracks refs created by stateful methods (btck_context_create, btck_chainstate_manager_create).
22+
// Tests using these refs depend on mutable state.
23+
statefulRefs map[string]bool
24+
25+
// depChains maps test index to its dependency chain (tests it depends on via ref usage)
26+
depChains map[int][]int
27+
28+
// stateDependencies is a cumulative list of all tests affecting state (state-mutating
29+
// tests and their complete dependency chains)
30+
stateDependencies []int
31+
}
32+
33+
// NewDependencyTracker creates a new dependency tracker
34+
func NewDependencyTracker() *DependencyTracker {
35+
return &DependencyTracker{
36+
refCreators: make(map[string]int),
37+
statefulRefs: make(map[string]bool),
38+
depChains: make(map[int][]int),
39+
stateDependencies: []int{},
40+
}
41+
}
42+
43+
// BuildDependenciesForTest analyzes a test's parameters to build its complete transitive
44+
// dependency chain. When a test uses refs created by earlier tests, this records all direct
45+
// dependencies (tests that created those refs) and indirect dependencies (their dependencies).
46+
// Must be called after all previous tests have been processed.
47+
func (dt *DependencyTracker) BuildDependenciesForTest(testIndex int, test *TestCase) {
48+
// Build dependency chain for current test based on refs it uses
49+
var parentChains [][]int
50+
for _, ref := range extractRefsFromParams(test.Request.Params) {
51+
if creatorIdx, exists := dt.refCreators[ref]; exists {
52+
// Add the creator as a direct dependency
53+
parentChains = append(parentChains, []int{creatorIdx})
54+
// Add transitive dependencies (creator's dependencies)
55+
if chain, hasChain := dt.depChains[creatorIdx]; hasChain {
56+
parentChains = append(parentChains, chain)
57+
}
58+
} else {
59+
panic(fmt.Sprintf("test %d (%s) uses undefined reference %s - no prior test created this ref",
60+
testIndex, test.Request.ID, ref))
61+
}
62+
}
63+
dt.depChains[testIndex] = mergeSortedUnique(parentChains...)
64+
}
65+
66+
// OnTestExecuted is called after a test executes successfully. It tracks the ref
67+
// created by the test, marks it as stateful if needed, and updates state dependencies
68+
// for state-mutating methods.
69+
func (dt *DependencyTracker) OnTestExecuted(testIndex int, test *TestCase) {
70+
if ref := extractRefFromExpected(test.ExpectedResponse); ref != "" {
71+
dt.refCreators[ref] = testIndex
72+
73+
// Mark refs from stateful methods
74+
if test.Request.Method == "btck_context_create" || test.Request.Method == "btck_chainstate_manager_create" {
75+
dt.statefulRefs[ref] = true
76+
}
77+
}
78+
79+
// Track state-mutating tests and their dependencies
80+
if stateMutatingMethods[test.Request.Method] {
81+
mutatorChain := append(dt.depChains[testIndex], testIndex)
82+
dt.stateDependencies = mergeSortedUnique(dt.stateDependencies, mutatorChain)
83+
}
84+
}
85+
86+
// BuildRequestChain builds the complete dependency chain for a test
87+
func (dt *DependencyTracker) BuildRequestChain(testIndex int, allTests []TestCase) []int {
88+
refDepChain := dt.depChains[testIndex]
89+
90+
// Only include state dependencies if the test's dep chain contains any stateful refs
91+
if dt.testUsesStatefulRefs(testIndex, allTests) {
92+
return mergeSortedUnique(refDepChain, dt.stateDependencies)
93+
}
94+
95+
return refDepChain
96+
}
97+
98+
// testUsesStatefulRefs checks if a test's dependency chain includes any stateful refs
99+
func (dt *DependencyTracker) testUsesStatefulRefs(testIndex int, allTests []TestCase) bool {
100+
// Check all tests in the dependency chain
101+
for _, depIdx := range dt.depChains[testIndex] {
102+
for _, ref := range extractRefsFromParams(allTests[depIdx].Request.Params) {
103+
if dt.statefulRefs[ref] {
104+
return true
105+
}
106+
}
107+
}
108+
109+
// Check the test itself
110+
for _, ref := range extractRefsFromParams(allTests[testIndex].Request.Params) {
111+
if dt.statefulRefs[ref] {
112+
return true
113+
}
114+
}
115+
return false
116+
}
117+
118+
// extractRefFromExpected extracts a reference name from the expected result if it's a
119+
// string starting with "$". Returns empty string if not a reference.
120+
func extractRefFromExpected(expected Response) string {
121+
var resultStr string
122+
if err := json.Unmarshal(expected.Result, &resultStr); err != nil {
123+
return ""
124+
}
125+
if len(resultStr) > 1 && resultStr[0] == '$' {
126+
return resultStr
127+
}
128+
return ""
129+
}
130+
131+
// extractRefsFromParams extracts all reference names from params JSON.
132+
func extractRefsFromParams(params json.RawMessage) []string {
133+
var refs []string
134+
135+
// Parse params as a generic structure
136+
var paramsMap map[string]interface{}
137+
if err := json.Unmarshal(params, &paramsMap); err != nil {
138+
return refs
139+
}
140+
141+
// Find all string values starting with "$" (assumes refs are at first level only)
142+
for _, value := range paramsMap {
143+
if str, ok := value.(string); ok && len(str) > 1 && str[0] == '$' {
144+
refs = append(refs, str)
145+
}
146+
}
147+
return refs
148+
}

0 commit comments

Comments
 (0)