Skip to content

Commit 9558afd

Browse files
committed
Add verbose mode with dependency tracking for test debugging
Add -v and -vv flags to print complete request chains that can be piped directly to a handler for manual debugging and reproduction. - Introduces DependencyTracker to build transitive dependency chains from ref usage - Switches to pflag for better CLI flag support - Adds unit tests for dependency tracking logic
1 parent 54a8e75 commit 9558afd

File tree

9 files changed

+647
-28
lines changed

9 files changed

+647
-28
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ test:
2222
@echo "Running runner unit tests..."
2323
go test -v ./runner/...
2424
@echo "Running conformance tests with mock handler..."
25-
$(RUNNER_BIN) -handler $(MOCK_HANDLER_BIN)
25+
$(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv
2626

2727
clean:
2828
@echo "Cleaning build artifacts..."

README.md

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,33 @@ make runner
6666

6767
The runner automatically detects and recovers from crashed/unresponsive handlers, allowing remaining tests to continue.
6868

69+
#### Verbose Flags
70+
71+
- **`-v, --verbose`**: Shows request chains and responses for **failed tests only**
72+
- **`-vv`**: Shows request chains and responses for **all tests** (passed and failed)
73+
74+
The request chains printed by verbose mode can be directly piped to the handler binary for manual debugging:
75+
76+
```bash
77+
# Example output from -vv mode:
78+
# ✓ chain#4 (Get active chain reference from chainstate manager)
79+
#
80+
# Request chain
81+
# ────────────────────────────────────────
82+
# {"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
83+
# {"id":"chain#2","method":"btck_chainstate_manager_create","params":{"context":"$context_ref"},"ref":"$chainstate_manager_ref"}
84+
# {"id":"chain#4","method":"btck_chainstate_manager_get_active_chain","params":{"chainstate_manager":"$chainstate_manager_ref"},"ref":"$chain_ref"}
85+
#
86+
# Response:
87+
# ────────────────────────────────────────
88+
# {"result":"$chain_ref"}
89+
90+
# Copy the request chain and pipe it to your handler for debugging:
91+
echo '{"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
92+
{"id":"chain#2","method":"btck_chainstate_manager_create","params":{"context":"$context_ref"},"ref":"$chainstate_manager_ref"}
93+
{"id":"chain#4","method":"btck_chainstate_manager_get_active_chain","params":{"chainstate_manager":"$chainstate_manager_ref"},"ref":"$chain_ref"}' | ./path/to/your/handler
94+
```
95+
6996
### Testing the Runner
7097

7198
Build and test the runner:

cmd/runner/main.go

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,36 @@ package main
22

33
import (
44
"context"
5-
"flag"
65
"fmt"
76
"io/fs"
87
"os"
98
"sort"
109
"strings"
1110
"time"
1211

12+
"github.com/spf13/pflag"
1313
"github.com/stringintech/kernel-bindings-tests/runner"
1414
"github.com/stringintech/kernel-bindings-tests/testdata"
1515
)
1616

1717
func main() {
18-
handlerPath := flag.String("handler", "", "Path to handler binary")
19-
handlerTimeout := flag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
20-
timeout := flag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)")
21-
flag.Parse()
18+
handlerPath := pflag.String("handler", "", "Path to handler binary")
19+
handlerTimeout := pflag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
20+
timeout := pflag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)")
21+
verboseCount := pflag.CountP("verbose", "v", "Verbose mode: -v shows all requests needed to reproduce failed tests, plus received/expected responses; -vv shows this for all tests (passed and failed)")
22+
pflag.Parse()
23+
24+
// Convert verbose count to verbosity level
25+
verbosity := runner.VerbosityQuiet
26+
if *verboseCount >= 2 {
27+
verbosity = runner.VerbosityAlways
28+
} else if *verboseCount == 1 {
29+
verbosity = runner.VerbosityOnFailure
30+
}
2231

2332
if *handlerPath == "" {
24-
fmt.Fprintf(os.Stderr, "Error: -handler flag is required\n")
25-
flag.Usage()
33+
fmt.Fprintf(os.Stderr, "Error: --handler flag is required\n")
34+
pflag.Usage()
2635
os.Exit(1)
2736
}
2837

@@ -69,7 +78,7 @@ func main() {
6978
}
7079

7180
// Run suite
72-
result := testRunner.RunTestSuite(ctx, *suite)
81+
result := testRunner.RunTestSuite(ctx, *suite, verbosity)
7382
printResults(suite, result)
7483

7584
totalPassed += result.PassedTests

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
module github.com/stringintech/kernel-bindings-tests
22

33
go 1.23
4+
5+
require github.com/spf13/pflag v1.0.10

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
2+
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=

runner/dependency_tracker.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package runner
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
)
7+
8+
// statefulCreatorMethods contains methods that create stateful objects.
9+
// Refs created by these methods are tracked as stateful, meaning tests
10+
// using these refs depend on mutable state.
11+
var statefulCreatorMethods = map[string]bool{
12+
"btck_context_create": true,
13+
"btck_chainstate_manager_create": true,
14+
}
15+
16+
// stateMutatingMethods contains methods that mutate internal state.
17+
// Tests using these methods are assumed to affect all subsequent tests
18+
// and are included in dependency chains printed in verbose mode.
19+
var stateMutatingMethods = map[string]bool{
20+
"btck_chainstate_manager_process_block": true,
21+
}
22+
23+
// DependencyTracker manages test dependencies and builds request chains for verbose output.
24+
// It tracks both explicit ref dependencies and implicit state dependencies.
25+
type DependencyTracker struct {
26+
// refCreators maps reference names to the test index that created them
27+
refCreators map[string]int
28+
29+
// statefulRefs tracks refs created by stateful methods.
30+
// Tests using these refs depend on mutable state.
31+
statefulRefs map[string]bool
32+
33+
// depChains maps test index to its dependency chain (tests it depends on via ref usage)
34+
depChains map[int][]int
35+
36+
// stateDependencies is a cumulative list of all tests affecting state (state-mutating
37+
// tests and their complete dependency chains)
38+
stateDependencies []int
39+
}
40+
41+
// NewDependencyTracker creates a new dependency tracker
42+
func NewDependencyTracker() *DependencyTracker {
43+
return &DependencyTracker{
44+
refCreators: make(map[string]int),
45+
statefulRefs: make(map[string]bool),
46+
depChains: make(map[int][]int),
47+
stateDependencies: []int{},
48+
}
49+
}
50+
51+
// BuildDependenciesForTest analyzes a test's parameters to build its complete transitive
52+
// dependency chain. When a test uses refs created by earlier tests, this records all direct
53+
// dependencies (tests that created those refs) and indirect dependencies (their dependencies).
54+
// Must be called after all previous tests have been processed.
55+
func (dt *DependencyTracker) BuildDependenciesForTest(testIndex int, test *TestCase) {
56+
// Build dependency chain for current test based on refs it uses
57+
var parentChains [][]int
58+
for _, ref := range extractRefsFromParams(test.Request.Params) {
59+
if creatorIdx, exists := dt.refCreators[ref]; exists {
60+
// Add the creator as a direct dependency
61+
parentChains = append(parentChains, []int{creatorIdx})
62+
// Add transitive dependencies (creator's dependencies)
63+
if chain, hasChain := dt.depChains[creatorIdx]; hasChain {
64+
parentChains = append(parentChains, chain)
65+
}
66+
} else {
67+
panic(fmt.Sprintf("test %d (%s) uses undefined reference %s - no prior test created this ref",
68+
testIndex, test.Request.ID, ref))
69+
}
70+
}
71+
dt.depChains[testIndex] = mergeSortedUnique(parentChains...)
72+
}
73+
74+
// OnTestExecuted is called after a test executes successfully. It tracks the ref
75+
// created by the test, marks it as stateful if needed, and updates state dependencies
76+
// for state-mutating methods.
77+
func (dt *DependencyTracker) OnTestExecuted(testIndex int, test *TestCase) {
78+
// Track ref creation using the request's ref field
79+
if test.Request.Ref != "" {
80+
dt.refCreators[test.Request.Ref] = testIndex
81+
82+
// Mark refs from stateful methods
83+
if statefulCreatorMethods[test.Request.Method] {
84+
dt.statefulRefs[test.Request.Ref] = true
85+
}
86+
}
87+
88+
// Track state-mutating tests and their dependencies
89+
if stateMutatingMethods[test.Request.Method] {
90+
mutatorChain := append(dt.depChains[testIndex], testIndex)
91+
dt.stateDependencies = mergeSortedUnique(dt.stateDependencies, mutatorChain)
92+
}
93+
}
94+
95+
// BuildRequestChain builds the complete dependency chain for a test
96+
func (dt *DependencyTracker) BuildRequestChain(testIndex int, allTests []TestCase) []int {
97+
refDepChain := dt.depChains[testIndex]
98+
99+
// Only include state dependencies if the test's dep chain contains any stateful refs
100+
if dt.testUsesStatefulRefs(testIndex, allTests) {
101+
return mergeSortedUnique(refDepChain, dt.stateDependencies)
102+
}
103+
104+
return refDepChain
105+
}
106+
107+
// testUsesStatefulRefs checks if a test's dependency chain includes any stateful refs
108+
func (dt *DependencyTracker) testUsesStatefulRefs(testIndex int, allTests []TestCase) bool {
109+
// Check all tests in the dependency chain
110+
for _, depIdx := range dt.depChains[testIndex] {
111+
for _, ref := range extractRefsFromParams(allTests[depIdx].Request.Params) {
112+
if dt.statefulRefs[ref] {
113+
return true
114+
}
115+
}
116+
}
117+
118+
// Check the test itself
119+
for _, ref := range extractRefsFromParams(allTests[testIndex].Request.Params) {
120+
if dt.statefulRefs[ref] {
121+
return true
122+
}
123+
}
124+
return false
125+
}
126+
127+
// extractRefsFromParams extracts all reference names from params JSON.
128+
// Searches for ref objects with structure {"ref": "..."} at the first level of params.
129+
func extractRefsFromParams(params json.RawMessage) []string {
130+
var refs []string
131+
132+
var paramsMap map[string]json.RawMessage
133+
if err := json.Unmarshal(params, &paramsMap); err != nil {
134+
return refs
135+
}
136+
137+
for _, value := range paramsMap {
138+
if ref, ok := ParseRefObject(value); ok {
139+
refs = append(refs, ref)
140+
}
141+
}
142+
return refs
143+
}

0 commit comments

Comments
 (0)