Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ test:
@echo "Running runner unit tests..."
go test -v ./runner/...
@echo "Running conformance tests with mock handler..."
$(RUNNER_BIN) -handler $(MOCK_HANDLER_BIN)
$(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv

clean:
@echo "Cleaning build artifacts..."
Expand Down
27 changes: 27 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,33 @@ make runner

The runner automatically detects and recovers from crashed/unresponsive handlers, allowing remaining tests to continue.

#### Verbose Flags

- **`-v, --verbose`**: Shows request chains and responses for **failed tests only**
- **`-vv`**: Shows request chains and responses for **all tests** (passed and failed)

The request chains printed by verbose mode can be directly piped to the handler binary for manual debugging:

```bash
# Example output from -vv mode:
# ✓ chain#4 (Get active chain reference from chainstate manager)
#
# Request chain
# ────────────────────────────────────────
# {"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
# {"id":"chain#2","method":"btck_chainstate_manager_create","params":{"context":"$context_ref"},"ref":"$chainstate_manager_ref"}
# {"id":"chain#4","method":"btck_chainstate_manager_get_active_chain","params":{"chainstate_manager":"$chainstate_manager_ref"},"ref":"$chain_ref"}
#
# Response:
# ────────────────────────────────────────
# {"result":"$chain_ref"}

# Copy the request chain and pipe it to your handler for debugging:
echo '{"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
{"id":"chain#2","method":"btck_chainstate_manager_create","params":{"context":"$context_ref"},"ref":"$chainstate_manager_ref"}
{"id":"chain#4","method":"btck_chainstate_manager_get_active_chain","params":{"chainstate_manager":"$chainstate_manager_ref"},"ref":"$chain_ref"}' | ./path/to/your/handler
```

### Testing the Runner

Build and test the runner:
Expand Down
35 changes: 27 additions & 8 deletions cmd/runner/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,36 @@ package main

import (
"context"
"flag"
"fmt"
"io/fs"
"os"
"sort"
"strings"
"time"

"github.com/spf13/pflag"
"github.com/stringintech/kernel-bindings-tests/runner"
"github.com/stringintech/kernel-bindings-tests/testdata"
)

func main() {
handlerPath := flag.String("handler", "", "Path to handler binary")
handlerTimeout := flag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
timeout := flag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)")
flag.Parse()
handlerPath := pflag.String("handler", "", "Path to handler binary")
handlerTimeout := pflag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
timeout := pflag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)")
verboseCount := pflag.CountP("verbose", "v", "Verbose mode: -v shows all requests needed to reproduce failed tests, plus received/expected responses; -vv shows this for all tests (passed and failed)")
pflag.Parse()

// Convert verbose count to verbosity level
verbosity := runner.VerbosityQuiet
if *verboseCount >= 2 {
verbosity = runner.VerbosityAlways
} else if *verboseCount == 1 {
verbosity = runner.VerbosityOnFailure
}

if *handlerPath == "" {
fmt.Fprintf(os.Stderr, "Error: -handler flag is required\n")
flag.Usage()
fmt.Fprintf(os.Stderr, "Error: --handler flag is required\n")
pflag.Usage()
os.Exit(1)
}

Expand All @@ -37,6 +47,9 @@ func main() {
os.Exit(1)
}

// Sort test files alphabetically for deterministic execution order
sort.Strings(testFiles)

// Create test runner
testRunner, err := runner.NewTestRunner(*handlerPath, *handlerTimeout, *timeout)
if err != nil {
Expand Down Expand Up @@ -65,12 +78,18 @@ func main() {
}

// Run suite
result := testRunner.RunTestSuite(ctx, *suite)
result := testRunner.RunTestSuite(ctx, *suite, verbosity)
printResults(suite, result)

totalPassed += result.PassedTests
totalFailed += result.FailedTests
totalTests += result.TotalTests

// Close handler after stateful suites to prevent state leaks.
// A new handler process will be spawned on-demand when the next request is sent.
if suite.Stateful {
testRunner.CloseHandler()
}
}

fmt.Printf("\n" + strings.Repeat("=", 60) + "\n")
Expand Down
Loading