Skip to content

Commit 37aab78

Browse files
stringintechstringintech
committed
Restructure test format and expand script verification coverage
Split script_pubkey_verify.json into separate error and success test files with improved error handling. Remove chainstate_basic.json to postpone decision on whether to support stateful tests. The current focus is on stateless script verification tests.
1 parent 3a05255 commit 37aab78

File tree

9 files changed

+313
-198
lines changed

9 files changed

+313
-198
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: all build test clean runner mock-handler deps lint
1+
.PHONY: all build test clean runner mock-handler
22

33
BUILD_DIR := build
44
RUNNER_BIN := $(BUILD_DIR)/runner

‎cmd/mock-handler/main.go‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,9 @@ func handleRequest(line string, testIndex map[string]string) error {
8080
if !ok {
8181
resp := runner.Response{
8282
ID: req.ID,
83-
Error: &runner.ErrorObj{
84-
Code: "UNKNOWN_TEST",
85-
Message: fmt.Sprintf("No test case found with ID: %s", req.ID),
83+
Error: &runner.Error{
84+
Type: "Handler",
85+
Variant: "UnknownTest",
8686
},
8787
}
8888
return writeResponse(resp)
@@ -93,9 +93,9 @@ func handleRequest(line string, testIndex map[string]string) error {
9393
if err != nil {
9494
resp := runner.Response{
9595
ID: req.ID,
96-
Error: &runner.ErrorObj{
97-
Code: "LOAD_ERROR",
98-
Message: fmt.Sprintf("Failed to load test suite: %v", err),
96+
Error: &runner.Error{
97+
Type: "Handler",
98+
Variant: "LoadError",
9999
},
100100
}
101101
return writeResponse(resp)
@@ -112,9 +112,9 @@ func handleRequest(line string, testIndex map[string]string) error {
112112
if testCase == nil {
113113
resp := runner.Response{
114114
ID: req.ID,
115-
Error: &runner.ErrorObj{
116-
Code: "TEST_NOT_FOUND",
117-
Message: fmt.Sprintf("Test case %s not found in file %s", req.ID, filename),
115+
Error: &runner.Error{
116+
Type: "Handler",
117+
Variant: "TestNotFound",
118118
},
119119
}
120120
return writeResponse(resp)
@@ -124,27 +124,20 @@ func handleRequest(line string, testIndex map[string]string) error {
124124
if req.Method != testCase.Method {
125125
resp := runner.Response{
126126
ID: req.ID,
127-
Error: &runner.ErrorObj{
128-
Code: "METHOD_MISMATCH",
129-
Message: fmt.Sprintf("Expected method %s, got %s", testCase.Method, req.Method),
127+
Error: &runner.Error{
128+
Type: "Handler",
129+
Variant: "MethodMismatch",
130130
},
131131
}
132132
return writeResponse(resp)
133133
}
134134

135135
// Build response based on expected result
136-
var resp runner.Response
137-
resp.ID = req.ID
138-
if testCase.Expected.Error != nil {
139-
resp.Error = &runner.ErrorObj{
140-
Code: testCase.Expected.Error.Code,
141-
Message: testCase.Expected.Error.Message,
142-
}
143-
}
144-
if testCase.Expected.Success != nil {
145-
resp.Result = *testCase.Expected.Success
146-
}
147-
return writeResponse(resp)
136+
return writeResponse(runner.Response{
137+
ID: req.ID,
138+
Success: testCase.Expected.Success,
139+
Error: testCase.Expected.Error,
140+
})
148141
}
149142

150143
// writeResponse writes a response to stdout as JSON

‎cmd/runner/main.go‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func main() {
5959
result := testRunner.RunTestSuite(*suite)
6060
testRunner.Close()
6161

62-
printResults(result)
62+
printResults(suite, result)
6363

6464
totalPassed += result.PassedTests
6565
totalFailed += result.FailedTests
@@ -79,16 +79,28 @@ func main() {
7979
}
8080
}
8181

82-
func printResults(result runner.TestResult) {
82+
func printResults(suite *runner.TestSuite, result runner.TestResult) {
8383
fmt.Printf("\nTest Suite: %s\n", result.SuiteName)
84+
if suite.Description != "" {
85+
fmt.Printf("Description: %s\n", suite.Description)
86+
}
8487
fmt.Printf("Total: %d, Passed: %d, Failed: %d\n\n", result.TotalTests, result.PassedTests, result.FailedTests)
8588

86-
for _, tr := range result.TestResults {
89+
for i, tr := range result.TestResults {
8790
status := "✓"
8891
if !tr.Passed {
8992
status = "✗"
9093
}
91-
fmt.Printf(" %s %s: %s\n", status, tr.TestID, tr.Message)
94+
95+
// Print test ID and description if available
96+
if suite.Tests[i].Description != "" {
97+
fmt.Printf(" %s %s (%s)\n", status, tr.TestID, suite.Tests[i].Description)
98+
} else {
99+
fmt.Printf(" %s %s\n", status, tr.TestID)
100+
}
101+
102+
// Print message indented
103+
fmt.Printf(" %s\n", tr.Message)
92104
}
93105

94106
fmt.Printf("\n")

‎runner/runner.go‎

Lines changed: 26 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"os"
1212
"os/exec"
1313
"path/filepath"
14-
"strings"
1514
)
1615

1716
// TestRunner executes test suites against a handler binary
@@ -64,9 +63,8 @@ func (tr *TestRunner) Close() error {
6463
if tr.stdin != nil {
6564
tr.stdin.Close()
6665
}
67-
if tr.cmd != nil && tr.cmd.Process != nil {
68-
tr.cmd.Process.Kill()
69-
tr.cmd.Wait()
66+
if tr.cmd != nil {
67+
return tr.cmd.Wait()
7068
}
7169
return nil
7270
}
@@ -139,36 +137,36 @@ func (tr *TestRunner) runTest(test TestCase) SingleTestResult {
139137
Message: fmt.Sprintf("Response ID mismatch: expected %s, got %s", test.ID, resp.ID),
140138
}
141139
}
142-
return tr.validateResponse(test, resp)
140+
return validateResponse(test, resp)
143141
}
144142

145143
// validateResponse checks if response matches expected result
146-
func (tr *TestRunner) validateResponse(test TestCase, resp *Response) SingleTestResult {
144+
func validateResponse(test TestCase, resp *Response) SingleTestResult {
147145
// Check if we expected an error
148146
if test.Expected.Error != nil {
149147
if resp.Error == nil {
150148
return SingleTestResult{
151149
TestID: test.ID,
152150
Passed: false,
153-
Message: fmt.Sprintf("Expected error %s, but got success", test.Expected.Error.Code),
151+
Message: fmt.Sprintf("Expected error type %s, but got no error", test.Expected.Error.Type),
154152
}
155153
}
156154

157-
// Check error code
158-
if resp.Error.Code != test.Expected.Error.Code {
155+
// Check error type
156+
if resp.Error.Type != test.Expected.Error.Type {
159157
return SingleTestResult{
160158
TestID: test.ID,
161159
Passed: false,
162-
Message: fmt.Sprintf("Expected error code %s, got %s", test.Expected.Error.Code, resp.Error.Code),
160+
Message: fmt.Sprintf("Expected error type %s, got %s", test.Expected.Error.Type, resp.Error.Type),
163161
}
164162
}
165163

166-
// Check error message if specified
167-
if test.Expected.Error.Message != "" && !strings.Contains(resp.Error.Message, test.Expected.Error.Message) {
164+
// Check error variant if specified
165+
if test.Expected.Error.Variant != "" && resp.Error.Variant != test.Expected.Error.Variant {
168166
return SingleTestResult{
169167
TestID: test.ID,
170168
Passed: false,
171-
Message: fmt.Sprintf("Expected error message containing '%s', got '%s'", test.Expected.Error.Message, resp.Error.Message),
169+
Message: fmt.Sprintf("Expected error variant %s, got %s", test.Expected.Error.Variant, resp.Error.Variant),
172170
}
173171
}
174172

@@ -182,10 +180,22 @@ func (tr *TestRunner) validateResponse(test TestCase, resp *Response) SingleTest
182180
// Check if we expected success
183181
if test.Expected.Success != nil {
184182
if resp.Error != nil {
183+
errMsg := fmt.Sprintf("Expected success, but got error: %s", resp.Error.Type)
184+
if resp.Error.Variant != "" {
185+
errMsg += fmt.Sprintf(" (variant: %s)", resp.Error.Variant)
186+
}
187+
return SingleTestResult{
188+
TestID: test.ID,
189+
Passed: false,
190+
Message: errMsg,
191+
}
192+
}
193+
194+
if resp.Success == nil {
185195
return SingleTestResult{
186196
TestID: test.ID,
187197
Passed: false,
188-
Message: fmt.Sprintf("Expected success, but got error: %s - %s", resp.Error.Code, resp.Error.Message),
198+
Message: "Expected success, but response contained no success field",
189199
}
190200
}
191201

@@ -198,7 +208,7 @@ func (tr *TestRunner) validateResponse(test TestCase, resp *Response) SingleTest
198208
Message: fmt.Sprintf("Invalid expected JSON: %v", err),
199209
}
200210
}
201-
if err := json.Unmarshal(bytes.TrimSpace(resp.Result), &actualData); err != nil {
211+
if err := json.Unmarshal(bytes.TrimSpace(*resp.Success), &actualData); err != nil {
202212
return SingleTestResult{
203213
TestID: test.ID,
204214
Passed: false,
@@ -212,7 +222,7 @@ func (tr *TestRunner) validateResponse(test TestCase, resp *Response) SingleTest
212222
return SingleTestResult{
213223
TestID: test.ID,
214224
Passed: false,
215-
Message: fmt.Sprintf("Result mismatch:\nExpected: %s\nActual: %s", string(expectedNormalized), string(actualNormalized)),
225+
Message: fmt.Sprintf("Success mismatch:\nExpected: %s\nActual: %s", string(expectedNormalized), string(actualNormalized)),
216226
}
217227
}
218228
return SingleTestResult{
@@ -244,26 +254,6 @@ type SingleTestResult struct {
244254
Message string
245255
}
246256

247-
// LoadTestSuite loads a test suite from a JSON file
248-
func LoadTestSuite(filePath string) (*TestSuite, error) {
249-
data, err := os.ReadFile(filePath)
250-
if err != nil {
251-
return nil, fmt.Errorf("failed to read file: %w", err)
252-
}
253-
254-
var suite TestSuite
255-
if err := json.Unmarshal(data, &suite); err != nil {
256-
return nil, fmt.Errorf("failed to parse JSON: %w", err)
257-
}
258-
259-
// Set suite name from filename if not specified
260-
if suite.Name == "" {
261-
suite.Name = filepath.Base(filePath)
262-
}
263-
264-
return &suite, nil
265-
}
266-
267257
// LoadTestSuiteFromFS loads a test suite from an embedded filesystem
268258
func LoadTestSuiteFromFS(fsys embed.FS, filePath string) (*TestSuite, error) {
269259
data, err := fs.ReadFile(fsys, filePath)

‎runner/types.go‎

Lines changed: 17 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,27 @@ import (
66

77
// TestCase represents a single test case
88
type TestCase struct {
9-
ID string `json:"id"`
10-
Method string `json:"method"`
11-
Params json.RawMessage `json:"params"`
12-
Expected TestExpectation `json:"expected"`
9+
ID string `json:"id"`
10+
Description string `json:"description,omitempty"`
11+
Method string `json:"method"`
12+
Params json.RawMessage `json:"params"`
13+
Expected TestExpectation `json:"expected"`
1314
}
1415

1516
// TestExpectation defines what response is expected
1617
type TestExpectation struct {
1718
Success *json.RawMessage `json:"success,omitempty"` // Expected successful result
18-
Error *ErrorPattern `json:"error,omitempty"` // Expected error
19-
}
20-
21-
// ErrorPattern defines expected error fields
22-
type ErrorPattern struct {
23-
Code string `json:"code"`
24-
Message string `json:"message,omitempty"` // Optional - if empty, any message is accepted
19+
Error *Error `json:"error,omitempty"` // Expected error
2520
}
2621

2722
// TestSuite represents a collection of test cases
2823
type TestSuite struct {
29-
Name string `json:"name"`
30-
Tests []TestCase `json:"tests"`
24+
Name string `json:"name"`
25+
Description string `json:"description,omitempty"`
26+
Tests []TestCase `json:"tests"`
3127
}
3228

33-
// Request represents a JSON-RPC style request
29+
// Request represents a request sent to the handler
3430
type Request struct {
3531
ID string `json:"id"`
3632
Method string `json:"method"`
@@ -39,13 +35,13 @@ type Request struct {
3935

4036
// Response represents a response from the handler
4137
type Response struct {
42-
ID string `json:"id"`
43-
Result json.RawMessage `json:"result,omitempty"`
44-
Error *ErrorObj `json:"error,omitempty"`
38+
ID string `json:"id"`
39+
Success *json.RawMessage `json:"success,omitempty"`
40+
Error *Error `json:"error,omitempty"`
4541
}
4642

47-
// ErrorObj represents an error response
48-
type ErrorObj struct {
49-
Code string `json:"code"`
50-
Message string `json:"message"`
43+
// Error represents an error response
44+
type Error struct {
45+
Type string `json:"type"`
46+
Variant string `json:"variant,omitempty"`
5147
}

‎testdata/chainstate_basic.json‎

Lines changed: 0 additions & 59 deletions
This file was deleted.

0 commit comments

Comments
 (0)