Restructure expected response format and improve semantics

Also refactor response validation to be aligned with the improved semantics:
- Success case: result contains the return value (or null for void/nullptr), error must be null/omitted
- Error case: result must be null/omitted, error contains error details

Additional changes:
- Replace Error.type/variant with Error.code.type/member
- Split validation into separate validateResponseForSuccess/Error functions
- Update test cases to reflect new format

Author: stringintech

cmd/mock-handler/main.go

@@ -81,8 +81,10 @@ func handleRequest(line string, testIndex map[string]string) error {
 		resp := runner.Response{
 			ID: req.ID,
 			Error: &runner.Error{
-				Type:    "Handler",
-				Variant: "UnknownTest",
+				Code: &runner.ErrorCode{
+					Type:   "Handler",
+					Member: "UNKNOWN_TEST",
+				},
 			},
 		}
 		return writeResponse(resp)
@@ -94,8 +96,10 @@ func handleRequest(line string, testIndex map[string]string) error {
 		resp := runner.Response{
 			ID: req.ID,
 			Error: &runner.Error{
-				Type:    "Handler",
-				Variant: "LoadError",
+				Code: &runner.ErrorCode{
+					Type:   "Handler",
+					Member: "LOAD_ERROR",
+				},
 			},
 		}
 		return writeResponse(resp)
@@ -113,8 +117,10 @@ func handleRequest(line string, testIndex map[string]string) error {
 		resp := runner.Response{
 			ID: req.ID,
 			Error: &runner.Error{
-				Type:    "Handler",
-				Variant: "TestNotFound",
+				Code: &runner.ErrorCode{
+					Type:   "Handler",
+					Member: "TEST_NOT_FOUND",
+				},
 			},
 		}
 		return writeResponse(resp)
@@ -125,18 +131,20 @@ func handleRequest(line string, testIndex map[string]string) error {
 		resp := runner.Response{
 			ID: req.ID,
 			Error: &runner.Error{
-				Type:    "Handler",
-				Variant: "MethodMismatch",
+				Code: &runner.ErrorCode{
+					Type:   "Handler",
+					Member: "METHOD_MISMATCH",
+				},
 			},
 		}
 		return writeResponse(resp)
 	}
 
 	// Build response based on expected result
 	return writeResponse(runner.Response{
-		ID:      req.ID,
-		Success: testCase.Expected.Success,
-		Error:   testCase.Expected.Error,
+		ID:     req.ID,
+		Result: testCase.Expected.Result,
+		Error:  testCase.Expected.Error,
 	})
 }
 

runner/runner.go

@@ -112,7 +112,15 @@ func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult {
 	}
 
 	for _, test := range suite.Tests {
-		testResult := tr.runTest(test)
+		err := tr.runTest(test)
+		testResult := SingleTestResult{
+			TestID: test.ID,
+			Passed: err == nil,
+		}
+		if err != nil {
+			testResult.Message = err.Error()
+		}
+
 		result.TestResults = append(result.TestResults, testResult)
 		if testResult.Passed {
 			result.PassedTests++
@@ -124,8 +132,9 @@ func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult {
 	return result
 }
 
-// runTest executes a single test case
-func (tr *TestRunner) runTest(test TestCase) SingleTestResult {
+// runTest executes a single test case and validates the response.
+// Returns an error if communication with the handler fails or validation fails.
+func (tr *TestRunner) runTest(test TestCase) error {
 	req := Request{
 		ID:     test.ID,
 		Method: test.Method,
@@ -134,118 +143,102 @@ func (tr *TestRunner) runTest(test TestCase) SingleTestResult {
 
 	resp, err := tr.SendRequest(req)
 	if err != nil {
-		return SingleTestResult{
-			TestID:  test.ID,
-			Passed:  false,
-			Message: fmt.Sprintf("Failed to send request: %v", err),
-		}
-	}
-	if resp.ID != test.ID {
-		return SingleTestResult{
-			TestID:  test.ID,
-			Passed:  false,
-			Message: fmt.Sprintf("Response ID mismatch: expected %s, got %s", test.ID, resp.ID),
-		}
+		return err
 	}
+
 	return validateResponse(test, resp)
 }
 
-// validateResponse checks if response matches expected result
-func validateResponse(test TestCase, resp *Response) SingleTestResult {
-	// Check if we expected an error
+// validateResponse validates that a response matches the expected test outcome.
+// Returns an error if:
+//
+//	Response ID does not match the request ID
+//	Response does not match the expected outcome (error or success)
+func validateResponse(test TestCase, resp *Response) error {
+	if resp.ID != test.ID {
+		return fmt.Errorf("response ID mismatch: expected %s, got %s", test.ID, resp.ID)
+	}
+
 	if test.Expected.Error != nil {
-		if resp.Error == nil {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: fmt.Sprintf("Expected error type %s, but got no error", test.Expected.Error.Type),
-			}
-		}
+		return validateResponseForError(test, resp)
+	}
 
-		// Check error type
-		if resp.Error.Type != test.Expected.Error.Type {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: fmt.Sprintf("Expected error type %s, got %s", test.Expected.Error.Type, resp.Error.Type),
-			}
-		}
+	return validateResponseForSuccess(test, resp)
+}
 
-		// Check error variant if specified
-		if test.Expected.Error.Variant != "" && resp.Error.Variant != test.Expected.Error.Variant {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: fmt.Sprintf("Expected error variant %s, got %s", test.Expected.Error.Variant, resp.Error.Variant),
-			}
-		}
+// validateResponseForError validates that a response correctly represents an error case.
+// It ensures the response contains an error, the result is null or omitted, and if an
+// error code is expected, it matches the expected type and member.
+func validateResponseForError(test TestCase, resp *Response) error {
+	if test.Expected.Error == nil {
+		panic("validateResponseForError expects non-nil error")
+	}
 
-		return SingleTestResult{
-			TestID:  test.ID,
-			Passed:  true,
-			Message: "Test passed (expected error matched)",
+	if resp.Error == nil {
+		if test.Expected.Error.Code != nil {
+			return fmt.Errorf("expected error %s.%s, but got no error",
+				test.Expected.Error.Code.Type, test.Expected.Error.Code.Member)
 		}
+		return fmt.Errorf("expected error, but got no error")
 	}
 
-	// Check if we expected success
-	if test.Expected.Success != nil {
-		if resp.Error != nil {
-			errMsg := fmt.Sprintf("Expected success, but got error: %s", resp.Error.Type)
-			if resp.Error.Variant != "" {
-				errMsg += fmt.Sprintf(" (variant: %s)", resp.Error.Variant)
-			}
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: errMsg,
-			}
-		}
+	if !resp.Result.IsNullOrOmitted() {
+		return fmt.Errorf("expected result to be null or omitted when error is present, got: %s", string(resp.Result))
+	}
 
-		if resp.Success == nil {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: "Expected success, but response contained no success field",
-			}
+	if test.Expected.Error.Code != nil {
+		if resp.Error.Code == nil {
+			return fmt.Errorf("expected error code %s.%s, but got error with no code",
+				test.Expected.Error.Code.Type, test.Expected.Error.Code.Member)
 		}
 
-		// Normalize JSON for comparison
-		var expectedData, actualData interface{}
-		if err := json.Unmarshal(bytes.TrimSpace(*test.Expected.Success), &expectedData); err != nil {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: fmt.Sprintf("Invalid expected JSON: %v", err),
-			}
+		if resp.Error.Code.Type != test.Expected.Error.Code.Type {
+			return fmt.Errorf("expected error type %s, got %s", test.Expected.Error.Code.Type, resp.Error.Code.Type)
 		}
-		if err := json.Unmarshal(bytes.TrimSpace(*resp.Success), &actualData); err != nil {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: fmt.Sprintf("Invalid response JSON: %v", err),
-			}
+
+		if resp.Error.Code.Member != test.Expected.Error.Code.Member {
+			return fmt.Errorf("expected error member %s, got %s", test.Expected.Error.Code.Member, resp.Error.Code.Member)
 		}
-		expectedNormalized, _ := json.Marshal(expectedData)
-		actualNormalized, _ := json.Marshal(actualData)
-
-		if !bytes.Equal(expectedNormalized, actualNormalized) {
-			return SingleTestResult{
-				TestID:  test.ID,
-				Passed:  false,
-				Message: fmt.Sprintf("Success mismatch:\nExpected: %s\nActual:   %s", string(expectedNormalized), string(actualNormalized)),
-			}
+	}
+	return nil
+}
+
+// validateResponseForSuccess validates that a response correctly represents a success case.
+// It ensures the response contains no error, and if a result is expected, it matches the
+// expected value.
+func validateResponseForSuccess(test TestCase, resp *Response) error {
+	if test.Expected.Error != nil {
+		panic("validateResponseForSuccess expects nil error")
+	}
+
+	if resp.Error != nil {
+		if resp.Error.Code != nil {
+			return fmt.Errorf("expected success with no error, but got error: %s.%s", resp.Error.Code.Type, resp.Error.Code.Member)
 		}
-		return SingleTestResult{
-			TestID:  test.ID,
-			Passed:  true,
-			Message: "Test passed",
+		return fmt.Errorf("expected success with no error, but got error")
+	}
+
+	if test.Expected.Result.IsNullOrOmitted() {
+		if !resp.Result.IsNullOrOmitted() {
+			return fmt.Errorf("expected null or omitted result, got: %s", string(resp.Result))
 		}
+		return nil
 	}
-	return SingleTestResult{
-		TestID:  test.ID,
-		Passed:  false,
-		Message: "Test has no expected result defined",
+
+	expectedNorm, err := test.Expected.Result.Normalize()
+	if err != nil {
+		return fmt.Errorf("failed to normalize expected result: %w", err)
 	}
+
+	actualNorm, err := resp.Result.Normalize()
+	if err != nil {
+		return fmt.Errorf("failed to normalize actual result: %w", err)
+	}
+
+	if expectedNorm != actualNorm {
+		return fmt.Errorf("result mismatch: expected %s, got %s", expectedNorm, actualNorm)
+	}
+	return nil
 }
 
 // TestResult contains results from running a test suite

runner/types.go

@@ -13,10 +13,12 @@ type TestCase struct {
 	Expected    TestExpectation `json:"expected"`
 }
 
-// TestExpectation defines what response is expected
+// TestExpectation defines what response is expected.
+// If expecting success, result contains the expected value (or null for void/nullptr) and error must be null.
+// If expecting failure, result must be null and error contains the expected error details.
 type TestExpectation struct {
-	Success *json.RawMessage `json:"success,omitempty"` // Expected successful result
-	Error   *Error           `json:"error,omitempty"`   // Expected error
+	Result Result `json:"result"`          // Expected return value (null for void/nullptr/error cases)
+	Error  *Error `json:"error,omitempty"` // Expected error (null for success cases)
 }
 
 // TestSuite represents a collection of test cases
@@ -33,15 +35,54 @@ type Request struct {
 	Params json.RawMessage `json:"params"`
 }
 
-// Response represents a response from the handler
+// Response represents a response from the handler.
+// If the operation succeeds, result contains the return value (or null for void/nullptr) and error must be null.
+// If the operation fails, result must be null and error contains error details.
 type Response struct {
-	ID      string           `json:"id"`
-	Success *json.RawMessage `json:"success,omitempty"`
-	Error   *Error           `json:"error,omitempty"`
+	ID     string `json:"id"`
+	Result Result `json:"result"`          // Return value (null for void/nullptr/error cases)
+	Error  *Error `json:"error,omitempty"` // Error details (null for success cases)
 }
 
-// Error represents an error response
+// Error represents an error response.
+// Code can be null for generic errors without specific error codes.
 type Error struct {
-	Type    string `json:"type"`
-	Variant string `json:"variant,omitempty"`
+	Code *ErrorCode `json:"code,omitempty"`
+}
+
+type ErrorCode struct {
+	Type   string `json:"type"`   // e.g., "btck_ScriptVerifyStatus"
+	Member string `json:"member"` // e.g., "ERROR_INVALID_FLAGS_COMBINATION"
+}
+
+// Result is a type alias for json.RawMessage with helper methods.
+type Result json.RawMessage
+
+// MarshalJSON implements json.Marshaler by delegating to json.RawMessage.
+func (r Result) MarshalJSON() ([]byte, error) {
+	return (json.RawMessage)(r).MarshalJSON()
+}
+
+// UnmarshalJSON implements json.Unmarshaler by delegating to json.RawMessage.
+func (r *Result) UnmarshalJSON(data []byte) error {
+	return (*json.RawMessage)(r).UnmarshalJSON(data)
+}
+
+// IsNullOrOmitted checks if the result is nil or represents a JSON null value.
+func (r Result) IsNullOrOmitted() bool {
+	return r == nil || string(r) == "null"
+}
+
+// Normalize normalizes JSON data by parsing and re-marshaling it.
+// This ensures consistent formatting and key ordering for comparison.
+func (r Result) Normalize() (string, error) {
+	var v interface{}
+	if err := json.Unmarshal(r, &v); err != nil {
+		return "", err
+	}
+	normalized, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(normalized), nil
 }