Add stateful suite support

Introduces stateful test suites where tests depend on each other and execute sequentially. If any test fails in a stateful suite, all subsequent tests are automatically skipped and marked as failed. Handler processes are closed after stateful suites to force re-spawn and prevent state leaks to the remaining tests.

Author: stringintech

cmd/runner/main.go

@@ -75,6 +75,12 @@ func main() {
 		totalPassed += result.PassedTests
 		totalFailed += result.FailedTests
 		totalTests += result.TotalTests
+
+		// Close handler after stateful suites to prevent state leaks.
+		// A new handler process will be spawned on-demand when the next request is sent.
+		if suite.Stateful {
+			testRunner.CloseHandler()
+		}
 	}
 
 	fmt.Printf("\n" + strings.Repeat("=", 60) + "\n")

runner/runner.go

@@ -106,13 +106,30 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite) TestRes
 		TotalTests: len(suite.Tests),
 	}
 
+	skipTests := false
+
 	for _, test := range suite.Tests {
-		testResult := tr.runTest(ctx, test)
+		var testResult SingleTestResult
+
+		if !skipTests {
+			testResult = tr.runTest(ctx, test)
+		} else {
+			// In stateful suites, if any previous test failed, fail all subsequent tests
+			testResult = SingleTestResult{
+				TestID:  test.Request.ID,
+				Passed:  false,
+				Message: "Skipped due to previous test failure in stateful suite",
+			}
+		}
+
 		result.TestResults = append(result.TestResults, testResult)
 		if testResult.Passed {
 			result.PassedTests++
 		} else {
 			result.FailedTests++
+			if suite.Stateful {
+				skipTests = true
+			}
 		}
 	}
 

runner/types.go

@@ -16,6 +16,13 @@ type TestSuite struct {
 	Name        string     `json:"name"`
 	Description string     `json:"description,omitempty"`
 	Tests       []TestCase `json:"tests"`
+
+	// Stateful indicates that tests in this suite depend on each other and must
+	// execute sequentially. If any test fails in a stateful suite, all subsequent
+	// tests are automatically skipped and considered as failed. Use this for test
+	// suites where later tests depend on the success of earlier tests
+	// (e.g., setup -> operation -> verification).
+	Stateful bool `json:"stateful,omitempty"`
 }
 
 // Request represents a request sent to the handler