Add configurable timeout for handler response to each test case

Add --handler-timeout flag to runner binary allowing users to configure max response wait time per test case (default: 10s).

Add TestHandler_Unresponsive to verify timeout behavior when handler becomes unresponsive.

Author: stringintech

cmd/runner/main.go

@@ -6,13 +6,15 @@ import (
 	"io/fs"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/stringintech/kernel-bindings-tests/runner"
 	"github.com/stringintech/kernel-bindings-tests/testdata"
 )
 
 func main() {
 	handlerPath := flag.String("handler", "", "Path to handler binary")
+	handlerTimeout := flag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)")
 	flag.Parse()
 
 	if *handlerPath == "" {
@@ -34,7 +36,7 @@ func main() {
 	}
 
 	// Create test runner
-	testRunner, err := runner.NewTestRunner(*handlerPath)
+	testRunner, err := runner.NewTestRunner(*handlerPath, *handlerTimeout)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error creating test runner: %v\n", err)
 		os.Exit(1)

runner/handler.go

@@ -23,18 +23,23 @@ type HandlerConfig struct {
 	Path string
 	Args []string
 	Env  []string
+	// Timeout specifies the maximum duration to wait when reading from the handler's
+	// stdout. If zero, defaults to 10 seconds. The handler is killed if it fails to
+	// write output within this timeout.
+	Timeout time.Duration
 }
 
 // Handler manages a conformance handler process communicating via stdin/stdout
 type Handler struct {
-	cmd    *exec.Cmd
-	stdin  io.WriteCloser
-	stdout *bufio.Scanner
-	stderr io.ReadCloser
+	cmd     *exec.Cmd
+	stdin   io.WriteCloser
+	stdout  *bufio.Scanner
+	stderr  io.ReadCloser
+	timeout time.Duration
 }
 
 // NewHandler spawns a new handler process with the given configuration
-func NewHandler(cfg HandlerConfig) (*Handler, error) {
+func NewHandler(cfg *HandlerConfig) (*Handler, error) {
 	cmd := exec.Command(cfg.Path, cfg.Args...)
 	if cfg.Env != nil {
 		cmd.Env = append(cmd.Environ(), cfg.Env...)
@@ -60,11 +65,17 @@ func NewHandler(cfg HandlerConfig) (*Handler, error) {
 		return nil, fmt.Errorf("failed to start handler: %w", err)
 	}
 
+	timeout := cfg.Timeout
+	if timeout == 0 {
+		timeout = 10 * time.Second
+	}
+
 	return &Handler{
-		cmd:    cmd,
-		stdin:  stdin,
-		stdout: bufio.NewScanner(stdout),
-		stderr: stderr,
+		cmd:     cmd,
+		stdin:   stdin,
+		stdout:  bufio.NewScanner(stdout),
+		stderr:  stderr,
+		timeout: timeout,
 	}, nil
 }
 
@@ -74,7 +85,7 @@ func (h *Handler) SendLine(line []byte) error {
 	return err
 }
 
-// ReadLine reads a line from the handler's stdout with a 10-second timeout
+// ReadLine reads a line from the handler's stdout with a configurable timeout
 func (h *Handler) ReadLine() ([]byte, error) {
 	// Use a timeout for Scan() in case the handler hangs
 	scanDone := make(chan bool, 1)
@@ -93,7 +104,7 @@ func (h *Handler) ReadLine() ([]byte, error) {
 		}
 		// EOF - handler closed stdout prematurely, fall through to kill and capture stderr
 		baseErr = ErrHandlerClosed
-	case <-time.After(10 * time.Second):
+	case <-time.After(h.timeout):
 		// Timeout - handler didn't respond, fall through to kill and capture stderr
 		baseErr = ErrHandlerTimeout
 	}

runner/handler_test.go

@@ -2,9 +2,11 @@ package runner
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"os"
 	"testing"
+	"time"
 )
 
 const (
@@ -14,12 +16,14 @@ const (
 	// envTestHelperName specifies which helper function to execute in subprocess mode.
 	envTestHelperName = "TEST_HELPER_NAME"
 
-	helperNameNormal = "normal"
+	helperNameNormal       = "normal"
+	helperNameUnresponsive = "unresponsive"
 )
 
 // testHelpers maps helper names to functions that simulate different handler behaviors.
 var testHelpers = map[string]func(){
-	helperNameNormal: helperNormal,
+	helperNameNormal:       helperNormal,
+	helperNameUnresponsive: helperUnresponsive,
 }
 
 // TestMain allows the test binary to serve two purposes:
@@ -46,7 +50,7 @@ func TestMain(m *testing.M) {
 
 // TestHandler_NormalOperation tests that a well-behaved handler works correctly
 func TestHandler_NormalOperation(t *testing.T) {
-	h, err := newHandlerForTest(t, helperNameNormal)
+	h, err := newHandlerForTest(t, helperNameNormal, 0)
 	if err != nil {
 		t.Fatalf("Failed to create handler: %v", err)
 	}
@@ -86,13 +90,60 @@ func helperNormal() {
 	}
 }
 
+// TestHandler_Unresponsive tests that the runner correctly handles an unresponsive handler
+func TestHandler_Unresponsive(t *testing.T) {
+	h, err := newHandlerForTest(t, helperNameUnresponsive, 100*time.Millisecond)
+	if err != nil {
+		t.Fatalf("Failed to create handler: %v", err)
+	}
+	defer h.Close()
+
+	// Send a request to the handler
+	request := `{"id":1,"method":"test"}`
+	if err := h.SendLine([]byte(request)); err != nil {
+		t.Fatalf("Failed to send request: %v", err)
+	}
+
+	// Try to read the response - should Timeout
+	start := time.Now()
+	_, err = h.ReadLine()
+	elapsed := time.Since(start)
+
+	if err == nil {
+		t.Fatal("Expected error from unresponsive handler, got nil")
+	}
+
+	// Verify it's the Timeout error we expect
+	if !errors.Is(err, ErrHandlerTimeout) {
+		t.Errorf("Expected ErrHandlerTimeout, got: %v", err)
+	}
+
+	// Verify Timeout happened quickly (within reasonable margin)
+	if elapsed > 200*time.Millisecond {
+		t.Errorf("Timeout took too long: %v (expected ~100ms)", elapsed)
+	}
+}
+
+// helperUnresponsive simulates a handler that receives requests but never responds,
+// triggering the Timeout mechanism in the runner.
+func helperUnresponsive() {
+	// Read from stdin to prevent broken pipe, but never write responses
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		// Sleep indefinitely to simulate unresponsiveness
+		time.Sleep(1 * time.Hour)
+	}
+}
+
 // newHandlerForTest creates a Handler that runs a test helper as a subprocess.
 // The helperName identifies which helper to run (e.g., "normal", "crash", "hang").
-func newHandlerForTest(t *testing.T, helperName string) (*Handler, error) {
+// The timeout parameter sets the per-request timeout (0 uses default).
+func newHandlerForTest(t *testing.T, helperName string, timeout time.Duration) (*Handler, error) {
 	t.Helper()
 
-	return NewHandler(HandlerConfig{
-		Path: os.Args[0],
-		Env:  []string{"TEST_AS_SUBPROCESS=1", "TEST_HELPER_NAME=" + helperName},
+	return NewHandler(&HandlerConfig{
+		Path:    os.Args[0],
+		Env:     []string{"TEST_AS_SUBPROCESS=1", "TEST_HELPER_NAME=" + helperName},
+		Timeout: timeout,
 	})
 }

runner/runner.go

@@ -8,35 +8,44 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"time"
 )
 
 // TestRunner executes test suites against a handler binary
 type TestRunner struct {
-	handlerPath string
-	handler     *Handler
+	handler       *Handler
+	handlerConfig *HandlerConfig
 }
 
-// NewTestRunner creates a new test runner
-func NewTestRunner(handlerPath string) (*TestRunner, error) {
+// NewTestRunner creates a new test runner for executing test suites against a handler binary.
+// The handlerTimeout parameter specifies the maximum duration to wait for the handler to
+// respond to each test case. If zero, defaults to 10 seconds.
+func NewTestRunner(handlerPath string, handlerTimeout time.Duration) (*TestRunner, error) {
 	if _, err := os.Stat(handlerPath); os.IsNotExist(err) {
 		return nil, fmt.Errorf("handler binary not found: %s", handlerPath)
 	}
 
-	handler, err := NewHandler(HandlerConfig{Path: handlerPath})
+	handler, err := NewHandler(&HandlerConfig{
+		Path:    handlerPath,
+		Timeout: handlerTimeout,
+	})
 	if err != nil {
 		return nil, err
 	}
 
 	return &TestRunner{
-		handlerPath: handlerPath,
-		handler:     handler,
+		handler: handler,
+		handlerConfig: &HandlerConfig{
+			Path:    handlerPath,
+			Timeout: handlerTimeout,
+		},
 	}, nil
 }
 
 // SendRequest sends a request to the handler, spawning a new handler if needed
 func (tr *TestRunner) SendRequest(req Request) error {
 	if tr.handler == nil {
-		handler, err := NewHandler(HandlerConfig{Path: tr.handlerPath})
+		handler, err := NewHandler(tr.handlerConfig)
 		if err != nil {
 			return fmt.Errorf("failed to spawn new handler: %w", err)
 		}