Add exception framework for better error reporting (#114)

* Add exception framework for better error reporting

* resolve comments

* fix ci

* Add new error codes for runtime component

* resolve comments

* Update config and transport

Author: Agent-Hellboy

docs/error-codes.md

@@ -0,0 +1,54 @@
+# MCP Fuzzer Error Codes
+
+The MCP Fuzzer surfaces every user-facing failure through numbered error codes.
+Each code follows the `CCTTT` format:
+
+- `CC` (two digits) identifies the category (e.g., `10` for Transport, `20` for Auth)
+- `TTT` (three digits) identifies the specific error within that category
+
+This keeps diagnostics easy to scan while preserving enough room for future error
+types. Refer to the table below for the current registry.
+
+| Code  | Category  | Description                                        |
+|-------|-----------|----------------------------------------------------|
+| 10001 | Transport | Transport failure                                  |
+| 10002 | Transport | Unable to establish connection with the server     |
+| 10003 | Transport | Malformed or unexpected server response            |
+| 10004 | Transport | Authentication with the server failed              |
+| 10005 | Transport | Network connectivity or policy failure             |
+| 10006 | Transport | Invalid transport payload                          |
+| 10007 | Transport | Transport registration or selection error          |
+| 20001 | Auth      | Authentication subsystem error                     |
+| 20002 | Auth      | Authentication configuration is invalid            |
+| 20003 | Auth      | Authentication provider is misconfigured           |
+| 30001 | Timeout   | Operation timed out                                |
+| 30002 | Timeout   | Subprocess execution timed out                     |
+| 30003 | Timeout   | Network request timed out                          |
+| 40001 | Safety    | Safety policy violated                             |
+| 40002 | Safety    | Network access blocked by safety policy            |
+| 40003 | Safety    | System command blocked by safety policy            |
+| 40004 | Safety    | Filesystem access blocked by safety policy         |
+| 50001 | Server    | Server returned an error                           |
+| 50002 | Server    | Server is unavailable or not responding            |
+| 50003 | Server    | Protocol negotiation failed                        |
+| 60001 | CLI       | CLI error                                          |
+| 60002 | CLI       | Invalid CLI arguments                              |
+| 70001 | Reporting | Reporting error                                    |
+| 70002 | Reporting | Report validation failed                           |
+| 80001 | Config    | Configuration error                                |
+| 80002 | Config    | Configuration file could not be read               |
+| 80003 | Config    | Configuration validation failed                    |
+| 90001 | Fuzzing   | Fuzzing engine error                               |
+| 90002 | Fuzzing   | Fuzzing strategy failed                            |
+| 90003 | Fuzzing   | Async executor encountered an error                |
+| 95001 | Runtime   | Runtime management error                           |
+| 95002 | Runtime   | Failed to start managed process                    |
+| 95003 | Runtime   | Failed to stop managed process                     |
+| 95004 | Runtime   | Failed to send process signal                      |
+| 95005 | Runtime   | Process registration failed                        |
+| 95006 | Runtime   | Process watchdog failed to start                   |
+
+Each exception class in `mcp_fuzzer/exceptions.py` sets its `code` and
+`description` fields so CLI output, logs, and structured reports can reference
+these values. Use `MCPError.to_metadata()` or `get_error_registry()` to fetch
+the mapping programmatically.

mcp_fuzzer/auth/loaders.py

@@ -2,6 +2,7 @@
 import logging
 import os
 
+from ..exceptions import AuthConfigError, AuthProviderError
 from .manager import AuthManager
 from .providers import (
     create_api_key_auth,
@@ -81,7 +82,7 @@ def load_auth_config(config_file: str) -> AuthManager:
     providers = config.get("providers", {})
     for name, provider_config in providers.items():
         if not isinstance(provider_config, dict):
-            raise ValueError(
+            raise AuthProviderError(
                 f"Error configuring auth provider '{name}': "
                 f"expected an object, got {type(provider_config).__name__}"
             )
@@ -90,7 +91,7 @@ def load_auth_config(config_file: str) -> AuthManager:
         try:
             if provider_type == "api_key":
                 if "api_key" not in provider_config:
-                    raise ValueError(
+                    raise AuthProviderError(
                         f"Provider '{name}' is type 'api_key' but missing "
                         "required field 'api_key'. Expected: "
                         "{'type': 'api_key', 'api_key': 'YOUR_API_KEY'}"
@@ -105,13 +106,13 @@ def load_auth_config(config_file: str) -> AuthManager:
                 )
             elif provider_type == "basic":
                 if "username" not in provider_config:
-                    raise ValueError(
+                    raise AuthProviderError(
                         f"Provider '{name}' is type 'basic' but missing "
                         "required field 'username'. Expected: "
                         "{'type': 'basic', 'username': 'user', 'password': 'pass'}"
                     )
                 if "password" not in provider_config:
-                    raise ValueError(
+                    raise AuthProviderError(
                         f"Provider '{name}' is type 'basic' but missing "
                         "required field 'password'. Expected: "
                         "{'type': 'basic', 'username': 'user', 'password': 'pass'}"
@@ -124,7 +125,7 @@ def load_auth_config(config_file: str) -> AuthManager:
                 )
             elif provider_type == "oauth":
                 if "token" not in provider_config:
-                    raise ValueError(
+                    raise AuthProviderError(
                         f"Provider '{name}' is type 'oauth' but missing "
                         "required field 'token'. Expected: "
                         "{'type': 'oauth', 'token': 'YOUR_TOKEN'}"
@@ -139,13 +140,13 @@ def load_auth_config(config_file: str) -> AuthManager:
             elif provider_type == "custom":
                 headers = provider_config.get("headers")
                 if not headers:
-                    raise ValueError(
+                    raise AuthProviderError(
                         f"Provider '{name}' is type 'custom' but missing "
                         "required field 'headers'. Expected: "
                         "{'type': 'custom', 'headers': {'X-Header': 'value'}}"
                     )
                 if not isinstance(headers, dict):
-                    raise ValueError(
+                    raise AuthProviderError(
                         f"Provider '{name}' custom headers must be a dict, "
                         f"got {type(headers).__name__}"
                     )
@@ -156,24 +157,28 @@ def load_auth_config(config_file: str) -> AuthManager:
                     name, create_custom_header_auth(headers_str)
                 )
             else:
-                raise ValueError(
+                raise AuthProviderError(
                     f"Unknown provider type: '{provider_type}' for provider '{name}'. "
                     f"Supported types: api_key, basic, oauth, custom"
                 )
-        except (KeyError, ValueError) as e:
-            raise ValueError(f"Error configuring auth provider '{name}': {str(e)}")
+        except AuthProviderError:
+            raise
+        except (KeyError, ValueError, TypeError) as e:
+            raise AuthProviderError(
+                f"Error configuring auth provider '{name}': {str(e)}"
+            ) from e
 
     tool_mappings = config.get("tool_mapping")
     legacy_tool_mappings = config.get("tool_mappings")
     if tool_mappings and legacy_tool_mappings:
-        raise ValueError(
+        raise AuthConfigError(
             "Both 'tool_mapping' and legacy 'tool_mappings' are defined. "
             "Please use only 'tool_mapping'."
         )
 
     final_tool_mappings = tool_mappings or legacy_tool_mappings or {}
     if final_tool_mappings and not isinstance(final_tool_mappings, dict):
-        raise ValueError(
+        raise AuthConfigError(
             f"'tool_mapping' must be a dict, "
             f"got {type(final_tool_mappings).__name__}"
         )

mcp_fuzzer/cli/args.py

@@ -7,6 +7,7 @@
 from rich.console import Console
 
 from ..config import config, load_config_file, apply_config_file
+from ..exceptions import ArgumentValidationError
 
 def create_argument_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
@@ -643,26 +644,28 @@ def validate_arguments(args: argparse.Namespace) -> None:
 
     # Require endpoint for non-utility commands
     if not is_utility_command and not getattr(args, 'endpoint', None):
-        raise ValueError("--endpoint is required for fuzzing operations")
+        raise ArgumentValidationError("--endpoint is required for fuzzing operations")
 
     if args.mode == "protocol" and not args.protocol_type:
         pass
 
     if args.protocol_type and args.mode != "protocol":
-        raise ValueError("--protocol-type can only be used with --mode protocol")
+        raise ArgumentValidationError(
+            "--protocol-type can only be used with --mode protocol"
+        )
 
     if hasattr(args, "runs") and args.runs is not None:
         if not isinstance(args.runs, int) or args.runs < 1:
-            raise ValueError("--runs must be at least 1")
+            raise ArgumentValidationError("--runs must be at least 1")
 
     if hasattr(args, "runs_per_type") and args.runs_per_type is not None:
         if not isinstance(args.runs_per_type, int) or args.runs_per_type < 1:
-            raise ValueError("--runs-per-type must be at least 1")
+            raise ArgumentValidationError("--runs-per-type must be at least 1")
 
     if hasattr(args, "timeout") and args.timeout is not None:
         if not isinstance(args.timeout, (int, float)) or args.timeout <= 0:
-            raise ValueError("--timeout must be positive")
+            raise ArgumentValidationError("--timeout must be positive")
 
     if hasattr(args, "endpoint") and args.endpoint is not None:
         if not args.endpoint.strip():
-            raise ValueError("--endpoint cannot be empty")
+            raise ArgumentValidationError("--endpoint cannot be empty")

mcp_fuzzer/cli/main.py

@@ -19,6 +19,12 @@
     start_safety_if_enabled,
     stop_safety_if_started,
 )
+from ..exceptions import (
+    ArgumentValidationError,
+    CLIError,
+    MCPError,
+    TransportError,
+)
 
 
 def _get_cli_helpers() -> tuple[Any, Any, Any, Any, Any, Any]:
@@ -59,24 +65,15 @@ def _get_cli_helpers() -> tuple[Any, Any, Any, Any, Any, Any]:
 def _handle_validate_config(args) -> None:
     """Handle --validate-config flag."""
     from ..config import load_config_file
-    try:
-        load_config_file(args.validate_config)
-        console = Console()
-        config_file = args.validate_config
-        success_msg = (
-            "[green]:heavy_check_mark: Configuration file '"
-            f"{config_file}' is valid[/green]"
-        )
-        console.print(emoji.emojize(success_msg, language='alias'))
-        sys.exit(0)
-    except Exception as e:
-        console = Console()
-        error_msg = (
-            "[red]:heavy_multiplication_x: Configuration validation failed: "
-            f"{e}[/red]"
-        )
-        console.print(emoji.emojize(error_msg, language='alias'))
-        sys.exit(1)
+    load_config_file(args.validate_config)
+    console = Console()
+    config_file = args.validate_config
+    success_msg = (
+        "[green]:heavy_check_mark: Configuration file "
+        f"'{config_file}' is valid[/green]"
+    )
+    console.print(emoji.emojize(success_msg, language='alias'))
+    sys.exit(0)
 
 
 def _handle_check_env() -> None:
@@ -122,12 +119,10 @@ def _handle_check_env() -> None:
 
     if all_valid:
         console.print("[green]All environment variables are valid[/green]")
-    else:
-        console.print(
-            "[red]Some environment variables have invalid values[/red]"
-        )
-        sys.exit(1)
-    sys.exit(0)
+        sys.exit(0)
+
+    console.print("[red]Some environment variables have invalid values[/red]")
+    raise ArgumentValidationError("Invalid environment variable values")
 
 
 def _validate_transport(args, cli_module) -> None:
@@ -156,10 +151,13 @@ def _validate_transport(args, cli_module) -> None:
             args.endpoint,
             timeout=args.timeout,
         )
+    except MCPError:
+        raise
     except Exception as transport_error:
-        console = Console()
-        console.print(f"[bold red]Error:[/bold red] {transport_error}")
-        sys.exit(1)
+        raise TransportError(
+            "Failed to initialize transport",
+            context={"protocol": args.protocol, "endpoint": args.endpoint},
+        ) from transport_error
 
 
 def _run_fuzzing(args, cli_module) -> None:
@@ -209,19 +207,33 @@ def run_cli() -> None:
             _validate_transport(args, cli_module)
             _run_fuzzing(args, cli_module)
 
-    except ValueError as e:
-        console = Console()
-        console.print(f"[bold red]Error:[/bold red] {e}")
-        sys.exit(1)
     except KeyboardInterrupt:
         console = Console()
         console.print("\n[yellow]Fuzzing interrupted by user[/yellow]")
         sys.exit(0)
-    except Exception as e:
-        console = Console()
-        console.print(f"[bold red]Unexpected error:[/bold red] {e}")
+    except MCPError as err:
+        _print_mcp_error(err)
+        sys.exit(1)
+    except ValueError as exc:
+        error = ArgumentValidationError(str(exc))
+        _print_mcp_error(error)
+        sys.exit(1)
+    except Exception as exc:
+        error = CLIError(
+            "Unexpected CLI failure",
+            context={"stage": "run_cli", "details": str(exc)},
+        )
+        _print_mcp_error(error)
         if logging.getLogger().level <= logging.DEBUG:
             import traceback
 
-            console.print(traceback.format_exc())
+            Console().print(traceback.format_exc())
         sys.exit(1)
+
+
+def _print_mcp_error(error: MCPError) -> None:
+    """Render MCP errors consistently for the CLI."""
+    console = Console()
+    console.print(f"[bold red]Error ({error.code}):[/bold red] {error}")
+    if error.context:
+        console.print(f"[dim]Context: {error.context}[/dim]")

mcp_fuzzer/config/loader.py

@@ -12,7 +12,7 @@
 import yaml
 
 from .manager import config
-from ..exceptions import ConfigFileError
+from ..exceptions import ConfigFileError, MCPError
 from ..transport.custom import register_custom_transport
 from ..transport.base import TransportProtocol
 import importlib
@@ -114,26 +114,21 @@ def apply_config_file(
     Returns:
         True if configuration was loaded and applied, False otherwise
     """
+    # Find config file
+    file_path = find_config_file(config_path, search_paths, file_names)
+    if not file_path:
+        logger.debug("No configuration file found")
+        return False
+
+    logger.info(f"Loading configuration from {file_path}")
     try:
-        # Find config file
-        file_path = find_config_file(config_path, search_paths, file_names)
-        if not file_path:
-            logger.debug("No configuration file found")
-            return False
-
-        # Load config file
-        logger.info(f"Loading configuration from {file_path}")
         config_data = load_config_file(file_path)
-
-        # Load custom transports if configured
         load_custom_transports(config_data)
-
-        # Apply configuration
-        config.update(config_data)
-        return True
-    except Exception as e:
-        logger.error(f"Error loading configuration file: {str(e)}")
+    except (ConfigFileError, MCPError):
+        logger.exception("Failed to load configuration from %s", file_path)
         return False
+    config.update(config_data)
+    return True
 
 def get_config_schema() -> dict[str, Any]:
     """Get the configuration schema.
@@ -380,8 +375,10 @@ def load_custom_transports(config_data: dict[str, Any]) -> None:
                 f"{module_path}.{class_name}"
             )
 
+        except MCPError:
+            raise
         except Exception as e:
             logger.error(f"Failed to load custom transport '{transport_name}': {e}")
             raise ConfigFileError(
                 f"Failed to load custom transport '{transport_name}': {e}"
-            )
+            ) from e