From 4ac81d49aa0bdee57bccd7e0be600a019f553fab Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Wed, 3 Dec 2025 17:35:12 +0000 Subject: [PATCH 1/8] add content filtering exceptions --- pydantic_ai_slim/pydantic_ai/exceptions.py | 25 +++++++ .../pydantic_ai/models/anthropic.py | 13 +++- pydantic_ai_slim/pydantic_ai/models/google.py | 16 +++-- pydantic_ai_slim/pydantic_ai/models/openai.py | 72 ++++++++++++++++++- tests/models/test_anthropic.py | 57 ++++++++++++++- tests/models/test_google.py | 60 +++++++++++++++- tests/models/test_openai.py | 53 ++++++++++++++ 7 files changed, 283 insertions(+), 13 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index 0b4500502c..1d81a6f319 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -24,6 +24,9 @@ 'UsageLimitExceeded', 'ModelAPIError', 'ModelHTTPError', + 'ContentFilterError', + 'PromptContentFilterError', + 'ResponseContentFilterError', 'IncompleteToolCall', 'FallbackExceptionGroup', ) @@ -179,6 +182,28 @@ def __init__(self, status_code: int, model_name: str, body: object | None = None super().__init__(model_name=model_name, message=message) +class ContentFilterError(ModelAPIError): + """Raised when content filtering is triggered by the model provider.""" + + +class PromptContentFilterError(ContentFilterError, ModelHTTPError): + """Raised when the prompt triggers a content filter.""" + + def __init__(self, status_code: int, model_name: str, body: object | None = None): + self.status_code = status_code + self.body = body + message = f'Prompt content filtered, status_code: {status_code}, model_name: {model_name}' + ModelAPIError.__init__(self, model_name, message) + + +class ResponseContentFilterError(ContentFilterError): + """Raised when the generated response triggers a content filter.""" + + def __init__(self, message: str, model_name: str, body: object | None = None): + self.body = body + super().__init__(model_name, message) + + class FallbackExceptionGroup(ExceptionGroup[Any]): """A group of exceptions that can be raised when all fallback models fail.""" diff --git a/pydantic_ai_slim/pydantic_ai/models/anthropic.py b/pydantic_ai_slim/pydantic_ai/models/anthropic.py index b23da276e2..627e67392f 100644 --- a/pydantic_ai_slim/pydantic_ai/models/anthropic.py +++ b/pydantic_ai_slim/pydantic_ai/models/anthropic.py @@ -14,7 +14,7 @@ from .._run_context import RunContext from .._utils import guard_tool_call_id as _guard_tool_call_id from ..builtin_tools import CodeExecutionTool, MCPServerTool, MemoryTool, WebFetchTool, WebSearchTool -from ..exceptions import ModelAPIError, UserError +from ..exceptions import ModelAPIError, ResponseContentFilterError, UserError from ..messages import ( BinaryContent, BuiltinToolCallPart, @@ -526,6 +526,12 @@ def _process_response(self, response: BetaMessage) -> ModelResponse: if raw_finish_reason := response.stop_reason: # pragma: no branch provider_details = {'finish_reason': raw_finish_reason} finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) + if finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {response.model}', + model_name=response.model, + body=response.model_dump(), + ) return ModelResponse( parts=items, @@ -1241,6 +1247,11 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason := event.delta.stop_reason: # pragma: no branch self.provider_details = {'finish_reason': raw_finish_reason} self.finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) + if self.finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {self.model_name}', + model_name=self.model_name, + ) elif isinstance(event, BetaRawContentBlockStopEvent): # pragma: no branch if isinstance(current_block, BetaMCPToolUseBlock): diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index 89290ea3ce..5c3625efe1 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -14,7 +14,7 @@ from .._output import OutputObjectDefinition from .._run_context import RunContext from ..builtin_tools import CodeExecutionTool, ImageGenerationTool, WebFetchTool, WebSearchTool -from ..exceptions import ModelAPIError, ModelHTTPError, UserError +from ..exceptions import ModelAPIError, ModelHTTPError, ResponseContentFilterError, UserError from ..messages import ( BinaryContent, BuiltinToolCallPart, @@ -495,8 +495,10 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: if candidate.content is None or candidate.content.parts is None: if finish_reason == 'content_filter' and raw_finish_reason: - raise UnexpectedModelBehavior( - f'Content filter {raw_finish_reason.value!r} triggered', response.model_dump_json() + raise ResponseContentFilterError( + f'Content filter {raw_finish_reason.value!r} triggered', + model_name=response.model_version or self._model_name, + body=response.model_dump_json(), ) parts = [] # pragma: no cover else: @@ -697,9 +699,11 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: yield self._parts_manager.handle_part(vendor_part_id=uuid4(), part=web_fetch_return) if candidate.content is None or candidate.content.parts is None: - if self.finish_reason == 'content_filter' and raw_finish_reason: # pragma: no cover - raise UnexpectedModelBehavior( - f'Content filter {raw_finish_reason.value!r} triggered', chunk.model_dump_json() + if self.finish_reason == 'content_filter' and raw_finish_reason: + raise ResponseContentFilterError( + f'Content filter {raw_finish_reason.value!r} triggered', + model_name=self.model_name, + body=chunk.model_dump_json(), ) else: # pragma: no cover continue diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index 3c5c184a76..44a2bb7c0b 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -20,7 +20,7 @@ from .._thinking_part import split_content_into_text_and_thinking from .._utils import guard_tool_call_id as _guard_tool_call_id, now_utc as _now_utc, number_to_datetime from ..builtin_tools import CodeExecutionTool, ImageGenerationTool, MCPServerTool, WebSearchTool -from ..exceptions import UserError +from ..exceptions import PromptContentFilterError, ResponseContentFilterError, UserError from ..messages import ( AudioUrl, BinaryContent, @@ -552,7 +552,27 @@ async def _completions_create( ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e + # Handle Azure Prompt Filter + err_body: Any = e.body + + if status_code == 400 and isinstance(err_body, dict): + err_dict = cast(dict[str, Any], err_body) + error = err_dict.get('error') + + if isinstance(error, dict): + error_dict = cast(dict[str, Any], error) + if error_dict.get('code') == 'content_filter': + raise PromptContentFilterError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e + + raise ModelHTTPError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -598,6 +618,14 @@ def _process_response(self, response: chat.ChatCompletion | str) -> ModelRespons raise UnexpectedModelBehavior(f'Invalid response from {self.system} chat completions endpoint: {e}') from e choice = response.choices[0] + + if choice.finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {response.model}', + model_name=response.model, + body=response.model_dump(), + ) + items: list[ModelResponsePart] = [] if thinking_parts := self._process_thinking(choice.message): @@ -1234,6 +1262,12 @@ def _process_response( # noqa: C901 finish_reason: FinishReason | None = None provider_details: dict[str, Any] | None = None raw_finish_reason = details.reason if (details := response.incomplete_details) else response.status + if raw_finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {response.model}', + model_name=response.model, + body=response.model_dump(), + ) if raw_finish_reason: provider_details = {'finish_reason': raw_finish_reason} finish_reason = _RESPONSES_FINISH_REASON_MAP.get(raw_finish_reason) @@ -1390,7 +1424,27 @@ async def _responses_create( # noqa: C901 ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e + # Handle Azure Prompt Filter + err_body: Any = e.body + + if status_code == 400 and isinstance(err_body, dict): + err_dict = cast(dict[str, Any], err_body) + error = err_dict.get('error') + + if isinstance(error, dict): + error_dict = cast(dict[str, Any], error) + if error_dict.get('code') == 'content_filter': + raise PromptContentFilterError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e + + raise ModelHTTPError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -1875,6 +1929,11 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: continue if raw_finish_reason := choice.finish_reason: + if raw_finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {self.model_name}', + model_name=self.model_name, + ) self.finish_reason = self._map_finish_reason(raw_finish_reason) if provider_details := self._map_provider_details(chunk): @@ -2020,6 +2079,13 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: raw_finish_reason = ( details.reason if (details := chunk.response.incomplete_details) else chunk.response.status ) + + if raw_finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {self.model_name}', + model_name=self.model_name, + ) + if raw_finish_reason: # pragma: no branch self.provider_details = {'finish_reason': raw_finish_reason} self.finish_reason = _RESPONSES_FINISH_REASON_MAP.get(raw_finish_reason) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index ad65735d38..2bc99c12cc 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -46,7 +46,7 @@ UserPromptPart, ) from pydantic_ai.builtin_tools import CodeExecutionTool, MCPServerTool, MemoryTool, WebFetchTool, WebSearchTool -from pydantic_ai.exceptions import UserError +from pydantic_ai.exceptions import ResponseContentFilterError, UserError from pydantic_ai.messages import ( BuiltinToolCallEvent, # pyright: ignore[reportDeprecated] BuiltinToolResultEvent, # pyright: ignore[reportDeprecated] @@ -7858,3 +7858,58 @@ async def test_anthropic_cache_messages_real_api(allow_model_requests: None, ant assert usage2.cache_read_tokens > 0 assert usage2.cache_write_tokens > 0 assert usage2.output_tokens > 0 + + +async def test_anthropic_response_filter_error_sync(allow_model_requests: None): + c = completion_message( + [BetaTextBlock(text='partial', type='text')], + usage=BetaUsage(input_tokens=5, output_tokens=10), + ) + # 'refusal' maps to 'content_filter' in _FINISH_REASON_MAP + c.stop_reason = 'refusal' + + mock_client = MockAnthropic.create_mock(c) + m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + await agent.run('hello') + + # The mock completion_message uses this model name hardcoded + assert exc_info.value.model_name == 'claude-3-5-haiku-123' + assert 'Content filter triggered' in str(exc_info.value) + + +async def test_anthropic_response_filter_error_stream(allow_model_requests: None): + stream = [ + BetaRawMessageStartEvent( + type='message_start', + message=BetaMessage( + id='msg_123', + model='claude-3-5-haiku-123', + role='assistant', + type='message', + content=[], + stop_reason=None, + usage=BetaUsage(input_tokens=20, output_tokens=0), + ), + ), + BetaRawMessageDeltaEvent( + type='message_delta', + delta=Delta(stop_reason='refusal'), # maps to content_filter + usage=BetaMessageDeltaUsage(input_tokens=20, output_tokens=5), + ), + BetaRawMessageStopEvent(type='message_stop'), + ] + + mock_client = MockAnthropic.create_stream_mock([stream]) + m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + async with agent.run_stream('hello') as result: + async for _ in result.stream_text(): + pass + + assert exc_info.value.model_name == 'claude-3-5-haiku-123' + assert 'Content filter triggered' in str(exc_info.value) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 3ef8cd5dda..3e40f822b5 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -51,7 +51,13 @@ WebFetchTool, WebSearchTool, ) -from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, ModelRetry, UnexpectedModelBehavior, UserError +from pydantic_ai.exceptions import ( + ModelAPIError, + ModelHTTPError, + ModelRetry, + ResponseContentFilterError, + UserError, +) from pydantic_ai.messages import ( BuiltinToolCallEvent, # pyright: ignore[reportDeprecated] BuiltinToolResultEvent, # pyright: ignore[reportDeprecated] @@ -67,6 +73,7 @@ with try_import() as imports_successful: from google.genai import errors from google.genai.types import ( + Candidate, FinishReason as GoogleFinishReason, GenerateContentResponse, GenerateContentResponseUsageMetadata, @@ -982,7 +989,8 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p ) agent = Agent(m, instructions='You hate the world!', model_settings=settings) - with pytest.raises(UnexpectedModelBehavior, match="Content filter 'SAFETY' triggered"): + # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError + with pytest.raises(ResponseContentFilterError, match="Content filter 'SAFETY' triggered"): await agent.run('Tell me a joke about a Brazilians.') @@ -4425,3 +4433,51 @@ def test_google_missing_tool_call_thought_signature(): ], } ) + + +async def test_google_response_filter_error_sync( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + model = GoogleModel('gemini-1.5-flash', provider=google_provider) + + candidate = Candidate( + finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None + ) + + response = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') + mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response) + + agent = Agent(model=model) + + with pytest.raises(ResponseContentFilterError) as exc_info: + await agent.run('bad content') + + assert exc_info.value.model_name == 'gemini-1.5-flash' + assert 'Content filter' in str(exc_info.value) + + +async def test_google_response_filter_error_stream( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + model = GoogleModel('gemini-1.5-flash', provider=google_provider) + + candidate = Candidate( + finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None + ) + + chunk = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') + + async def stream_iterator(): + yield chunk + + mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) + + agent = Agent(model=model) + + with pytest.raises(ResponseContentFilterError) as exc_info: + async with agent.run_stream('bad content') as result: + async for _ in result.stream_text(): + pass + + assert exc_info.value.model_name == 'gemini-1.5-flash' + assert 'Content filter' in str(exc_info.value) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index ed68edd94f..e82f0ce96e 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -38,6 +38,7 @@ ) from pydantic_ai._json_schema import InlineDefsJsonSchemaTransformer from pydantic_ai.builtin_tools import WebSearchTool +from pydantic_ai.exceptions import PromptContentFilterError, ResponseContentFilterError from pydantic_ai.models import ModelRequestParameters from pydantic_ai.output import NativeOutput, PromptedOutput, TextOutput, ToolOutput from pydantic_ai.profiles.openai import OpenAIModelProfile, openai_model_profile @@ -3296,3 +3297,55 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): """, } ) + + +def test_azure_prompt_filter_error(allow_model_requests: None) -> None: + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'content filter', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, + ) + ) + m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + with pytest.raises(PromptContentFilterError) as exc_info: + agent.run_sync('bad prompt') + + assert exc_info.value.status_code == 400 + assert exc_info.value.model_name == 'gpt-4o' + assert 'Prompt content filtered' in str(exc_info.value) + + +async def test_openai_response_filter_error_sync(allow_model_requests: None): + c = completion_message( + ChatCompletionMessage(content='partial', role='assistant'), + ) + # Simulate content filter finish reason + c.choices[0].finish_reason = 'content_filter' + + mock_client = MockOpenAI.create_mock(c) + m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + await agent.run('hello') + + # assertion: matches the mock's default model name + assert exc_info.value.model_name == 'gpt-4o-123' + assert 'Content filter triggered' in str(exc_info.value) + + +async def test_openai_response_filter_error_stream(allow_model_requests: None): + stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] + mock_client = MockOpenAI.create_mock_stream(stream) + m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + async with agent.run_stream('hello') as result: + async for _ in result.stream_text(): + pass + + assert exc_info.value.model_name == 'gpt-4o-123' + assert 'Content filter triggered' in str(exc_info.value) From 404a833661e2ee0b9cc66ebe7832eb797036ae7f Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 09:23:18 +0000 Subject: [PATCH 2/8] resolve issues --- pydantic_ai_slim/pydantic_ai/exceptions.py | 20 +++--- .../pydantic_ai/models/anthropic.py | 2 - pydantic_ai_slim/pydantic_ai/models/google.py | 10 +-- pydantic_ai_slim/pydantic_ai/models/openai.py | 71 +++++++------------ tests/models/test_anthropic.py | 12 +--- tests/models/test_google.py | 43 ++++++----- tests/models/test_openai.py | 30 ++++---- 7 files changed, 79 insertions(+), 109 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index 1d81a6f319..d9d1745d4d 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -182,26 +182,28 @@ def __init__(self, status_code: int, model_name: str, body: object | None = None super().__init__(model_name=model_name, message=message) -class ContentFilterError(ModelAPIError): +class ContentFilterError(ModelHTTPError): """Raised when content filtering is triggered by the model provider.""" + def __init__(self, message: str, status_code: int, model_name: str, body: object | None = None): + super().__init__(status_code, model_name, body) + self.message = message + -class PromptContentFilterError(ContentFilterError, ModelHTTPError): +class PromptContentFilterError(ContentFilterError): """Raised when the prompt triggers a content filter.""" def __init__(self, status_code: int, model_name: str, body: object | None = None): - self.status_code = status_code - self.body = body - message = f'Prompt content filtered, status_code: {status_code}, model_name: {model_name}' - ModelAPIError.__init__(self, model_name, message) + message = f"Prompt content filtered by model '{model_name}'" + super().__init__(message, status_code, model_name, body) class ResponseContentFilterError(ContentFilterError): """Raised when the generated response triggers a content filter.""" - def __init__(self, message: str, model_name: str, body: object | None = None): - self.body = body - super().__init__(model_name, message) + def __init__(self, model_name: str, body: object | None = None, status_code: int = 200): + message = f"Response content filtered by model '{model_name}'" + super().__init__(message, status_code, model_name, body) class FallbackExceptionGroup(ExceptionGroup[Any]): diff --git a/pydantic_ai_slim/pydantic_ai/models/anthropic.py b/pydantic_ai_slim/pydantic_ai/models/anthropic.py index 627e67392f..fc57c487c0 100644 --- a/pydantic_ai_slim/pydantic_ai/models/anthropic.py +++ b/pydantic_ai_slim/pydantic_ai/models/anthropic.py @@ -528,7 +528,6 @@ def _process_response(self, response: BetaMessage) -> ModelResponse: finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {response.model}', model_name=response.model, body=response.model_dump(), ) @@ -1249,7 +1248,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: self.finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if self.finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {self.model_name}', model_name=self.model_name, ) diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index 5c3625efe1..532772d5ed 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -496,9 +496,7 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: if candidate.content is None or candidate.content.parts is None: if finish_reason == 'content_filter' and raw_finish_reason: raise ResponseContentFilterError( - f'Content filter {raw_finish_reason.value!r} triggered', - model_name=response.model_version or self._model_name, - body=response.model_dump_json(), + model_name=response.model_version or self._model_name, body=response.model_dump_json() ) parts = [] # pragma: no cover else: @@ -700,11 +698,7 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if candidate.content is None or candidate.content.parts is None: if self.finish_reason == 'content_filter' and raw_finish_reason: - raise ResponseContentFilterError( - f'Content filter {raw_finish_reason.value!r} triggered', - model_name=self.model_name, - body=chunk.model_dump_json(), - ) + raise ResponseContentFilterError(model_name=self.model_name, body=chunk.model_dump_json()) else: # pragma: no cover continue diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index 44a2bb7c0b..2b204fe161 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -157,6 +157,24 @@ } +def _check_azure_content_filter(e: APIStatusError, model_name: str) -> None: + """Check if the error is an Azure content filter error and raise PromptContentFilterError if so.""" + if e.status_code == 400: + body_any: Any = e.body + + if isinstance(body_any, dict): + body_dict = cast(dict[str, Any], body_any) + + if (error := body_dict.get('error')) and isinstance(error, dict): + error_dict = cast(dict[str, Any], error) + if error_dict.get('code') == 'content_filter': + raise PromptContentFilterError( + status_code=e.status_code, + model_name=model_name, + body=body_dict, + ) from e + + class OpenAIChatModelSettings(ModelSettings, total=False): """Settings used for an OpenAI model request.""" @@ -552,27 +570,9 @@ async def _completions_create( ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - # Handle Azure Prompt Filter - err_body: Any = e.body - - if status_code == 400 and isinstance(err_body, dict): - err_dict = cast(dict[str, Any], err_body) - error = err_dict.get('error') - - if isinstance(error, dict): - error_dict = cast(dict[str, Any], error) - if error_dict.get('code') == 'content_filter': - raise PromptContentFilterError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e - - raise ModelHTTPError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e + _check_azure_content_filter(e, self.model_name) + + raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -621,7 +621,6 @@ def _process_response(self, response: chat.ChatCompletion | str) -> ModelRespons if choice.finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {response.model}', model_name=response.model, body=response.model_dump(), ) @@ -1264,7 +1263,6 @@ def _process_response( # noqa: C901 raw_finish_reason = details.reason if (details := response.incomplete_details) else response.status if raw_finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {response.model}', model_name=response.model, body=response.model_dump(), ) @@ -1424,27 +1422,10 @@ async def _responses_create( # noqa: C901 ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - # Handle Azure Prompt Filter - err_body: Any = e.body - - if status_code == 400 and isinstance(err_body, dict): - err_dict = cast(dict[str, Any], err_body) - error = err_dict.get('error') - - if isinstance(error, dict): - error_dict = cast(dict[str, Any], error) - if error_dict.get('code') == 'content_filter': - raise PromptContentFilterError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e - - raise ModelHTTPError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e + _check_azure_content_filter(e, self.model_name) + + # Reverted cast + raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -1931,7 +1912,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason := choice.finish_reason: if raw_finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {self.model_name}', model_name=self.model_name, ) self.finish_reason = self._map_finish_reason(raw_finish_reason) @@ -2082,7 +2062,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {self.model_name}', model_name=self.model_name, ) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index 2bc99c12cc..54a04cf4d3 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -7872,13 +7872,10 @@ async def test_anthropic_response_filter_error_sync(allow_model_requests: None): m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + # Mock uses hardcoded 'claude-3-5-haiku-123' + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): await agent.run('hello') - # The mock completion_message uses this model name hardcoded - assert exc_info.value.model_name == 'claude-3-5-haiku-123' - assert 'Content filter triggered' in str(exc_info.value) - async def test_anthropic_response_filter_error_stream(allow_model_requests: None): stream = [ @@ -7906,10 +7903,7 @@ async def test_anthropic_response_filter_error_stream(allow_model_requests: None m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass - - assert exc_info.value.model_name == 'claude-3-5-haiku-123' - assert 'Content filter triggered' in str(exc_info.value) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 3e40f822b5..7b3f8e3199 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -73,7 +73,6 @@ with try_import() as imports_successful: from google.genai import errors from google.genai.types import ( - Candidate, FinishReason as GoogleFinishReason, GenerateContentResponse, GenerateContentResponseUsageMetadata, @@ -990,7 +989,7 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p agent = Agent(m, instructions='You hate the world!', model_settings=settings) # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError - with pytest.raises(ResponseContentFilterError, match="Content filter 'SAFETY' triggered"): + with pytest.raises(ResponseContentFilterError, match="Response content filtered by model 'gemini-1.5-flash'"): await agent.run('Tell me a joke about a Brazilians.') @@ -4438,46 +4437,54 @@ def test_google_missing_tool_call_thought_signature(): async def test_google_response_filter_error_sync( allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture ): - model = GoogleModel('gemini-1.5-flash', provider=google_provider) + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) - candidate = Candidate( + # Create a Candidate mock with the specific failure condition + candidate_mock = mocker.Mock( finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None ) - response = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') - mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response) + # Create the Response mock containing the candidate + response_mock = mocker.Mock(candidates=[candidate_mock], model_version=model_name, usage_metadata=None) + + response_mock.model_dump_json.return_value = '{"mock": "json"}' + + # Patch the client + mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response_mock) agent = Agent(model=model) - with pytest.raises(ResponseContentFilterError) as exc_info: + # Verify the exception is raised + with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): await agent.run('bad content') - assert exc_info.value.model_name == 'gemini-1.5-flash' - assert 'Content filter' in str(exc_info.value) - async def test_google_response_filter_error_stream( allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture ): - model = GoogleModel('gemini-1.5-flash', provider=google_provider) + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) - candidate = Candidate( + # Create Candidate mock + candidate_mock = mocker.Mock( finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None ) - chunk = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') + # Create Chunk mock + chunk_mock = mocker.Mock( + candidates=[candidate_mock], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + ) + chunk_mock.model_dump_json.return_value = '{"mock": "json"}' async def stream_iterator(): - yield chunk + yield chunk_mock mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) agent = Agent(model=model) - with pytest.raises(ResponseContentFilterError) as exc_info: + with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): async with agent.run_stream('bad content') as result: async for _ in result.stream_text(): pass - - assert exc_info.value.model_name == 'gemini-1.5-flash' - assert 'Content filter' in str(exc_info.value) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index e82f0ce96e..ca41ff19ad 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3307,14 +3307,13 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, ) ) - m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(PromptContentFilterError) as exc_info: - agent.run_sync('bad prompt') - assert exc_info.value.status_code == 400 - assert exc_info.value.model_name == 'gpt-4o' - assert 'Prompt content filtered' in str(exc_info.value) + # Asserting the full error message structure via match + with pytest.raises(PromptContentFilterError, match=r"Prompt content filtered by model 'gpt-5-mini'"): + agent.run_sync('bad prompt') async def test_openai_response_filter_error_sync(allow_model_requests: None): @@ -3325,27 +3324,24 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): c.choices[0].finish_reason = 'content_filter' mock_client = MockOpenAI.create_mock(c) - m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + # The mock message uses 'gpt-4o-123' by default + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): await agent.run('hello') - # assertion: matches the mock's default model name - assert exc_info.value.model_name == 'gpt-4o-123' - assert 'Content filter triggered' in str(exc_info.value) - async def test_openai_response_filter_error_stream(allow_model_requests: None): stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] mock_client = MockOpenAI.create_mock_stream(stream) - m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + # The mock chunks use 'gpt-4o-123' by default + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass - - assert exc_info.value.model_name == 'gpt-4o-123' - assert 'Content filter triggered' in str(exc_info.value) From 3771c799afbdeaf769741ed9fd71bc19fbf122dc Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 10:00:05 +0000 Subject: [PATCH 3/8] add docs --- docs/models/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/models/overview.md b/docs/models/overview.md index a568a20161..131caa5679 100644 --- a/docs/models/overview.md +++ b/docs/models/overview.md @@ -229,7 +229,7 @@ contains all the exceptions encountered during the `run` execution. By default, the `FallbackModel` only moves on to the next model if the current model raises a [`ModelAPIError`][pydantic_ai.exceptions.ModelAPIError], which includes -[`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError]. You can customize this behavior by +[`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError] and [`ContentFilterError`][pydantic_ai.exceptions.ContentFilterError]. You can customize this behavior by passing a custom `fallback_on` argument to the `FallbackModel` constructor. !!! note From 70bcb7455a97cf8753f3d451682a3c0f400727a1 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 10:29:40 +0000 Subject: [PATCH 4/8] test coverage --- tests/models/test_openai.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index ca41ff19ad..85be0d1961 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3299,6 +3299,23 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): ) +def test_openai_generic_400_error(allow_model_requests: None) -> None: + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'bad request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'error': {'code': 'invalid_parameter', 'message': 'Invalid param.'}}, + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert not isinstance(exc_info.value, PromptContentFilterError) + assert exc_info.value.status_code == 400 + + def test_azure_prompt_filter_error(allow_model_requests: None) -> None: mock_client = MockOpenAI.create_mock( APIStatusError( From e861a508a20064c2950693393a44ed2bdfb3cf8c Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 11:04:47 +0000 Subject: [PATCH 5/8] improve tests --- tests/models/test_anthropic.py | 5 ++--- tests/models/test_openai.py | 11 +++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index 926d15d314..fbecd3dd88 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -7912,10 +7912,9 @@ async def test_anthropic_response_filter_error_sync(allow_model_requests: None): c.stop_reason = 'refusal' mock_client = MockAnthropic.create_mock(c) - m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - # Mock uses hardcoded 'claude-3-5-haiku-123' with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): await agent.run('hello') @@ -7943,7 +7942,7 @@ async def test_anthropic_response_filter_error_stream(allow_model_requests: None ] mock_client = MockAnthropic.create_stream_mock([stream]) - m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 85be0d1961..829c52f645 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3339,26 +3339,29 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): ) # Simulate content filter finish reason c.choices[0].finish_reason = 'content_filter' + c.model = 'gpt-5-mini' mock_client = MockOpenAI.create_mock(c) m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # The mock message uses 'gpt-4o-123' by default - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): await agent.run('hello') async def test_openai_response_filter_error_stream(allow_model_requests: None): stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] + + for chunk in stream: + chunk.model = 'gpt-5-mini' + mock_client = MockOpenAI.create_mock_stream(stream) m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # The mock chunks use 'gpt-4o-123' by default - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass From 51506b2a6a52e486d8bf46762d8a50dd575644bc Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 6 Dec 2025 05:13:17 +0000 Subject: [PATCH 6/8] resolve issues --- pydantic_ai_slim/pydantic_ai/exceptions.py | 4 +- pydantic_ai_slim/pydantic_ai/models/openai.py | 1 - tests/models/test_anthropic.py | 10 +- tests/models/test_google.py | 15 ++- tests/models/test_openai.py | 108 ++++++++++++++---- 5 files changed, 109 insertions(+), 29 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index d9d1745d4d..c91c967c46 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -194,7 +194,7 @@ class PromptContentFilterError(ContentFilterError): """Raised when the prompt triggers a content filter.""" def __init__(self, status_code: int, model_name: str, body: object | None = None): - message = f"Prompt content filtered by model '{model_name}'" + message = f"Model '{model_name}' content filter was triggered by the user's prompt" super().__init__(message, status_code, model_name, body) @@ -202,7 +202,7 @@ class ResponseContentFilterError(ContentFilterError): """Raised when the generated response triggers a content filter.""" def __init__(self, model_name: str, body: object | None = None, status_code: int = 200): - message = f"Response content filtered by model '{model_name}'" + message = f"Model '{model_name}' triggered its content filter while generating a response" super().__init__(message, status_code, model_name, body) diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index d0b3c82d93..f12d93a1ab 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -1432,7 +1432,6 @@ async def _responses_create( # noqa: C901 if (status_code := e.status_code) >= 400: _check_azure_content_filter(e, self.model_name) - # Reverted cast raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index fbecd3dd88..ca900ab62a 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -7915,7 +7915,10 @@ async def test_anthropic_response_filter_error_sync(allow_model_requests: None): m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): + with pytest.raises( + ResponseContentFilterError, + match=r"Model 'claude-3-5-haiku-123' triggered its content filter while generating a response", + ): await agent.run('hello') @@ -7945,7 +7948,10 @@ async def test_anthropic_response_filter_error_stream(allow_model_requests: None m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): + with pytest.raises( + ResponseContentFilterError, + match=r"Model 'claude-3-5-haiku-123' triggered its content filter while generating a response", + ): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 7b3f8e3199..38ec043c20 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -989,7 +989,10 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p agent = Agent(m, instructions='You hate the world!', model_settings=settings) # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError - with pytest.raises(ResponseContentFilterError, match="Response content filtered by model 'gemini-1.5-flash'"): + with pytest.raises( + ResponseContentFilterError, + match="Model 'gemini-1.5-flash' triggered its content filter while generating a response", + ): await agent.run('Tell me a joke about a Brazilians.') @@ -4456,7 +4459,10 @@ async def test_google_response_filter_error_sync( agent = Agent(model=model) # Verify the exception is raised - with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): + with pytest.raises( + ResponseContentFilterError, + match="Model 'gemini-2.5-flash' triggered its content filter while generating a response", + ): await agent.run('bad content') @@ -4484,7 +4490,10 @@ async def stream_iterator(): agent = Agent(model=model) - with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): + with pytest.raises( + ResponseContentFilterError, + match="Model 'gemini-2.5-flash' triggered its content filter while generating a response", + ): async with agent.run_stream('bad content') as result: async for _ in result.stream_text(): pass diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 829c52f645..8f49509e78 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -11,6 +11,7 @@ import pytest from inline_snapshot import snapshot from pydantic import AnyUrl, BaseModel, ConfigDict, Discriminator, Field, Tag +from pytest_mock import MockerFixture from typing_extensions import NotRequired, TypedDict from pydantic_ai import ( @@ -72,6 +73,7 @@ from openai.types.chat.chat_completion_message_tool_call import Function from openai.types.chat.chat_completion_token_logprob import ChatCompletionTokenLogprob from openai.types.completion_usage import CompletionUsage, PromptTokensDetails + from openai.types.responses import Response, ResponseCompletedEvent, ResponseCreatedEvent from pydantic_ai.models.google import GoogleModel from pydantic_ai.models.openai import ( @@ -3299,23 +3301,6 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): ) -def test_openai_generic_400_error(allow_model_requests: None) -> None: - mock_client = MockOpenAI.create_mock( - APIStatusError( - 'bad request', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), - body={'error': {'code': 'invalid_parameter', 'message': 'Invalid param.'}}, - ) - ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - with pytest.raises(ModelHTTPError) as exc_info: - agent.run_sync('hello') - - assert not isinstance(exc_info.value, PromptContentFilterError) - assert exc_info.value.status_code == 400 - - def test_azure_prompt_filter_error(allow_model_requests: None) -> None: mock_client = MockOpenAI.create_mock( APIStatusError( @@ -3328,8 +3313,9 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # Asserting the full error message structure via match - with pytest.raises(PromptContentFilterError, match=r"Prompt content filtered by model 'gpt-5-mini'"): + with pytest.raises( + PromptContentFilterError, match=r"Model 'gpt-5-mini' content filter was triggered by the user's prompt" + ): agent.run_sync('bad prompt') @@ -3346,7 +3332,9 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): await agent.run('hello') @@ -3361,7 +3349,85 @@ async def test_openai_response_filter_error_stream(allow_model_requests: None): m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): + async with agent.run_stream('hello') as result: + async for _ in result.stream_text(): + pass + + +def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None: + """Test ResponsesModel (Azure) prompt filter.""" + mock_client = MockOpenAIResponses.create_mock( + APIStatusError( + 'content filter', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, + ) + ) + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises( + PromptContentFilterError, match=r"Model 'gpt-5-mini' content filter was triggered by the user's prompt" + ): + agent.run_sync('bad prompt') + + +async def test_responses_response_filter_error_sync(allow_model_requests: None, mocker: MockerFixture): + """Test ResponsesModel sync response filter.""" + mock_openai_client = mocker.Mock() + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_openai_client)) + agent = Agent(m) + + mock_response = mocker.Mock() + mock_response.model = 'gpt-5-mini' + mock_response.model_dump.return_value = {} + mock_response.created_at = 1234567890 + mock_response.id = 'resp_123' + mock_response.incomplete_details.reason = 'content_filter' + mock_response.output = [] + + mock_openai_client.responses.create = mocker.AsyncMock(return_value=mock_response) + + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): + await agent.run('hello') + + +async def test_responses_response_filter_error_stream(allow_model_requests: None, mocker: MockerFixture): + """Test ResponsesModel stream response filter.""" + mock_openai_client = mocker.Mock() + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_openai_client)) + agent = Agent(m) + + resp = Response( + id='resp_123', + model='gpt-5-mini', + created_at=1234567890, + object='response', + status='incomplete', + incomplete_details=cast(Any, {'reason': 'content_filter'}), + output=[], + parallel_tool_calls=False, + tool_choice='none', + tools=[], + ) + + stream = [ + ResponseCreatedEvent(response=resp, type='response.created', sequence_number=0), + ResponseCompletedEvent(response=resp, type='response.completed', sequence_number=1), + ] + + mock_client = MockOpenAIResponses.create_mock_stream(stream) + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass From b67d216bc2a3dd12b5d1d13c09436226c3495df0 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 6 Dec 2025 05:43:48 +0000 Subject: [PATCH 7/8] test coverage --- tests/models/test_openai.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 8f49509e78..dd9c7556b5 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3431,3 +3431,42 @@ async def test_responses_response_filter_error_stream(allow_model_requests: None async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass + + +def test_openai_400_non_content_filter(allow_model_requests: None) -> None: + """Test a standard 400 error (not content filter) raises ModelHTTPError.""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + body={'error': {'code': 'invalid_parameter', 'message': 'Invalid parameter'}}, + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert exc_info.value.status_code == 400 + + assert not isinstance(exc_info.value, PromptContentFilterError) + + +def test_openai_400_non_dict_body(allow_model_requests: None) -> None: + """Test a 400 error with a non-dict body raises ModelHTTPError.""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + body='Raw string body', + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert exc_info.value.status_code == 400 + assert not isinstance(exc_info.value, PromptContentFilterError) From 4ac16089d9aff6b34f6edc5dc567812449b53af6 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 6 Dec 2025 06:06:38 +0000 Subject: [PATCH 8/8] fix ci fails --- pydantic_ai_slim/pydantic_ai/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/_utils.py b/pydantic_ai_slim/pydantic_ai/_utils.py index 97230bb046..d0fc819e73 100644 --- a/pydantic_ai_slim/pydantic_ai/_utils.py +++ b/pydantic_ai_slim/pydantic_ai/_utils.py @@ -215,7 +215,7 @@ async def async_iter_groups() -> AsyncIterator[list[T]]: try: yield async_iter_groups() - finally: # pragma: no cover + finally: # after iteration if a tasks still exists, cancel it, this will only happen if an error occurred if task: task.cancel('Cancelling due to error in iterator')