-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Add examples field to ToolDefinition and send it to Anthropic
#3619
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -262,6 +262,7 @@ class Tool(Generic[ToolAgentDepsT]): | |
| sequential: bool | ||
| requires_approval: bool | ||
| metadata: dict[str, Any] | None | ||
| input_examples: list[dict[str, Any]] | None | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should also add this to |
||
| function_schema: _function_schema.FunctionSchema | ||
| """ | ||
| The base JSON schema for the tool's parameters. | ||
|
|
@@ -285,6 +286,7 @@ def __init__( | |
| sequential: bool = False, | ||
| requires_approval: bool = False, | ||
| metadata: dict[str, Any] | None = None, | ||
| input_examples: list[dict[str, Any]] | None = None, | ||
| function_schema: _function_schema.FunctionSchema | None = None, | ||
| ): | ||
| """Create a new tool instance. | ||
|
|
@@ -341,6 +343,8 @@ async def prep_my_tool( | |
| requires_approval: Whether this tool requires human-in-the-loop approval. Defaults to False. | ||
| See the [tools documentation](../deferred-tools.md#human-in-the-loop-tool-approval) for more info. | ||
| metadata: Optional metadata for the tool. This is not sent to the model but can be used for filtering and tool behavior customization. | ||
| input_examples: Example inputs demonstrating correct tool usage. Defaults to None. | ||
| See [`ToolDefinition.input_examples`][pydantic_ai.tools.ToolDefinition.input_examples] for more info. | ||
| function_schema: The function schema to use for the tool. If not provided, it will be generated. | ||
| """ | ||
| self.function = function | ||
|
|
@@ -362,6 +366,7 @@ async def prep_my_tool( | |
| self.sequential = sequential | ||
| self.requires_approval = requires_approval | ||
| self.metadata = metadata | ||
| self.input_examples = input_examples | ||
|
|
||
| @classmethod | ||
| def from_schema( | ||
|
|
@@ -418,6 +423,7 @@ def tool_def(self): | |
| sequential=self.sequential, | ||
| metadata=self.metadata, | ||
| kind='unapproved' if self.requires_approval else 'function', | ||
| input_examples=self.input_examples, | ||
| ) | ||
|
|
||
| async def prepare_tool_def(self, ctx: RunContext[ToolAgentDepsT]) -> ToolDefinition | None: | ||
|
|
@@ -503,6 +509,18 @@ class ToolDefinition: | |
| For MCP tools, this contains the `meta`, `annotations`, and `output_schema` fields from the tool definition. | ||
| """ | ||
|
|
||
| input_examples: list[dict[str, Any]] | None = None | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Anthropic calls them |
||
| """Example inputs demonstrating correct tool usage patterns. | ||
|
|
||
| Provide 1-5 realistic examples showing parameter conventions, optional field patterns, | ||
| nested structures, and API-specific conventions. Each example must validate against | ||
| the tool's `parameters_json_schema`. | ||
|
|
||
| Supported by: | ||
|
|
||
| * [Anthropic](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/tool-use-examples) | ||
| """ | ||
|
|
||
| @property | ||
| def defer(self) -> bool: | ||
| """Whether calls to this tool will be deferred. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| """Tests for Anthropic input_examples feature on ToolDefinition. | ||
|
|
||
| This feature allows providing example inputs to help the model understand | ||
| correct tool usage patterns. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from typing import Any, cast | ||
|
|
||
| import pytest | ||
|
|
||
| from pydantic_ai import Agent, Tool | ||
| from pydantic_ai.tools import ToolDefinition | ||
|
|
||
| from ...conftest import try_import | ||
| from ..test_anthropic import MockAnthropic, completion_message | ||
|
|
||
| with try_import() as imports_successful: | ||
| from anthropic.types.beta import BetaTextBlock, BetaUsage | ||
|
|
||
| from pydantic_ai.models.anthropic import AnthropicModel | ||
| from pydantic_ai.providers.anthropic import AnthropicProvider | ||
|
|
||
|
|
||
| pytestmark = [ | ||
| pytest.mark.skipif(not imports_successful(), reason='anthropic not installed'), | ||
| ] | ||
|
|
||
|
|
||
| class TestToolDefinitionInputExamples: | ||
| """Tests for ToolDefinition.input_examples field.""" | ||
|
|
||
| def test_input_examples_defaults_to_none(self): | ||
| """Test that input_examples defaults to None.""" | ||
| tool_def = ToolDefinition(name='test_tool') | ||
| assert tool_def.input_examples is None | ||
|
|
||
| def test_input_examples_can_be_set(self): | ||
| """Test setting input_examples.""" | ||
| examples = [ | ||
| {'param1': 'value1', 'param2': 123}, | ||
| {'param1': 'value2'}, | ||
| ] | ||
| tool_def = ToolDefinition(name='test_tool', input_examples=examples) | ||
| assert tool_def.input_examples == examples | ||
|
|
||
|
|
||
| class TestToolInputExamples: | ||
| """Tests for Tool class with input_examples.""" | ||
|
|
||
| def test_tool_input_examples_default(self): | ||
| """Test that Tool.input_examples defaults to None.""" | ||
|
|
||
| def my_tool(x: int) -> str: | ||
| return str(x) | ||
|
|
||
| tool = Tool(my_tool) | ||
| assert tool.input_examples is None | ||
| assert tool.tool_def.input_examples is None | ||
|
|
||
| def test_tool_input_examples_set(self): | ||
| """Test setting Tool.input_examples.""" | ||
|
|
||
| def my_tool(x: int) -> str: | ||
| return str(x) | ||
|
|
||
| examples = [{'x': 1}, {'x': 42}] | ||
| tool = Tool(my_tool, input_examples=examples) | ||
| assert tool.input_examples == examples | ||
| assert tool.tool_def.input_examples == examples | ||
|
|
||
|
|
||
| class TestAnthropicMapToolDefinition: | ||
| """Tests for AnthropicModel._map_tool_definition with input_examples.""" | ||
|
|
||
| def test_map_tool_definition_without_input_examples(self): | ||
| """Test tool definition mapping without input_examples.""" | ||
| tool_def = ToolDefinition( | ||
| name='test_tool', | ||
| description='A test tool', | ||
| parameters_json_schema={'type': 'object', 'properties': {'x': {'type': 'integer'}}}, | ||
| ) | ||
| c = completion_message( | ||
| [BetaTextBlock(text='Hello', type='text')], | ||
| BetaUsage(input_tokens=5, output_tokens=10), | ||
| ) | ||
| mock_client = MockAnthropic.create_mock(c) | ||
| model = AnthropicModel('claude-sonnet-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) | ||
| result = model._map_tool_definition(tool_def) # pyright: ignore[reportPrivateUsage] | ||
| result_dict = cast(dict[str, Any], result) | ||
| assert result_dict['name'] == 'test_tool' | ||
| assert result_dict['description'] == 'A test tool' | ||
| assert 'input_examples' not in result_dict | ||
|
|
||
| def test_map_tool_definition_with_input_examples(self): | ||
| """Test tool definition mapping with input_examples.""" | ||
| examples = [{'x': 1}, {'x': 2}] | ||
| tool_def = ToolDefinition( | ||
| name='test_tool', | ||
| description='A test tool', | ||
| input_examples=examples, | ||
| ) | ||
| c = completion_message( | ||
| [BetaTextBlock(text='Hello', type='text')], | ||
| BetaUsage(input_tokens=5, output_tokens=10), | ||
| ) | ||
| mock_client = MockAnthropic.create_mock(c) | ||
| model = AnthropicModel('claude-sonnet-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) | ||
| result = model._map_tool_definition(tool_def) # pyright: ignore[reportPrivateUsage] | ||
| result_dict = cast(dict[str, Any], result) | ||
| assert result_dict['input_examples'] == examples | ||
|
|
||
|
|
||
| class TestAgentWithInputExamples: | ||
| """Tests for Agent with input_examples on tools.""" | ||
|
|
||
| def test_agent_with_input_examples_tool(self): | ||
| """Test creating an agent with a tool that has input_examples.""" | ||
|
|
||
| def my_tool(x: int) -> str: | ||
| """A test tool.""" | ||
| return str(x) | ||
|
|
||
| examples = [{'x': 1}, {'x': 42}] | ||
| agent = Agent( | ||
| 'test', | ||
| tools=[Tool(my_tool, input_examples=examples)], | ||
| ) | ||
|
|
||
| # Verify the tool was registered with input_examples | ||
| tool = agent._function_toolset.tools.get('my_tool') # pyright: ignore[reportPrivateUsage] | ||
| assert tool is not None | ||
| assert tool.input_examples == examples | ||
|
|
||
| def test_agent_tool_plain_decorator_with_input_examples(self): | ||
| """Test the @agent.tool_plain decorator with input_examples.""" | ||
| agent: Agent[None, str] = Agent('test') | ||
|
|
||
| examples = [{'x': 1}, {'x': 42}] | ||
|
|
||
| @agent.tool_plain(input_examples=examples) | ||
| def my_example_tool(x: int) -> str: | ||
| """A tool with examples.""" | ||
| return str(x) | ||
|
|
||
| tool = agent._function_toolset.tools.get('my_example_tool') # pyright: ignore[reportPrivateUsage] | ||
| assert tool is not None | ||
| assert tool.input_examples == examples |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There's a different beta header for Vertex and Bedrock: https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use#providing-tool-use-examples
That's relevant when using
AnthropicModelwithAnthropicProviderand a customanthropic_clientlikeAsyncAnthropicVertex. So we should check the type ofself._clientto send the right header.