From 57bad218222c3b48e10ff4db4c6d1b63872b0721 Mon Sep 17 00:00:00 2001 From: Christopher Hertel Date: Sat, 15 Feb 2025 02:01:32 +0100 Subject: [PATCH] feat: introduce optional fault tolerant toolbox --- README.md | 22 +++++- .../ToolBox/Exception/ExceptionInterface.php | 11 +++ .../Exception/ToolConfigurationException.php | 21 +++++ .../Exception/ToolExecutionException.php | 20 +++++ .../Exception/ToolNotFoundException.php | 20 +++++ src/Chain/ToolBox/FaultTolerantToolBox.php | 38 +++++++++ src/Chain/ToolBox/ParameterAnalyzer.php | 4 +- src/Chain/ToolBox/ToolAnalyzer.php | 4 +- src/Chain/ToolBox/ToolBox.php | 7 +- src/Chain/ToolBox/ToolBoxInterface.php | 6 ++ src/Exception/InvalidToolImplementation.php | 15 ---- src/Exception/ToolBoxException.php | 33 -------- .../ToolBox/FaultTolerantToolBoxTest.php | 78 +++++++++++++++++++ tests/Chain/ToolBox/ToolAnalyzerTest.php | 6 +- tests/Chain/ToolBox/ToolBoxTest.php | 11 +-- 15 files changed, 229 insertions(+), 67 deletions(-) create mode 100644 src/Chain/ToolBox/Exception/ExceptionInterface.php create mode 100644 src/Chain/ToolBox/Exception/ToolConfigurationException.php create mode 100644 src/Chain/ToolBox/Exception/ToolExecutionException.php create mode 100644 src/Chain/ToolBox/Exception/ToolNotFoundException.php create mode 100644 src/Chain/ToolBox/FaultTolerantToolBox.php delete mode 100644 src/Exception/InvalidToolImplementation.php delete mode 100644 src/Exception/ToolBoxException.php create mode 100644 tests/Chain/ToolBox/FaultTolerantToolBoxTest.php diff --git a/README.md b/README.md index fb94a39e..0b77e60b 100644 --- a/README.md +++ b/README.md @@ -139,9 +139,6 @@ Tool calling can be enabled by registering the processors in the chain: use PhpLlm\LlmChain\Chain\ToolBox\ChainProcessor; use PhpLlm\LlmChain\Chain\ToolBox\ToolAnalyzer; use PhpLlm\LlmChain\Chain\ToolBox\ToolBox; -use Symfony\Component\Serializer\Encoder\JsonEncoder; -use Symfony\Component\Serializer\Normalizer\ObjectNormalizer; -use Symfony\Component\Serializer\Serializer; // Platform & LLM instantiation @@ -180,7 +177,6 @@ You can configure the method to be called by the LLM with the `#[AsTool]` attrib ```php use PhpLlm\LlmChain\ToolBox\Attribute\AsTool; - #[AsTool(name: 'weather_current', description: 'get current weather for a location', method: 'current')] #[AsTool(name: 'weather_forecast', description: 'get weather forecast for a location', method: 'forecast')] final readonly class OpenMeteo @@ -231,6 +227,24 @@ See attribute class [ToolParameter](src/Chain/ToolBox/Attribute/ToolParameter.ph > [!NOTE] > Please be aware, that this is only converted in a JSON Schema for the LLM to respect, but not validated by LLM Chain. +#### Fault Tolerance + +To gracefully handle errors that occur during tool calling, e.g. wrong tool names or runtime errors, you can use the +`FaultTolerantToolBox` as a decorator for the `ToolBox`. It will catch the exceptions and return readable error messages +to the LLM. + +```php +use PhpLlm\LlmChain\Chain\ToolBox\ChainProcessor; +use PhpLlm\LlmChain\Chain\ToolBox\FaultTolerantToolBox; + +// Platform, LLM & ToolBox instantiation + +$toolBox = new FaultTolerantToolBox($innerToolBox); +$toolProcessor = new ChainProcessor($toolBox); + +$chain = new Chain($platform, $llm, inputProcessor: [$toolProcessor], outputProcessor: [$toolProcessor]); +``` + #### Tool Result Interception To react to the result of a tool, you can implement an EventListener or EventSubscriber, that listens to the diff --git a/src/Chain/ToolBox/Exception/ExceptionInterface.php b/src/Chain/ToolBox/Exception/ExceptionInterface.php new file mode 100644 index 00000000..7962cbf5 --- /dev/null +++ b/src/Chain/ToolBox/Exception/ExceptionInterface.php @@ -0,0 +1,11 @@ +name, $previous->getMessage()), previous: $previous); + $exception->toolCall = $toolCall; + + return $exception; + } +} diff --git a/src/Chain/ToolBox/Exception/ToolNotFoundException.php b/src/Chain/ToolBox/Exception/ToolNotFoundException.php new file mode 100644 index 00000000..f64473a8 --- /dev/null +++ b/src/Chain/ToolBox/Exception/ToolNotFoundException.php @@ -0,0 +1,20 @@ +name)); + $exception->toolCall = $toolCall; + + return $exception; + } +} diff --git a/src/Chain/ToolBox/FaultTolerantToolBox.php b/src/Chain/ToolBox/FaultTolerantToolBox.php new file mode 100644 index 00000000..adfd1001 --- /dev/null +++ b/src/Chain/ToolBox/FaultTolerantToolBox.php @@ -0,0 +1,38 @@ +innerToolBox->getMap(); + } + + public function execute(ToolCall $toolCall): mixed + { + try { + return $this->innerToolBox->execute($toolCall); + } catch (ToolExecutionException $e) { + return sprintf('An error occurred while executing tool "%s".', $e->toolCall->name); + } catch (ToolNotFoundException) { + $names = array_map(fn (Metadata $metadata) => $metadata->name, $this->getMap()); + + return sprintf('Tool "%s" was not found, please use one of these: %s', $toolCall->name, implode(', ', $names)); + } + } +} diff --git a/src/Chain/ToolBox/ParameterAnalyzer.php b/src/Chain/ToolBox/ParameterAnalyzer.php index 4d95f1e2..598a00eb 100644 --- a/src/Chain/ToolBox/ParameterAnalyzer.php +++ b/src/Chain/ToolBox/ParameterAnalyzer.php @@ -5,7 +5,7 @@ namespace PhpLlm\LlmChain\Chain\ToolBox; use PhpLlm\LlmChain\Chain\ToolBox\Attribute\ToolParameter; -use PhpLlm\LlmChain\Exception\ToolBoxException; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException; /** * @phpstan-type ParameterDefinition array{ @@ -46,7 +46,7 @@ public function getDefinition(string $className, string $methodName): ?array try { $reflection = new \ReflectionMethod($className, $methodName); } catch (\ReflectionException) { - throw ToolBoxException::invalidMethod($className, $methodName); + throw ToolConfigurationException::invalidMethod($className, $methodName); } $parameters = $reflection->getParameters(); diff --git a/src/Chain/ToolBox/ToolAnalyzer.php b/src/Chain/ToolBox/ToolAnalyzer.php index 1d02e216..3a41065c 100644 --- a/src/Chain/ToolBox/ToolAnalyzer.php +++ b/src/Chain/ToolBox/ToolAnalyzer.php @@ -5,7 +5,7 @@ namespace PhpLlm\LlmChain\Chain\ToolBox; use PhpLlm\LlmChain\Chain\ToolBox\Attribute\AsTool; -use PhpLlm\LlmChain\Exception\InvalidToolImplementation; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException; final readonly class ToolAnalyzer { @@ -25,7 +25,7 @@ public function getMetadata(string $className): iterable $attributes = $reflectionClass->getAttributes(AsTool::class); if (0 === count($attributes)) { - throw InvalidToolImplementation::missingAttribute($className); + throw ToolConfigurationException::missingAttribute($className); } foreach ($attributes as $attribute) { diff --git a/src/Chain/ToolBox/ToolBox.php b/src/Chain/ToolBox/ToolBox.php index bbb128e4..e9dfd575 100644 --- a/src/Chain/ToolBox/ToolBox.php +++ b/src/Chain/ToolBox/ToolBox.php @@ -4,7 +4,8 @@ namespace PhpLlm\LlmChain\Chain\ToolBox; -use PhpLlm\LlmChain\Exception\ToolBoxException; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException; use PhpLlm\LlmChain\Model\Response\ToolCall; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; @@ -61,13 +62,13 @@ public function execute(ToolCall $toolCall): mixed $result = $tool->{$metadata->method}(...$toolCall->arguments); } catch (\Throwable $e) { $this->logger->warning(sprintf('Failed to execute tool "%s".', $metadata->name), ['exception' => $e]); - throw ToolBoxException::executionFailed($toolCall, $e); + throw ToolExecutionException::executionFailed($toolCall, $e); } return $result; } } - throw ToolBoxException::notFoundForToolCall($toolCall); + throw ToolNotFoundException::notFoundForToolCall($toolCall); } } diff --git a/src/Chain/ToolBox/ToolBoxInterface.php b/src/Chain/ToolBox/ToolBoxInterface.php index 4cb8c7df..99454c69 100644 --- a/src/Chain/ToolBox/ToolBoxInterface.php +++ b/src/Chain/ToolBox/ToolBoxInterface.php @@ -4,6 +4,8 @@ namespace PhpLlm\LlmChain\Chain\ToolBox; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException; use PhpLlm\LlmChain\Model\Response\ToolCall; interface ToolBoxInterface @@ -13,5 +15,9 @@ interface ToolBoxInterface */ public function getMap(): array; + /** + * @throws ToolExecutionException if the tool execution fails + * @throws ToolNotFoundException if the tool is not found + */ public function execute(ToolCall $toolCall): mixed; } diff --git a/src/Exception/InvalidToolImplementation.php b/src/Exception/InvalidToolImplementation.php deleted file mode 100644 index 703633a0..00000000 --- a/src/Exception/InvalidToolImplementation.php +++ /dev/null @@ -1,15 +0,0 @@ -name)); - $exception->toolCall = $toolCall; - - return $exception; - } - - public static function invalidMethod(string $toolClass, string $methodName): self - { - return new self(sprintf('Method "%s" not found in tool "%s".', $methodName, $toolClass)); - } - - public static function executionFailed(ToolCall $toolCall, \Throwable $previous): self - { - $exception = new self(sprintf('Execution of tool "%s" failed with error: %s', $toolCall->name, $previous->getMessage()), previous: $previous); - $exception->toolCall = $toolCall; - - return $exception; - } -} diff --git a/tests/Chain/ToolBox/FaultTolerantToolBoxTest.php b/tests/Chain/ToolBox/FaultTolerantToolBoxTest.php new file mode 100644 index 00000000..c13b8b08 --- /dev/null +++ b/tests/Chain/ToolBox/FaultTolerantToolBoxTest.php @@ -0,0 +1,78 @@ +createFaultyToolBox( + fn (ToolCall $toolCall) => ToolExecutionException::executionFailed($toolCall, new \Exception('error')) + ); + + $faultTolerantToolBox = new FaultTolerantToolBox($faultyToolBox); + $expected = 'An error occurred while executing tool "tool_foo".'; + + $toolCall = new ToolCall('987654321', 'tool_foo'); + $actual = $faultTolerantToolBox->execute($toolCall); + + self::assertSame($expected, $actual); + } + + #[Test] + public function faultyToolCall(): void + { + $faultyToolBox = $this->createFaultyToolBox( + fn (ToolCall $toolCall) => ToolNotFoundException::notFoundForToolCall($toolCall) + ); + + $faultTolerantToolBox = new FaultTolerantToolBox($faultyToolBox); + $expected = 'Tool "tool_xyz" was not found, please use one of these: tool_no_params, tool_required_params'; + + $toolCall = new ToolCall('123456789', 'tool_xyz'); + $actual = $faultTolerantToolBox->execute($toolCall); + + self::assertSame($expected, $actual); + } + + private function createFaultyToolBox(\Closure $exceptionFactory): ToolBoxInterface + { + return new class($exceptionFactory) implements ToolBoxInterface { + public function __construct(private readonly \Closure $exceptionFactory) + { + } + + /** + * @return Metadata[] + */ + public function getMap(): array + { + return [ + new Metadata(ToolNoParams::class, 'tool_no_params', 'A tool without parameters', '__invoke', null), + new Metadata(ToolRequiredParams::class, 'tool_required_params', 'A tool with required parameters', 'bar', null), + ]; + } + + public function execute(ToolCall $toolCall): mixed + { + throw ($this->exceptionFactory)($toolCall); + } + }; + } +} diff --git a/tests/Chain/ToolBox/ToolAnalyzerTest.php b/tests/Chain/ToolBox/ToolAnalyzerTest.php index 30314e7f..fc2952ac 100644 --- a/tests/Chain/ToolBox/ToolAnalyzerTest.php +++ b/tests/Chain/ToolBox/ToolAnalyzerTest.php @@ -5,10 +5,10 @@ namespace PhpLlm\LlmChain\Tests\Chain\ToolBox; use PhpLlm\LlmChain\Chain\ToolBox\Attribute\AsTool; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException; use PhpLlm\LlmChain\Chain\ToolBox\Metadata; use PhpLlm\LlmChain\Chain\ToolBox\ParameterAnalyzer; use PhpLlm\LlmChain\Chain\ToolBox\ToolAnalyzer; -use PhpLlm\LlmChain\Exception\InvalidToolImplementation; use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolMultiple; use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolRequiredParams; use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolWrong; @@ -21,7 +21,7 @@ #[UsesClass(AsTool::class)] #[UsesClass(Metadata::class)] #[UsesClass(ParameterAnalyzer::class)] -#[UsesClass(InvalidToolImplementation::class)] +#[UsesClass(ToolConfigurationException::class)] final class ToolAnalyzerTest extends TestCase { private ToolAnalyzer $toolAnalyzer; @@ -34,7 +34,7 @@ protected function setUp(): void #[Test] public function withoutAttribute(): void { - $this->expectException(InvalidToolImplementation::class); + $this->expectException(ToolConfigurationException::class); iterator_to_array($this->toolAnalyzer->getMetadata(ToolWrong::class)); } diff --git a/tests/Chain/ToolBox/ToolBoxTest.php b/tests/Chain/ToolBox/ToolBoxTest.php index 90603abf..e75b9b51 100644 --- a/tests/Chain/ToolBox/ToolBoxTest.php +++ b/tests/Chain/ToolBox/ToolBoxTest.php @@ -5,11 +5,13 @@ namespace PhpLlm\LlmChain\Tests\Chain\ToolBox; use PhpLlm\LlmChain\Chain\ToolBox\Attribute\AsTool; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolConfigurationException; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolExecutionException; +use PhpLlm\LlmChain\Chain\ToolBox\Exception\ToolNotFoundException; use PhpLlm\LlmChain\Chain\ToolBox\Metadata; use PhpLlm\LlmChain\Chain\ToolBox\ParameterAnalyzer; use PhpLlm\LlmChain\Chain\ToolBox\ToolAnalyzer; use PhpLlm\LlmChain\Chain\ToolBox\ToolBox; -use PhpLlm\LlmChain\Exception\ToolBoxException; use PhpLlm\LlmChain\Model\Response\ToolCall; use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolException; use PhpLlm\LlmChain\Tests\Fixture\Tool\ToolMisconfigured; @@ -28,7 +30,6 @@ #[UsesClass(Metadata::class)] #[UsesClass(ParameterAnalyzer::class)] #[UsesClass(ToolAnalyzer::class)] -#[UsesClass(ToolBoxException::class)] final class ToolBoxTest extends TestCase { private ToolBox $toolBox; @@ -117,7 +118,7 @@ public function toolsMap(): void #[Test] public function executeWithUnknownTool(): void { - self::expectException(ToolBoxException::class); + self::expectException(ToolNotFoundException::class); self::expectExceptionMessage('Tool not found for call: foo_bar_baz'); $this->toolBox->execute(new ToolCall('call_1234', 'foo_bar_baz')); @@ -126,7 +127,7 @@ public function executeWithUnknownTool(): void #[Test] public function executeWithMisconfiguredTool(): void { - self::expectException(ToolBoxException::class); + self::expectException(ToolConfigurationException::class); self::expectExceptionMessage('Method "foo" not found in tool "PhpLlm\LlmChain\Tests\Fixture\Tool\ToolMisconfigured".'); $toolBox = new ToolBox(new ToolAnalyzer(), [new ToolMisconfigured()]); @@ -137,7 +138,7 @@ public function executeWithMisconfiguredTool(): void #[Test] public function executeWithException(): void { - self::expectException(ToolBoxException::class); + self::expectException(ToolExecutionException::class); self::expectExceptionMessage('Execution of tool "tool_exception" failed with error: Tool error.'); $this->toolBox->execute(new ToolCall('call_1234', 'tool_exception'));