From 6ad1789a2761b79e2a7b19d01a984685be2bee8f Mon Sep 17 00:00:00 2001 From: TJ Miller Date: Sat, 9 Nov 2024 18:06:26 -0500 Subject: [PATCH] Test utilities (#54) --- docs/.vitepress/config.mts | 4 + docs/core-concepts/testing.md | 162 ++++++++++++++++++++++++++++ src/Concerns/BuildsTextRequests.php | 1 + src/Prism.php | 25 ++++- src/Requests/TextRequest.php | 1 + src/Testing/PrismFake.php | 90 ++++++++++++++++ tests/Testing/PrismFakeTest.php | 61 +++++++++++ 7 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 docs/core-concepts/testing.md create mode 100644 src/Testing/PrismFake.php create mode 100644 tests/Testing/PrismFakeTest.php diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 6284a85..613b05d 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -102,6 +102,10 @@ export default defineConfig({ text: "Prism Server", link: "/core-concepts/prism-server", }, + { + text: "Testing", + link: "/core-concepts/testing", + }, ], }, { diff --git a/docs/core-concepts/testing.md b/docs/core-concepts/testing.md new file mode 100644 index 0000000..079da54 --- /dev/null +++ b/docs/core-concepts/testing.md @@ -0,0 +1,162 @@ +# Testing + +Want to make sure your Prism integrations work flawlessly? Let's dive into testing! Prism provides a powerful fake implementation that makes it a breeze to test your AI-powered features. + +## Basic Test Setup + +First, let's look at how to set up basic response faking: + +```php +use EchoLabs\Prism\Facades\Prism; +use EchoLabs\Prism\ValueObjects\Usage; +use EchoLabs\Prism\Enums\FinishReason; +use EchoLabs\Prism\Providers\ProviderResponse; + +public function test_can_generate_text(): void +{ + // Create a fake provider response + $fakeResponse = new ProviderResponse( + text: 'Hello, I am Claude!', + toolCalls: [], + usage: new Usage(10, 20), + finishReason: FinishReason::Stop, + response: ['id' => 'fake-1', 'model' => 'fake-model'] + ); + + // Set up the fake + $fake = Prism::fake([$fakeResponse]); + + // Run your code + $response = Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('Who are you?') + ->generate(); + + // Make assertions + $this->assertEquals('Hello, I am Claude!', $response->text); +} +``` + +## Testing Multiple Responses + +When testing conversations or tool usage, you might need to simulate multiple responses: + +```php +public function test_can_handle_tool_calls(): void +{ + $responses = [ + new ProviderResponse( + text: '', + toolCalls: [ + new ToolCall( + id: 'call_1', + name: 'search', + arguments: ['query' => 'Latest news'] + ) + ], + usage: new Usage(15, 25), + finishReason: FinishReason::ToolCalls, + response: ['id' => 'fake-1', 'model' => 'fake-model'] + ), + new ProviderResponse( + text: 'Here are the latest news...', + toolCalls: [], + usage: new Usage(20, 30), + finishReason: FinishReason::Stop, + response: ['id' => 'fake-2', 'model' => 'fake-model'] + ), + ]; + + $fake = Prism::fake($responses); +} +``` + +## Assertions + +Prism's fake implementation provides several helpful assertion methods: + +```php +// Assert specific prompt was sent +$fake->assertPrompt('Who are you?'); + +// Assert number of calls made +$fake->assertCallCount(2); + +// Assert detailed request properties +$fake->assertRequest(function ($requests) { + $this->assertEquals('anthropic', $requests[0]->provider); + $this->assertEquals('claude-3-sonnet', $requests[0]->model); +}); +``` + +## Testing Tools + +When testing tools, you'll want to verify both the tool calls and their results. Here's a complete example: + +```php +public function test_can_use_weather_tool(): void +{ + // Define the expected tool call and response sequence + $responses = [ + // First response: AI decides to use the weather tool + new ProviderResponse( + text: '', // Empty text since the AI is using a tool + toolCalls: [ + new ToolCall( + id: 'call_123', + name: 'weather', + arguments: ['city' => 'Paris'] + ) + ], + usage: new Usage(15, 25), + finishReason: FinishReason::ToolCalls, + response: ['id' => 'fake-1', 'model' => 'fake-model'] + ), + // Second response: AI uses the tool result to form a response + new ProviderResponse( + text: 'Based on current conditions, the weather in Paris is sunny with a temperature of 72°F.', + toolCalls: [], + usage: new Usage(20, 30), + finishReason: FinishReason::Stop, + response: ['id' => 'fake-2', 'model' => 'fake-model'] + ), + ]; + + // Set up the fake + $fake = Prism::fake($responses); + + // Create the weather tool + $weatherTool = Tool::as('weather') + ->for('Get weather information') + ->withStringParameter('city', 'City name') + ->using(fn (string $city) => "The weather in {$city} is sunny with a temperature of 72°F"); + + // Run the actual test + $response = Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('What\'s the weather in Paris?') + ->withTools([$weatherTool]) + ->generate(); + + // Assert the correct number of API calls were made + $fake->assertCallCount(2); + + // Assert tool calls were made correctly + $this->assertCount(1, $response->steps[0]->toolCalls); + $this->assertEquals('weather', $response->steps[0]->toolCalls[0]->name); + $this->assertEquals(['city' => 'Paris'], $response->steps[0]->toolCalls[0]->arguments()); + + // Assert tool results were processed + $this->assertCount(1, $response->toolResults); + $this->assertEquals( + 'The weather in Paris is sunny with a temperature of 72°F', + $response->toolResults[0]->result + ); + + // Assert final response + $this->assertEquals( + 'Based on current conditions, the weather in Paris is sunny with a temperature of 72°F.', + $response->text + ); +} +``` diff --git a/src/Concerns/BuildsTextRequests.php b/src/Concerns/BuildsTextRequests.php index c9213b8..f0f56a8 100644 --- a/src/Concerns/BuildsTextRequests.php +++ b/src/Concerns/BuildsTextRequests.php @@ -151,6 +151,7 @@ protected function textRequest(): TextRequest return new TextRequest( model: $this->model, systemPrompt: $this->systemPrompt, + prompt: $this->prompt, messages: $this->state->messages()->toArray(), temperature: $this->temperature, maxTokens: $this->maxTokens, diff --git a/src/Prism.php b/src/Prism.php index aa4a6b3..6534af8 100644 --- a/src/Prism.php +++ b/src/Prism.php @@ -5,11 +5,34 @@ namespace EchoLabs\Prism; use EchoLabs\Prism\Contracts\Provider; +use EchoLabs\Prism\Enums\Provider as ProviderEnum; use EchoLabs\Prism\Generators\TextGenerator; +use EchoLabs\Prism\Providers\ProviderResponse; +use EchoLabs\Prism\Testing\PrismFake; class Prism { - protected Provider $provider; + /** + * @param array $responses + */ + public static function fake(array $responses = []): PrismFake + { + $fake = new PrismFake($responses); + + app()->instance(PrismManager::class, new class($fake) extends PrismManager + { + public function __construct( + private readonly PrismFake $fake + ) {} + + public function resolve(ProviderEnum|string $name): Provider + { + return $this->fake; + } + }); + + return $fake; + } public static function text(): TextGenerator { diff --git a/src/Requests/TextRequest.php b/src/Requests/TextRequest.php index 5f02802..3e3ee1a 100644 --- a/src/Requests/TextRequest.php +++ b/src/Requests/TextRequest.php @@ -18,6 +18,7 @@ class TextRequest public function __construct( public readonly string $model, public readonly ?string $systemPrompt, + public readonly ?string $prompt, public readonly array $messages, public readonly ?int $maxTokens, public readonly int|float|null $temperature, diff --git a/src/Testing/PrismFake.php b/src/Testing/PrismFake.php new file mode 100644 index 0000000..7847e8b --- /dev/null +++ b/src/Testing/PrismFake.php @@ -0,0 +1,90 @@ + */ + protected array $recorded = []; + + /** + * @param array $responses + */ + public function __construct(protected array $responses = []) {} + + #[\Override] + public function text(TextRequest $request): ProviderResponse + { + $this->recorded[] = $request; + + return $this->nextResponse() ?? new ProviderResponse( + text: '', + toolCalls: [], + usage: new Usage(0, 0), + finishReason: FinishReason::Stop, + response: ['id' => 'fake', 'model' => 'fake'] + ); + } + + /** + * @param Closure(array):void $fn + */ + public function assertRequest(Closure $fn): void + { + $fn($this->recorded); + } + + public function assertPrompt(string $prompt): void + { + $prompts = collect($this->recorded) + ->flatten() + ->map + ->prompt; + + PHPUnit::assertTrue( + $prompts->contains($prompt), + "Could not find the prompt {$prompt} in the recorded requests" + ); + } + + /** + * Assert number of calls made + */ + public function assertCallCount(int $expectedCount): void + { + $actualCount = count($this->recorded ?? []); + + PHPUnit::assertEquals($expectedCount, $actualCount, "Expected {$expectedCount} calls, got {$actualCount}"); + } + + protected function nextResponse(): ?ProviderResponse + { + if (! isset($this->responses)) { + return null; + } + + $responses = $this->responses; + $sequence = $this->responseSequence; + + if (! isset($responses[$sequence])) { + throw new Exception('Could not find a response for the request'); + } + + $this->responseSequence++; + + return $responses[$sequence]; + } +} diff --git a/tests/Testing/PrismFakeTest.php b/tests/Testing/PrismFakeTest.php new file mode 100644 index 0000000..1040cdc --- /dev/null +++ b/tests/Testing/PrismFakeTest.php @@ -0,0 +1,61 @@ + 'cpl_1234', 'model' => 'claude-3-sonnet'], + ), + ]); + + Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('What is the meaning of life?') + ->generate(); + + $fake->assertCallCount(1); + $fake->assertPrompt('What is the meaning of life?'); + $fake->assertRequest(function (array $requests): void { + expect($requests)->toHaveCount(1); + expect($requests[0])->toBeInstanceOf(TextRequest::class); + }); +}); + +it("throws an exception when it can't runs out of responses", function (): void { + $this->expectException(Exception::class); + $this->expectExceptionMessage('Could not find a response for the request'); + + Prism::fake([ + new ProviderResponse( + text: 'The meaning of life is 42', + toolCalls: [], + usage: new Usage(42, 42), + finishReason: FinishReason::Stop, + response: ['id' => 'cpl_1234', 'model' => 'claude-3-sonnet'], + ), + ]); + + Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('What is the meaning of life?') + ->generate(); + + Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('What is the meaning of life?') + ->generate(); +});