Skip to content
Snippets Groups Projects
Commit 37dbc842 authored by Andrew Belcher's avatar Andrew Belcher Committed by Marcus Johansson
Browse files

Issue #3493865: Provide a framework for testing with real providers

parent 591c0ee5
No related branches found
No related tags found
1 merge request!338Issue #3493865: Provide a framework for testing with real providers
Pipeline #369748 passed
<?xml version="1.0" encoding="UTF-8"?>
<!-- PHPUnit configuration for AI LLM tests. -->
<!-- Copy this into your project root, removing the .dist. -->
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
bootstrap="web/core/tests/bootstrap.php"
colors="true"
beStrictAboutTestsThatDoNotTestAnything="true"
beStrictAboutOutputDuringTests="true"
beStrictAboutChangesToGlobalState="true"
failOnWarning="true"
displayDetailsOnTestsThatTriggerErrors="true"
displayDetailsOnTestsThatTriggerWarnings="true"
displayDetailsOnTestsThatTriggerDeprecations="true"
cacheResult="false"
xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/10.5/phpunit.xsd"
cacheDirectory=".phpunit.cache">
<php>
<!-- Set error reporting to E_ALL. -->
<ini name="error_reporting" value="32767"/>
<!-- Do not limit the amount of memory tests take to run. -->
<ini name="memory_limit" value="-1"/>
<!-- Example SIMPLETEST_BASE_URL value: http://localhost -->
<env name="SIMPLETEST_DB" value=""/>
<!-- Override to run tests with a the comma separated list of models. -->
<!-- This overrides any test level configuration. -->
<env name="AI_PHPUNIT_TARGET_MODELS" value=""/>
<!-- Provide authentication credentials for any provider being run. -->
<!-- This can a string for a simple API key, or a JSON object. -->
<env name="AI_PHPUNIT_AUTH_<PROVIDER>" value=""/>
</php>
<testsuites>
<testsuite name="ai_llm">
<directory>web/modules/*/*/tests/src/AiLlm</directory>
</testsuite>
</testsuites>
</phpunit>
<?php
declare(strict_types=1);
namespace Drupal\Tests\ai\AiLlm;
use Drupal\ai\AiProviderInterface;
use Drupal\ai\Plugin\ProviderProxy;
use Drupal\KernelTests\KernelTestBase;
/**
* Base class for AI tests that run against real providers.
*/
abstract class AiProviderTestBase extends KernelTestBase {
/**
* {@inheritdoc}
*/
protected static $modules = [
'ai',
'key',
];
/**
* The default models to target for the test.
*
* @var string[]
*/
public static array $targetModels = [];
/**
* Get the target models, allowing env override via AI_PHPUNIT_TARGET_MODELS.
*
* @return string[]
* The models (provider__model).
*/
final protected static function getModels(): array {
if ($models = getenv('AI_PHPUNIT_TARGET_MODELS')) {
return explode(',', $models);
}
return static::$targetModels;
}
/**
* Get the provider.
*
* If there is no authentication, or the provider is not usable, the test will
* be marked as skipped.
*
* @param string $provider_id
* The provider ID.
* @param string $model
* The model being used.
*
* @return \Drupal\ai\AiProviderInterface|\Drupal\ai\Plugin\ProviderProxy
* The instantiated provider with authentication configured.
*
* @todo Allow providers to provide a test setup callback. See
* https://www.drupal.org/project/ai/issues/3494192.
*/
final protected function getProvider(string $provider_id, string $model): ProviderProxy|AiProviderInterface {
// Get any model and provider auth configuration.
$model_auth = getenv('AI_PHPUNIT_AUTH_' . strtoupper($provider_id) . '_' . strtoupper($model));
$provider_auth = getenv('AI_PHPUNIT_AUTH_' . strtoupper($provider_id));
// Attempt to decode json, falling back to a simple _auth string. We merge
// the provider and model auth, with model overriding provider.
$auth = array_merge(
json_decode($provider_auth ?: 'null', TRUE) ?:
array_filter(['_key' => $provider_auth]),
json_decode($model_auth ?: 'null', TRUE) ?:
array_filter(['_key' => $model_auth]),
);
// If we have no auth, we have to skip the test.
if (empty($auth)) {
$this->markTestSkipped("Provider {$provider_id} has no authentication.");
}
// If no modules have been provider, make an educated guess.
$auth['_modules'] ??= ['ai_provider_' . $provider_id];
$this->enableModules($auth['_modules']);
$manager = $this->container->get('ai.provider');
// Instantiate the plugin and check it's usable.
/** @var \Drupal\ai\AiProviderInterface|\Drupal\ai\Plugin\ProviderProxy $provider */
$provider = $manager->createInstance($provider_id);
$provider->setAuthentication($auth['_key']);
if (!$provider->isUsable()) {
$this->markTestSkipped("Provider {$provider_id} is not usable.");
}
return $provider;
}
}
<?php
namespace Drupal\Tests\ai\AiLlm;
use Drupal\Core\Form\FormStateInterface;
/**
* An interface for tests that can be modified and run via the AI Test UI.
*
* @phpstan-require-extends \Drupal\Tests\ai\AiLlm\AiProviderTestBase
*/
interface AiTestUiInterface {
/**
* The model agnostic data provider.
*
* @param string|null $model
* The model to generate test cases for.
*
* @return \Generator<int|string, array>
* The test cases.
*/
public static function dataProvider(?string $model): \Generator;
/**
* The configuration form for the AI Test UI.
*
* @param array<int, string> $config
* The existing config (may be empty).
*
* @return array<string, mixed>
* The config for elements.
*/
public static function getRunConfigForm(array $config): array;
/**
* Extract the run data fom the submitted form values.
*
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The form state.
*
* @return array<int, string>
* The run data.
*/
public static function getSubmittedRunData(FormStateInterface $form_state): array;
}
<?php
namespace Drupal\Tests\ai\AiLlm;
/**
* A trait to support tests that can be modified via the AI Test UI.
*
* @phpstan-require-implements \Drupal\Tests\ai\AiLlm\AiTestUiInterface
* @phpstan-require-extends \Drupal\Tests\ai\AiLlm\AiProviderTestBase
*/
trait AiTestUiTrait {
/**
* Data provider that handles UI provided data and the available models.
*
* @return \Generator<int|string, array>
* The full set of run data.
*/
public static function dataProviderWithModels(): \Generator {
if ($config = getenv('AI_PHPUNIT_UI_DATA')) {
yield json_decode($config, flags: \JSON_THROW_ON_ERROR);
}
else {
foreach (self::getModels() as $model) {
yield from static::dataProvider($model);
}
}
}
}
<?php
namespace Drupal\Tests\ai\Attributes;
use Drupal\Core\StringTranslation\TranslatableMarkup;
/**
* Defines an attribute class for metadata around tests exposed in the UI.
*/
#[\Attribute(\Attribute::TARGET_CLASS)]
class AiTestUi {
/**
* Construct the attribute.
*
* @param \Drupal\Core\StringTranslation\TranslatableMarkup $title
* The human friendly title for the test, used in menus and page titles.
* @param \Drupal\Core\StringTranslation\TranslatableMarkup|null $description
* An optional description for the test, used in menus.
* @param \Drupal\Core\StringTranslation\TranslatableMarkup|null $intro
* An optional intro to the test, shown in the run test UI.
*/
public function __construct(
public readonly TranslatableMarkup $title,
public readonly ?TranslatableMarkup $description = NULL,
public readonly ?TranslatableMarkup $intro = NULL,
) {}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment