Skip to content
Snippets Groups Projects
Commit 30fa3079 authored by Marcus Johansson's avatar Marcus Johansson
Browse files

Issue #3487158: Make sure that the agents logs are stored as well

parent 172b8333
No related branches found
No related tags found
1 merge request!1Issue #3487158: Make sure that the agents logs are stored as well
......@@ -6,6 +6,31 @@ use Drupal\Core\Link;
use Drupal\Core\StringTranslation\TranslatableMarkup;
use Drupal\Core\Url;
/**
* Implements hook_cron.
*/
function ai_evaluations_cron() {
// Only run once per day.
$key = 'ai_evaluations_clean_agent_responses';
$current_time = \Drupal::time()->getCurrentTime();
$last_run = \Drupal::state()->get($key, 0);
if (($current_time - $last_run) >= 86400) {
// Delete all agent responses older than 1 day that has no evaluation.
$storage = \Drupal::entityTypeManager()->getStorage('ai_agent_result');
$query = $storage->getQuery();
$query->condition('created', $current_time - 86400, '<');
$query->condition('ai_evaluation', NULL, 'IS NULL');
$query->accessCheck(FALSE);
$ids = $query->execute();
if (!empty($ids)) {
$storage->delete($storage->loadMultiple($ids));
}
// Store the last time it ran.
\Drupal::state()->set($key, $current_time);
}
}
/**
* Implements hook_preprocess_HOOK() for ai_chatbot_message.
*/
......
services:
ai_evaluations.post_request_subscriber:
class: '\Drupal\ai_evaluations\EventSubscriber\PostRequestEventSubscriber'
arguments: ['@entity_type.manager', '@config.factory', '@module_handler', '@ai.prompt_json_decode']
tags:
- { name: 'event_subscriber' }
<?php
declare(strict_types=1);
namespace Drupal\ai_evaluations;
use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\Core\Entity\EntityChangedInterface;
use Drupal\user\EntityOwnerInterface;
/**
* Provides an interface defining an AI Agent Result entity type.
*/
interface AIAgentResultInterface extends ContentEntityInterface, EntityOwnerInterface, EntityChangedInterface {
}
......@@ -43,6 +43,7 @@ class Vote extends ControllerBase {
'preprompt_instructions' => $assistant->get('preprompt_instructions'),
'pre_action_prompt' => $assistant->get('pre_action_prompt'),
'assistant_message' => $assistant->get('assistant_message'),
'thread_id' => $thread_id,
'model_config' => [
'provider' => $assistant->get('llm_provider'),
'model' => $assistant->get('llm_model'),
......
<?php
declare(strict_types=1);
namespace Drupal\ai_evaluations\Entity;
use Drupal\Core\Entity\ContentEntityBase;
use Drupal\Core\Entity\EntityChangedTrait;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Field\BaseFieldDefinition;
use Drupal\Core\StringTranslation\TranslatableMarkup;
use Drupal\ai_evaluations\AIAgentResultInterface;
use Drupal\user\EntityOwnerTrait;
/**
* Defines the AI Agent Result entity class.
*
* @ContentEntityType(
* id = "ai_agent_result",
* label = @Translation("AI Agent Result"),
* label_collection = @Translation("AI Agent Results"),
* label_singular = @Translation("AI Agent Result"),
* label_plural = @Translation("AI Agent Results"),
* label_count = @PluralTranslation(
* singular = "@count AI Agent Results",
* plural = "@count AI Agent Results",
* ),
* handlers = {
* "views_data" = "Drupal\views\EntityViewsData",
* "form" = {
* "add" = "Drupal\ai_evaluations\Form\AIAgentResultForm",
* "edit" = "Drupal\ai_evaluations\Form\AIAgentResultForm",
* "delete" = "Drupal\Core\Entity\ContentEntityDeleteForm",
* "delete-multiple-confirm" = "Drupal\Core\Entity\Form\DeleteMultipleForm",
* },
* "route_provider" = {
* "html" = "Drupal\ai_evaluations\Routing\AIAgentResultHtmlRouteProvider",
* },
* },
* base_table = "ai_agent_result",
* admin_permission = "administer ai_agent_result",
* entity_keys = {
* "id" = "id",
* "label" = "label",
* "uuid" = "uuid",
* "owner" = "uid",
* },
* links = {
* "collection" = "/admin/content/ai-agent-result",
* "add-form" = "/admin/config/ai/ai-agent-results/add",
* "canonical" = "/admin/config/ai/ai-agent-results/{ai_agent_result}",
* "edit-form" = "/admin/config/ai/ai-agent-results/{ai_agent_result}",
* "delete-form" = "/admin/config/ai/ai-agent-results/{ai_agent_result}/delete",
* "delete-multiple-form" = "/admin/content/ai-agent-result/delete-multiple",
* },
* field_ui_base_route = "entity.ai_agent_result.settings",
* )
*/
final class AIAgentResult extends ContentEntityBase implements AIAgentResultInterface {
use EntityChangedTrait;
use EntityOwnerTrait;
/**
* {@inheritdoc}
*/
public function preSave(EntityStorageInterface $storage): void {
parent::preSave($storage);
if (!$this->getOwnerId()) {
// If no owner has been set explicitly, make the anonymous user the owner.
$this->setOwnerId(0);
}
}
/**
* {@inheritdoc}
*/
public static function baseFieldDefinitions(EntityTypeInterface $entity_type): array {
$fields = parent::baseFieldDefinitions($entity_type);
$fields['label'] = BaseFieldDefinition::create('string')
->setLabel(t('Label'))
->setDescription(t('The label of the Agent Decision entity.'))
->setRequired(TRUE)
->setSettings([
'default_value' => '',
'max_length' => 255,
])
->setDisplayOptions('form', [
'type' => 'string_textfield',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['uid'] = BaseFieldDefinition::create('entity_reference')
->setLabel(new TranslatableMarkup('Author'))
->setSetting('target_type', 'user')
->setDefaultValueCallback(self::class . '::getDefaultEntityOwner')
->setDisplayOptions('form', [
'type' => 'entity_reference_autocomplete',
'settings' => [
'match_operator' => 'CONTAINS',
'size' => 60,
'placeholder' => '',
],
'weight' => 15,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'author',
'weight' => 15,
])
->setDisplayConfigurable('view', TRUE);
$fields['created'] = BaseFieldDefinition::create('created')
->setLabel(new TranslatableMarkup('Authored on'))
->setDescription(new TranslatableMarkup('The time that the AI Agent Result was created.'))
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'timestamp',
'weight' => 20,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayOptions('form', [
'type' => 'datetime_timestamp',
'weight' => 20,
])
->setDisplayConfigurable('view', TRUE);
$fields['changed'] = BaseFieldDefinition::create('changed')
->setLabel(new TranslatableMarkup('Changed'))
->setDescription(new TranslatableMarkup('The time that the AI Agent Result was last edited.'));
$fields['ai_evaluation'] = BaseFieldDefinition::create('entity_reference')
->setLabel(new TranslatableMarkup('AI Evaluation'))
->setSetting('target_type', 'ai_evaluation')
->setDisplayOptions('form', [
'type' => 'entity_reference_autocomplete',
'settings' => [
'match_operator' => 'CONTAINS',
'size' => 60,
'placeholder' => '',
],
'weight' => 15,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'author',
'weight' => 15,
])
->setDisplayConfigurable('view', TRUE);
$fields['log_status'] = BaseFieldDefinition::create('list_string')
->setLabel(t('Log Status'))
->setRequired(TRUE)
->setDescription(t('The status of the log.'))
->setSettings([
'default_value' => 'notice',
'allowed_values' => [
'notice' => 'Notice',
'warning' => 'Warning',
'error' => 'Error',
],
])
->setDisplayOptions('form', [
'type' => 'list_string',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['thread_id'] = BaseFieldDefinition::create('string')
->setLabel(t('Thread ID'))
->setDescription(t('The thread id.'))
->setSettings([
'default_value' => '',
])
->setDisplayOptions('form', [
'type' => 'string_textfield',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['ai_provider_info'] = BaseFieldDefinition::create('string_long')
->setLabel(t('AI Provider Info'))
->setDescription(t('The provider and model etc.'))
->setSettings([
'default_value' => '',
])
->setDisplayOptions('form', [
'type' => 'string_textarea',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['prompt_used'] = BaseFieldDefinition::create('string_long')
->setLabel(t('Prompt Used'))
->setDescription(t('The prompt used for the decision.'))
->setSettings([
'default_value' => '',
])
->setDisplayOptions('form', [
'type' => 'string_textarea',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['question'] = BaseFieldDefinition::create('string_long')
->setLabel(t('Question'))
->setDescription(t('The question to the agent.'))
->setSettings([
'default_value' => '',
])
->setDisplayOptions('form', [
'type' => 'string_textarea',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['response_given'] = BaseFieldDefinition::create('string_long')
->setLabel(t('Response Given'))
->setDescription(t('The response given by the agent.'))
->setSettings([
'default_value' => '',
])
->setDisplayOptions('form', [
'type' => 'string_textarea',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
$fields['detailed_output'] = BaseFieldDefinition::create('string_long')
->setLabel(t('Detailed Output'))
->setDescription(t('The detailed output of the decision.'))
->setSettings([
'default_value' => '',
])
->setDisplayOptions('form', [
'type' => 'string_textarea',
'weight' => 0,
])
->setDisplayOptions('view', [
'label' => 'above',
'type' => 'string',
'weight' => 0,
])
->setDisplayConfigurable('form', TRUE)
->setDisplayConfigurable('view', FALSE);
return $fields;
}
}
......@@ -4,13 +4,13 @@ declare(strict_types=1);
namespace Drupal\ai_evaluations\Entity;
use Drupal\ai_evaluations\AIEvaluationInterface;
use Drupal\Core\Entity\ContentEntityBase;
use Drupal\Core\Entity\EntityChangedTrait;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Field\BaseFieldDefinition;
use Drupal\Core\StringTranslation\TranslatableMarkup;
use Drupal\ai_evaluations\AIEvaluationInterface;
use Drupal\user\EntityOwnerTrait;
/**
......@@ -146,10 +146,35 @@ final class AIEvaluation extends ContentEntityBase implements AIEvaluationInterf
$fields['model_config'] = BaseFieldDefinition::create('map')
->setLabel(new TranslatableMarkup('Model config'));
$fields['thread_id'] = BaseFieldDefinition::create('string')
->setLabel(new TranslatableMarkup('Thread ID'));
$fields['value'] = BaseFieldDefinition::create('integer')
->setLabel(new TranslatableMarkup('Value'));
return $fields;
}
/**
* {@inheritdoc}
*/
public function postSave(EntityStorageInterface $storage, $update = TRUE) {
parent::postSave($storage, $update);
// Connect the agent records to the AI evaluation.
if ($this->thread_id->value) {
$agent_storage = $this->entityTypeManager()->getStorage('ai_agent_result');
$query = $agent_storage->getQuery();
$query->condition('thread_id', $this->thread_id->value);
$query->accessCheck(FALSE);
$result = $query->execute();
foreach ($result as $id) {
/** @var \Drupal\ai_evaluations\Entity\AIAgentResult */
$agent = $agent_storage->load($id);
$agent->set('ai_evaluation', $this->id());
$agent->save();
}
}
}
}
<?php
namespace Drupal\ai_evaluations\EventSubscriber;
use Drupal\Component\Serialization\Json;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Extension\ModuleHandlerInterface;
use Drupal\ai\Event\PostGenerateResponseEvent;
use Drupal\ai\Service\PromptJsonDecoder\PromptJsonDecoderInterface;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* The event that is triggered after a response is generated.
*
* @package Drupal\ai_agents_explorer\EventSubscriber
*/
class PostRequestEventSubscriber implements EventSubscriberInterface {
/**
* The entity type manager.
*
* @var \Drupal\Core\Entity\EntityTypeManagerInterface
*/
protected $entityTypeManager;
/**
* The AI settings.
*
* @var ImmutableConfig
*/
protected $aiSettings;
/**
* The module handler.
*
* @var \Drupal\Core\Extension\ModuleHandlerInterface
*/
protected $moduleHandler;
/**
* The JSON Prompt handler.
*
* @var \Drupal\ai\Service\PromptJsonDecoderInterface
* The JSON Prompt handler.
*/
protected $jsonPromptHandler;
/**
* Constructor.
*/
public function __construct(EntityTypeManagerInterface $entityTypeManager, ConfigFactoryInterface $configFactory, ModuleHandlerInterface $moduleHandler, PromptJsonDecoderInterface $jsonPromptHandler) {
$this->entityTypeManager = $entityTypeManager;
$this->aiSettings = $configFactory->get('ai_logging.settings');
$this->moduleHandler = $moduleHandler;
$this->jsonPromptHandler = $jsonPromptHandler;
}
/**
* {@inheritdoc}
*
* @return array
* The post generate response event.
*/
public static function getSubscribedEvents(): array {
return [
PostGenerateResponseEvent::EVENT_NAME => 'logAgentPostRequest',
];
}
/**
* Log if needed after running an AI request.
*
* @param \Drupal\ai\Event\PostGenerateResponseEvent $event
* The event to log.
*/
public function logAgentPostRequest(PostGenerateResponseEvent $event) {
// It has to be the assistant_action_ai_agent tag.
if (in_array('assistant_action_ai_agent', $event->getTags())) {
$file = '';
$thread_id = '';
foreach ($event->getTags() as $tag) {
if (strpos($tag, 'ai_agents_prompt_') !== FALSE) {
$file = str_replace('ai_agents_prompt_', '', $tag);
}
if (strpos($tag, 'thread_id__') !== FALSE) {
$thread_id = str_replace('thread_id__', '', $tag);
}
}
$storage = $this->entityTypeManager->getStorage('ai_agent_result');
$response = $this->jsonPromptHandler->decode($event->getOutput()->getNormalized());
$formatted = is_array($response) ? Json::encode($response) : $response;
$provider_info = $event->getProviderId() . "\n" . $event->getModelId() . "\n" . Json::encode($event->getConfiguration());
$result = $storage->create([
'label' => $file,
'thread_id' => $thread_id,
'action' => 'sub_agent',
'log_status' => 'notice',
'ai_provider_info' => $provider_info,
'question' => $event->getInput()->getMessages()[0]->getText(),
'prompt_used' => $event->getDebugData()['chat_system_role'] ?? '',
'response_given' => is_string($formatted) ?? $formatted->getText(),
'detailed_output' => Json::encode($event->getOutput()->getRawOutput()),
]);
$result->save();
}
}
}
<?php
declare(strict_types=1);
namespace Drupal\ai_evaluations\Form;
use Drupal\Core\Entity\ContentEntityForm;
use Drupal\Core\Form\FormStateInterface;
/**
* Form controller for the AI Agent Result entity edit forms.
*/
final class AIAgentResultForm extends ContentEntityForm {
/**
* {@inheritdoc}
*/
public function save(array $form, FormStateInterface $form_state): int {
$result = parent::save($form, $form_state);
$message_args = ['%label' => $this->entity->toLink()->toString()];
$logger_args = [
'%label' => $this->entity->label(),
'link' => $this->entity->toLink($this->t('View'))->toString(),
];
switch ($result) {
case SAVED_NEW:
$this->messenger()->addStatus($this->t('New AI agent result %label has been created.', $message_args));
$this->logger('ai_agent_result')->notice('New AI agent result %label has been created.', $logger_args);
break;
case SAVED_UPDATED:
$this->messenger()->addStatus($this->t('The AI agent result %label has been updated.', $message_args));
$this->logger('ai_agent_result')->notice('The AI agent result %label has been updated.', $logger_args);
break;
default:
throw new \LogicException('Could not save the entity.');
}
$form_state->setRedirectUrl($this->entity->toUrl('collection'));
return $result;
}
}
<?php
declare(strict_types=1);
namespace Drupal\ai_evaluations\Form;
use Drupal\Core\Form\FormBase;
use Drupal\Core\Form\FormStateInterface;
/**
* Configuration form for an AI agent result entity type.
*/
final class AIAgentResultSettingsForm extends FormBase {
/**
* {@inheritdoc}
*/
public function getFormId(): string {
return 'ai_agent_result_settings';
}
/**
* {@inheritdoc}
*/
public function buildForm(array $form, FormStateInterface $form_state): array {
$form['settings'] = [
'#markup' => $this->t('Settings form for an AI agent result entity type.'),
];
$form['actions'] = [
'#type' => 'actions',
'submit' => [
'#type' => 'submit',
'#value' => $this->t('Save'),
],
];
return $form;
}
/**
* {@inheritdoc}
*/
public function submitForm(array &$form, FormStateInterface $form_state): void {
$this->messenger()->addStatus($this->t('The configuration has been updated.'));
}
}
<?php
declare(strict_types=1);
namespace Drupal\ai_evaluations\Routing;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Entity\Routing\AdminHtmlRouteProvider;
use Symfony\Component\Routing\Route;
/**
* Provides HTML routes for entities with administrative pages.
*/
final class AIAgentResultHtmlRouteProvider extends AdminHtmlRouteProvider {
/**
* {@inheritdoc}
*/
protected function getCanonicalRoute(EntityTypeInterface $entity_type): ?Route {
return $this->getEditFormRoute($entity_type);
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment