Skip to content
Snippets Groups Projects
Commit 2ba23f6b authored by Scott Euser's avatar Scott Euser Committed by Kevin Quillen
Browse files

Issue #3400295 by kevinquillen, scott_euser: Prepare openai_embeddings module...

Issue #3400295 by kevinquillen, scott_euser: Prepare openai_embeddings module for multiple plugins beyond pinecone
parent ec91562e
No related branches found
Tags 10.0.0-rc3
1 merge request!55Remove hard-coding of pinecone in queue worker and instead use selected vector client plugin
......@@ -20,6 +20,9 @@ openai_embeddings.settings:
model:
type: string
label: 'OpenAI model to use'
vector_client_plugin:
type: string
label: 'Vector client plugin to use.'
openai_embeddings.pinecone_client:
type: config_object
......
......@@ -96,4 +96,4 @@ function openai_embeddings_update_10001() {
$config->set('disable_namespace', FALSE);
$config->save();
}
}
\ No newline at end of file
}
......@@ -2,3 +2,6 @@ services:
openai_embeddings.pinecone_client:
class: Drupal\openai_embeddings\Http\PineconeClient
arguments: ['@http_client_factory', '@config.factory']
plugin.manager.vector_client:
class: Drupal\openai_embeddings\VectorClientPluginManager
arguments: ['@container.namespaces', '@cache.discovery', '@module_handler']
......@@ -56,7 +56,7 @@ class DeleteConfirmForm extends ConfirmFormBase {
* {@inheritdoc}
*/
public function getDescription() {
return $this->t('This will delete all items in your Pinecone instance.');
return $this->t('This will delete all items in your Pinecone instance. Note that this action is NOT permitted if you are on the Starter plan with Pinecone.');
}
/**
......
......@@ -71,7 +71,7 @@ class SearchForm extends FormBase {
$form['namespace'] = [
'#type' => 'textfield',
'#title' => $this->t('Namespace'),
'#description' => $this->t('Enter the namespace to search through. You can find the namespaces on the Stats tab.'),
'#description' => $this->t('Enter the namespace to search through. You can find the namespaces on the Stats tab. Leave blank if you are on the Starter plan.'),
'#maxlength' => 64,
];
......
......@@ -29,6 +29,13 @@ class SettingsForm extends ConfigFormBase {
*/
protected $entityTypeBundleInfo;
/**
* The vector client plugin manager.
*
* @var \Drupal\openai_embeddings\VectorClientPluginManager
*/
protected $pluginManager;
/**
* {@inheritdoc}
*/
......@@ -53,6 +60,7 @@ class SettingsForm extends ConfigFormBase {
$instance = parent::create($container);
$instance->entityTypeManager = $container->get('entity_type.manager');
$instance->entityTypeBundleInfo = $container->get('entity_type.bundle.info');
$instance->pluginManager = $container->get('plugin.manager.vector_client');
return $instance;
}
......@@ -123,10 +131,29 @@ class SettingsForm extends ConfigFormBase {
'#description' => $this->t('Searching vector/embedding data is only available one of these services.... TBD'),
];
// Generate an array of annotated plugins for generating PDF.
$plugins = [];
foreach ($this->pluginManager->getDefinitions() as $pid => $plugin) {
$plugins[$pid] = $plugin['label'];
}
$form['connections']['vector_client_plugin'] = [
'#title' => $this->t('Vector client plugin'),
'#type' => 'select',
'#required' => TRUE,
'#options' => $plugins,
'#description' => $this->t('Choose the vector database to store embeddings in.'),
'#default_value' => $this->config('openai_embeddings.settings')->get('vector_client_plugin'),
];
$form['connections']['pinecone'] = [
'#type' => 'details',
'#title' => $this->t('Pinecone'),
'#description' => $this->t('Configure Pinecone settings (need links + description)'),
'#states' => [
'visible' => [
'select[name="connections[vector_client_plugin]"]' => ['value' => 'pinecone'],
],
],
];
$form['connections']['pinecone']['api_key'] = [
......@@ -134,6 +161,11 @@ class SettingsForm extends ConfigFormBase {
'#title' => $this->t('API Key'),
'#default_value' => $this->config('openai_embeddings.pinecone_client')->get('api_key'),
'#description' => $this->t('The API key is required to make calls to Pinecone for vector searching.'),
'#states' => [
'visible' => [
'select[name="connections[vector_client_plugin]"]' => ['value' => 'pinecone'],
],
],
];
$form['connections']['pinecone']['hostname'] = [
......@@ -141,6 +173,11 @@ class SettingsForm extends ConfigFormBase {
'#title' => $this->t('Hostname'),
'#default_value' => $this->config('openai_embeddings.pinecone_client')->get('hostname'),
'#description' => $this->t('The hostname or base URI where your Pinecone instance is located.'),
'#states' => [
'visible' => [
'select[name="connections[vector_client_plugin]"]' => ['value' => 'pinecone'],
],
],
];
$form['connections']['pinecone']['disable_namespace'] = [
......@@ -172,14 +209,16 @@ class SettingsForm extends ConfigFormBase {
$stopwords = explode(', ', mb_strtolower($form_state->getValue('stopwords')));
sort($stopwords);
$connections = $form_state->getValue('connections');
$this->config('openai_embeddings.settings')
->set('entity_types', $entity_types)
->set('stopwords', $stopwords)
->set('model', $form_state->getValue('model'))
->set('vector_client_plugin', $connections['vector_client_plugin'])
->save();
$pinecone = $form_state->getValue('connections')['pinecone'];
$pinecone = $connections['pinecone'];
$this->config('openai_embeddings.pinecone_client')
->set('api_key', $pinecone['api_key'])
->set('hostname', $pinecone['hostname'])
......
......@@ -63,6 +63,7 @@ class PineconeClient {
'includeValues' => $include_values,
'namespace' => $namespace,
];
if (!empty($namespace)) {
$payload['namespace'] = $namespace;
}
......@@ -92,9 +93,11 @@ class PineconeClient {
$payload = [
'vectors' => $vectors,
];
if (!empty($namespace)) {
$payload['namespace'] = $namespace;
}
return $this->client->post(
'/vectors/upsert',
[
......@@ -118,9 +121,11 @@ class PineconeClient {
$payload = [
'ids' => $ids,
];
if (!empty($namespace)) {
$payload['namespace'] = $namespace;
}
return $this->client->get(
'/vectors/fetch',
[
......
......@@ -13,6 +13,7 @@ use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\Core\Queue\QueueWorkerBase;
use Drupal\openai\Utility\StringHelper;
use Drupal\openai_embeddings\Http\PineconeClient;
use Drupal\openai_embeddings\VectorClientPluginManager;
use Symfony\Component\DependencyInjection\ContainerInterface;
use OpenAI\Client;
use Drupal\Core\Entity\EntityTypeManagerInterface;
......@@ -65,11 +66,11 @@ final class EmbeddingQueueWorker extends QueueWorkerBase implements ContainerFac
protected $client;
/**
* The Pinecone client.
* The vector client plugin manager.
*
* @var \Drupal\openai_embeddings\Http\PineconeClient
* @var \Drupal\openai_embeddings\VectorClientPluginManager
*/
protected $pinecone;
protected $pluginManager;
/**
* The pinecone configuration object.
......@@ -88,14 +89,14 @@ final class EmbeddingQueueWorker extends QueueWorkerBase implements ContainerFac
/**
* {@inheritdoc}
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, EntityTypeManagerInterface $entity_type_manager, EntityFieldManagerInterface $entity_field_manager, Connection $connection, ConfigFactoryInterface $config_factory, Client $client, PineconeClient $pinecone_client, LoggerChannelFactoryInterface $logger_channel_factory) {
public function __construct(array $configuration, $plugin_id, $plugin_definition, EntityTypeManagerInterface $entity_type_manager, EntityFieldManagerInterface $entity_field_manager, Connection $connection, ConfigFactoryInterface $config_factory, Client $client, VectorClientPluginManager $plugin_manager, LoggerChannelFactoryInterface $logger_channel_factory) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->entityTypeManager = $entity_type_manager;
$this->entityFieldManager = $entity_field_manager;
$this->database = $connection;
$this->config = $config_factory->get('openai_embeddings.settings');
$this->client = $client;
$this->pinecone = $pinecone_client;
$this->pluginManager = $plugin_manager;
$this->pineconeConfig = $config_factory->get('openai_embeddings.pinecone_client');
$this->logger = $logger_channel_factory->get('openai_embeddings');
}
......@@ -113,7 +114,7 @@ final class EmbeddingQueueWorker extends QueueWorkerBase implements ContainerFac
$container->get('database'),
$container->get('config.factory'),
$container->get('openai.client'),
$container->get('openai_embeddings.pinecone_client'),
$container->get('plugin.manager.vector_client'),
$container->get('logger.factory'),
);
}
......@@ -134,6 +135,8 @@ final class EmbeddingQueueWorker extends QueueWorkerBase implements ContainerFac
$field_types = $this->getFieldTypes();
$stopwords = $this->config->get('stopwords');
$model = $this->config->get('model');
$plugin_id = $this->config->get('vector_client_plugin');
$vector_client_plugin = $this->pluginManager->createInstance($plugin_id);
foreach ($fields as $field) {
if (in_array($field->getType(), $field_types)) {
......@@ -176,7 +179,7 @@ final class EmbeddingQueueWorker extends QueueWorkerBase implements ContainerFac
]
];
$this->pinecone->upsert($vectors, $namespace);
$vector_client_plugin->upsert($vectors, $namespace);
$this->database->merge('openai_embeddings')
->keys(
......
......@@ -47,7 +47,9 @@ class Pinecone extends VectorClientPluginBase {
public function __construct(array $configuration, $plugin_id, $plugin_definition, ClientFactory $factory, ConfigFactoryInterface $config_factory) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->config = $config_factory->getEditable('plugin.plugin_configuration.vector_client.' . $plugin_id);
// @todo: this should pull from the plugin configuration
//$this->config = $config_factory->get('plugin.plugin_configuration.vector_client.' . $plugin_id);
$this->config = $config_factory->get('openai_embeddings.pinecone_client');
$this->client = $factory->fromOptions([
'headers' => [
......@@ -115,13 +117,19 @@ class Pinecone extends VectorClientPluginBase {
* @return \Psr\Http\Message\ResponseInterface
* The API response.
*/
public function upsert(array $vectors) {
public function upsert(array $vectors, string $namespace = '') {
$payload = [
'vectors' => $vectors,
];
if (!empty($namespace)) {
$payload['namespace'] = $namespace;
}
return $this->client->post(
'/vectors/upsert',
[
'json' => [
'vectors' => $vectors,
]
'json' => $payload,
]
);
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment