Use the key module, and get embeddings working (#13)

* port openai_embeddings, refactor for key module

* composer update

* fix settings fetch

* fix settings

* get embeddings working with key

* use key in openai.module

Author: keiserjb

modules/openai_embeddings/includes/EmbeddingQueueWorker.php

@@ -18,84 +18,84 @@ public function processItem($data) {
       if (!$entity) {
         throw new Exception("Could not load entity with ID {$data['entity_id']}.");
       }
-  
+
       // Log the entity type.
-      /* watchdog('openai_embeddings', 'Processing entity ID: @id, Type: @type', [
+       watchdog('openai_embeddings', 'Processing entity ID: @id, Type: @type', [
         '@id' => $entity->nid,
         '@type' => $entity->type,
-      ], WATCHDOG_DEBUG); */
-  
+      ], WATCHDOG_DEBUG);
+
       // Get configuration.
       $config = config_get('openai_embeddings.settings');
       $stopwords = array_map('trim', explode(',', $config['stopwords'] ?? ''));
       $model = $config['model'] ?? 'text-embedding-ada-002';
       $plugin_id = $config['vector_client_plugin'] ?? NULL;
       $allowed_bundles = $config['content_types'] ?? [];
-  
+
       if (!$plugin_id) {
         throw new Exception('Vector client plugin ID is not configured.');
       }
-  
+
       // Skip entity if its bundle is not allowed.
       if (!in_array($entity->type, $allowed_bundles)) {
-        /* watchdog('openai_embeddings', 'Skipping entity ID: @id because its bundle (@bundle) is not allowed.', [
+         /*watchdog('openai_embeddings', 'Skipping entity ID: @id because its bundle (@bundle) is not allowed.', [
           '@id' => $entity->nid,
           '@bundle' => $entity->type,
-        ], WATCHDOG_INFO); */
+        ], WATCHDOG_INFO);*/
         return;
       }
-  
+
       // Load the vector client.
       $vector_client = openai_embeddings_get_vector_client($plugin_id);
-  
+
       // Supported field types.
       $supported_field_types = ['string', 'text', 'text_long', 'text_with_summary', 'text_textarea_with_summary'];
-  
+
       // Retrieve field definitions.
       $fields = field_info_instances('node', $entity->type);
-  
+
       foreach ($fields as $field_name => $field_info) {
         $field_type = $field_info['type'] ?? ($field_info['widget']['type'] ?? 'undefined');
-  
+
         // Check if the field type is supported.
         if (!in_array($field_type, $supported_field_types)) {
-          /* watchdog('openai_embeddings', 'Skipping unsupported field: @field_name, Type: @field_type', [
+           /*watchdog('openai_embeddings', 'Skipping unsupported field: @field_name, Type: @field_type', [
             '@field_name' => $field_name,
             '@field_type' => $field_type,
-          ], WATCHDOG_INFO); */
+          ], WATCHDOG_INFO);*/
           continue;
         }
-  
+
         // Retrieve field values.
         $field_items = field_get_items('node', $entity, $field_name);
         if (empty($field_items)) {
-          /* watchdog('openai_embeddings', 'Field @field_name has no items or is empty.', [
+          /*watchdog('openai_embeddings', 'Field @field_name has no items or is empty.', [
             '@field_name' => $field_name,
-          ], WATCHDOG_INFO); */
+          ], WATCHDOG_INFO);*/
           continue;
         }
-  
+
         foreach ($field_items as $delta => $item) {
           if (empty($item['value'])) {
             continue;
           }
-  
+
           // Prepare text and remove stopwords.
           $text = StringHelper::prepareText($item['value'], [], 8000);
           foreach ($stopwords as $word) {
             $text = $this->removeStopWord($word, $text);
           }
-  
+
           // Generate embedding via OpenAI.
           $response = openai_client_embed($text, $model);
           if (empty($response['data'][0]['embedding'])) {
             throw new Exception('Failed to generate embedding.');
           }
           $embedding = $response['data'][0]['embedding'];
-  
+
           // Dynamically determine the namespace based on entity type.
           $collection = $data['entity_type']; // Use the entity type directly.
-  
+
           // Prepare vector metadata.
           $unique_id = $this->generateUniqueId($entity, $field_name, $delta);
           $vectors = [
@@ -109,13 +109,13 @@ public function processItem($data) {
               'field_delta' => $delta,
             ],
           ];
-  
+
           // Upsert into vector database.
           $vector_client->upsert([
             'vectors' => [$vectors],
             'collection' => $collection,
           ]);
-  
+
           // Update the local database.
           db_merge('openai_embeddings')
             ->key([
@@ -139,8 +139,8 @@ public function processItem($data) {
       ], WATCHDOG_ERROR);
     }
   }
-  
-  
+
+
 
   /**
    * Generates a unique ID for the record in the vector database.

modules/openai_embeddings/includes/PineconeVectorClient.php

@@ -15,13 +15,31 @@ class PineconeVectorClient extends VectorClientBase {
    *   The configured HTTP client for Pinecone.
    */
   protected function getPineconeClient(): \GuzzleHttp\Client {
+    // Retrieve Pinecone API key and hostname from configuration.
     $api_key = $this->getEmbeddingConfig('pinecone_api_key');
-    $hostname = rtrim($this->getEmbeddingConfig('pinecone_hostname'), '/');
+    $hostname = $this->getEmbeddingConfig('pinecone_hostname');
 
-    if (empty($api_key) || empty($hostname)) {
-      throw new \Exception('Pinecone API key or hostname is not set. Please check your configuration.');
+    // If the Key module is used, resolve the values.
+    if (is_array($api_key)) {
+      $api_key = key_get_key_value($api_key);
+    }
+    if (is_array($hostname)) {
+      $hostname = key_get_key_value($hostname);
+    }
+
+    // Validate and sanitize the values.
+    if (empty($api_key) || !is_string($api_key)) {
+      throw new \Exception('Invalid or missing Pinecone API key. Please check your configuration.');
+    }
+    if (empty($hostname) || !is_string($hostname)) {
+      throw new \Exception('Invalid or missing Pinecone hostname. Please check your configuration.');
     }
 
+    // Trim and sanitize the hostname and API key.
+    $api_key = trim($api_key);
+    $hostname = rtrim(trim($hostname), '/');
+
+    // Return a configured HTTP client.
     return $this->getHttpClient([
       'headers' => [
         'API-Key' => $api_key,
@@ -81,9 +99,9 @@ public function upsert(array $parameters) {
     }
 
     // Log the payload being sent for upsert.
-    /* watchdog('openai_embeddings', 'Pinecone upsert payload: @payload', [
+     watchdog('openai_embeddings', 'Pinecone upsert payload: @payload', [
         '@payload' => json_encode($payload),
-    ], WATCHDOG_DEBUG); */
+    ], WATCHDOG_DEBUG);
 
     try {
         $response = $client->post('/vectors/upsert', ['json' => $payload]);
@@ -140,6 +158,125 @@ public function stats(): array {
       return [];
   }
 }
+
+  /**
+   * Delete records in Pinecone.
+   *
+   * @param array $parameters
+   *   An array with at least key 'source_ids'. The key
+   *   'collection' is required if not using the Pinecone free Starter plan.
+   *
+   * @return \Psr\Http\Message\ResponseInterface
+   *   The response object.
+   *
+   * @throws \Exception
+   *   If required parameters are missing or an error occurs.
+   */
+  public function delete(array $parameters): ResponseInterface {
+    // Ensure necessary parameters are provided.
+    if (empty($parameters['source_ids']) && empty($parameters['filter'])) {
+      throw new \Exception('Either "source_ids" to delete or a "filter" is required for Pinecone deletion.');
+    }
+
+    if (!empty($parameters['deleteAll'])) {
+      throw new \Exception('"deleteAll" must be handled by the deleteAll() method.');
+    }
+
+    // Prepare the payload for deletion.
+    $payload = [];
+    if (!empty($parameters['source_ids'])) {
+      $payload['ids'] = $parameters['source_ids'];
+    }
+    if (!empty($parameters['collection'])) {
+      $payload['namespace'] = $parameters['collection'];
+    }
+    if (!empty($parameters['filter'])) {
+      $payload['filter'] = $parameters['filter'];
+    }
+
+    try {
+      // Execute the delete request.
+      return $this->getClient()->post('/vectors/delete', ['json' => $payload]);
+    } catch (RequestException $e) {
+      watchdog('openai_embeddings', 'Error during Pinecone delete: @message', ['@message' => $e->getMessage()], WATCHDOG_ERROR);
+      throw $e;
+    }
+  }
 
+  /**
+   * Delete all records in Pinecone.
+   *
+   * @param array $parameters
+   *   An array with at least the key 'collection'. Full deletion
+   *   is not supported in the Pinecone free Starter plan.
+   *
+   * @throws \Exception
+   *   If required parameters are missing or an error occurs.
+   */
+  public function deleteAll(array $parameters): void {
+    // Use the configuration method provided by the VectorClientBase class.
+    $disable_namespace = $this->getEmbeddingConfig('disable_namespace');
+
+    // Validate the Pinecone plan and required parameters.
+    if (!empty($disable_namespace)) {
+      watchdog('openai_embeddings', 'Pinecone free starter plan does not support Delete All.', [], WATCHDOG_WARNING);
+      throw new \Exception('Pinecone free starter plan does not support full namespace deletion.');
+    }
+    if (empty($parameters['collection'])) {
+      throw new \Exception('Namespace (collection) is required for deleteAll in Pinecone.');
+    }
+
+    // Retrieve Pinecone API key and hostname from configuration.
+    $api_key = $this->getEmbeddingConfig('pinecone_api_key');
+    $hostname = $this->getEmbeddingConfig('pinecone_hostname');
+
+    // If the Key module is used, resolve the values.
+    if (is_array($api_key)) {
+      $api_key = key_get_key_value($api_key);
+    }
+    if (is_array($hostname)) {
+      $hostname = key_get_key_value($hostname);
+    }
+
+    // Validate the resolved values.
+    if (empty($api_key) || !is_string($api_key)) {
+      throw new \Exception('Invalid or missing Pinecone API key. Please check your configuration.');
+    }
+    if (empty($hostname) || !is_string($hostname)) {
+      throw new \Exception('Invalid or missing Pinecone hostname. Please check your configuration.');
+    }
+
+    // Trim and sanitize the hostname and API key.
+    $api_key = trim($api_key);
+    $hostname = rtrim(trim($hostname), '/');
+
+    // Prepare the payload for full deletion.
+    $payload = [
+      'deleteAll' => TRUE,
+      'namespace' => $parameters['collection'],
+    ];
+
+    try {
+      // Create a Guzzle client with the resolved configuration.
+      $client = $this->getHttpClient([
+        'headers' => [
+          'API-Key' => $api_key,
+        ],
+        'base_uri' => $hostname,
+      ]);
+
+      // Execute the deleteAll request.
+      $client->post('/vectors/delete', ['json' => $payload]);
+    } catch (RequestException $e) {
+      watchdog('openai_embeddings', 'Error during Pinecone deleteAll: @message', ['@message' => $e->getMessage()], WATCHDOG_ERROR);
+      throw $e;
+    } catch (\Exception $e) {
+      watchdog('openai_embeddings', 'Unexpected error during Pinecone deleteAll: @message', ['@message' => $e->getMessage()], WATCHDOG_ERROR);
+      throw $e;
+    }
+  }
+
+
+
 }
 

modules/openai_embeddings/includes/VectorClientBase.php

@@ -48,9 +48,17 @@ public function getOpenAIKey(): ?string {
    *   The configuration value or NULL if not set.
    */
   public function getEmbeddingConfig(string $key) {
-    return $this->embeddingsConfig[$key] ?? NULL;
+    $value = $this->embeddingsConfig[$key] ?? NULL;
+
+    // If this key references a managed key, resolve its value.
+    if ($key === 'pinecone_api_key' || $key === 'pinecone_hostname') {
+      return key_get_key_value($value);
+    }
+
+    return $value;
   }
 
+
   /**
    * Initialize an HTTP client with appropriate headers and base URI.
    *