Issue #2430219 by alexpott: Implement a key value store to optimise config entity lookups

core/core.services.yml

@@ -721,7 +721,9 @@ services:
       - [setContainer, ['@service_container']]
   entity.query.config:
     class: Drupal\Core\Config\Entity\Query\QueryFactory
-    arguments: ['@config.factory']
+    arguments: ['@config.factory', '@keyvalue', '@config.manager']
+    tags:
+      - { name: event_subscriber }
   entity.query.sql:
     class: Drupal\Core\Entity\Query\Sql\QueryFactory
     arguments: ['@database']

core/lib/Drupal/Core/Config/Entity/ConfigEntityType.php

@@ -34,6 +34,13 @@ class ConfigEntityType extends EntityType implements ConfigEntityTypeInterface {
    */
   protected $static_cache = FALSE;
 
+  /**
+   * Keys that are stored key value store for fast lookup.
+   *
+   * @var array
+   */
+  protected $lookup_keys = [];
+
   /**
    * The list of configuration entity properties to export from the annotation.
    *
@@ -74,6 +81,7 @@ public function __construct($definition) {
     $this->handlers += array(
       'storage' => 'Drupal\Core\Config\Entity\ConfigEntityStorage',
     );
+    $this->lookup_keys[] = 'uuid';
   }
 
   /**
@@ -190,4 +198,11 @@ public function getPropertiesToExport() {
     return NULL;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getLookupKeys() {
+    return $this->lookup_keys;
+  }
+
 }

core/lib/Drupal/Core/Config/Entity/ConfigEntityTypeInterface.php

@@ -70,4 +70,12 @@ public function getConfigPrefix();
    */
   public function getPropertiesToExport();
 
+  /**
+   * Gets the keys that are available for fast lookup.
+   *
+   * @return string[]
+   *   The list of lookup keys.
+   */
+  public function getLookupKeys();
+
 }

core/lib/Drupal/Core/Config/Entity/Query/InvalidLookupKeyException.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Config\Entity\Query\InvalidLookupKeyException.
+ */
+
+namespace Drupal\Core\Config\Entity\Query;
+
+/**
+ * Exception thrown when a config entity uses an invalid lookup key.
+ */
+class InvalidLookupKeyException extends \LogicException {
+}

core/lib/Drupal/Core/Config/Entity/Query/Query.php

@@ -11,12 +11,20 @@
 use Drupal\Core\Entity\EntityTypeInterface;
 use Drupal\Core\Entity\Query\QueryBase;
 use Drupal\Core\Entity\Query\QueryInterface;
+use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
 
 /**
  * Defines the entity query for configuration entities.
  */
 class Query extends QueryBase implements QueryInterface {
 
+  /**
+   * Information about the entity type.
+   *
+   * @var \Drupal\Core\Config\Entity\ConfigEntityTypeInterface
+   */
+  protected $entityType;
+
   /**
    * The config factory used by the config entity query.
    *
@@ -24,6 +32,13 @@ class Query extends QueryBase implements QueryInterface {
    */
   protected $configFactory;
 
+  /**
+   * The key value factory.
+   *
+   * @var \Drupal\Core\KeyValueStore\KeyValueFactoryInterface
+   */
+  protected $keyValueFactory;
+
   /**
    * Constructs a Query object.
    *
@@ -34,12 +49,15 @@ class Query extends QueryBase implements QueryInterface {
    *   - OR: at least one of the conditions on the query need to match.
    * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
    *   The config factory.
+   * @param \Drupal\Core\KeyValueStore\KeyValueFactoryInterface $key_value_factory
+   *   The key value factory.
    * @param array $namespaces
    *   List of potential namespaces of the classes belonging to this query.
    */
-  function __construct(EntityTypeInterface $entity_type, $conjunction, ConfigFactoryInterface $config_factory, array $namespaces) {
+  function __construct(EntityTypeInterface $entity_type, $conjunction, ConfigFactoryInterface $config_factory, KeyValueFactoryInterface $key_value_factory, array $namespaces) {
     parent::__construct($entity_type, $conjunction, $namespaces);
     $this->configFactory = $config_factory;
+    $this->keyValueFactory = $key_value_factory;
   }
 
   /**
@@ -108,37 +126,47 @@ protected function loadRecords() {
     $prefix = $this->entityType->getConfigPrefix() . '.';
     $prefix_length = strlen($prefix);
 
-    // Search the conditions for restrictions on entity IDs.
-    $ids = array();
+    // Search the conditions for restrictions on configuration object names.
+    $names = FALSE;
     if ($this->condition->getConjunction() == 'AND') {
-      foreach ($this->condition->conditions() as $condition) {
-        if (is_string($condition['field']) && $condition['field'] == $this->entityType->getKey('id')) {
-          $operator = $condition['operator'] ?: (is_array($condition['value']) ? 'IN' : '=');
-          if ($operator == '=') {
-            $ids = array($condition['value']);
+      $lookup_keys = $this->entityType->getLookupKeys();
+      $conditions = $this->condition->conditions();
+      foreach ($conditions as $condition_key => $condition) {
+        $operator = $condition['operator'] ?: (is_array($condition['value']) ? 'IN' : '=');
+        if (is_string($condition['field']) && ($operator == 'IN' || $operator == '=')) {
+          // Special case ID lookups.
+          if ($condition['field'] == $this->entityType->getKey('id')) {
+            $ids = (array) $condition['value'];
+            $names = array_map(function ($id) use ($prefix) {
+              return $prefix . $id;
+            }, $ids);
           }
-          elseif ($operator == 'IN') {
-            $ids = $condition['value'];
-          }
-          // We stop at the first restricting condition on ID. In the (weird)
-          // case where there are additional restricting conditions, results
-          // will be eliminated when the conditions are checked on the loaded
-          // records.
-          if ($ids) {
-            break;
+          elseif (in_array($condition['field'], $lookup_keys)) {
+            // If we don't find anything then there are no matches. No point in
+            // listing anything.
+            $names = array();
+            $keys = (array) $condition['value'];
+            $keys = array_map(function ($value) use ($condition) {
+              return $condition['field'] . ':' . $value;
+            }, $keys);
+            foreach ($this->getConfigKeyStore()->getMultiple($keys) as $list) {
+              $names = array_merge($names, $list);
+            }
           }
         }
+        // We stop at the first restricting condition on name. In the case where
+        // there are additional restricting conditions, results will be
+        // eliminated when the conditions are checked on the loaded records.
+        if ($names !== FALSE) {
+          // If the condition has been responsible for narrowing the list of
+          // configuration to check there is no point in checking it further.
+          unset($conditions[$condition_key]);
+          break;
+        }
       }
     }
-    // If there are conditions restricting config ID, we can narrow the list of
-    // records to load and parse.
-    if ($ids) {
-      $names = array_map(function ($id) use ($prefix) {
-        return $prefix . $id;
-      }, $ids);
-    }
     // If no restrictions on IDs were found, we need to parse all records.
-    else {
+    if ($names === FALSE) {
       $names = $this->configFactory->listAll($prefix);
     }
 
@@ -150,4 +178,14 @@ protected function loadRecords() {
     return $records;
   }
 
+  /**
+   * Gets the key value store used to store fast lookups.
+   *
+   * @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface
+   *   The key value store used to store fast lookups.
+   */
+  protected function getConfigKeyStore() {
+    return $this->keyValueFactory->get(QueryFactory::CONFIG_LOOKUP_PREFIX . $this->entityTypeId);
+  }
+
 }

core/lib/Drupal/Core/Config/Entity/Query/QueryFactory.php

@@ -7,17 +7,28 @@
 
 namespace Drupal\Core\Config\Entity\Query;
 
+use Drupal\Core\Config\Config;
+use Drupal\Core\Config\ConfigCrudEvent;
+use Drupal\Core\Config\ConfigEvents;
 use Drupal\Core\Config\ConfigFactoryInterface;
-use Drupal\Core\Entity\EntityManagerInterface;
+use Drupal\Core\Config\ConfigManagerInterface;
+use Drupal\Core\Config\Entity\ConfigEntityTypeInterface;
 use Drupal\Core\Entity\EntityTypeInterface;
 use Drupal\Core\Entity\Query\QueryBase;
 use Drupal\Core\Entity\Query\QueryException;
 use Drupal\Core\Entity\Query\QueryFactoryInterface;
+use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 
 /**
  * Provides a factory for creating entity query objects for the config backend.
  */
-class QueryFactory implements QueryFactoryInterface {
+class QueryFactory implements QueryFactoryInterface, EventSubscriberInterface {
+
+  /**
+   * The prefix for the key value collection for fast lookups.
+   */
+  const CONFIG_LOOKUP_PREFIX = 'config.entity.key_store.';
 
   /**
    * The config factory used by the config entity query.
@@ -36,13 +47,17 @@ class QueryFactory implements QueryFactoryInterface {
   /**
    * Constructs a QueryFactory object.
    *
-   * @param \Drupal\Core\Config\StorageInterface $config_storage
-   *   The config storage used by the config entity query.
    * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
    *   The config storage used by the config entity query.
+   * @param \Drupal\Core\KeyValueStore\KeyValueFactoryInterface $key_value
+   *   The key value factory.
+   * @param \Drupal\Core\Config\ConfigManagerInterface $config_manager
+   *   The configuration manager.
    */
-  public function __construct(ConfigFactoryInterface $config_factory) {
+  public function __construct(ConfigFactoryInterface $config_factory, KeyValueFactoryInterface $key_value, ConfigManagerInterface $config_manager) {
     $this->configFactory = $config_factory;
+    $this->keyValueFactory = $key_value;
+    $this->configManager = $config_manager;
     $this->namespaces = QueryBase::getNamespaces($this);
   }
 
@@ -50,7 +65,7 @@ public function __construct(ConfigFactoryInterface $config_factory) {
    * {@inheritdoc}
    */
   public function get(EntityTypeInterface $entity_type, $conjunction) {
-    return new Query($entity_type, $conjunction, $this->configFactory, $this->namespaces);
+    return new Query($entity_type, $conjunction, $this->configFactory, $this->keyValueFactory, $this->namespaces);
   }
 
   /**
@@ -60,4 +75,189 @@ public function getAggregate(EntityTypeInterface $entity_type, $conjunction) {
       throw new QueryException('Aggregation over configuration entities is not supported');
   }
 
+  /**
+   * Gets the key value store used to store fast lookups.
+   *
+   * @param \Drupal\Core\Entity\EntityTypeInterface $entity_type
+   *   The entity type.
+   *
+   * @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface
+   *   The key value store used to store fast lookups.
+   */
+  protected function getConfigKeyStore(EntityTypeInterface $entity_type) {
+    return $this->keyValueFactory->get(static::CONFIG_LOOKUP_PREFIX . $entity_type->id());
+  }
+
+  /**
+   * Updates or adds lookup data.
+   *
+   * @param \Drupal\Core\Config\Entity\ConfigEntityTypeInterface $entity_type
+   *   The entity type.
+   * @param \Drupal\Core\Config\Config $config
+   *   The configuration object that is being saved.
+   */
+  protected function updateConfigKeyStore(ConfigEntityTypeInterface $entity_type, Config $config) {
+    $config_key_store = $this->getConfigKeyStore($entity_type);
+    foreach ($entity_type->getLookupKeys() as $lookup_key) {
+      foreach ($this->getKeys($config, $lookup_key, 'get', $entity_type) as $key) {
+        $values = $config_key_store->get($key, []);
+        if (!in_array($config->getName(), $values, TRUE)) {
+          $values[] = $config->getName();
+          $config_key_store->set($key, $values);
+        }
+      }
+    }
+  }
+
+  /**
+   * Deletes lookup data.
+   *
+   * @param \Drupal\Core\Config\Entity\ConfigEntityTypeInterface $entity_type
+   *   The entity type.
+   * @param \Drupal\Core\Config\Config $config
+   *   The configuration object that is being deleted.
+   */
+  protected function deleteConfigKeyStore(ConfigEntityTypeInterface $entity_type, Config $config) {
+    $config_key_store = $this->getConfigKeyStore($entity_type);
+    foreach ($entity_type->getLookupKeys() as $lookup_key) {
+      foreach ($this->getKeys($config, $lookup_key, 'getOriginal', $entity_type) as $key) {
+        $values = $config_key_store->get($key, []);
+        $pos = array_search($config->getName(), $values, TRUE);
+        if ($pos !== FALSE) {
+          unset($values[$pos]);
+        }
+        if (empty($values)) {
+          $config_key_store->delete($key);
+        }
+        else {
+          $config_key_store->set($key, $values);
+        }
+      }
+    }
+  }
+
+  /**
+   * Creates lookup keys for configuration data.
+   *
+   * @param \Drupal\Core\Config\Config $config
+   *   The configuration object.
+   *  @param string $key
+   *   The configuration key to look for.
+   * @param string $get_method
+   *   Which method on the config object to call to get the value. Either 'get'
+   *   or 'getOriginal'.
+   * @param \Drupal\Core\Config\Entity\ConfigEntityTypeInterface $entity_type
+   *   The configuration entity type.
+   *
+   * @return array
+   *   An array of lookup keys concatenated to the configuration values.
+   *
+   * @throws \Drupal\Core\Config\Entity\Query\InvalidLookupKeyException
+   *   The provided $key cannot end with a wildcard. This makes no sense since
+   *   you cannot do fast lookups against this.
+   */
+  protected function getKeys(Config $config, $key, $get_method, ConfigEntityTypeInterface $entity_type) {
+    if (substr($key, -1) == '*') {
+      throw new InvalidLookupKeyException(strtr('%entity_type lookup key %key ends with a wildcard this can not be used as a lookup', ['%entity_type' => $entity_type->id(), '%key' => $key]));
+    }
+    $parts = explode('.*', $key);
+    // Remove leading dots.
+    array_walk($parts, function (&$value) {
+      $value = trim($value, '.');
+    });
+
+    $values = (array) $this->getValues($config, $parts[0], $get_method, $parts);
+
+    $output = array();
+    // Flatten the array to a single dimension and add the key to all the
+    // values.
+    array_walk_recursive($values, function ($current) use (&$output, $key) {
+      if (is_scalar($current)) {
+        $current = $key . ':' . $current;
+      }
+      $output[] = $current;
+    });
+    return $output;
+  }
+
+  /**
+   * Finds all the values for a configuration key in a configuration object.
+   *
+   * @param \Drupal\Core\Config\Config $config
+   *   The configuration object.
+   * @param string $key
+   *   The current key being checked.
+   * @param string $get_method
+   *   Which method on the config object to call to get the value.
+   * @param array $parts
+   *   All the parts of a configuration key we are checking.
+   * @param int $start
+   *   Which position of $parts we are processing. Defaults to 0.
+   *
+   * @return array|NULL
+   *   The array of configuration values the match the provided key. NULL if
+   *   the configuration object does not have a value that corresponds to the
+   *   key.
+   */
+  protected function getValues(Config $config, $key, $get_method, array $parts, $start = 0) {
+    $value = $config->$get_method($key);
+    if (is_array($value)) {
+      $new_value = [];
+      $start++;
+      if (!isset($parts[$start])) {
+        // The configuration object does not have a value that corresponds to
+        // the key.
+        return NULL;
+      }
+      foreach (array_keys($value) as $key_bit) {
+        $new_key = $key . '.' . $key_bit;
+        if (!empty($parts[$start])) {
+          $new_key .= '.' . $parts[$start];
+        }
+        $new_value[] = $this->getValues($config, $new_key, $get_method, $parts, $start);
+      }
+      $value = $new_value;
+    }
+    return $value;
+  }
+
+  /**
+   * Updates configuration entity in the key store.
+   *
+   * @param ConfigCrudEvent $event
+   *   The configuration event.
+   */
+  public function onConfigSave(ConfigCrudEvent $event) {
+    $saved_config = $event->getConfig();
+    $entity_type_id = $this->configManager->getEntityTypeIdByName($saved_config->getName());
+    if ($entity_type_id) {
+      $entity_type = $this->configManager->getEntityManager()->getDefinition($entity_type_id);
+      $this->updateConfigKeyStore($entity_type, $saved_config);
+    }
+  }
+
+  /**
+   * Removes configuration entity from key store.
+   *
+   * @param \Drupal\Core\Config\ConfigCrudEvent $event
+   *   The configuration event.
+   */
+  public function onConfigDelete(ConfigCrudEvent $event) {
+    $saved_config = $event->getConfig();
+    $entity_type_id = $this->configManager->getEntityTypeIdByName($saved_config->getName());
+    if ($entity_type_id) {
+      $entity_type = $this->configManager->getEntityManager()->getDefinition($entity_type_id);
+      $this->deleteConfigKeyStore($entity_type, $saved_config);
+    }
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  static function getSubscribedEvents() {
+    $events[ConfigEvents::SAVE][] = array('onConfigSave', 128);
+    $events[ConfigEvents::DELETE][] = array('onConfigDelete', 128);
+    return $events;
+  }
+
 }

core/modules/block/src/Entity/Block.php

@@ -48,6 +48,9 @@
  *     "plugin",
  *     "settings",
  *     "visibility",
+ *   },
+ *   lookup_keys = {
+ *     "theme"
  *   }
  * )
  */

core/modules/config/tests/config_test/config_test.module

@@ -58,4 +58,7 @@ function config_test_entity_type_alter(array &$entity_types) {
   $config_test_no_status->set('id', 'config_test_no_status');
   $config_test_no_status->set('entity_keys', $keys);
   $config_test_no_status->set('config_prefix', 'no_status');
+  if (\Drupal::service('state')->get('config_test.lookup_keys', FALSE)) {
+    $entity_types['config_test']->set('lookup_keys', ['uuid', 'style']);
+  }
 }

core/modules/system/src/Tests/Entity/ConfigEntityQueryTest.php

@@ -7,6 +7,7 @@
 
 namespace Drupal\system\Tests\Entity;
 
+use Drupal\Core\Config\Entity\Query\QueryFactory;
 use Drupal\simpletest\KernelTestBase;
 
 /**
@@ -469,6 +470,64 @@ public function testCaseSensitivity() {
     $this->assertResults(array('3', '4', '5'));
   }
 
+  /**
+   * Tests lookup keys are added to the key value store.
+   */
+  public function testLookupKeys() {
+    \Drupal::service('state')->set('config_test.lookup_keys', TRUE);
+    \Drupal::entityManager()->clearCachedDefinitions();
+    $key_value = $this->container->get('keyvalue')->get(QueryFactory::CONFIG_LOOKUP_PREFIX . 'config_test');
+
+    $test_entities = [];
+    $entity = entity_create('config_test', array(
+      'label' => $this->randomMachineName(),
+      'id' => '1',
+      'style' => 'test',
+    ));
+    $test_entities[$entity->getConfigDependencyName()] = $entity;
+    $entity->enforceIsNew();
+    $entity->save();
+
+
+    $expected[] = $entity->getConfigDependencyName();
+    $this->assertEqual($expected, $key_value->get('style:test'));
+
+    $entity = entity_create('config_test', array(
+      'label' => $this->randomMachineName(),
+      'id' => '2',
+      'style' => 'test',
+    ));
+    $test_entities[$entity->getConfigDependencyName()] = $entity;
+    $entity->enforceIsNew();
+    $entity->save();
+    $expected[] = $entity->getConfigDependencyName();
+    $this->assertEqual($expected, $key_value->get('style:test'));
+
+    $entity = entity_create('config_test', array(
+      'label' => $this->randomMachineName(),
+      'id' => '3',
+      'style' => 'blah',
+    ));
+    $entity->enforceIsNew();
+    $entity->save();
+    // Do not add this entity to the list of expected result as it has a
+    // different value.
+    $this->assertEqual($expected, $key_value->get('style:test'));
+    $this->assertEqual([$entity->getConfigDependencyName()], $key_value->get('style:blah'));
+
+    // Ensure that a delete clears a key.
+    $entity->delete();
+    $this->assertEqual([], $key_value->get('style:blah'));
+
+    // Ensure that delete only clears one key.
+    $entity_id = array_pop($expected);
+    $test_entities[$entity_id]->delete();
+    $this->assertEqual($expected, $key_value->get('style:test'));
+    $entity_id = array_pop($expected);
+    $test_entities[$entity_id]->delete();
+    $this->assertEqual($expected, $key_value->get('style:test'));
+  }
+
   /**
    * Asserts the results as expected regardless of order.
    *

core/modules/tour/src/Entity/Tour.php

@@ -31,6 +31,9 @@
  *     "module",
  *     "routes",
  *     "tips",
+ *   },
+ *   lookup_keys = {
+ *     "routes.*.route_name"
  *   }
  * )
  */

core/tests/Drupal/Tests/Core/Config/Entity/Query/QueryFactoryTest.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Tests\Core\Config\Entity\Query\QueryFactoryTest.
+ */
+
+namespace Drupal\Tests\Core\Config\Entity\Query;
+
+use Drupal\Core\Config\Config;
+use Drupal\Core\Config\Entity\Query\QueryFactory;
+use Drupal\Tests\UnitTestCase;
+
+/**
+ * @coversDefaultClass \Drupal\Core\Config\Entity\Query\QueryFactory
+ * @group Config
+ */
+class QueryFactoryTest extends UnitTestCase {
+
+  /**
+   * @covers ::getKeys
+   * @covers ::getValues
+   *
+   * @dataProvider providerTestGetKeys
+   */
+  public function testGetKeys(array $expected, $key, Config $config) {
+    $config_factory = $this->getMock('Drupal\Core\Config\ConfigFactoryInterface');
+    $key_value_factory = $this->getMock('Drupal\Core\KeyValueStore\KeyValueFactoryInterface');
+    $config_manager = $this->getMock('Drupal\Core\Config\ConfigManagerInterface');
+    $config_entity_type = $this->getMock('Drupal\Core\Config\Entity\ConfigEntityTypeInterface');
+    $query_factory = new QueryFactory($config_factory, $key_value_factory, $config_manager);
+    $method = new \ReflectionMethod($query_factory, 'getKeys');
+    $method->setAccessible(TRUE);
+