Issue #1971158 by Berdir, beejeebus, alexpott, amateescu: Add loadMultiple()...

Issue #1971158 by Berdir, beejeebus, alexpott, amateescu: Add loadMultiple() and listAll() caching to (cached) config storage.

core/core.services.yml

@@ -65,6 +65,8 @@ services:
   config.storage:
     class: Drupal\Core\Config\CachedStorage
     arguments: ['@config.cachedstorage.storage', '@cache.config']
+    tags:
+      - { name: persist }
   config.context.factory:
     class: Drupal\Core\Config\Context\ConfigContextFactory
     arguments: ['@event_dispatcher']

core/lib/Drupal/Core/Config/CachedStorage.php

@@ -16,7 +16,7 @@
  * the cache and delegates the read to the storage on a cache miss. It also
  * handles cache invalidation.
  */
-class CachedStorage implements StorageInterface {
+class CachedStorage implements StorageInterface, StorageCacheInterface {
 
   /**
    * The configuration storage to be cached.
@@ -32,6 +32,13 @@ class CachedStorage implements StorageInterface {
    */
   protected $cache;
 
+  /**
+   * List of listAll() prefixes with their results.
+   *
+   * @var array
+   */
+  protected $findByPrefixCache = array();
+
   /**
    * Constructs a new CachedStorage controller.
    *
@@ -79,6 +86,32 @@ public function read($name) {
     return $data;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function readMultiple(array $names) {
+    $list = array();
+    // The names array is passed by reference and will only contain the names of
+    // config object not found after the method call.
+    // @see Drupal\Core\Cache\CacheBackendInterface::getMultiple()
+    $cached_list = $this->cache->getMultiple($names);
+
+    if (!empty($names)) {
+      $list = $this->storage->readMultiple($names);
+      // Cache configuration objects that were loaded from the storage.
+      foreach ($list as $name => $data) {
+        $this->cache->set($name, $data, CacheBackendInterface::CACHE_PERMANENT);
+      }
+    }
+
+    // Add the configuration objects from the cache to the list.
+    foreach ($cached_list as $name => $cache) {
+      $list[$name] = $cache->data;
+    }
+
+    return $list;
+  }
+
   /**
    * Implements Drupal\Core\Config\StorageInterface::write().
    */
@@ -87,6 +120,8 @@ public function write($name, array $data) {
       // While not all written data is read back, setting the cache instead of
       // just deleting it avoids cache rebuild stampedes.
       $this->cache->set($name, $data, CacheBackendInterface::CACHE_PERMANENT);
+      $this->cache->deleteTags(array($this::FIND_BY_PREFIX_CACHE_TAG => TRUE));
+      $this->findByPrefixCache = array();
       return TRUE;
     }
     return FALSE;
@@ -100,6 +135,8 @@ public function delete($name) {
     // rebuilding the cache before the storage is gone.
     if ($this->storage->delete($name)) {
       $this->cache->delete($name);
+      $this->cache->deleteTags(array($this::FIND_BY_PREFIX_CACHE_TAG => TRUE));
+      $this->findByPrefixCache = array();
       return TRUE;
     }
     return FALSE;
@@ -114,6 +151,8 @@ public function rename($name, $new_name) {
     if ($this->storage->rename($name, $new_name)) {
       $this->cache->delete($name);
       $this->cache->delete($new_name);
+      $this->cache->deleteTags(array($this::FIND_BY_PREFIX_CACHE_TAG => TRUE));
+      $this->findByPrefixCache = array();
       return TRUE;
     }
     return FALSE;
@@ -134,12 +173,50 @@ public function decode($raw) {
   }
 
   /**
-   * Implements Drupal\Core\Config\StorageInterface::listAll().
-   *
-   * Not supported by CacheBackendInterface.
+   * {@inheritdoc}
    */
   public function listAll($prefix = '') {
-    return $this->storage->listAll($prefix);
+    // Do not cache when a prefix is not provided.
+    if ($prefix) {
+      return $this->findByPrefix($prefix);
+    }
+    return $this->storage->listAll();
+  }
+
+  /**
+   * Finds configuration object names starting with a given prefix.
+   *
+   * Given the following configuration objects:
+   * - node.type.article
+   * - node.type.page
+   *
+   * Passing the prefix 'node.type.' will return an array containing the above
+   * names.
+   *
+   * @param string $prefix
+   *   The prefix to search for
+   *
+   * @return array
+   *   An array containing matching configuration object names.
+   */
+  protected function findByPrefix($prefix) {
+    if (!isset($this->findByPrefixCache[$prefix])) {
+      // The : character is not allowed in config file names, so this can not
+      // conflict.
+      if ($cache = $this->cache->get('find:' . $prefix)) {
+        $this->findByPrefixCache[$prefix] = $cache->data;
+      }
+      else {
+        $this->findByPrefixCache[$prefix] = $this->storage->listAll($prefix);
+        $this->cache->set(
+          'find:' . $prefix,
+          $this->findByPrefixCache[$prefix],
+          CacheBackendInterface::CACHE_PERMANENT,
+          array($this::FIND_BY_PREFIX_CACHE_TAG => TRUE)
+        );
+      }
+    }
+    return $this->findByPrefixCache[$prefix];
   }
 
   /**
@@ -155,4 +232,11 @@ public function deleteAll($prefix = '') {
     }
     return FALSE;
   }
+
+  /**
+   * Clears the static list cache.
+   */
+  public function resetListCache() {
+    $this->findByPrefixCache = array();
+  }
 }

core/lib/Drupal/Core/Config/Config.php

@@ -107,6 +107,25 @@ public function init() {
     return $this;
   }
 
+  /**
+   * Initializes a configuration object with pre-loaded data.
+   *
+   * @param array $data
+   *   Array of loaded data for this configuration object.
+   *
+   * @return Drupal\Core\Config\Config
+   *   The configuration object.
+   */
+  public function initWithData(array $data) {
+    $this->isLoaded = TRUE;
+    $this->overrides = array();
+    $this->isNew = FALSE;
+    $this->notify('init');
+    $this->replaceData($data);
+    $this->notify('load');
+    return $this;
+  }
+
   /**
    * Returns the name of this configuration object.
    *

core/lib/Drupal/Core/Config/ConfigFactory.php

@@ -84,6 +84,45 @@ public function get($name) {
     return $this->cache[$cache_key]->init();
   }
 
+  /**
+   * Returns a list of configuration objects for a given names and context.
+   *
+   * This will pre-load all requested configuration objects does not create
+   * new configuration objects.
+   *
+   * @param array $names
+   *   List of names of configuration objects.
+   *
+   * @return array
+   *   List of successfully loaded configuration objects, keyed by name.
+   */
+  public function loadMultiple(array $names) {
+    $context = $this->getContext();
+
+    $list = array();
+    foreach ($names as $key => $name) {
+      $cache_key = $this->getCacheKey($name, $context);
+      // @todo: Deleted configuration stays in $this->cache, only return
+      //   config entities that are not new.
+      if (isset($this->cache[$cache_key]) && !$this->cache[$cache_key]->isNew()) {
+        $list[$name] = $this->cache[$cache_key];
+        unset($names[$key]);
+      }
+    }
+
+    // Pre-load remaining configuration files.
+    if (!empty($names)) {
+      $storage_data = $this->storage->readMultiple($names);
+      foreach ($storage_data as $name => $data) {
+        $cache_key = $this->getCacheKey($name, $context);
+        $this->cache[$cache_key] = new Config($name, $this->storage, $context);
+        $this->cache[$cache_key]->initWithData($data);
+        $list[$name] = $this->cache[$cache_key];
+      }
+    }
+    return $list;
+  }
+
   /**
    * Resets and re-initializes configuration objects. Internal use only.
    *
@@ -104,6 +143,11 @@ public function reset($name = NULL) {
     else {
       $this->cache = array();
     }
+
+    // Clear the static list cache if supported by the storage.
+    if ($this->storage instanceof StorageCacheInterface) {
+      $this->storage->resetListCache();
+    }
     return $this;
   }
 

core/lib/Drupal/Core/Config/DatabaseStorage.php

@@ -85,6 +85,26 @@ public function read($name) {
     return $data;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function readMultiple(array $names) {
+    // There are situations, like in the installer, where we may attempt a
+    // read without actually having the database available. In this case,
+    // catch the exception and just return an empty array so the caller can
+    // handle it if need be.
+    $list = array();
+    try {
+      $list = $this->connection->query('SELECT name, data FROM {' . $this->connection->escapeTable($this->table) . '} WHERE name IN (:names)', array(':names' => $names), $this->options)->fetchAllKeyed();
+      foreach ($list as &$data) {
+        $data = $this->decode($data);
+      }
+    }
+    catch (Exception $e) {
+    }
+    return $list;
+  }
+
   /**
    * Implements Drupal\Core\Config\StorageInterface::write().
    *

core/lib/Drupal/Core/Config/Entity/ConfigStorageController.php

@@ -254,27 +254,24 @@ protected function buildQuery($ids, $revision_id = FALSE) {
     $config_class = $this->entityInfo['class'];
     $prefix = $this->getConfigPrefix();
 
-    // Load all of the configuration entities.
+    // Get the names of the configuration entities we are going to load.
     if ($ids === NULL) {
       $names = $this->configStorage->listAll($prefix);
-      $result = array();
-      foreach ($names as $name) {
-        $config = $this->configFactory->get($name);
-        $result[$config->get($this->idKey)] = new $config_class($config->get(), $this->entityType);
-      }
-      return $result;
     }
     else {
-      $result = array();
+      $names = array();
       foreach ($ids as $id) {
         // Add the prefix to the ID to serve as the configuration object name.
-        $config = $this->configFactory->get($prefix . $id);
-        if (!$config->isNew()) {
-          $result[$id] = new $config_class($config->get(), $this->entityType);
-        }
+        $names[] = $prefix . $id;
       }
-      return $result;
     }
+
+    // Load all of the configuration entities.
+    $result = array();
+    foreach ($this->configFactory->loadMultiple($names) as $config) {
+      $result[$config->get($this->idKey)] = new $config_class($config->get(), $this->entityType);
+    }
+    return $result;
   }
 
   /**

core/lib/Drupal/Core/Config/Entity/Query/Query.php

@@ -83,8 +83,8 @@ public function condition($property, $value = NULL, $operator = NULL, $langcode
   public function execute() {
     // Load all config files.
     $entity_info = $this->entityManager->getDefinition($this->getEntityType());
-    $prefix = $entity_info['config_prefix'];
-    $prefix_length = strlen($prefix) + 1;
+    $prefix = $entity_info['config_prefix'] . '.';
+    $prefix_length = strlen($prefix);
     $names = $this->configStorage->listAll($prefix);
     $configs = array();
     foreach ($names as $name) {

core/lib/Drupal/Core/Config/FileStorage.php

@@ -89,6 +89,19 @@ public function read($name) {
     return $data;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function readMultiple(array $names) {
+    $list = array();
+    foreach ($names as $name) {
+      if ($data = $this->read($name)) {
+        $list[$name] = $data;
+      }
+    }
+    return $list;
+  }
+
   /**
    * Implements Drupal\Core\Config\StorageInterface::write().
    *

core/lib/Drupal/Core/Config/NullStorage.php

@@ -37,6 +37,13 @@ public function read($name) {
     return array();
   }
 
+  /**
+   * Implements Drupal\Core\Config\StorageInterface::readMultiple().
+   */
+  public function readMultiple(array $names) {
+    return array();
+  }
+
   /**
    * Implements Drupal\Core\Config\StorageInterface::write().
    */

core/lib/Drupal/Core/Config/StorageCacheInterface.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Config\StorageCacheInterface.
+ */
+
+namespace Drupal\Core\Config;
+
+/**
+ * Defines an interface for cached configuration storage controllers.
+ */
+interface StorageCacheInterface {
+
+  /**
+   * Cache tag.
+   *
+   * Used by Drupal\Core\Config\CachedStorage::findByPrefix so that cached items
+   * can be cleared during writes, deletes and renames.
+   */
+  const FIND_BY_PREFIX_CACHE_TAG = 'configFindByPrefix';
+
+  /**
+   * Reset the static cache of the listAll() cache.
+   */
+  public function resetListCache();
+
+}

core/lib/Drupal/Core/Config/StorageInterface.php

@@ -38,6 +38,18 @@ public function exists($name);
    */
   public function read($name);
 
+  /**
+   * Reads configuration data from the storage.
+   *
+   * @param array $name
+   *   List of names of the configuration objects to load.
+   *
+   * @return array
+   *   A list of the configuration data stored for the configuration object name
+   *   that could be loaded for the passed list of names.
+   */
+  public function readMultiple(array $names);
+
   /**
    * Writes configuration data to the storage.
    *

core/modules/field/field.module

@@ -403,7 +403,7 @@ function field_sync_field_status() {
 
   // Get both deleted and non-deleted field definitions.
   $fields = array();
-  foreach (config_get_storage_names_with_prefix('field.field') as $name) {
+  foreach (config_get_storage_names_with_prefix('field.field.') as $name) {
     $field = Drupal::config($name)->get();
     $fields[$field['uuid']] = $field;
   }

core/modules/field/lib/Drupal/field/FieldInfo.php

@@ -178,14 +178,14 @@ public function getFieldMap() {
     $map = array();
 
     // Get active fields.
-    foreach (config_get_storage_names_with_prefix('field.field') as $config_id) {
+    foreach (config_get_storage_names_with_prefix('field.field.') as $config_id) {
       $field_config = $this->config->get($config_id)->get();
       if ($field_config['active'] && $field_config['storage']['active']) {
         $fields[$field_config['uuid']] = $field_config;
       }
     }
     // Get field instances.
-    foreach (config_get_storage_names_with_prefix('field.instance') as $config_id) {
+    foreach (config_get_storage_names_with_prefix('field.instance.') as $config_id) {
       $instance_config = $this->config->get($config_id)->get();
       $field_uuid = $instance_config['field_uuid'];
       // Filter out instances of inactive fields, and instances on unknown

core/modules/simpletest/lib/Drupal/simpletest/WebTestBase.php

@@ -934,6 +934,8 @@ protected function refreshVariables() {
     global $conf;
     cache('bootstrap')->delete('variables');
     $conf = variable_initialize();
+    // Clear the tag cache.
+    drupal_static_reset('Drupal\Core\Cache\CacheBackendInterface::tagCache');
     drupal_container()->get('config.factory')->reset();
   }
 

core/modules/user/tests/Drupal/user/Tests/Views/Argument/RolesRidTest.php

@@ -10,6 +10,7 @@
 use Drupal\Component\Utility\String;
 use Drupal\Core\DependencyInjection\ContainerBuilder;
 use Drupal\Tests\UnitTestCase;
+use Drupal\user\Plugin\Core\Entity\Role;
 use Drupal\user\Plugin\views\argument\RolesRid;
 
 /**
@@ -47,28 +48,24 @@ public static function getInfo() {
    * @see \Drupal\user\Plugin\views\argument\RolesRid::title_query()
    */
   public function testTitleQuery() {
-    $config = array(
-      'user.role.test_rid_1' => array(
-        'id' => 'test_rid_1',
-        'label' => 'test rid 1'
-      ),
-      'user.role.test_rid_2' => array(
-        'id' => 'test_rid_2',
-        'label' => 'test <strong>rid 2</strong>',
-      ),
-    );
-    $config_factory = $this->getConfigFactoryStub($config);
-    $config_storage = $this->getConfigStorageStub($config);
-
-    $entity_query_factory = $this->getMockBuilder('Drupal\Core\Entity\Query\QueryFactory')
-      ->disableOriginalConstructor()
-      ->getMock();
-
-    // Creates a stub role storage controller and replace the attachLoad()
-    // method with an empty version, because attachLoad() calls
-    // module_implements().
-    $role_storage_controller = $this->getMock('Drupal\user\RoleStorageController', array('attachLoad'), array('user_role', static::$entityInfo, $config_factory, $config_storage, $entity_query_factory));
-
+    $role1 = new Role(array(
+      'id' => 'test_rid_1',
+      'label' => 'test rid 1'
+    ), 'user_role');
+    $role2 = new Role(array(
+      'id' => 'test_rid_2',
+      'label' => 'test <strong>rid 2</strong>',
+    ), 'user_role');
+
+    // Creates a stub entity storage controller;
+    $role_storage_controller = $this->getMockForAbstractClass('Drupal\Core\Entity\EntityStorageControllerInterface');
+    $role_storage_controller->expects($this->any())
+      ->method('loadMultiple')
+      ->will($this->returnValueMap(array(
+        array(array(), array()),
+        array(array('test_rid_1'), array('test_rid_1' => $role1)),
+        array(array('test_rid_1', 'test_rid_2'), array('test_rid_1' => $role1, 'test_rid_2' => $role2)),
+      )));
 
     $entity_manager = $this->getMockBuilder('Drupal\Core\Entity\EntityManager')
       ->disableOriginalConstructor()
@@ -92,7 +89,7 @@ public function testTitleQuery() {
     $container->set('plugin.manager.entity', $entity_manager);
     \Drupal::setContainer($container);
 
-    $roles_rid_argument = new RolesRid($config, 'users_roles_rid', array(), $entity_manager);
+    $roles_rid_argument = new RolesRid(array(), 'users_roles_rid', array(), $entity_manager);
 
     $roles_rid_argument->value = array();
     $titles = $roles_rid_argument->title_query();

core/tests/Drupal/Tests/Core/Config/CachedStorageTest.php

+<?php
+
+namespace Drupal\Tests\Core\Config;
+
+use Drupal\Tests\UnitTestCase;
+use Drupal\Core\Config\CachedStorage;
+use Drupal\Core\Cache\MemoryBackend;
+use Drupal\Core\Cache\NullBackend;
+use Drupal\Core\Cache\CacheBackendInterface;
+
+/**
+ * Tests the interaction of cache and file storage in CachedStorage.
+ *
+ * @group Config
+ */
+class CachedStorageTest extends UnitTestCase {
+
+  public static function getInfo() {
+    return array(
+      'name' => 'Config cached storage test',
+      'description' => 'Tests the interaction of cache and file storage in CachedStorage.',
+      'group' => 'Configuration'
+    );
+  }
+
+  /**
+   * Test listAll static cache.
+   */
+  public function testListAllStaticCache() {
+    $prefix = __FUNCTION__;
+    $storage = $this->getMock('Drupal\Core\Config\StorageInterface');
+
+    $response = array("$prefix." . $this->randomName(), "$prefix." . $this->randomName());
+    $storage->expects($this->once())
+      ->method('listAll')
+      ->with($prefix)
+      ->will($this->returnValue($response));
+
+    $cache = new NullBackend(__FUNCTION__);
+    $cachedStorage = new CachedStorage($storage, $cache);
+    $this->assertEquals($response, $cachedStorage->listAll($prefix));
+    $this->assertEquals($response, $cachedStorage->listAll($prefix));
+  }
+
+  /**
+   * Test CachedStorage::listAll() persistent cache.
+   */
+  public function testListAllPrimedPersistentCache() {
+    $prefix = __FUNCTION__;
+    $storage = $this->getMock('Drupal\Core\Config\StorageInterface');
+    $storage->expects($this->never())->method('listAll');
+
+    $response = array("$prefix." . $this->randomName(), "$prefix." . $this->randomName());
+    $cache = new MemoryBackend(__FUNCTION__);
+    $cache->set('find:' . $prefix, $response);
+    $cachedStorage = new CachedStorage($storage, $cache);
+    $this->assertEquals($response, $cachedStorage->listAll($prefix));
+  }
+
+  /**
+   * Test that we don't fall through to file storage with a primed cache.
+   */
+  public function testGetMultipleOnPrimedCache() {
+    $configNames = array(
+      'foo.bar',
+      'baz.back',
+    );
+    $configCacheValues = array(
+      'foo.bar' => (object) array(
+        'data' => array('foo' => 'bar'),
+      ),
+      'baz.back' => (object) array(
+        'data' => array('foo' => 'bar'),
+      ),
+    );
+    $storage = $this->getMock('Drupal\Core\Config\StorageInterface');
+    $storage->expects($this->never())->method('readMultiple');
+    $cache = new MemoryBackend(__FUNCTION__);
+    foreach ($configCacheValues as $key => $value) {
+      $cache->set($key, $value);
+    }
+    $cachedStorage = new CachedStorage($storage, $cache);
+    $this->assertEquals($configCacheValues, $cachedStorage->readMultiple($configNames));
+  }
+
+  /**
+   * Test fall through to file storage on a cache miss.
+   */
+  public function testGetMultipleOnPartiallyPrimedCache() {
+    $configNames = array(
+      'foo.bar',
+      'baz.back',
+      $this->randomName() . '. ' . $this->randomName(),
+    );
+    $configCacheValues = array(
+      'foo.bar' => (object) array(
+        'data' => array('foo' => 'bar'),
+      ),
+      'baz.back' => (object) array(
+        'data' => array('foo' => 'bar'),
+      ),
+    );
+    $cache = new MemoryBackend(__FUNCTION__);
+    foreach ($configCacheValues as $key => $value) {
+      $cache->set($key, $value);
+    }
+
+    $response = array($configNames[2] => array($this->randomName()));
+    $storage = $this->getMock('Drupal\Core\Config\StorageInterface');
+    $storage->expects($this->once())
+      ->method('readMultiple')
+      ->with(array(2 => $configNames[2]))
+      ->will($this->returnValue($response));
+
+    $cachedStorage = new CachedStorage($storage, $cache);
+    $this->assertEquals($configCacheValues + $response, $cachedStorage->readMultiple($configNames));
+  }
+}