Commit fea7acc2 authored by catch's avatar catch

Issue #1547008 by Berdir, Sutharsan: Replace Update's cache system with the...

Issue #1547008 by Berdir, Sutharsan: Replace Update's cache system with the (expirable) key value store.
parent 980cb9c5
...@@ -642,6 +642,44 @@ function update_fix_d8_requirements() { ...@@ -642,6 +642,44 @@ function update_fix_d8_requirements() {
// picture upgrade path. // picture upgrade path.
update_module_enable(array('file')); update_module_enable(array('file'));
$schema = array(
'description' => 'Generic key/value storage table with an expiration.',
'fields' => array(
'collection' => array(
'description' => 'A named collection of key and value pairs.',
'type' => 'varchar',
'length' => 128,
'not null' => TRUE,
'default' => '',
),
'name' => array(
// KEY is an SQL reserved word, so use 'name' as the key's field name.
'description' => 'The key of the key/value pair.',
'type' => 'varchar',
'length' => 128,
'not null' => TRUE,
'default' => '',
),
'value' => array(
'description' => 'The value of the key/value pair.',
'type' => 'blob',
'not null' => TRUE,
'size' => 'big',
),
'expire' => array(
'description' => 'The time since Unix epoch in seconds when this item expires. Defaults to the maximum possible time.',
'type' => 'int',
'not null' => TRUE,
'default' => 2147483647,
),
),
'primary key' => array('collection', 'name'),
'indexes' => array(
'all' => array('name', 'collection', 'expire'),
),
);
db_create_table('key_value_expire', $schema);
update_variable_set('update_d8_requirements', TRUE); update_variable_set('update_d8_requirements', TRUE);
} }
} }
......
...@@ -131,6 +131,19 @@ public static function database() { ...@@ -131,6 +131,19 @@ public static function database() {
return static::$container->get('database'); return static::$container->get('database');
} }
/**
* Returns an expirable key value store collection.
*
* @param string $collection
* The name of the collection holding key and value pairs.
*
* @return \Drupal\Core\KeyValueStore\KeyValueStoreExpirableInterface
* An expirable key value store collection.
*/
public static function keyValueExpirable($collection) {
return static::$container->get('keyvalue.expirable')->get($collection);
}
/** /**
* Returns the locking layer instance. * Returns the locking layer instance.
* *
......
...@@ -105,6 +105,14 @@ public function build(ContainerBuilder $container) { ...@@ -105,6 +105,14 @@ public function build(ContainerBuilder $container) {
$container $container
->register('keyvalue.database', 'Drupal\Core\KeyValueStore\KeyValueDatabaseFactory') ->register('keyvalue.database', 'Drupal\Core\KeyValueStore\KeyValueDatabaseFactory')
->addArgument(new Reference('database')); ->addArgument(new Reference('database'));
// Register the KeyValueStoreExpirable factory.
$container
->register('keyvalue.expirable', 'Drupal\Core\KeyValueStore\KeyValueExpirableFactory')
->addArgument(new Reference('service_container'));
$container
->register('keyvalue.expirable.database', 'Drupal\Core\KeyValueStore\KeyValueDatabaseExpirableFactory')
->addArgument(new Reference('database'))
->addTag('needs_destruction');
$container->register('settings', 'Drupal\Component\Utility\Settings') $container->register('settings', 'Drupal\Component\Utility\Settings')
->setFactoryClass('Drupal\Component\Utility\Settings') ->setFactoryClass('Drupal\Component\Utility\Settings')
......
...@@ -125,4 +125,12 @@ public function deleteMultiple(array $keys) { ...@@ -125,4 +125,12 @@ public function deleteMultiple(array $keys) {
while (count($keys)); while (count($keys));
} }
/**
* Implements Drupal\Core\KeyValueStore\KeyValueStoreInterface::deleteAll().
*/
public function deleteAll() {
$this->connection->delete($this->table)
->condition('collection', $this->collection)
->execute();
}
} }
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
namespace Drupal\Core\KeyValueStore; namespace Drupal\Core\KeyValueStore;
use Drupal\Core\DestructableInterface;
use Drupal\Core\Database\Connection; use Drupal\Core\Database\Connection;
use Drupal\Core\Database\Query\Merge; use Drupal\Core\Database\Query\Merge;
...@@ -16,7 +17,7 @@ ...@@ -16,7 +17,7 @@
* This key/value store implementation uses the database to store key/value * This key/value store implementation uses the database to store key/value
* data with an expire date. * data with an expire date.
*/ */
class DatabaseStorageExpirable extends DatabaseStorage implements KeyValueStoreExpirableInterface { class DatabaseStorageExpirable extends DatabaseStorage implements KeyValueStoreExpirableInterface, DestructableInterface {
/** /**
* The connection object for this storage. * The connection object for this storage.
...@@ -55,15 +56,6 @@ public function __construct($collection, Connection $connection, $table = 'key_v ...@@ -55,15 +56,6 @@ public function __construct($collection, Connection $connection, $table = 'key_v
parent::__construct($collection, $connection, $table); parent::__construct($collection, $connection, $table);
} }
/**
* Performs garbage collection as needed when destructing the storage object.
*/
public function __destruct() {
if ($this->needsGarbageCollection) {
$this->garbageCollection();
}
}
/** /**
* Implements Drupal\Core\KeyValueStore\KeyValueStoreInterface::getMultiple(). * Implements Drupal\Core\KeyValueStore\KeyValueStoreInterface::getMultiple().
*/ */
...@@ -158,4 +150,13 @@ protected function garbageCollection() { ...@@ -158,4 +150,13 @@ protected function garbageCollection() {
->execute(); ->execute();
} }
/**
* Implements Drupal\Core\DestructableInterface::destruct().
*/
public function destruct() {
if ($this->needsGarbageCollection) {
$this->garbageCollection();
}
}
} }
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
namespace Drupal\Core\KeyValueStore; namespace Drupal\Core\KeyValueStore;
use Drupal\Core\DestructableInterface;
use Drupal\Core\Database\Connection; use Drupal\Core\Database\Connection;
use Drupal\Core\Database\Database; use Drupal\Core\Database\Database;
use Drupal\Core\KeyValueStore\KeyValueDatabaseFactory; use Drupal\Core\KeyValueStore\KeyValueDatabaseFactory;
...@@ -14,7 +15,14 @@ ...@@ -14,7 +15,14 @@
/** /**
* Defines the key/value store factory for the database backend. * Defines the key/value store factory for the database backend.
*/ */
class KeyValueDatabaseExpirableFactory extends KeyValueDatabaseFactory { class KeyValueDatabaseExpirableFactory extends KeyValueDatabaseFactory implements DestructableInterface {
/**
* Holds references to each instantiation so they can be terminated.
*
* @var array
*/
protected $storages;
/** /**
* Constructs a new key/value expirable database storage object for a given * Constructs a new key/value expirable database storage object for a given
...@@ -28,6 +36,17 @@ class KeyValueDatabaseExpirableFactory extends KeyValueDatabaseFactory { ...@@ -28,6 +36,17 @@ class KeyValueDatabaseExpirableFactory extends KeyValueDatabaseFactory {
* A key/value store implementation for the given $collection. * A key/value store implementation for the given $collection.
*/ */
public function get($collection) { public function get($collection) {
return new DatabaseStorageExpirable($collection, $this->connection); $storage = new DatabaseStorageExpirable($collection, $this->connection);
$this->storages[] = $storage;
return $storage;
}
/**
* Implements Drupal\Core\DestructableInterface::terminate().
*/
public function destruct() {
foreach ($this->storages as $storage) {
$storage->destruct();
}
} }
} }
...@@ -99,4 +99,9 @@ public function delete($key); ...@@ -99,4 +99,9 @@ public function delete($key);
*/ */
public function deleteMultiple(array $keys); public function deleteMultiple(array $keys);
/**
* Deletes all items from the key/value store.
*/
public function deleteAll();
} }
...@@ -81,4 +81,10 @@ public function deleteMultiple(array $keys) { ...@@ -81,4 +81,10 @@ public function deleteMultiple(array $keys) {
} }
} }
/**
* Implements Drupal\Core\KeyValueStore\KeyValueStoreInterface::deleteAll().
*/
public function deleteAll() {
$this->data = array();
}
} }
...@@ -62,10 +62,10 @@ public function testGarbageCollection() { ...@@ -62,10 +62,10 @@ public function testGarbageCollection() {
->execute(); ->execute();
} }
// Perform a new set operation and then manually unset the object to // Perform a new set operation and then manually destruct the object to
// trigger garbage collection. // trigger garbage collection.
$store->setWithExpire('autumn', 'winter', rand(500, 1000000)); $store->setWithExpire('autumn', 'winter', rand(500, 1000000));
unset($store); $store->destruct();
// Query the database and confirm that the stale records were deleted. // Query the database and confirm that the stale records were deleted.
$result = db_query( $result = db_query(
......
...@@ -1735,44 +1735,7 @@ function system_update_8022() { ...@@ -1735,44 +1735,7 @@ function system_update_8022() {
* Create the 'key_value_expire' table. * Create the 'key_value_expire' table.
*/ */
function system_update_8023() { function system_update_8023() {
$table = array( // Moved to update_fix_d8_requirements() as it is required early.
'description' => 'Generic key/value storage table with an expiration.',
'fields' => array(
'collection' => array(
'description' => 'A named collection of key and value pairs.',
'type' => 'varchar',
'length' => 128,
'not null' => TRUE,
'default' => '',
),
'name' => array(
// KEY is an SQL reserved word, so use 'name' as the key's field name.
'description' => 'The key of the key/value pair.',
'type' => 'varchar',
'length' => 128,
'not null' => TRUE,
'default' => '',
),
'value' => array(
'description' => 'The value of the key/value pair.',
'type' => 'blob',
'not null' => TRUE,
'size' => 'big',
),
'expire' => array(
'description' => 'The time since Unix epoch in seconds when this item expires. Defaults to the maximum possible time.',
'type' => 'int',
'not null' => TRUE,
'default' => 2147483647,
),
),
'primary key' => array('collection', 'name'),
'indexes' => array(
'all' => array('name', 'collection', 'expire'),
),
);
db_create_table('key_value_expire', $table);
} }
/** /**
......
...@@ -234,5 +234,5 @@ function hook_themes_enabled($theme_list) { ...@@ -234,5 +234,5 @@ function hook_themes_enabled($theme_list) {
*/ */
function hook_themes_disabled($theme_list) { function hook_themes_disabled($theme_list) {
// Clear all update module caches. // Clear all update module caches.
_update_cache_clear(); update_storage_clear();
} }
...@@ -211,8 +211,8 @@ function testFetchTasks() { ...@@ -211,8 +211,8 @@ function testFetchTasks() {
update_create_fetch_task($projecta); update_create_fetch_task($projecta);
$this->assertEqual($queue->numberOfItems(), 2, 'Queue still contains two items'); $this->assertEqual($queue->numberOfItems(), 2, 'Queue still contains two items');
// Clear cache and try again. // Clear storage and try again.
_update_cache_clear(); update_storage_clear();
drupal_static_reset('_update_create_fetch_task'); drupal_static_reset('_update_create_fetch_task');
update_create_fetch_task($projecta); update_create_fetch_task($projecta);
$this->assertEqual($queue->numberOfItems(), 2, 'Queue contains two items'); $this->assertEqual($queue->numberOfItems(), 2, 'Queue contains two items');
......
...@@ -106,9 +106,9 @@ public function validateForm(array &$form, array &$form_state) { ...@@ -106,9 +106,9 @@ public function validateForm(array &$form, array &$form_state) {
public function submitForm(array &$form, array &$form_state) { public function submitForm(array &$form, array &$form_state) {
$config = $this->configFactory->get('update.settings'); $config = $this->configFactory->get('update.settings');
// See if the update_check_disabled setting is being changed, and if so, // See if the update_check_disabled setting is being changed, and if so,
// invalidate all cached update status data. // invalidate all update status data.
if ($form_state['values']['update_check_disabled'] != $config->get('check.disabled_extensions')) { if ($form_state['values']['update_check_disabled'] != $config->get('check.disabled_extensions')) {
_update_cache_clear(); update_storage_clear();
} }
$config $config
......
...@@ -178,7 +178,7 @@ function update_authorize_batch_copy_project($project, $updater_name, $local_url ...@@ -178,7 +178,7 @@ function update_authorize_batch_copy_project($project, $updater_name, $local_url
* *
* This processes the results and stashes them into SESSION such that * This processes the results and stashes them into SESSION such that
* authorize.php will render a report. Also responsible for putting the site * authorize.php will render a report. Also responsible for putting the site
* back online and clearing the update status cache after a successful update. * back online and clearing the update status storage after a successful update.
* *
* @param $success * @param $success
* TRUE if the batch operation was successful; FALSE if there were errors. * TRUE if the batch operation was successful; FALSE if there were errors.
...@@ -193,8 +193,8 @@ function update_authorize_update_batch_finished($success, $results) { ...@@ -193,8 +193,8 @@ function update_authorize_update_batch_finished($success, $results) {
} }
$offline = config('system.maintenance')->get('enabled'); $offline = config('system.maintenance')->get('enabled');
if ($success) { if ($success) {
// Now that the update completed, we need to clear the cache of available // Now that the update completed, we need to clear the available update data
// update data and recompute our status, so prevent show bogus results. // and recompute our status, so prevent show bogus results.
_update_authorize_clear_update_status(); _update_authorize_clear_update_status();
// Take the site out of maintenance mode if it was previously that way. // Take the site out of maintenance mode if it was previously that way.
...@@ -315,25 +315,19 @@ function _update_batch_create_message(&$project_results, $message, $success = TR ...@@ -315,25 +315,19 @@ function _update_batch_create_message(&$project_results, $message, $success = TR
} }
/** /**
* Clears cached available update status data. * Clears available update status data.
* *
* Since this function is run at such a low bootstrap level, the Update Manager * Since this function is run at such a low bootstrap level, the Update Manager
* module is not loaded. So, we can't just call _update_cache_clear(). However, * module is not loaded. So, we can't just call update_storage_clear(). However,
* the database is bootstrapped, so we can do a query ourselves to clear out * the key-value backend is available, so we just call that.
* what we want to clear.
* *
* Note that we do not want to just truncate the table, since that would remove * Note that we do not want to delete items related to currently pending fetch
* items related to currently pending fetch attempts. * attempts.
* *
* @see update_authorize_update_batch_finished() * @see update_authorize_update_batch_finished()
* @see _update_cache_clear() * @see update_storage_clear()
*/ */
function _update_authorize_clear_update_status() { function _update_authorize_clear_update_status() {
$query = db_delete('cache_update'); Drupal::keyValueExpirable('update')->deleteAll();
$query->condition( Drupal::keyValueExpirable('update_available_release')->deleteAll();
db_or()
->condition('cid', 'update_project_%', 'LIKE')
->condition('cid', 'available_releases::%', 'LIKE')
);
$query->execute();
} }
...@@ -16,13 +16,12 @@ ...@@ -16,13 +16,12 @@
* fetching the available release data. * fetching the available release data.
* *
* This array is fairly expensive to construct, since it involves a lot of disk * This array is fairly expensive to construct, since it involves a lot of disk
* I/O, so we cache the results into the {cache_update} table using the * I/O, so we store the results. However, since this is not the data about
* 'update_project_projects' cache ID. However, since this is not the data about
* available updates fetched from the network, it is acceptable to invalidate it * available updates fetched from the network, it is acceptable to invalidate it
* somewhat quickly. If we keep this data for very long, site administrators are * somewhat quickly. If we keep this data for very long, site administrators are
* more likely to see incorrect results if they upgrade to a newer version of a * more likely to see incorrect results if they upgrade to a newer version of a
* module or theme but do not visit certain pages that automatically clear this * module or theme but do not visit certain pages that automatically clear this
* cache. * data.
* *
* @return * @return
* An associative array of currently enabled projects keyed by the * An associative array of currently enabled projects keyed by the
...@@ -51,15 +50,15 @@ ...@@ -51,15 +50,15 @@
* *
* @see update_process_project_info() * @see update_process_project_info()
* @see update_calculate_project_data() * @see update_calculate_project_data()
* @see update_project_cache() * @see update_project_storage()
*/ */
function update_get_projects() { function update_get_projects() {
$projects = &drupal_static(__FUNCTION__, array()); $projects = &drupal_static(__FUNCTION__, array());
if (empty($projects)) { if (empty($projects)) {
// Retrieve the projects from cache, if present. // Retrieve the projects from storage, if present.
$projects = update_project_cache('update_project_projects'); $projects = update_project_storage('update_project_projects');
if (empty($projects)) { if (empty($projects)) {
// Still empty, so we have to rebuild the cache. // Still empty, so we have to rebuild.
$module_data = system_rebuild_module_data(); $module_data = system_rebuild_module_data();
$theme_data = system_rebuild_theme_data(); $theme_data = system_rebuild_theme_data();
update_process_info_list($projects, $module_data, 'module', TRUE); update_process_info_list($projects, $module_data, 'module', TRUE);
...@@ -70,8 +69,8 @@ function update_get_projects() { ...@@ -70,8 +69,8 @@ function update_get_projects() {
} }
// Allow other modules to alter projects before fetching and comparing. // Allow other modules to alter projects before fetching and comparing.
drupal_alter('update_projects', $projects); drupal_alter('update_projects', $projects);
// Cache the site's project data for at most 1 hour. // Store the site's project data for at most 1 hour.
_update_cache_set('update_project_projects', $projects, REQUEST_TIME + 3600); Drupal::keyValueExpirable('update')->setWithExpire('update_project_projects', $projects, 3600);
} }
} }
return $projects; return $projects;
...@@ -90,7 +89,7 @@ function update_get_projects() { ...@@ -90,7 +89,7 @@ function update_get_projects() {
* 'Hidden' themes are ignored only if they have no enabled sub-themes. * 'Hidden' themes are ignored only if they have no enabled sub-themes.
* This function also records the latest change time on the .info.yml * This function also records the latest change time on the .info.yml
* files for each module or theme, which is important data which is used when * files for each module or theme, which is important data which is used when
* deciding if the cached available update data should be invalidated. * deciding if the available update data should be invalidated.
* *
* @param $projects * @param $projects
* Reference to the array of project data of what's installed on this site. * Reference to the array of project data of what's installed on this site.
...@@ -318,13 +317,12 @@ function update_process_project_info(&$projects) { ...@@ -318,13 +317,12 @@ function update_process_project_info(&$projects) {
* *
* The results of this function are expensive to compute, especially on sites * The results of this function are expensive to compute, especially on sites
* with lots of modules or themes, since it involves a lot of comparisons and * with lots of modules or themes, since it involves a lot of comparisons and
* other operations. Therefore, we cache the results into the {cache_update} * other operations. Therefore, we store the results. However, since this is not
* table using the 'update_project_data' cache ID. However, since this is not
* the data about available updates fetched from the network, it is ok to * the data about available updates fetched from the network, it is ok to
* invalidate it somewhat quickly. If we keep this data for very long, site * invalidate it somewhat quickly. If we keep this data for very long, site
* administrators are more likely to see incorrect results if they upgrade to a * administrators are more likely to see incorrect results if they upgrade to a
* newer version of a module or theme but do not visit certain pages that * newer version of a module or theme but do not visit certain pages that
* automatically clear this cache. * automatically clear this.
* *
* @param array $available * @param array $available
* Data about available project releases. * Data about available project releases.
...@@ -335,13 +333,13 @@ function update_process_project_info(&$projects) { ...@@ -335,13 +333,13 @@ function update_process_project_info(&$projects) {
* @see update_get_available() * @see update_get_available()
* @see update_get_projects() * @see update_get_projects()
* @see update_process_project_info() * @see update_process_project_info()
* @see update_project_cache() * @see update_project_storage()
*/ */
function update_calculate_project_data($available) { function update_calculate_project_data($available) {
// Retrieve the projects from cache, if present. // Retrieve the projects from storage, if present.
$projects = update_project_cache('update_project_data'); $projects = update_project_storage('update_project_data');
// If $projects is empty, then the cache must be rebuilt. // If $projects is empty, then the data must be rebuilt.
// Otherwise, return the cached data and skip the rest of the function. // Otherwise, return the data and skip the rest of the function.
if (!empty($projects)) { if (!empty($projects)) {
return $projects; return $projects;
} }
...@@ -361,8 +359,8 @@ function update_calculate_project_data($available) { ...@@ -361,8 +359,8 @@ function update_calculate_project_data($available) {
// projects or releases). // projects or releases).
drupal_alter('update_status', $projects); drupal_alter('update_status', $projects);
// Cache the site's update status for at most 1 hour. // Store the site's update status for at most 1 hour.
_update_cache_set('update_project_data', $projects, REQUEST_TIME + 3600); Drupal::keyValueExpirable('update')->setWithExpire('update_project_data', $projects, 3600);
return $projects; return $projects;
} }
...@@ -752,37 +750,36 @@ function update_calculate_project_update_status(&$project_data, $available) { ...@@ -752,37 +750,36 @@ function update_calculate_project_update_status(&$project_data, $available) {
} }
/** /**
* Retrieves data from {cache_update} or empties the cache when necessary. * Retrieves update storage data or empties it.
* *
* Two very expensive arrays computed by this module are the list of all * Two very expensive arrays computed by this module are the list of all
* installed modules and themes (and .info.yml data, project associations, etc), * installed modules and themes (and .info.yml data, project associations, etc), and
* and the current status of the site relative to the currently available * the current status of the site relative to the currently available releases.
* releases. These two arrays are cached in the {cache_update} table and used * These two arrays are stored and used whenever possible. The data is cleared
* whenever possible. The cache is cleared whenever the administrator visits the * whenever the administrator visits the status report, available updates
* status report, available updates report, or the module or theme * report, or the module or theme administration pages, since we should always
* administration pages, since we should always recompute the most current * recompute the most current values on any of those pages.
* values on any of those pages.
* *
* Note: while both of these arrays are expensive to compute (in terms of disk * Note: while both of these arrays are expensive to compute (in terms of disk
* I/O and some fairly heavy CPU processing), neither of these is the actual * I/O and some fairly heavy CPU processing), neither of these is the actual
* data about available updates that we have to fetch over the network from * data about available updates that we have to fetch over the network from
* updates.drupal.org. That information is stored with the * updates.drupal.org. That information is stored in the
* 'update_available_releases' cache ID -- it needs to persist longer than 1 * 'update_available_releases' collection -- it needs to persist longer than 1
* hour and never get invalidated just by visiting a page on the site. * hour and never get invalidated just by visiting a page on the site.
* *
* @param $cid * @param $key
* The cache ID of data to return from the cache. Valid options are * The key of data to return. Valid options are 'update_project_data' and
* 'update_project_data' and 'update_project_projects'. * 'update_project_projects'.
* *
* @return * @return
* The cached value of the $projects array generated by * The stored value of the $projects array generated by
* update_calculate_project_data() or update_get_projects(), or an empty array * update_calculate_project_data() or update_get_projects(), or an empty array
* when the cache is cleared. * when the storage is cleared.
*/ */
function update_project_cache($cid) { function update_project_storage($key) {
$projects = array(); $projects = array();
// On certain paths, we should clear the cache and recompute the projects for // On certain paths, we should clear the data and recompute the projects for
// update status of the site to avoid presenting stale information. // update status of the site to avoid presenting stale information.
$paths = array( $paths = array(
'admin/modules', 'admin/modules',
...@@ -796,13 +793,10 @@ function update_project_cache($cid) { ...@@ -796,13 +793,10 @@ function update_project_cache($cid) {
'admin/reports/updates/check', 'admin/reports/updates/check',
); );
if (in_array(current_path(), $paths)) { if (in_array(current_path(), $paths)) {
_update_cache_clear($cid); Drupal::keyValueExpirable('update')->delete($key);
} }
else { else {
$cache = _update_cache_get($cid); $projects = Drupal::keyValueExpirable('update')->get($key);
if (!empty($cache->data) && $cache->expire > REQUEST_TIME) {
$projects = $cache->data;
}
} }
return $projects; return $projects;
} }
......