Commit 1a71b1a9 authored by catch's avatar catch

Issue #2526150 by catch, Wim Leers, Denchev, hussainweb, dawehner, mpdonadio,...

Issue #2526150 by catch, Wim Leers, Denchev, hussainweb, dawehner, mpdonadio, borisson_, fgm, olli, alexpott, flocondetoile, Berdir, Fabianx: Database cache bins allow unlimited growth: cache DB tables of gigabytes!
parent 517619e9
......@@ -606,6 +606,30 @@
* $settings['cache']['default'] = 'cache.custom';
* @endcode
*
* For cache bins that are stored in the database, the number of rows is limited
* to 5000 by default. This can be changed for all database cache bins. For
* example, to instead limit the number of rows to 50000:
* @code
* $settings['database_cache_max_rows']['default'] = 50000;
* @endcode
*
* Or per bin (in this example we allow infinite entries):
* @code
* $settings['database_cache_max_rows']['bins']['dynamic_page_cache'] = -1;
* @endcode
*
* For monitoring reasons it might be useful to figure out the amount of data
* stored in tables. The following SQL snippet can be used for that:
* @code
* SELECT table_name AS `Table`, table_rows AS 'Num. of Rows',
* ROUND(((data_length + index_length) / 1024 / 1024), 2) `Size in MB` FROM
* information_schema.TABLES WHERE table_schema = '***DATABASE_NAME***' AND
* table_name LIKE 'cache_%' ORDER BY (data_length + index_length) DESC
* LIMIT 10;
* @encode
*
* @see \Drupal\Core\Cache\DatabaseBackend
*
* Finally, you can chain multiple cache backends together, see
* \Drupal\Core\Cache\ChainedFastBackend and \Drupal\Core\Cache\BackendChain.
*
......
......@@ -192,7 +192,7 @@ services:
- [setContainer, ['@service_container']]
cache.backend.database:
class: Drupal\Core\Cache\DatabaseBackendFactory
arguments: ['@database', '@cache_tags.invalidator.checksum']
arguments: ['@database', '@cache_tags.invalidator.checksum', '@settings']
cache.backend.apcu:
class: Drupal\Core\Cache\ApcuBackendFactory
arguments: ['@app.root', '@site.path', '@cache_tags.invalidator.checksum']
......
......@@ -16,6 +16,30 @@
*/
class DatabaseBackend implements CacheBackendInterface {
/**
* The default maximum number of rows that this cache bin table can store.
*
* This maximum is introduced to ensure that the database is not filled with
* hundred of thousand of cache entries with gigabytes in size.
*
* Read about how to change it in the @link cache Cache API topic. @endlink
*/
const DEFAULT_MAX_ROWS = 5000;
/**
* -1 means infinite allows numbers of rows for the cache backend.
*/
const MAXIMUM_NONE = -1;
/**
* The maximum number of rows that this cache bin table is allowed to store.
*
* * @see ::MAXIMUM_NONE
*
* @var int
*/
protected $maxRows;
/**
* @var string
*/
......@@ -45,14 +69,18 @@ class DatabaseBackend implements CacheBackendInterface {
* The cache tags checksum provider.
* @param string $bin
* The cache bin for which the object is created.
* @param int $max_rows
* (optional) The maximum number of rows that are allowed in this cache bin
* table.
*/
public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, $bin) {
public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, $bin, $max_rows = NULL) {
// All cache tables should be prefixed with 'cache_'.
$bin = 'cache_' . $bin;
$this->bin = $bin;
$this->connection = $connection;
$this->checksumProvider = $checksum_provider;
$this->maxRows = $max_rows === NULL ? static::DEFAULT_MAX_ROWS : $max_rows;
}
/**
......@@ -326,6 +354,22 @@ public function invalidateAll() {
*/
public function garbageCollection() {
try {
// Bounded size cache bin, using FIFO.
if ($this->maxRows !== static::MAXIMUM_NONE) {
$first_invalid_create_time = $this->connection->select($this->bin)
->fields($this->bin, ['created'])
->orderBy("{$this->bin}.created", 'DESC')
->range($this->maxRows, $this->maxRows + 1)
->execute()
->fetchField();
if ($first_invalid_create_time) {
$this->connection->delete($this->bin)
->condition('created', $first_invalid_create_time, '<=')
->execute();
}
}
$this->connection->delete($this->bin)
->condition('expire', Cache::PERMANENT, '<>')
->condition('expire', REQUEST_TIME, '<')
......@@ -472,10 +516,20 @@ public function schemaDefinition() {
],
'indexes' => [
'expire' => ['expire'],
'created' => ['created'],
],
'primary key' => ['cid'],
];
return $schema;
}
/**
* The maximum number of rows that this cache bin table is allowed to store.
*
* @return int
*/
public function getMaxRows() {
return $this->maxRows;
}
}
......@@ -3,6 +3,7 @@
namespace Drupal\Core\Cache;
use Drupal\Core\Database\Connection;
use Drupal\Core\Site\Settings;
class DatabaseBackendFactory implements CacheFactoryInterface {
......@@ -20,6 +21,13 @@ class DatabaseBackendFactory implements CacheFactoryInterface {
*/
protected $checksumProvider;
/**
* The settings array.
*
* @var \Drupal\Core\Site\Settings
*/
protected $settings;
/**
* Constructs the DatabaseBackendFactory object.
*
......@@ -27,10 +35,13 @@ class DatabaseBackendFactory implements CacheFactoryInterface {
* Database connection
* @param \Drupal\Core\Cache\CacheTagsChecksumInterface $checksum_provider
* The cache tags checksum provider.
* @param \Drupal\Core\Site\Settings $settings
* (optional) The settings array.
*/
public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider) {
public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, Settings $settings = NULL) {
$this->connection = $connection;
$this->checksumProvider = $checksum_provider;
$this->settings = $settings ?: Settings::getInstance();
}
/**
......@@ -43,7 +54,35 @@ public function __construct(Connection $connection, CacheTagsChecksumInterface $
* The cache backend object for the specified cache bin.
*/
public function get($bin) {
return new DatabaseBackend($this->connection, $this->checksumProvider, $bin);
$max_rows = $this->getMaxRowsForBin($bin);
return new DatabaseBackend($this->connection, $this->checksumProvider, $bin, $max_rows);
}
/**
* Gets the max rows for the specified cache bin.
*
* @param string $bin
* The cache bin for which the object is created.
*
* @return int
* The maximum number of rows for the given bin. Defaults to
* DatabaseBackend::DEFAULT_MAX_ROWS.
*/
protected function getMaxRowsForBin($bin) {
$max_rows_settings = $this->settings->get('database_cache_max_rows');
// First, look for a cache bin specific setting.
if (isset($max_rows_settings['bins'][$bin])) {
$max_rows = $max_rows_settings['bins'][$bin];
}
// Third, use configured default backend.
elseif (isset($max_rows_settings['default'])) {
$max_rows = $max_rows_settings['default'];
}
else {
// Fall back to the default max rows if nothing else is configured.
$max_rows = DatabaseBackend::DEFAULT_MAX_ROWS;
}
return $max_rows;
}
}
......@@ -7,6 +7,7 @@
use Drupal\Component\FileCache\FileCacheFactory;
use Drupal\Component\Utility\Unicode;
use Drupal\Component\Utility\UrlHelper;
use Drupal\Core\Cache\DatabaseBackend;
use Drupal\Core\Config\BootstrapConfigStorageFactory;
use Drupal\Core\Config\NullStorage;
use Drupal\Core\Database\Database;
......@@ -77,7 +78,7 @@ class DrupalKernel implements DrupalKernelInterface, TerminableInterface {
],
'cache.container' => [
'class' => 'Drupal\Core\Cache\DatabaseBackend',
'arguments' => ['@database', '@cache_tags_provider.container', 'container'],
'arguments' => ['@database', '@cache_tags_provider.container', 'container', DatabaseBackend::MAXIMUM_NONE],
],
'cache_tags_provider.container' => [
'class' => 'Drupal\Core\Cache\DatabaseCacheTagsChecksum',
......
......@@ -10,6 +10,7 @@
use Drupal\Component\FileSystem\FileSystem;
use Drupal\Component\Utility\OpCodeCache;
use Drupal\Component\Utility\Unicode;
use Drupal\Core\Cache\Cache;
use Drupal\Core\Path\AliasStorage;
use Drupal\Core\Url;
use Drupal\Core\Database\Database;
......@@ -2025,3 +2026,19 @@ function system_update_8402() {
}
}
}
/**
* Delete all cache_* tables. They are recreated on demand with the new schema.
*/
function system_update_8403() {
foreach (Cache::getBins() as $bin => $cache_backend) {
// Try to delete the table regardless of which cache backend is handling it.
// This is to ensure the new schema is used if the configuration for the
// backend class is changed after the update hook runs.
$table_name = "cache_$bin";
$schema = Database::getConnection()->schema();
if ($schema->tableExists($table_name)) {
$schema->dropTable($table_name);
}
}
}
......@@ -20,7 +20,7 @@ class ChainedFastBackendTest extends GenericCacheBackendUnitTestBase {
* A new ChainedFastBackend object.
*/
protected function createCacheBackend($bin) {
$consistent_backend = new DatabaseBackend(\Drupal::service('database'), \Drupal::service('cache_tags.invalidator.checksum'), $bin);
$consistent_backend = new DatabaseBackend(\Drupal::service('database'), \Drupal::service('cache_tags.invalidator.checksum'), $bin, 100);
$fast_backend = new PhpBackend($bin, \Drupal::service('cache_tags.invalidator.checksum'));
$backend = new ChainedFastBackend($consistent_backend, $fast_backend, $bin);
// Explicitly register the cache bin as it can not work through the
......
......@@ -11,6 +11,13 @@
*/
class DatabaseBackendTest extends GenericCacheBackendUnitTestBase {
/**
* The max rows to use for test bins.
*
* @var int
*/
protected static $maxRows = 100;
/**
* Modules to enable.
*
......@@ -25,7 +32,7 @@ class DatabaseBackendTest extends GenericCacheBackendUnitTestBase {
* A new DatabaseBackend object.
*/
protected function createCacheBackend($bin) {
return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin);
return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin, static::$maxRows);
}
/**
......@@ -48,4 +55,47 @@ public function testSetGet() {
$this->assertIdentical($cached_value_short, $backend->get($cid_short)->data, "Backend contains the correct value for short, non-ASCII cache id.");
}
/**
* Tests the row count limiting of cache bin database tables.
*/
public function testGarbageCollection() {
$backend = $this->getCacheBackend();
$max_rows = static::$maxRows;
$this->assertSame(0, (int) $this->getNumRows());
// Fill to just the limit.
for ($i = 0; $i < $max_rows; $i++) {
$backend->set("test$i", $i);
}
$this->assertSame($max_rows, $this->getNumRows());
// Garbage collection has no effect.
$backend->garbageCollection();
$this->assertSame($max_rows, $this->getNumRows());
// Go one row beyond the limit.
$backend->set('test' . ($max_rows + 1), $max_rows + 1);
$this->assertSame($max_rows + 1, $this->getNumRows());
// Garbage collection removes one row: the oldest.
$backend->garbageCollection();
$this->assertSame($max_rows, $this->getNumRows());
$this->assertFalse($backend->get('test0'));
}
/**
* Gets the number of rows in the test cache bin database table.
*
* @return int
* The number of rows in the test cache bin database table.
*/
protected function getNumRows() {
$table = 'cache_' . $this->testBin;
$connection = $this->container->get('database');
$query = $connection->select($table);
$query->addExpression('COUNT(cid)', 'cid');
return (int) $query->execute()->fetchField();
}
}
......@@ -70,7 +70,8 @@ public function register(ContainerBuilder $container) {
parent::register($container);
$container->register('cache_factory', 'Drupal\Core\Cache\DatabaseBackendFactory')
->addArgument(new Reference('database'))
->addArgument(new Reference('cache_tags.invalidator.checksum'));
->addArgument(new Reference('cache_tags.invalidator.checksum'))
->addArgument(new Reference('settings'));
}
/**
......
<?php
namespace Drupal\Tests\Core\Cache;
use Drupal\Core\Cache\CacheTagsChecksumInterface;
use Drupal\Core\Cache\DatabaseBackend;
use Drupal\Core\Cache\DatabaseBackendFactory;
use Drupal\Core\Database\Connection;
use Drupal\Core\Site\Settings;
use Drupal\Tests\UnitTestCase;
/**
* @coversDefaultClass \Drupal\Core\Cache\DatabaseBackendFactory
* @group Cache
*/
class DatabaseBackendFactoryTest extends UnitTestCase {
/**
* @covers ::__construct
* @covers ::get
* @dataProvider getProvider
*/
public function testGet(array $settings, $expected_max_rows_foo, $expected_max_rows_bar) {
$database_backend_factory = new DatabaseBackendFactory(
$this->prophesize(Connection::class)->reveal(),
$this->prophesize(CacheTagsChecksumInterface::class)->reveal(),
new Settings($settings)
);
$this->assertSame($expected_max_rows_foo, $database_backend_factory->get('foo')->getMaxRows());
$this->assertSame($expected_max_rows_bar, $database_backend_factory->get('bar')->getMaxRows());
}
public function getProvider() {
return [
'default' => [
[],
DatabaseBackend::DEFAULT_MAX_ROWS,
DatabaseBackend::DEFAULT_MAX_ROWS,
],
'default overridden' => [
[
'database_cache_max_rows' => [
'default' => 99,
],
],
99,
99,
],
'default + foo bin overridden' => [
[
'database_cache_max_rows' => [
'bins' => [
'foo' => 13,
],
],
],
13,
DatabaseBackend::DEFAULT_MAX_ROWS,
],
'default + bar bin overridden' => [
[
'database_cache_max_rows' => [
'bins' => [
'bar' => 13,
],
],
],
DatabaseBackend::DEFAULT_MAX_ROWS,
13,
],
'default overridden + bar bin overridden' => [
[
'database_cache_max_rows' => [
'default' => 99,
'bins' => [
'bar' => 13,
],
],
],
99,
13,
],
'default + both bins overridden' => [
[
'database_cache_max_rows' => [
'bins' => [
'foo' => 13,
'bar' => 31,
],
],
],
13,
31,
],
'default overridden + both bins overridden' => [
[
'database_cache_max_rows' => [
'default' => 99,
'bins' => [
'foo' => 13,
'bar' => 31,
],
],
],
13,
31,
],
];
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment