From 9b674084d3b75565ab38184db8ff2d1c10a49901 Mon Sep 17 00:00:00 2001
From: Nathaniel Catchpole <catch@35733.no-reply.drupal.org>
Date: Tue, 12 Sep 2017 09:22:02 +0100
Subject: [PATCH] Issue #2526150 by catch, Wim Leers, Denchev, hussainweb,
dawehner, mpdonadio, borisson_, fgm, olli, alexpott, vaplas, flocondetoile,
Berdir, Fabianx, cilefen: Database cache bins allow unlimited growth: cache
DB tables of gigabytes!
---
core/core.api.php | 24 ++++
core/core.services.yml | 2 +-
.../lib/Drupal/Core/Cache/DatabaseBackend.php | 56 ++++++++-
.../Core/Cache/DatabaseBackendFactory.php | 45 ++++++-
core/lib/Drupal/Core/DrupalKernel.php | 3 +-
core/modules/system/system.install | 17 +++
.../Core/Cache/ChainedFastBackendTest.php | 2 +-
.../Core/Cache/DatabaseBackendTest.php | 57 ++++++++-
.../KernelTests/Core/Command/DbDumpTest.php | 3 +-
.../Core/Cache/DatabaseBackendFactoryTest.php | 112 ++++++++++++++++++
10 files changed, 313 insertions(+), 8 deletions(-)
create mode 100644 core/tests/Drupal/Tests/Core/Cache/DatabaseBackendFactoryTest.php
diff --git a/core/core.api.php b/core/core.api.php
index 161ddb9146cf..f537882a5a50 100644
--- a/core/core.api.php
+++ b/core/core.api.php
@@ -606,6 +606,30 @@
* $settings['cache']['default'] = 'cache.custom';
* @endcode
*
+ * For cache bins that are stored in the database, the number of rows is limited
+ * to 5000 by default. This can be changed for all database cache bins. For
+ * example, to instead limit the number of rows to 50000:
+ * @code
+ * $settings['database_cache_max_rows']['default'] = 50000;
+ * @endcode
+ *
+ * Or per bin (in this example we allow infinite entries):
+ * @code
+ * $settings['database_cache_max_rows']['bins']['dynamic_page_cache'] = -1;
+ * @endcode
+ *
+ * For monitoring reasons it might be useful to figure out the amount of data
+ * stored in tables. The following SQL snippet can be used for that:
+ * @code
+ * SELECT table_name AS `Table`, table_rows AS 'Num. of Rows',
+ * ROUND(((data_length + index_length) / 1024 / 1024), 2) `Size in MB` FROM
+ * information_schema.TABLES WHERE table_schema = '***DATABASE_NAME***' AND
+ * table_name LIKE 'cache_%' ORDER BY (data_length + index_length) DESC
+ * LIMIT 10;
+ * @encode
+ *
+ * @see \Drupal\Core\Cache\DatabaseBackend
+ *
* Finally, you can chain multiple cache backends together, see
* \Drupal\Core\Cache\ChainedFastBackend and \Drupal\Core\Cache\BackendChain.
*
diff --git a/core/core.services.yml b/core/core.services.yml
index a7997ea66b16..21a3e590bafe 100644
--- a/core/core.services.yml
+++ b/core/core.services.yml
@@ -192,7 +192,7 @@ services:
- [setContainer, ['@service_container']]
cache.backend.database:
class: Drupal\Core\Cache\DatabaseBackendFactory
- arguments: ['@database', '@cache_tags.invalidator.checksum']
+ arguments: ['@database', '@cache_tags.invalidator.checksum', '@settings']
cache.backend.apcu:
class: Drupal\Core\Cache\ApcuBackendFactory
arguments: ['@app.root', '@site.path', '@cache_tags.invalidator.checksum']
diff --git a/core/lib/Drupal/Core/Cache/DatabaseBackend.php b/core/lib/Drupal/Core/Cache/DatabaseBackend.php
index d53c51c2fc8f..82f0054093ed 100644
--- a/core/lib/Drupal/Core/Cache/DatabaseBackend.php
+++ b/core/lib/Drupal/Core/Cache/DatabaseBackend.php
@@ -16,6 +16,30 @@
*/
class DatabaseBackend implements CacheBackendInterface {
+ /**
+ * The default maximum number of rows that this cache bin table can store.
+ *
+ * This maximum is introduced to ensure that the database is not filled with
+ * hundred of thousand of cache entries with gigabytes in size.
+ *
+ * Read about how to change it in the @link cache Cache API topic. @endlink
+ */
+ const DEFAULT_MAX_ROWS = 5000;
+
+ /**
+ * -1 means infinite allows numbers of rows for the cache backend.
+ */
+ const MAXIMUM_NONE = -1;
+
+ /**
+ * The maximum number of rows that this cache bin table is allowed to store.
+ *
+ * @see ::MAXIMUM_NONE
+ *
+ * @var int
+ */
+ protected $maxRows;
+
/**
* @var string
*/
@@ -45,14 +69,18 @@ class DatabaseBackend implements CacheBackendInterface {
* The cache tags checksum provider.
* @param string $bin
* The cache bin for which the object is created.
+ * @param int $max_rows
+ * (optional) The maximum number of rows that are allowed in this cache bin
+ * table.
*/
- public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, $bin) {
+ public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, $bin, $max_rows = NULL) {
// All cache tables should be prefixed with 'cache_'.
$bin = 'cache_' . $bin;
$this->bin = $bin;
$this->connection = $connection;
$this->checksumProvider = $checksum_provider;
+ $this->maxRows = $max_rows === NULL ? static::DEFAULT_MAX_ROWS : $max_rows;
}
/**
@@ -326,6 +354,22 @@ public function invalidateAll() {
*/
public function garbageCollection() {
try {
+ // Bounded size cache bin, using FIFO.
+ if ($this->maxRows !== static::MAXIMUM_NONE) {
+ $first_invalid_create_time = $this->connection->select($this->bin)
+ ->fields($this->bin, ['created'])
+ ->orderBy("{$this->bin}.created", 'DESC')
+ ->range($this->maxRows, $this->maxRows + 1)
+ ->execute()
+ ->fetchField();
+
+ if ($first_invalid_create_time) {
+ $this->connection->delete($this->bin)
+ ->condition('created', $first_invalid_create_time, '<=')
+ ->execute();
+ }
+ }
+
$this->connection->delete($this->bin)
->condition('expire', Cache::PERMANENT, '<>')
->condition('expire', REQUEST_TIME, '<')
@@ -472,10 +516,20 @@ public function schemaDefinition() {
],
'indexes' => [
'expire' => ['expire'],
+ 'created' => ['created'],
],
'primary key' => ['cid'],
];
return $schema;
}
+ /**
+ * The maximum number of rows that this cache bin table is allowed to store.
+ *
+ * @return int
+ */
+ public function getMaxRows() {
+ return $this->maxRows;
+ }
+
}
diff --git a/core/lib/Drupal/Core/Cache/DatabaseBackendFactory.php b/core/lib/Drupal/Core/Cache/DatabaseBackendFactory.php
index 8aa018ec4533..b86390e19656 100644
--- a/core/lib/Drupal/Core/Cache/DatabaseBackendFactory.php
+++ b/core/lib/Drupal/Core/Cache/DatabaseBackendFactory.php
@@ -3,6 +3,7 @@
namespace Drupal\Core\Cache;
use Drupal\Core\Database\Connection;
+use Drupal\Core\Site\Settings;
class DatabaseBackendFactory implements CacheFactoryInterface {
@@ -20,6 +21,13 @@ class DatabaseBackendFactory implements CacheFactoryInterface {
*/
protected $checksumProvider;
+ /**
+ * The settings array.
+ *
+ * @var \Drupal\Core\Site\Settings
+ */
+ protected $settings;
+
/**
* Constructs the DatabaseBackendFactory object.
*
@@ -27,10 +35,15 @@ class DatabaseBackendFactory implements CacheFactoryInterface {
* Database connection
* @param \Drupal\Core\Cache\CacheTagsChecksumInterface $checksum_provider
* The cache tags checksum provider.
+ * @param \Drupal\Core\Site\Settings $settings
+ * (optional) The settings array.
+ *
+ * @throws \BadMethodCallException
*/
- public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider) {
+ public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, Settings $settings = NULL) {
$this->connection = $connection;
$this->checksumProvider = $checksum_provider;
+ $this->settings = $settings ?: Settings::getInstance();
}
/**
@@ -43,7 +56,35 @@ public function __construct(Connection $connection, CacheTagsChecksumInterface $
* The cache backend object for the specified cache bin.
*/
public function get($bin) {
- return new DatabaseBackend($this->connection, $this->checksumProvider, $bin);
+ $max_rows = $this->getMaxRowsForBin($bin);
+ return new DatabaseBackend($this->connection, $this->checksumProvider, $bin, $max_rows);
+ }
+
+ /**
+ * Gets the max rows for the specified cache bin.
+ *
+ * @param string $bin
+ * The cache bin for which the object is created.
+ *
+ * @return int
+ * The maximum number of rows for the given bin. Defaults to
+ * DatabaseBackend::DEFAULT_MAX_ROWS.
+ */
+ protected function getMaxRowsForBin($bin) {
+ $max_rows_settings = $this->settings->get('database_cache_max_rows');
+ // First, look for a cache bin specific setting.
+ if (isset($max_rows_settings['bins'][$bin])) {
+ $max_rows = $max_rows_settings['bins'][$bin];
+ }
+ // Second, use configured default backend.
+ elseif (isset($max_rows_settings['default'])) {
+ $max_rows = $max_rows_settings['default'];
+ }
+ else {
+ // Fall back to the default max rows if nothing else is configured.
+ $max_rows = DatabaseBackend::DEFAULT_MAX_ROWS;
+ }
+ return $max_rows;
}
}
diff --git a/core/lib/Drupal/Core/DrupalKernel.php b/core/lib/Drupal/Core/DrupalKernel.php
index 260e4772f3cf..217d4dc781ed 100644
--- a/core/lib/Drupal/Core/DrupalKernel.php
+++ b/core/lib/Drupal/Core/DrupalKernel.php
@@ -7,6 +7,7 @@
use Drupal\Component\FileCache\FileCacheFactory;
use Drupal\Component\Utility\Unicode;
use Drupal\Component\Utility\UrlHelper;
+use Drupal\Core\Cache\DatabaseBackend;
use Drupal\Core\Config\BootstrapConfigStorageFactory;
use Drupal\Core\Config\NullStorage;
use Drupal\Core\Database\Database;
@@ -77,7 +78,7 @@ class DrupalKernel implements DrupalKernelInterface, TerminableInterface {
],
'cache.container' => [
'class' => 'Drupal\Core\Cache\DatabaseBackend',
- 'arguments' => ['@database', '@cache_tags_provider.container', 'container'],
+ 'arguments' => ['@database', '@cache_tags_provider.container', 'container', DatabaseBackend::MAXIMUM_NONE],
],
'cache_tags_provider.container' => [
'class' => 'Drupal\Core\Cache\DatabaseCacheTagsChecksum',
diff --git a/core/modules/system/system.install b/core/modules/system/system.install
index 3c8332e36738..33fa077e2ce3 100644
--- a/core/modules/system/system.install
+++ b/core/modules/system/system.install
@@ -10,6 +10,7 @@
use Drupal\Component\FileSystem\FileSystem;
use Drupal\Component\Utility\OpCodeCache;
use Drupal\Component\Utility\Unicode;
+use Drupal\Core\Cache\Cache;
use Drupal\Core\Path\AliasStorage;
use Drupal\Core\Url;
use Drupal\Core\Database\Database;
@@ -2025,3 +2026,19 @@ function system_update_8402() {
}
}
}
+
+/**
+ * Delete all cache_* tables. They are recreated on demand with the new schema.
+ */
+function system_update_8403() {
+ foreach (Cache::getBins() as $bin => $cache_backend) {
+ // Try to delete the table regardless of which cache backend is handling it.
+ // This is to ensure the new schema is used if the configuration for the
+ // backend class is changed after the update hook runs.
+ $table_name = "cache_$bin";
+ $schema = Database::getConnection()->schema();
+ if ($schema->tableExists($table_name)) {
+ $schema->dropTable($table_name);
+ }
+ }
+}
diff --git a/core/tests/Drupal/KernelTests/Core/Cache/ChainedFastBackendTest.php b/core/tests/Drupal/KernelTests/Core/Cache/ChainedFastBackendTest.php
index 3021b041fe8d..a93b674d5cf0 100644
--- a/core/tests/Drupal/KernelTests/Core/Cache/ChainedFastBackendTest.php
+++ b/core/tests/Drupal/KernelTests/Core/Cache/ChainedFastBackendTest.php
@@ -20,7 +20,7 @@ class ChainedFastBackendTest extends GenericCacheBackendUnitTestBase {
* A new ChainedFastBackend object.
*/
protected function createCacheBackend($bin) {
- $consistent_backend = new DatabaseBackend(\Drupal::service('database'), \Drupal::service('cache_tags.invalidator.checksum'), $bin);
+ $consistent_backend = new DatabaseBackend(\Drupal::service('database'), \Drupal::service('cache_tags.invalidator.checksum'), $bin, 100);
$fast_backend = new PhpBackend($bin, \Drupal::service('cache_tags.invalidator.checksum'));
$backend = new ChainedFastBackend($consistent_backend, $fast_backend, $bin);
// Explicitly register the cache bin as it can not work through the
diff --git a/core/tests/Drupal/KernelTests/Core/Cache/DatabaseBackendTest.php b/core/tests/Drupal/KernelTests/Core/Cache/DatabaseBackendTest.php
index 04bbdc65dbba..358b92530c6f 100644
--- a/core/tests/Drupal/KernelTests/Core/Cache/DatabaseBackendTest.php
+++ b/core/tests/Drupal/KernelTests/Core/Cache/DatabaseBackendTest.php
@@ -11,6 +11,13 @@
*/
class DatabaseBackendTest extends GenericCacheBackendUnitTestBase {
+ /**
+ * The max rows to use for test bins.
+ *
+ * @var int
+ */
+ protected static $maxRows = 100;
+
/**
* Modules to enable.
*
@@ -25,7 +32,7 @@ class DatabaseBackendTest extends GenericCacheBackendUnitTestBase {
* A new DatabaseBackend object.
*/
protected function createCacheBackend($bin) {
- return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin);
+ return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin, static::$maxRows);
}
/**
@@ -48,4 +55,52 @@ public function testSetGet() {
$this->assertSame($cached_value_short, $backend->get($cid_short)->data, "Backend contains the correct value for short, non-ASCII cache id.");
}
+ /**
+ * Tests the row count limiting of cache bin database tables.
+ */
+ public function testGarbageCollection() {
+ $backend = $this->getCacheBackend();
+ $max_rows = static::$maxRows;
+
+ $this->assertSame(0, (int) $this->getNumRows());
+
+ // Fill to just the limit.
+ for ($i = 0; $i < $max_rows; $i++) {
+ // Ensure that each cache item created happens in a different millisecond,
+ // by waiting 1 ms (1000 microseconds). The garbage collection might
+ // otherwise keep less than exactly 100 records (which is acceptable for
+ // real-world cases, but not for this test).
+ usleep(1000);
+ $backend->set("test$i", $i);
+ }
+ $this->assertSame($max_rows, $this->getNumRows());
+
+ // Garbage collection has no effect.
+ $backend->garbageCollection();
+ $this->assertSame($max_rows, $this->getNumRows());
+
+ // Go one row beyond the limit.
+ $backend->set('test' . ($max_rows + 1), $max_rows + 1);
+ $this->assertSame($max_rows + 1, $this->getNumRows());
+
+ // Garbage collection removes one row: the oldest.
+ $backend->garbageCollection();
+ $this->assertSame($max_rows, $this->getNumRows());
+ $this->assertFalse($backend->get('test0'));
+ }
+
+ /**
+ * Gets the number of rows in the test cache bin database table.
+ *
+ * @return int
+ * The number of rows in the test cache bin database table.
+ */
+ protected function getNumRows() {
+ $table = 'cache_' . $this->testBin;
+ $connection = $this->container->get('database');
+ $query = $connection->select($table);
+ $query->addExpression('COUNT(cid)', 'cid');
+ return (int) $query->execute()->fetchField();
+ }
+
}
diff --git a/core/tests/Drupal/KernelTests/Core/Command/DbDumpTest.php b/core/tests/Drupal/KernelTests/Core/Command/DbDumpTest.php
index 3bdfbc72e631..17b6ae41da10 100644
--- a/core/tests/Drupal/KernelTests/Core/Command/DbDumpTest.php
+++ b/core/tests/Drupal/KernelTests/Core/Command/DbDumpTest.php
@@ -70,7 +70,8 @@ public function register(ContainerBuilder $container) {
parent::register($container);
$container->register('cache_factory', 'Drupal\Core\Cache\DatabaseBackendFactory')
->addArgument(new Reference('database'))
- ->addArgument(new Reference('cache_tags.invalidator.checksum'));
+ ->addArgument(new Reference('cache_tags.invalidator.checksum'))
+ ->addArgument(new Reference('settings'));
}
/**
diff --git a/core/tests/Drupal/Tests/Core/Cache/DatabaseBackendFactoryTest.php b/core/tests/Drupal/Tests/Core/Cache/DatabaseBackendFactoryTest.php
new file mode 100644
index 000000000000..9d5ac4bdf924
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Cache/DatabaseBackendFactoryTest.php
@@ -0,0 +1,112 @@
+<?php
+
+namespace Drupal\Tests\Core\Cache;
+
+use Drupal\Core\Cache\CacheTagsChecksumInterface;
+use Drupal\Core\Cache\DatabaseBackend;
+use Drupal\Core\Cache\DatabaseBackendFactory;
+use Drupal\Core\Database\Connection;
+use Drupal\Core\Site\Settings;
+use Drupal\Tests\UnitTestCase;
+
+/**
+ * @coversDefaultClass \Drupal\Core\Cache\DatabaseBackendFactory
+ * @group Cache
+ */
+class DatabaseBackendFactoryTest extends UnitTestCase {
+
+ /**
+ * @covers ::__construct
+ * @covers ::get
+ * @dataProvider getProvider
+ */
+ public function testGet(array $settings, $expected_max_rows_foo, $expected_max_rows_bar) {
+ $database_backend_factory = new DatabaseBackendFactory(
+ $this->prophesize(Connection::class)->reveal(),
+ $this->prophesize(CacheTagsChecksumInterface::class)->reveal(),
+ new Settings($settings)
+ );
+
+ $this->assertSame($expected_max_rows_foo, $database_backend_factory->get('foo')->getMaxRows());
+ $this->assertSame($expected_max_rows_bar, $database_backend_factory->get('bar')->getMaxRows());
+ }
+
+ public function getProvider() {
+ return [
+ 'default' => [
+ [],
+ DatabaseBackend::DEFAULT_MAX_ROWS,
+ DatabaseBackend::DEFAULT_MAX_ROWS,
+ ],
+ 'default overridden' => [
+ [
+ 'database_cache_max_rows' => [
+ 'default' => 99,
+ ],
+ ],
+ 99,
+ 99,
+ ],
+ 'default + foo bin overridden' => [
+ [
+ 'database_cache_max_rows' => [
+ 'bins' => [
+ 'foo' => 13,
+ ],
+ ],
+ ],
+ 13,
+ DatabaseBackend::DEFAULT_MAX_ROWS,
+ ],
+ 'default + bar bin overridden' => [
+ [
+ 'database_cache_max_rows' => [
+ 'bins' => [
+ 'bar' => 13,
+ ],
+ ],
+ ],
+ DatabaseBackend::DEFAULT_MAX_ROWS,
+ 13,
+ ],
+ 'default overridden + bar bin overridden' => [
+ [
+ 'database_cache_max_rows' => [
+ 'default' => 99,
+ 'bins' => [
+ 'bar' => 13,
+ ],
+ ],
+ ],
+ 99,
+ 13,
+ ],
+ 'default + both bins overridden' => [
+ [
+ 'database_cache_max_rows' => [
+ 'bins' => [
+ 'foo' => 13,
+ 'bar' => 31,
+ ],
+ ],
+ ],
+ 13,
+ 31,
+ ],
+ 'default overridden + both bins overridden' => [
+ [
+ 'database_cache_max_rows' => [
+ 'default' => 99,
+ 'bins' => [
+ 'foo' => 13,
+ 'bar' => 31,
+ ],
+ ],
+ ],
+ 13,
+ 31,
+ ],
+ ];
+ }
+
+}
--
GitLab