Issue #3327118 by Defcon0, acbramley, iSampo, poker10, Nitin shrivastava,...

Issue #3327118 by Defcon0, acbramley, iSampo, poker10, Nitin shrivastava, catch, fgm, longwave: Chunk multiple cache sets into groups of 100 to avoid OOM/max_allowed_packet issues

(cherry picked from commit a04bdb08)

core/lib/Drupal/Core/Cache/DatabaseBackend.php

@@ -32,6 +32,11 @@ class DatabaseBackend implements CacheBackendInterface {
    */
   const MAXIMUM_NONE = -1;
 
+  /**
+   * The chunk size for inserting cache entities.
+   */
+  const MAX_ITEMS_PER_CACHE_SET = 100;
+
   /**
    * The maximum number of rows that this cache bin table is allowed to store.
    *
@@ -215,62 +220,68 @@ public function setMultiple(array $items) {
    * @see \Drupal\Core\Cache\CacheBackendInterface::setMultiple()
    */
   protected function doSetMultiple(array $items) {
-    $values = [];
-
-    foreach ($items as $cid => $item) {
-      $item += [
-        'expire' => CacheBackendInterface::CACHE_PERMANENT,
-        'tags' => [],
-      ];
-
-      assert(Inspector::assertAllStrings($item['tags']), 'Cache Tags must be strings.');
-      $item['tags'] = array_unique($item['tags']);
-      // Sort the cache tags so that they are stored consistently in the DB.
-      sort($item['tags']);
-
-      $fields = [
-        'cid' => $this->normalizeCid($cid),
-        'expire' => $item['expire'],
-        'created' => round(microtime(TRUE), 3),
-        'tags' => implode(' ', $item['tags']),
-        'checksum' => $this->checksumProvider->getCurrentChecksum($item['tags']),
-      ];
-
-      // Avoid useless writes.
-      if ($fields['checksum'] === CacheTagsChecksumInterface::INVALID_CHECKSUM_WHILE_IN_TRANSACTION) {
-        continue;
-      }
+    // Chunk the items as the database might not be able to receive thousands
+    // of items in a single query.
+    $chunks = array_chunk($items, self::MAX_ITEMS_PER_CACHE_SET, TRUE);
+
+    foreach ($chunks as $chunk_items) {
+      $values = [];
+
+      foreach ($chunk_items as $cid => $item) {
+        $item += [
+          'expire' => CacheBackendInterface::CACHE_PERMANENT,
+          'tags' => [],
+        ];
+
+        assert(Inspector::assertAllStrings($item['tags']), 'Cache Tags must be strings.');
+        $item['tags'] = array_unique($item['tags']);
+        // Sort the cache tags so that they are stored consistently in the DB.
+        sort($item['tags']);
+
+        $fields = [
+          'cid' => $this->normalizeCid($cid),
+          'expire' => $item['expire'],
+          'created' => round(microtime(TRUE), 3),
+          'tags' => implode(' ', $item['tags']),
+          'checksum' => $this->checksumProvider->getCurrentChecksum($item['tags']),
+        ];
+
+        // Avoid useless writes.
+        if ($fields['checksum'] === CacheTagsChecksumInterface::INVALID_CHECKSUM_WHILE_IN_TRANSACTION) {
+          continue;
+        }
 
-      if (!is_string($item['data'])) {
-        $fields['data'] = serialize($item['data']);
-        $fields['serialized'] = 1;
+        if (!is_string($item['data'])) {
+          $fields['data'] = serialize($item['data']);
+          $fields['serialized'] = 1;
+        }
+        else {
+          $fields['data'] = $item['data'];
+          $fields['serialized'] = 0;
+        }
+        $values[] = $fields;
       }
-      else {
-        $fields['data'] = $item['data'];
-        $fields['serialized'] = 0;
+
+      // If all $items were useless writes, we may end up with zero writes.
+      if (count($values) === 0) {
+        return;
       }
-      $values[] = $fields;
-    }
 
-    // If all $items were useless writes, we may end up with zero writes.
-    if (empty($values)) {
-      return;
-    }
+      // Use an upsert query which is atomic and optimized for multiple-row
+      // merges.
+      $query = $this->connection
+        ->upsert($this->bin)
+        ->key('cid')
+        ->fields(['cid', 'expire', 'created', 'tags', 'checksum', 'data', 'serialized']);
+      foreach ($values as $fields) {
+        // Only pass the values since the order of $fields matches the order of
+        // the insert fields. This is a performance optimization to avoid
+        // unnecessary loops within the method.
+        $query->values(array_values($fields));
+      }
 
-    // Use an upsert query which is atomic and optimized for multiple-row
-    // merges.
-    $query = $this->connection
-      ->upsert($this->bin)
-      ->key('cid')
-      ->fields(['cid', 'expire', 'created', 'tags', 'checksum', 'data', 'serialized']);
-    foreach ($values as $fields) {
-      // Only pass the values since the order of $fields matches the order of
-      // the insert fields. This is a performance optimization to avoid
-      // unnecessary loops within the method.
-      $query->values(array_values($fields));
+      $query->execute();
     }
-
-    $query->execute();
   }
 
   /**

core/tests/Drupal/KernelTests/Core/Cache/DatabaseBackendTest.php

@@ -53,6 +53,15 @@ public function testSetGet() {
     $cached_value_short = $this->randomMachineName();
     $backend->set($cid_short, $cached_value_short);
     $this->assertSame($cached_value_short, $backend->get($cid_short)->data, "Backend contains the correct value for short, non-ASCII cache id.");
+
+    // Set multiple items to test exceeding the chunk size.
+    $backend->deleteAll();
+    $items = [];
+    for ($i = 0; $i <= DatabaseBackend::MAX_ITEMS_PER_CACHE_SET; $i++) {
+      $items["test$i"]['data'] = $i;
+    }
+    $backend->setMultiple($items);
+    $this->assertSame(DatabaseBackend::MAX_ITEMS_PER_CACHE_SET + 1, $this->getNumRows());
   }
 
   /**