Issue #1446932 by timmillwood, mallezie, RobLoach, hussainweb, Wim Leers,...

Issue #1446932 by timmillwood, mallezie, RobLoach, hussainweb, Wim Leers, rpayanm, kostyashupenko, mparker17, dawehner, Berdir, Crell: Improve statistics performance by adding a swappable backend

core/modules/statistics/src/NodeStatisticsDatabaseStorage.php

+<?php
+
+namespace Drupal\statistics;
+
+use Drupal\Core\Database\Connection;
+use Drupal\Core\State\StateInterface;
+use Symfony\Component\HttpFoundation\RequestStack;
+
+/**
+ * Provides the default database storage backend for statistics.
+ */
+class NodeStatisticsDatabaseStorage implements StatisticsStorageInterface {
+
+  /**
+  * The database connection used.
+  *
+  * @var \Drupal\Core\Database\Connection
+  */
+  protected $connection;
+
+  /**
+   * The state service.
+   *
+   * @var \Drupal\Core\State\StateInterface
+   */
+  protected $state;
+
+  /**
+   * The request stack.
+   *
+   * @var \Symfony\Component\HttpFoundation\RequestStack
+   */
+  protected $requestStack;
+
+  /**
+   * Constructs the statistics storage.
+   *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection for the node view storage.
+   * @param \Drupal\Core\State\StateInterface $state
+   *   The state service.
+   */
+  public function __construct(Connection $connection, StateInterface $state, RequestStack $request_stack) {
+    $this->connection = $connection;
+    $this->state = $state;
+    $this->requestStack = $request_stack;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function recordView($id) {
+    return (bool) $this->connection
+      ->merge('node_counter')
+      ->key('nid', $id)
+      ->fields([
+        'daycount' => 1,
+        'totalcount' => 1,
+        'timestamp' => $this->getRequestTime(),
+      ])
+      ->expression('daycount', 'daycount + 1')
+      ->expression('totalcount', 'totalcount + 1')
+      ->execute();
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fetchViews($ids) {
+    $views = $this->connection
+      ->select('node_counter', 'nc')
+      ->fields('nc', ['totalcount', 'daycount', 'timestamp'])
+      ->condition('nid', $ids, 'IN')
+      ->execute()
+      ->fetchAll();
+    foreach ($views as $id => $view) {
+      $views[$id] = new StatisticsViewsResult($view->totalcount, $view->daycount, $view->timestamp);
+    }
+    return $views;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fetchView($id) {
+    $views = $this->fetchViews(array($id));
+    return reset($views);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fetchAll($order = 'totalcount', $limit = 5) {
+    assert(in_array($order, ['totalcount', 'daycount', 'timestamp']), "Invalid order argument.");
+
+    return $this->connection
+      ->select('node_counter', 'nc')
+      ->fields('nc', ['nid'])
+      ->orderBy($order, 'DESC')
+      ->range(0, $limit)
+      ->execute()
+      ->fetchCol();
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function deleteViews($id) {
+    return (bool) $this->connection
+      ->delete('node_counter')
+      ->condition('nid', $id)
+      ->execute();
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function resetDayCount() {
+    $statistics_timestamp = $this->state->get('statistics.day_timestamp') ?: 0;
+    if (($this->getRequestTime() - $statistics_timestamp) >= 86400) {
+      $this->state->set('statistics.day_timestamp', $this->getRequestTime());
+      $this->connection->update('node_counter')
+        ->fields(['daycount' => 0])
+        ->execute();
+    }
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function maxTotalCount() {
+    $query = $this->connection->select('node_counter', 'nc');
+    $query->addExpression('MAX(totalcount)');
+    $max_total_count = (int)$query->execute()->fetchField();
+    return $max_total_count;
+  }
+
+  /**
+   * Get current request time.
+   *
+   * @return int
+   *   Unix timestamp for current server request time.
+   */
+  protected function getRequestTime() {
+    return $this->requestStack->getCurrentRequest()->server->get('REQUEST_TIME');
+  }
+
+}

core/modules/statistics/src/Plugin/Block/StatisticsPopularBlock.php

@@ -4,8 +4,14 @@
 
 use Drupal\Core\Access\AccessResult;
 use Drupal\Core\Block\BlockBase;
+use Drupal\Core\Entity\EntityRepositoryInterface;
+use Drupal\Core\Entity\EntityTypeManagerInterface;
 use Drupal\Core\Form\FormStateInterface;
+use Drupal\Core\Render\RendererInterface;
 use Drupal\Core\Session\AccountInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\statistics\StatisticsStorageInterface;
 
 /**
  * Provides a 'Popular content' block.
@@ -15,7 +21,72 @@
  *   admin_label = @Translation("Popular content")
  * )
  */
-class StatisticsPopularBlock extends BlockBase {
+class StatisticsPopularBlock extends BlockBase implements ContainerFactoryPluginInterface {
+
+  /**
+   * The entity type manager.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
+   */
+  protected $entityTypeManager;
+
+  /**
+   * The entity repository service.
+   *
+   * @var \Drupal\Core\Entity\EntityRepositoryInterface
+   */
+  protected $entityRepository;
+
+  /**
+   * The storage for statistics.
+   *
+   * @var \Drupal\statistics\StatisticsStorageInterface
+   */
+  protected $statisticsStorage;
+
+  /**
+   * @var \Drupal\Core\Render\RendererInterface
+   */
+  protected $renderer;
+
+  /**
+   * Constructs an StatisticsPopularBlock object.
+   *
+   * @param array $configuration
+   *   A configuration array containing information about the plugin instance.
+   * @param string $plugin_id
+   *   The plugin_id for the plugin instance.
+   * @param mixed $plugin_definition
+   *   The plugin implementation definition.
+   * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager
+   *   The entity type manager.
+   * @param \Drupal\Core\Entity\EntityRepositoryInterface $entity_repository
+   *   The entity repository service
+   * @param \Drupal\statistics\StatisticsStorageInterface $statistics_storage
+   *   The storage for statistics.
+   */
+  public function __construct(array $configuration, $plugin_id, $plugin_definition, EntityTypeManagerInterface $entity_type_manager, EntityRepositoryInterface $entity_repository, StatisticsStorageInterface $statistics_storage, RendererInterface $renderer) {
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+    $this->entityTypeManager = $entity_type_manager;
+    $this->entityRepository = $entity_repository;
+    $this->statisticsStorage = $statistics_storage;
+    $this->renderer = $renderer;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
+    return new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $container->get('entity_type.manager'),
+      $container->get('entity.repository'),
+      $container->get('statistics.storage.node'),
+      $container->get('renderer')
+    );
+  }
 
   /**
    * {@inheritdoc}
@@ -82,28 +153,64 @@ public function build() {
     $content = array();
 
     if ($this->configuration['top_day_num'] > 0) {
-      $result = statistics_title_list('daycount', $this->configuration['top_day_num']);
-      if ($result) {
-        $content['top_day'] = node_title_list($result, $this->t("Today's:"));
+      $nids = $this->statisticsStorage->fetchAll('daycount', $this->configuration['top_day_num']);
+      if ($nids) {
+        $content['top_day'] = $this->nodeTitleList($nids, $this->t("Today's:"));
         $content['top_day']['#suffix'] = '<br />';
       }
     }
 
     if ($this->configuration['top_all_num'] > 0) {
-      $result = statistics_title_list('totalcount', $this->configuration['top_all_num']);
-      if ($result) {
-        $content['top_all'] = node_title_list($result, $this->t('All time:'));
+      $nids = $this->statisticsStorage->fetchAll('totalcount', $this->configuration['top_all_num']);
+      if ($nids) {
+        $content['top_all'] = $this->nodeTitleList($nids, $this->t('All time:'));
         $content['top_all']['#suffix'] = '<br />';
       }
     }
 
     if ($this->configuration['top_last_num'] > 0) {
-      $result = statistics_title_list('timestamp', $this->configuration['top_last_num']);
-      $content['top_last'] = node_title_list($result, $this->t('Last viewed:'));
+      $nids = $this->statisticsStorage->fetchAll('timestamp', $this->configuration['top_last_num']);
+      $content['top_last'] = $this->nodeTitleList($nids, $this->t('Last viewed:'));
       $content['top_last']['#suffix'] = '<br />';
     }
 
     return $content;
   }
 
+  /**
+   * Generates the ordered array of node links for build().
+   *
+   * @param int[] $nids
+   *   An ordered array of node ids.
+   * @param string $title
+   *   The title for the list.
+   *
+   * @return array
+   *   A render array for the list.
+   */
+  protected function nodeTitleList(array $nids, $title) {
+    $nodes = $this->entityTypeManager->getStorage('node')->loadMultiple($nids);
+
+    $items = [];
+    foreach ($nids as $nid) {
+      $node = $this->entityRepository->getTranslationFromContext($nodes[$nid]);
+      $item = [
+        '#type' => 'link',
+        '#title' => $node->getTitle(),
+        '#url' => $node->urlInfo('canonical'),
+      ];
+      $this->renderer->addCacheableDependency($item, $node);
+      $items[] = $item;
+    }
+
+    return [
+      '#theme' => 'item_list__node',
+      '#items' => $items,
+      '#title' => $title,
+      '#cache' => [
+        'tags' => $this->entityTypeManager->getDefinition('node')->getListCacheTags(),
+      ],
+    ];
+  }
+
 }

core/modules/statistics/src/StatisticsStorageInterface.php

+<?php
+
+namespace Drupal\statistics;
+
+/**
+ * Provides an interface defining Statistics Storage.
+ *
+ * Stores the views per day, total views and timestamp of last view
+ * for entities.
+ */
+interface StatisticsStorageInterface {
+
+  /**
+   * Count a entity view.
+   *
+   * @param int $id
+   *   The ID of the entity to count.
+   *
+   * @return bool
+   *   TRUE if the entity view has been counted.
+   */
+  public function recordView($id);
+
+  /**
+   * Returns the number of times entities have been viewed.
+   *
+   * @param array $ids
+   *   An array of IDs of entities to fetch the views for.
+   *
+   * @return array \Drupal\statistics\StatisticsViewsResult
+   */
+  public function fetchViews($ids);
+
+  /**
+   * Returns the number of times a single entity has been viewed.
+   *
+   * @param int $id
+   *   The ID of the entity to fetch the views for.
+   *
+   * @return \Drupal\statistics\StatisticsViewsResult
+   */
+  public function fetchView($id);
+
+  /**
+   * Returns the number of times a entity has been viewed.
+   *
+   * @param string $order
+   *   The counter name to order by:
+   *   - 'totalcount' The total number of views.
+   *   - 'daycount' The number of views today.
+   *   - 'timestamp' The unix timestamp of the last view.
+   *
+   * @param int $limit
+   *   The number of entity IDs to return.
+   *
+   * @return array
+   *   An ordered array of entity IDs.
+   */
+  public function fetchAll($order = 'totalcount', $limit = 5);
+
+  /**
+   * Delete counts for a specific entity.
+   *
+   * @param int $id
+   *   The ID of the entity which views to delete.
+   *
+   * @return bool
+   *   TRUE if the entity views have been deleted.
+   */
+  public function deleteViews($id);
+
+  /**
+   * Reset the day counter for all entities once every day.
+   */
+  public function resetDayCount();
+
+  /**
+   * Returns the highest 'totalcount' value.
+   *
+   * @return int
+   *   The highest 'totalcount' value.
+   */
+  public function maxTotalCount();
+
+}

core/modules/statistics/src/StatisticsViewsResult.php

+<?php
+
+namespace Drupal\statistics;
+
+/**
+ * Value object for passing statistic results.
+ */
+class StatisticsViewsResult {
+
+  /**
+   * @var int
+   */
+  protected $totalCount;
+
+  /**
+   * @var int
+   */
+  protected $dayCount;
+
+  /**
+   * @var int
+   */
+  protected $timestamp;
+
+  public function __construct($total_count, $day_count, $timestamp) {
+    $this->totalCount = $total_count;
+    $this->dayCount = $day_count;
+    $this->timestamp = $timestamp;
+  }
+
+  /**
+   * Total number of times the entity has been viewed.
+   *
+   * @return int
+   */
+  public function getTotalCount() {
+    return $this->totalCount;
+  }
+
+
+  /**
+   * Total number of times the entity has been viewed "today".
+   *
+   * @return int
+   */
+  public function getDayCount() {
+    return $this->dayCount;
+  }
+
+
+  /**
+   * Timestamp of when the entity was last viewed.
+   *
+   * @return int
+   */
+  public function getTimestamp() {
+    return $this->timestamp;
+  }
+
+}

core/modules/statistics/src/Tests/StatisticsReportsTest.php

@@ -2,6 +2,9 @@
 
 namespace Drupal\statistics\Tests;
 
+use Drupal\Core\Cache\Cache;
+use Drupal\system\Tests\Cache\AssertPageCacheContextsAndTagsTrait;
+
 /**
  * Tests display of statistics report blocks.
  *
@@ -9,6 +12,8 @@
  */
 class StatisticsReportsTest extends StatisticsTestBase {
 
+  use AssertPageCacheContextsAndTagsTrait;
+
   /**
    * Tests the "popular content" block.
    */
@@ -30,7 +35,7 @@ function testPopularContentBlock() {
     $client->post($stats_path, array('headers' => $headers, 'body' => $post));
 
     // Configure and save the block.
-    $this->drupalPlaceBlock('statistics_popular_block', array(
+    $block = $this->drupalPlaceBlock('statistics_popular_block', array(
       'label' => 'Popular content',
       'top_day_num' => 3,
       'top_all_num' => 3,
@@ -44,9 +49,16 @@ function testPopularContentBlock() {
     $this->assertText('All time', 'Found the all time popular content.');
     $this->assertText('Last viewed', 'Found the last viewed popular content.');
 
-    // statistics.module doesn't use node entities, prevent the node language
-    // from being added to the options.
-    $this->assertRaw(\Drupal::l($node->label(), $node->urlInfo('canonical', ['language' => NULL])), 'Found link to visited node.');
+    $tags = Cache::mergeTags($node->getCacheTags(), $block->getCacheTags());
+    $tags = Cache::mergeTags($tags, $this->blockingUser->getCacheTags());
+    $tags = Cache::mergeTags($tags, ['block_view', 'config:block_list', 'node_list', 'rendered', 'user_view']);
+    $this->assertCacheTags($tags);
+    $contexts = Cache::mergeContexts($node->getCacheContexts(), $block->getCacheContexts());
+    $contexts = Cache::mergeContexts($contexts, ['url.query_args:_wrapper_format']);
+    $this->assertCacheContexts($contexts);
+
+    // Check if the node link is displayed.
+    $this->assertRaw(\Drupal::l($node->label(), $node->urlInfo('canonical')), 'Found link to visited node.');
   }
 
 }

core/modules/statistics/statistics.module

@@ -52,9 +52,9 @@ function statistics_node_links_alter(array &$links, NodeInterface $entity, array
   if ($context['view_mode'] != 'rss') {
     $links['#cache']['contexts'][] = 'user.permissions';
     if (\Drupal::currentUser()->hasPermission('view post access counter')) {
-      $statistics = statistics_get($entity->id());
+      $statistics = \Drupal::service('statistics.storage.node')->fetchView($entity->id());
       if ($statistics) {
-        $statistics_links['statistics_counter']['title'] = \Drupal::translation()->formatPlural($statistics['totalcount'], '1 view', '@count views');
+        $statistics_links['statistics_counter']['title'] = \Drupal::translation()->formatPlural($statistics->getTotalCount(), '1 view', '@count views');
         $links['statistics'] = array(
           '#theme' => 'links__node__statistics',
           '#links' => $statistics_links,
@@ -70,18 +70,10 @@ function statistics_node_links_alter(array &$links, NodeInterface $entity, array
  * Implements hook_cron().
  */
 function statistics_cron() {
-  $statistics_timestamp = \Drupal::state()->get('statistics.day_timestamp') ?: 0;
-
-  if ((REQUEST_TIME - $statistics_timestamp) >= 86400) {
-    // Reset day counts.
-    db_update('node_counter')
-      ->fields(array('daycount' => 0))
-      ->execute();
-    \Drupal::state()->set('statistics.day_timestamp', REQUEST_TIME);
-  }
-
-  // Calculate the maximum of node views, for node search ranking.
-  \Drupal::state()->set('statistics.node_counter_scale', 1.0 / max(1.0, db_query('SELECT MAX(totalcount) FROM {node_counter}')->fetchField()));
+  $storage = \Drupal::service('statistics.storage.node');
+  $storage->resetDayCount();
+  $max_total_count = $storage->maxTotalCount();
+  \Drupal::state()->set('statistics.node_counter_scale', 1.0 / max(1.0, $max_total_count));
 }
 
 /**
@@ -123,26 +115,21 @@ function statistics_title_list($dbfield, $dbrows) {
   return FALSE;
 }
 
-
 /**
  * Retrieves a node's "view statistics".
  *
- * @param int $nid
- *   The node ID.
- *
- * @return array
- *   An associative array containing:
- *   - totalcount: Integer for the total number of times the node has been
- *     viewed.
- *   - daycount: Integer for the total number of times the node has been viewed
- *     "today". For the daycount to be reset, cron must be enabled.
- *   - timestamp: Integer for the timestamp of when the node was last viewed.
+ * @deprecated in Drupal 8.2.x, will be removed before Drupal 9.0.0.
+ *   Use \Drupal::service('statistics.storage.node')->fetchView($id).
  */
-function statistics_get($nid) {
-
-  if ($nid > 0) {
-    // Retrieve an array with both totalcount and daycount.
-    return db_query('SELECT totalcount, daycount, timestamp FROM {node_counter} WHERE nid = :nid', array(':nid' => $nid), array('target' => 'replica'))->fetchAssoc();
+function statistics_get($id) {
+  if ($id > 0) {
+    /** @var \Drupal\statistics\StatisticsViewsResult $statistics */
+    $statistics = \Drupal::service('statistics.storage.node')->fetchView($id);
+    return [
+      'totalcount' => $statistics->getTotalCount(),
+      'daycount' => $statistics->getDayCount(),
+      'timestamp' => $statistics->getTimestamp(),
+    ];
   }
 }
 
@@ -151,9 +138,8 @@ function statistics_get($nid) {
  */
 function statistics_node_predelete(EntityInterface $node) {
   // Clean up statistics table when node is deleted.
-  db_delete('node_counter')
-    ->condition('nid', $node->id())
-    ->execute();
+  $id = $node->id();
+  return \Drupal::service('statistics.storage.node')->deleteViews($id);
 }
 
 /**

core/modules/statistics/statistics.php

@@ -14,8 +14,9 @@
 
 $kernel = DrupalKernel::createFromRequest(Request::createFromGlobals(), $autoloader, 'prod');
 $kernel->boot();
+$container = $kernel->getContainer();
 
-$views = $kernel->getContainer()
+$views = $container
   ->get('config.factory')
   ->get('statistics.settings')
   ->get('count_content_views');
@@ -23,15 +24,7 @@
 if ($views) {
   $nid = filter_input(INPUT_POST, 'nid', FILTER_VALIDATE_INT);
   if ($nid) {
-    \Drupal::database()->merge('node_counter')
-      ->key('nid', $nid)
-      ->fields(array(
-        'daycount' => 1,
-        'totalcount' => 1,
-        'timestamp' => REQUEST_TIME,
-      ))
-      ->expression('daycount', 'daycount + 1')
-      ->expression('totalcount', 'totalcount + 1')
-      ->execute();
+    $container->get('request_stack')->push(Request::createFromGlobals());
+    $container->get('statistics.storage.node')->recordView($nid);
   }
 }

core/modules/statistics/statistics.services.yml

+services:
+  statistics.storage.node:
+    class: Drupal\statistics\NodeStatisticsDatabaseStorage
+    arguments: ['@database', '@state', '@request_stack']
+    tags:
+      - { name: backend_overridable }