QueueWorker.php 11.1 KB
Newer Older
1 2 3 4 5
<?php

namespace Drupal\simple_sitemap\Queue;

use Drupal\Component\Utility\Timer;
6
use Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator\SitemapGeneratorBase;
7 8
use Drupal\simple_sitemap\SimplesitemapSettings;
use Drupal\simple_sitemap\SimplesitemapManager;
9
use Drupal\Core\State\StateInterface;
10
use Drupal\simple_sitemap\Logger;
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29


class QueueWorker {

  use BatchTrait;

  const REBUILD_QUEUE_CHUNK_ITEM_SIZE = 5000;

  /**
   * @var \Drupal\simple_sitemap\SimplesitemapSettings
   */
  protected $settings;

  /**
   * @var \Drupal\simple_sitemap\SimplesitemapManager
   */
  protected $manager;

  /**
30
   * @var \Drupal\Core\State\StateInterface
31 32 33 34
   */
  protected $state;

  /**
gbyte.co's avatar
gbyte.co committed
35
   * @var \Drupal\simple_sitemap\Queue\SimplesitemapQueue
36
   */
gbyte.co's avatar
gbyte.co committed
37
  protected $queue;
38

39 40 41 42 43
  /**
   * @var \Drupal\simple_sitemap\Logger
   */
  protected $logger;

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
  /**
   * @var string|null
   */
  protected $variantProcessedNow;

  /**
   * @var string|null
   */
  protected $generatorProcessedNow;

  /**
   * @var array
   */
  protected $results = [];

59 60 61 62 63
  /**
   * @var array
   */
  protected $processedPaths = [];

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
  /**
   * @var array
   */
  protected $generatorSettings;

  /**
   * @var int|null
   */
  protected $maxLinks;

  /**
   * @var int|null
   */
  protected $elementsRemaining;

  /**
   * @var int|null
   */
  protected $elementsTotal;

  /**
   * QueueWorker constructor.
   * @param \Drupal\simple_sitemap\SimplesitemapSettings $settings
   * @param \Drupal\simple_sitemap\SimplesitemapManager $manager
88
   * @param \Drupal\Core\State\StateInterface $state
gbyte.co's avatar
gbyte.co committed
89
   * @param \Drupal\simple_sitemap\Queue\SimplesitemapQueue $element_queue
90
   * @param \Drupal\simple_sitemap\Logger $logger
91 92 93
   */
  public function __construct(SimplesitemapSettings $settings,
                              SimplesitemapManager $manager,
94
                              StateInterface $state,
95 96
                              SimplesitemapQueue $element_queue,
                              Logger $logger) {
97 98 99
    $this->settings = $settings;
    $this->manager = $manager;
    $this->state = $state;
gbyte.co's avatar
gbyte.co committed
100
    $this->queue = $element_queue;
101
    $this->logger = $logger;
102 103
  }

gbyte.co's avatar
gbyte.co committed
104 105 106
  /**
   * @return $this
   */
107
  public function deleteQueue() {
gbyte.co's avatar
gbyte.co committed
108
    $this->queue->deleteQueue();
109
    SitemapGeneratorBase::purgeSitemapVariants(NULL, 'unpublished');
110 111 112
    $this->variantProcessedNow = NULL;
    $this->generatorProcessedNow = NULL;
    $this->results = [];
113
    $this->processedPaths = [];
114 115 116 117 118 119 120 121 122
    $this->state->set('simple_sitemap.queue_items_initial_amount', 0);
    $this->state->delete('simple_sitemap.queue_stashed_results');
    $this->elementsTotal = NULL;
    $this->elementsRemaining = NULL;

    return $this;
  }

  /**
123
   * @param array|null $variants
124 125 126 127
   * @return $this
   * @throws \Drupal\Component\Plugin\Exception\PluginException
   */
  public function rebuildQueue($variants = NULL) {
128
    $all_data_sets = [];
129 130 131 132 133 134 135
    $sitemap_variants = $this->manager->getSitemapVariants();
    $type_definitions = $this->manager->getSitemapTypes();
    $this->deleteQueue();

    foreach ($sitemap_variants as $variant_name => $variant_definition) {

      // Skipping unwanted sitemap variants.
gbyte.co's avatar
gbyte.co committed
136
      if (NULL !== $variants && !in_array($variant_name, (array) $variants)) {
137 138 139 140 141 142 143 144
        continue;
      }

      $type = $variant_definition['type'];

      // Adding generate_sitemap operations for all data sets.
      foreach ($type_definitions[$type]['urlGenerators'] as $url_generator_id) {

145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
        $data_sets = $this->manager->getUrlGenerator($url_generator_id)
          ->setSitemapVariant($variant_name)
          ->getDataSets();

        if (!empty($data_sets)) {
          $sitemap_variants[$variant_name]['data'] = TRUE;
          foreach ($data_sets as $data_set) {
            $all_data_sets[] = [
              'data' => $data_set,
              'sitemap_variant' => $variant_name,
              'url_generator' => $url_generator_id,
              'sitemap_generator' => $type_definitions[$type]['sitemapGenerator'],
            ];

            if (count($all_data_sets) === self::REBUILD_QUEUE_CHUNK_ITEM_SIZE) {
              $this->queueElements($all_data_sets);
              $all_data_sets = [];
            }
163 164 165 166 167
          }
        }
      }
    }

168 169
    if (!empty($all_data_sets)) {
      $this->queueElements($all_data_sets);
170
    }
171
    $this->getQueuedElementCount(TRUE);
172

173
    // todo: May not be clean to remove sitemap variants data when queuing elements.
174
    // todo: Add test.
175 176 177
    // Remove all sitemap variant instances where no results have been queued.
    $this->manager->removeSitemap(array_keys(array_filter($sitemap_variants, function($e) { return empty($e['data']); })));

178 179 180 181
    return $this;
  }

  protected function queueElements($elements) {
gbyte.co's avatar
gbyte.co committed
182
    $this->queue->createItems($elements);
183 184 185 186
    $this->state->set('simple_sitemap.queue_items_initial_amount', ($this->state->get('simple_sitemap.queue_items_initial_amount') + count($elements)));
  }

  /**
gbyte.co's avatar
gbyte.co committed
187
   * @param string $from
188 189 190
   * @return $this
   * @throws \Drupal\Component\Plugin\Exception\PluginException
   */
gbyte.co's avatar
gbyte.co committed
191
  public function generateSitemap($from = 'form') {
192 193 194

    $this->generatorSettings = [
      'base_url' => $this->settings->getSetting('base_url', ''),
195
      'xsl' => $this->settings->getSetting('xsl', TRUE),
196 197
      'default_variant' => $this->settings->getSetting('default_variant', NULL),
      'skip_untranslated' => $this->settings->getSetting('skip_untranslated', FALSE),
198
      'remove_duplicates' => $this->settings->getSetting('remove_duplicates', TRUE),
199 200 201 202 203 204 205 206
      'excluded_languages' => $this->settings->getSetting('excluded_languages', []),
    ];
    $this->maxLinks = $this->settings->getSetting('max_links');
    $max_execution_time = $this->settings->getSetting('generate_duration', 10000);
    Timer::start('simple_sitemap_generator');

    $this->unstashResults();

207
    if (!$this->generationInProgress()) {
gbyte.co's avatar
gbyte.co committed
208
      $this->rebuildQueue();
209 210
    }

gbyte.co's avatar
gbyte.co committed
211
    while ($element = $this->queue->claimItem()) {
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226

      if (!empty($max_execution_time) && Timer::read('simple_sitemap_generator') >= $max_execution_time) {
        break;
      }

      try {
        if ($element->data['sitemap_variant'] !== $this->variantProcessedNow) {

          if (NULL !== $this->variantProcessedNow) {
            $this->generateVariantChunksFromResults(TRUE);
            $this->publishCurrentVariant();
          }

          $this->variantProcessedNow = $element->data['sitemap_variant'];
          $this->generatorProcessedNow = $element->data['sitemap_generator'];
227
          $this->processedPaths = [];
228 229 230 231 232 233 234 235 236 237 238 239
        }

        $this->generateResultsFromElement($element);

        if (!empty($this->maxLinks) && count($this->results) >= $this->maxLinks) {
          $this->generateVariantChunksFromResults();
        }
      }
      catch (\Exception $e) {
        watchdog_exception('simple_sitemap', $e);
      }

gbyte.co's avatar
gbyte.co committed
240
      $this->queue->deleteItem($element); //todo May want to use deleteItems() instead.
241 242 243
      $this->elementsRemaining--;
    }

244
    if ($this->getQueuedElementCount() === 0) {
245 246 247 248 249 250 251 252 253 254
      $this->generateVariantChunksFromResults(TRUE);
      $this->publishCurrentVariant();
    }
    else {
      $this->stashResults();
    }

    return $this;
  }

255 256 257 258
  /**
   * @param $element
   * @throws \Drupal\Component\Plugin\Exception\PluginException
   */
259
  protected function generateResultsFromElement($element) {
260 261 262 263 264 265 266 267 268
    $results = $this->manager->getUrlGenerator($element->data['url_generator'])
      ->setSitemapVariant($this->variantProcessedNow)
      ->setSettings($this->generatorSettings)
      ->generate($element->data['data']);

    $this->removeDuplicates($results);
    $this->results = array_merge($this->results, $results);
  }

269 270 271
  /**
   * @param array $results
   */
272
  protected function removeDuplicates(&$results) {
273 274 275 276 277 278 279 280 281
    if ($this->generatorSettings['remove_duplicates'] && !empty($results)) {
      $result = $results[key($results)];
      if (!empty($result['meta']['path'])) {
        if (in_array($result['meta']['path'], $this->processedPaths)) {
          $results = [];
        }
        else {
          $this->processedPaths[] = $result['meta']['path'];
        }
282 283
      }
    }
284 285
  }

286 287 288 289
  /**
   * @param bool $complete
   * @throws \Drupal\Component\Plugin\Exception\PluginException
   */
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
  protected function generateVariantChunksFromResults($complete = FALSE) {
    if (!empty($this->results)) {
      $generator = $this->manager->getSitemapGenerator($this->generatorProcessedNow)
        ->setSitemapVariant($this->variantProcessedNow)
        ->setSettings($this->generatorSettings);

      if (empty($this->maxLinks) || $complete) {
        $generator->generate($this->results);
        $this->results = [];
      }
      else {
        foreach (array_chunk($this->results, $this->maxLinks, TRUE) as $chunk_links) {
          if (count($chunk_links) === $this->maxLinks || $complete) {
            $generator->generate($chunk_links);
            $this->results = array_diff_key($this->results, $chunk_links);
          }
        }
      }
    };
  }

  protected function publishCurrentVariant() {
    if ($this->variantProcessedNow !== NULL) {
      $this->manager->getSitemapGenerator($this->generatorProcessedNow)
        ->setSitemapVariant($this->variantProcessedNow)
        ->setSettings($this->generatorSettings)
        ->generateIndex()
        ->publish();
    }
  }

  protected function stashResults() {
    $this->state->set('simple_sitemap.queue_stashed_results', [
      'variant' => $this->variantProcessedNow,
      'generator' => $this->generatorProcessedNow,
325 326
      'results' => $this->results,
      'processed_paths' => $this->processedPaths,
327 328
    ]);
    $this->results = [];
329
    $this->processedPaths = [];
330 331 332 333 334
    $this->generatorProcessedNow = NULL;
    $this->variantProcessedNow = NULL;
  }

  protected function unstashResults() {
335
    if (NULL !== $results = $this->state->get('simple_sitemap.queue_stashed_results')) {
336
      $this->state->delete('simple_sitemap.queue_stashed_results');
gbyte.co's avatar
gbyte.co committed
337
      $this->results = !empty($results['results']) ? $results['results'] : [];
338
      $this->processedPaths = !empty($results['processed_paths']) ? $results['processed_paths'] : [];
339 340 341 342 343
      $this->variantProcessedNow = $results['variant'];
      $this->generatorProcessedNow = $results['generator'];
    }
  }

344
  public function getInitialElementCount() {
345
    if (NULL === $this->elementsTotal) {
346
      $this->elementsTotal = (int) $this->state->get('simple_sitemap.queue_items_initial_amount', 0);
347 348 349 350 351
    }

    return $this->elementsTotal;
  }

352 353 354 355
  /**
   * @param bool $force_recount
   * @return int
   */
356
  public function getQueuedElementCount($force_recount = FALSE) {
357
    if ($force_recount || NULL === $this->elementsRemaining) {
gbyte.co's avatar
gbyte.co committed
358
      $this->elementsRemaining = $this->queue->numberOfItems();
359 360 361 362 363
    }

    return $this->elementsRemaining;
  }

364 365 366
  /**
   * @return int
   */
367
  public function getStashedResultCount() {
368
    return count($this->state->get('simple_sitemap.queue_stashed_results', ['results' => []])['results']);
369 370
  }

371 372 373
  /**
   * @return int
   */
374
  public function getProcessedElementCount() {
375
    $initial = $this->getInitialElementCount();
376
    $remaining = $this->getQueuedElementCount();
377

378
    return $initial > $remaining ? ($initial - $remaining) : 0;
379
  }
380

381 382 383
  /**
   * @return bool
   */
384 385 386
  public function generationInProgress() {
    return 0 < ($this->getQueuedElementCount() + $this->getStashedResultCount());
  }
387 388
}