Commit b76c9320 authored by gbyte.co's avatar gbyte.co

Issue #2961235 by gbyte.co: Store sitemap indexes in the database

parent dff86b92
......@@ -110,26 +110,6 @@ function simple_sitemap_schema() {
'primary key' => ['id'],
];
// $schema['simple_sitemap_index'] = [
// 'description' => 'Holds XML sitemaps indexes as strings for quick retrieval.',
// 'fields' => [
// 'id' => [
// 'description' => 'Sitemap index unique identifier.',
// 'type' => 'varchar',
// 'length' => 50,
// 'not null' => TRUE,
// 'default' => '',
// ],
// 'sitemap_string' => [
// 'description' => 'XML sitemap index string.',
// 'type' => 'text',
// 'size' => 'big',
// 'not null' => TRUE,
// ]
// ],
// 'primary key' => ['id'],
// ];
$schema['simple_sitemap_entity_overrides'] = [
'description' => 'Holds sitemap settings overridden by entities.',
'fields' => [
......@@ -433,62 +413,37 @@ function simple_sitemap_update_8209() {
}
/**
* Adding simple_sitemap_index table and altering the simple_sitemap table.
* Adding 'type' and 'delta' fields to simple_sitemap table.
*/
function simple_sitemap_update_8210() {
$database = \Drupal::database();
$database->truncate('simple_sitemap')->execute();
// Create simple_sitemap_index table.
// if (!$database->schema()->tableExists('simple_sitemap_index')) {
// $database->schema()->createTable('simple_sitemap_index', [
// 'description' => 'Holds XML sitemaps indexes as strings for quick retrieval.',
// 'fields' => [
// 'id' => [
// 'description' => 'Sitemap index unique identifier.',
// 'type' => 'varchar',
// 'length' => 50,
// 'not null' => TRUE,
// 'default' => '',
// ],
// 'sitemap_string' => [
// 'description' => 'XML sitemap index string.',
// 'type' => 'text',
// 'size' => 'big',
// 'not null' => TRUE,
// ]
// ],
// 'primary key' => ['id'],
// ]);
// }
// Alter simple_sitemap table.
$database->truncate('simple_sitemap')->execute();
if (!$database->schema()->fieldExists('simple_sitemap', 'type')) {
$database->schema()->addField(
'simple_sitemap',
'type', [
'description' => 'Type of sitemap this chunk belongs to.',
'type' => 'varchar',
'length' => 50,
'not null' => TRUE,
'default' => '',
]
);
}
if (!$database->schema()->fieldExists('simple_sitemap', 'delta')) {
$database->schema()->addField(
'simple_sitemap',
'delta', [
'description' => 'Delta of the chunk within the type scope.',
'type' => 'int',
'size' => 'small',
'not null' => TRUE,
'unsigned' => TRUE,
]
);
}
if (!$database->schema()->fieldExists('simple_sitemap', 'type')) {
$database->schema()->addField(
'simple_sitemap',
'type', [
'description' => 'Type of sitemap this chunk belongs to.',
'type' => 'varchar',
'length' => 50,
'not null' => TRUE,
'default' => '',
]
);
}
if (!$database->schema()->fieldExists('simple_sitemap', 'delta')) {
$database->schema()->addField(
'simple_sitemap',
'delta', [
'description' => 'Delta of the chunk within the type scope.',
'type' => 'int',
'size' => 'small',
'not null' => TRUE,
'unsigned' => TRUE,
]
);
}
}
......@@ -61,7 +61,6 @@ class SitemapGenerator extends Plugin {
* @var array
*/
public $settings = [
'list' => TRUE,
'default' => FALSE,
];
}
......@@ -62,6 +62,5 @@ class UrlGenerator extends Plugin {
*/
public $settings = [
'default_sitemap_generator' => 'default',
'instantiate_for_each_data_set' => FALSE,
];
}
......@@ -21,11 +21,6 @@ class Batch {
*/
protected $batch;
/**
* @var array
*/
protected $batchSettings;
/**
* @var array
*/
......@@ -34,7 +29,7 @@ class Batch {
const BATCH_TITLE = 'Generating XML sitemap';
const BATCH_INIT_MESSAGE = 'Initializing batch...';
const BATCH_ERROR_MESSAGE = 'An error has occurred. This may result in an incomplete XML sitemap.';
const BATCH_PROGRESS_MESSAGE = 'Processing @current out of @total link types.';
const BATCH_PROGRESS_MESSAGE = 'Running @current out of @total operations.';
const REGENERATION_FINISHED_MESSAGE = 'The XML sitemaps have been regenerated.';
const REGENERATION_FINISHED_ERROR_MESSAGE = 'The sitemap generation finished with an error.';
......@@ -53,28 +48,112 @@ class Batch {
}
/**
* @param array $batch_settings
* @param array $batch_meta
*/
public function setBatchMeta(array $batch_meta) {
$this->batchMeta = $batch_meta;
}
/**
* Adds an operation to the batch.
*
* @param string $operation_name
* @param array $arguments
*/
public function setBatchSettings(array $batch_settings) {
$this->batchSettings = $batch_settings;
public function addOperation($operation_name, $arguments = []) {
$this->batch['operations'][] = [
__CLASS__ . '::' . $operation_name, [$arguments, $this->batchMeta]
];
}
/**
* Batch callback function which generates URLs.
*
* @param array $arguments
* @param array $batch_meta
* @param $context
*
* @see https://api.drupal.org/api/drupal/core!includes!form.inc/group/batch/8
*/
public function setBatchMeta(array $batch_meta) {
$this->batchMeta = $batch_meta;
public static function generateSitemap(array $arguments, array $batch_meta, &$context) {
\Drupal::service('plugin.manager.simple_sitemap.url_generator')
->createInstance($arguments['url_generator'])
->setContext($context)
->setSettings($arguments['settings'])
->setBatchMeta($batch_meta)
->generate($arguments['data_set']);
}
/**
* Batch callback function which generates URLs.
*
* @param array $arguments
* @param array $batch_meta
* @param $context
*
* @see https://api.drupal.org/api/drupal/core!includes!form.inc/group/batch/8
*/
public static function generateIndex(array $arguments, array $batch_meta, &$context) {
\Drupal::service('plugin.manager.simple_sitemap.sitemap_generator')
->createInstance($arguments['sitemap_generator'])
->setSettings($arguments['settings'])
->generateIndex();
}
/**
* Batch callback function which generates URLs.
*
* @param array $arguments
* @param array $batch_meta
* @param $context
*
* @see https://api.drupal.org/api/drupal/core!includes!form.inc/group/batch/8
*/
public static function removeSitemap(array $arguments, array $batch_meta, &$context) {
\Drupal::service('plugin.manager.simple_sitemap.sitemap_generator')
->createInstance($arguments['sitemap_generator'])
->remove();
}
/**
* Callback function called by the batch API when all operations are finished.
*
* @param $success
* @param $results
* @param $operations
*
* @return bool
*
* @see https://api.drupal.org/api/drupal/core!includes!form.inc/group/batch/8
*
* @todo Display success/failure message in Drush > 9.
*/
public static function finishGeneration($success, $results, $operations) {
if ($success) {
Cache::invalidateTags(['simple_sitemap']);
\Drupal::service('simple_sitemap.logger')->m(self::REGENERATION_FINISHED_MESSAGE)
// ['@url' => $this->sitemapGenerator->getCustomBaseUrl() . '/sitemap.xml']) //todo: Use actual base URL for message.
->display('status')
->log('info');
}
else {
\Drupal::service('simple_sitemap.logger')->m(self::REGENERATION_FINISHED_ERROR_MESSAGE)
->display('error', 'administer sitemap settings')
->log('error');
}
return $success;
}
/**
* Starts the batch process depending on where it was requested from.
*
* @return bool
*/
public function start() {
// Update total operation count for each operation.
foreach ($this->batch['operations'] as $i => $operation) {
$this->batch['operations'][$i][1][3]['operations_count'] = count($this->batch['operations']);
}
// Update last operation info for each operation.
$this->addAdditionalMetaInfo();
switch ($this->batchMeta['from']) {
......@@ -121,65 +200,18 @@ class Batch {
return FALSE;
}
/**
* Adds an operation to the batch.
*
* @param string $url_generator_id$data_sets
* @param array|null $data_sets
*/
public function addOperation($url_generator_id, $data_sets = NULL) {
$operation_no = count($this->batch['operations']) + 1;
$this->batch['operations'][$operation_no] = [
__CLASS__ . '::generate', [$url_generator_id, $data_sets, $this->batchSettings, $this->batchMeta + ['current_operation_no' => $operation_no]],
];
}
/**
* Batch callback function which generates URLs.
*
* @param string $url_generator_id
* @param array|null $data_sets
* @param array $batch_settings
* @param array $batch_meta
* @param $context
*
* @see https://api.drupal.org/api/drupal/core!includes!form.inc/group/batch/8
*/
public static function generate($url_generator_id, $data_sets, array $batch_settings, array $batch_meta, &$context) {
\Drupal::service('plugin.manager.simple_sitemap.url_generator')
->createInstance($url_generator_id)
->setContext($context)
->setBatchMeta($batch_meta)
->setBatchSettings($batch_settings)
->generate($data_sets);
}
/**
* Callback function called by the batch API when all operations are finished.
*
* @param $success
* @param $results
* @param $operations
*
* @return bool
*
* @see https://api.drupal.org/api/drupal/core!includes!form.inc/group/batch/8
*/
public static function finishGeneration($success, $results, $operations) {
if ($success) {
Cache::invalidateTags(['simple_sitemap']);
\Drupal::service('simple_sitemap.logger')->m(self::REGENERATION_FINISHED_MESSAGE)
// ['@url' => $this->sitemapGenerator->getCustomBaseUrl() . '/sitemap.xml']) //todo: Use actual base URL for message.
->display('status')
->log('info');
protected function addAdditionalMetaInfo() {
$last_operation_no = 0;
foreach ($this->batch['operations'] as $i => $operation) {
if ($operation[0] === __CLASS__ . '::generateSitemap') {
$this->batch['operations'][$i][1][1]['current_generate_sitemap_operation_no'] = $i;
$last_operation_no = $i;
}
}
else {
\Drupal::service('simple_sitemap.logger')->m(self::REGENERATION_FINISHED_ERROR_MESSAGE)
->display('error', 'administer sitemap settings')
->log('error');
foreach ($this->batch['operations'] as $i => $operation) {
if ($operation[0] === __CLASS__ . '::generateSitemap') {
$this->batch['operations'][$i][1][1]['last_generate_sitemap_operation_no'] = $last_operation_no;
}
}
return $success;
}
}
<?php
namespace Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator;
use Symfony\Component\DependencyInjection\ContainerInterface;
use Drupal\Core\Database\Connection;
use Drupal\Core\Extension\ModuleHandler;
use Drupal\Core\Language\LanguageManagerInterface;
use Drupal\Component\Datetime\Time;
/**
* Class DefaultSitemapGenerator
* @package Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator
*
* @SitemapGenerator(
* id = "index",
* title = @Translation("Sitemap index generator"),
* description = @Translation("Generates the sitemap index."),
* weight = 0,
* settings = {
* "list" = false,
* },
* )
*
* @todo Save index in DB instead of creating on the fly.
*/
class IndexSitemapGenerator extends SitemapGeneratorBase {
/**
* @var array
*/
protected static $indexAttributes = [
'xmlns' => self::XMLNS,
];
/**
* DefaultSitemapGenerator constructor.
* @param array $configuration
* @param string $plugin_id
* @param mixed $plugin_definition
* @param \Drupal\Core\Database\Connection $database
* @param \Drupal\Core\Extension\ModuleHandler $module_handler
* @param \Drupal\Core\Language\LanguageManagerInterface $language_manager
* @param \Drupal\Component\Datetime\Time $time
* @param \Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator\SitemapWriter $sitemapWriter
*/
public function __construct(
array $configuration,
$plugin_id,
$plugin_definition,
Connection $database,
ModuleHandler $module_handler,
LanguageManagerInterface $language_manager,
Time $time,
SitemapWriter $sitemapWriter
) {
parent::__construct(
$configuration,
$plugin_id,
$plugin_definition,
$database,
$module_handler,
$language_manager,
$time,
$sitemapWriter
);
}
public static function create(
ContainerInterface $container,
array $configuration,
$plugin_id,
$plugin_definition) {
return new static(
$configuration,
$plugin_id,
$plugin_definition,
$container->get('database'),
$container->get('module_handler'),
$container->get('language_manager'),
$container->get('datetime.time'),
$container->get('simple_sitemap.sitemap_writer')
);
}
// public function generateSitemapIndex(array $chunk_info) {
// $values = [
// 'id' => $this->configuration['id'],
// 'sitemap_string' => $this->getXml($chunk_info),
// ];
// $this->db->insert('simple_sitemap')->fields($values)->execute();
// }
public function getSitemapIndex($chunk_info) {
return $this->getXml($chunk_info);
}
/**
* Generates and returns the sitemap index for all sitemap chunks.
*
* @param array $chunk_info
* Array containing chunk creation timestamps keyed by chunk ID.
*
* @return string sitemap index
*/
protected function getXml(array $chunk_info) {
$this->writer->openMemory();
$this->writer->setIndent(TRUE);
$this->writer->startDocument(self::XML_VERSION, self::ENCODING);
$this->writer->writeComment(self::GENERATED_BY);
$this->writer->startElement('sitemapindex');
// Add attributes to document.
$this->moduleHandler->alter('simple_sitemap_index_attributes', self::$indexAttributes);
foreach (self::$indexAttributes as $name => $value) {
$this->writer->writeAttribute($name, $value);
}
// Add sitemap chunk locations to document.
foreach ($chunk_info as $chunk_data) {
$this->writer->startElement('sitemap');
$this->writer->writeElement('loc', $this->getCustomBaseUrl()
. '/sitemaps/' . $chunk_data->type . '/' . $chunk_data->delta . '/sitemap.xml');
$this->writer->writeElement('lastmod', date_iso8601($chunk_data->sitemap_created));
$this->writer->endElement();
}
$this->writer->endElement();
$this->writer->endDocument();
return $this->writer->outputMemory();
}
}
......@@ -12,12 +12,11 @@ use Drupal\Component\Datetime\Time;
/**
* Class SitemapGeneratorBase
* @package Drupal\simple_sitemap\Plugin\simple_sitemap\UrlGenerator
*
* @todo Add interface.
*/
abstract class SitemapGeneratorBase extends SimplesitemapPluginBase implements SitemapGeneratorInterface {
const FIRST_DELTA_INDEX = 1;
const FIRST_CHUNK_DELTA = 1;
const INDEX_DELTA = 0;
const DEFAULT_SITEMAP_TYPE = 'default';
const GENERATED_BY = 'Generated by the Simple XML sitemap Drupal module: https://drupal.org/project/simple_sitemap.';
const XML_VERSION = '1.0';
......@@ -59,6 +58,13 @@ abstract class SitemapGeneratorBase extends SimplesitemapPluginBase implements S
*/
protected $writer;
/**
* @var array
*/
protected static $indexAttributes = [
'xmlns' => self::XMLNS,
];
/**
* SitemapGeneratorBase constructor.
* @param array $configuration
......@@ -107,6 +113,48 @@ abstract class SitemapGeneratorBase extends SimplesitemapPluginBase implements S
*/
abstract protected function getXml(array $links);
protected function getChunkInfo() {
return $this->db->select('simple_sitemap', 's')
->fields('s', ['delta', 'sitemap_created', 'type'])
->condition('s.type', $this->getPluginId())
->condition('s.delta', self::INDEX_DELTA, '<>')
->execute()
->fetchAllAssoc('delta');
}
/**
* Returns the sitemap index for all sitemap chunks of this type.
*
* @return string
*/
protected function getIndexXml(array $chunk_info) {
$this->writer->openMemory();
$this->writer->setIndent(TRUE);
$this->writer->startDocument(self::XML_VERSION, self::ENCODING);
$this->writer->writeComment(self::GENERATED_BY);
$this->writer->startElement('sitemapindex');
// Add attributes to document.
$this->moduleHandler->alter('simple_sitemap_index_attributes', self::$indexAttributes);
foreach (self::$indexAttributes as $name => $value) {
$this->writer->writeAttribute($name, $value);
}
// Add sitemap chunk locations to document.
foreach ($chunk_info as $chunk_data) {
$this->writer->startElement('sitemap');
$this->writer->writeElement('loc', $this->getCustomBaseUrl()
. '/sitemaps/' . $chunk_data->type . '/' . $chunk_data->delta . '/sitemap.xml');
$this->writer->writeElement('lastmod', date_iso8601($chunk_data->sitemap_created));
$this->writer->endElement();
}
$this->writer->endElement();
$this->writer->endDocument();
return $this->writer->outputMemory();
}
/**
* @return $this
*/
......@@ -114,28 +162,24 @@ abstract class SitemapGeneratorBase extends SimplesitemapPluginBase implements S
$this->db->delete('simple_sitemap')
->condition('type', $this->getPluginId())
->execute();
// $this->db->delete('simple_sitemap_index')
// ->condition('id', $this->getPluginId())
// ->execute();
return $this;
}
/**
* Wrapper method which takes links along with their options and then
* generates and saves the sitemap.
* Takes links along with their options and then generates and saves the
* sitemap.
*
* @param array $links
* All links with their multilingual versions and settings.
*/
public function generate(array $links) {
$highest_id = $this->db->query('SELECT MAX(id) FROM {simple_sitemap}')->fetchField();
$highest_delta = $this->db->query('SELECT MAX(delta) FROM {simple_sitemap} WHERE type = :type', [':type' => $this->getPluginId()])
->fetchField();
$values = [
'id' => NULL === $highest_id ? 0 : $highest_id + 1,
'delta' => NULL === $highest_delta ? self::FIRST_DELTA_INDEX : $highest_delta + 1,
'delta' => NULL === $highest_delta ? self::FIRST_CHUNK_DELTA : $highest_delta + 1,
'type' => $this->getPluginId(),
'sitemap_string' => $this->getXml($links),
'sitemap_created' => $this->time->getRequestTime(),
......@@ -144,6 +188,28 @@ abstract class SitemapGeneratorBase extends SimplesitemapPluginBase implements S
$this->db->insert('simple_sitemap')->fields($values)->execute();
}
public function generateIndex() {
if (!empty($chunk_info = $this->getChunkInfo()) && count($chunk_info) > 1) {
$index_xml = $this->getIndexXml($chunk_info);
$highest_id = $this->db->query('SELECT MAX(id) FROM {simple_sitemap}')->fetchField();
$this->db->merge('simple_sitemap')
->key(['delta' => self::INDEX_DELTA, 'type' => $this->getPluginId()])
->insertFields([
'id' => NULL === $highest_id ? 0 : $highest_id + 1,
'delta' => self::INDEX_DELTA,
'type' => $this->getPluginId(),
'sitemap_string' => $index_xml,
'sitemap_created' => $this->time->getRequestTime(),
])
->updateFields([
'type' => $this->getPluginId(),
'sitemap_string' => $index_xml,
'sitemap_created' => $this->time->getRequestTime(),
])
->execute();
}
}
/**
* @return bool
*/
......@@ -166,7 +232,7 @@ abstract class SitemapGeneratorBase extends SimplesitemapPluginBase implements S
/**
* @return string
*/
public function getCustomBaseUrl() {
protected function getCustomBaseUrl() {
$customBaseUrl = $this->settings['base_url'];
return !empty($customBaseUrl) ? $customBaseUrl : $GLOBALS['base_url'];
}
......
......@@ -3,9 +3,15 @@
namespace Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator;
/**
* Interface UrlGeneratorInterface
* Interface SitemapGeneratorInterface
* @package Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator
*/
interface SitemapGeneratorInterface {
public function generate(array $links);
public function generateIndex();
public function remove();
}
......@@ -20,9 +20,6 @@ use Symfony\Component\DependencyInjection\ContainerInterface;
* title = @Translation("Arbitrary URL generator"),
* description = @Translation("Generates URLs from data sets collected in the hook_arbitrary_links_alter hook."),
* weight = 20,
* settings = {
* "default_sitemap_generator" = "default",
* },
* )
*/
class ArbitraryUrlGenerator extends UrlGeneratorBase {
......@@ -94,7 +91,7 @@ class ArbitraryUrlGenerator extends UrlGeneratorBase {
public function getDataSets() {
$arbitrary_links = [];
$this->moduleHandler->alter('simple_sitemap_arbitrary_links', $arbitrary_links);
return array_values($arbitrary_links);
return [$this->getPluginDefinition()['settings']['default_sitemap_generator'] => [0 => array_values($arbitrary_links)]];
}
/**
......
......@@ -22,9 +22,6 @@ use Symfony\Component\DependencyInjection\ContainerInterface;
* title = @Translation("Custom URL generator"),
* description = @Translation("Generates URLs set in admin/config/search/simplesitemap/custom."),
* weight = 0,
* settings = {
* "default_sitemap_generator" = "default",
* },
* enabled = true,
* )
*
......@@ -106,8 +103,7 @@ class CustomUrlGenerator extends UrlGeneratorBase {
*/
public function getDataSets() {
$this->includeImages = $this->generator->getSetting('custom_links_include_images', FALSE);
return array_values($this->generator->getCustomLinks());
return [$this->getPluginDefinition()['settings']['default_sitemap_generator'] => [0 => array_values($this->generator->getCustomLinks())]];
}
/**
......@@ -128,7 +124,7 @@ class CustomUrlGenerator extends UrlGeneratorBase {
$url_object = Url::fromUserInput($data_set['path'], ['absolute' => TRUE]);
$path = $url_object->getInternalPath();
if ($this->batchSettings['remove_duplicates'] && $this->pathProcessed($path)) {
if ($this->settings['remove_duplicates'] && $this->pathProcessed($path)) {
return FALSE;
}
......
......@@ -22,8 +22,6 @@ use Drupal\Core\Menu\MenuLinkTree;
* description = @Translation("Generates menu link URLs by overriding the 'entity' URL generator."),
* weight = 5,
* settings = {
* "default_sitemap_generator" = "default",
* "instantiate_for_each_data_set" = true,
* "overrides_entity_type" = "menu_link_content",
* },
* )
......@@ -97,17 +95,17 @@ class EntityMenuLinkContentUrlGenerator extends UrlGeneratorBase {
* @inheritdoc
*/
public function getDataSets() {
$menu_names = [];
$data_sets = [];
$bundle_settings = $this->generator->getBundleSettings();
if (!empty($bundle_settings['menu_link_content'])) {
foreach ($bundle_settings['menu_link_content'] as $bundle_name => $settings) {
if ($settings['index']) {
$menu_names[] = $bundle_name;
$data_sets[$this->getPluginDefinition()['settings']['default_sitemap_generator']][] = $bundle_name;
}
}
}
return $menu_names;