Unverified Commit 63dab7ca authored by Dave Reid's avatar Dave Reid Committed by Dave Reid

Issue #3150732 by Dave Reid: Fixed xmlsitemap_xmlsitemap_index_links() does...

Issue #3150732 by Dave Reid: Fixed xmlsitemap_xmlsitemap_index_links() does not subquery against the xmlsitemap table to only query for un-indexed links.
parent fde47049
...@@ -526,17 +526,18 @@ class XmlSitemapGenerator implements XmlSitemapGeneratorInterface { ...@@ -526,17 +526,18 @@ class XmlSitemapGenerator implements XmlSitemapGeneratorInterface {
} }
$info = $context['sandbox']['info']; $info = $context['sandbox']['info'];
$entity_type = $this->entityTypeManager->getDefinition($entity_type_id);
$query = $this->entityTypeManager->getStorage($entity_type_id)->getQuery(); $query = $this->entityTypeManager->getStorage($entity_type_id)->getQuery();
$query->condition($entity_type->getKey('id'), $context['sandbox']['last_id'], '>'); $query->condition($info['entity keys']['id'], $context['sandbox']['last_id'], '>');
if ($entity_type->hasKey('bundle')) { if (!empty($info['entity keys']['bundle'])) {
$query->condition($entity_type->getKey('bundle'), $context['sandbox']['bundles'], 'IN'); $query->condition($info['entity keys']['bundle'], $context['sandbox']['bundles'], 'IN');
} }
$query->addTag('xmlsitemap_link_bundle_access');
// Access for entities is checked individually for the anonymous user
// when each item is processed. We can skip the access check for the
// query.
$query->accessCheck(FALSE);
$query->addTag('xmlsitemap_rebuild'); $query->addTag('xmlsitemap_rebuild');
$query->addMetaData('entity_type_id', $entity_type_id);
$query->addMetaData('entity_info', $info);
if (!isset($context['sandbox']['max'])) { if (!isset($context['sandbox']['max'])) {
$count_query = clone $query; $count_query = clone $query;
...@@ -549,7 +550,7 @@ class XmlSitemapGenerator implements XmlSitemapGeneratorInterface { ...@@ -549,7 +550,7 @@ class XmlSitemapGenerator implements XmlSitemapGeneratorInterface {
} }
// PostgreSQL cannot have the ORDERED BY in the count query. // PostgreSQL cannot have the ORDERED BY in the count query.
$query->sort($entity_type->getKey('id')); $query->sort($info['entity keys']['id']);
// Get batch limit. // Get batch limit.
$limit = $this->config->get('batch_limit'); $limit = $this->config->get('batch_limit');
......
...@@ -21,7 +21,6 @@ use Drupal\Core\Entity\ContentEntityInterface; ...@@ -21,7 +21,6 @@ use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\Core\Entity\ContentEntityTypeInterface; use Drupal\Core\Entity\ContentEntityTypeInterface;
use Drupal\Core\Entity\EntityInterface; use Drupal\Core\Entity\EntityInterface;
use Drupal\Core\Entity\EntityTypeInterface; use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Entity\Query\QueryInterface;
use Drupal\Core\File\FileSystemInterface; use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\Form\FormStateInterface; use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Language\LanguageInterface; use Drupal\Core\Language\LanguageInterface;
...@@ -105,7 +104,7 @@ function xmlsitemap_hook_info() { ...@@ -105,7 +104,7 @@ function xmlsitemap_hook_info() {
'xmlsitemap_sitemap_delete', 'xmlsitemap_sitemap_delete',
'xmlsitemap_sitemap_link_url_options_alter', 'xmlsitemap_sitemap_link_url_options_alter',
'query_xmlsitemap_generate_alter', 'query_xmlsitemap_generate_alter',
'query_xmlsitemap_link_bundle_access_alter', 'query_xmlsitemap_index_links_alter',
'form_xmlsitemap_sitemap_edit_form_alter', 'form_xmlsitemap_sitemap_edit_form_alter',
'xmlsitemap_rebuild_clear', 'xmlsitemap_rebuild_clear',
]; ];
...@@ -735,6 +734,11 @@ function xmlsitemap_get_link_info($type = NULL, $reset = FALSE) { ...@@ -735,6 +734,11 @@ function xmlsitemap_get_link_info($type = NULL, $reset = FALSE) {
'id' => $entity_type->getKey('id'), 'id' => $entity_type->getKey('id'),
'bundle' => $entity_type->getKey('bundle'), 'bundle' => $entity_type->getKey('bundle'),
], ],
'xmlsitemap' => [
// Add in the default callbacks for entity types.
'process callback' => $entity_type->get('xmlsitemap')['process callback'] ?? 'xmlsitemap_xmlsitemap_process_entity_links',
'rebuild callback' => $entity_type->get('xmlsitemap')['process callback'] ?? 'xmlsitemap_rebuild_batch_fetch',
],
]; ];
} }
...@@ -745,12 +749,6 @@ function xmlsitemap_get_link_info($type = NULL, $reset = FALSE) { ...@@ -745,12 +749,6 @@ function xmlsitemap_get_link_info($type = NULL, $reset = FALSE) {
'base table' => FALSE, 'base table' => FALSE,
'bundles' => [], 'bundles' => [],
]; ];
if (!isset($info['xmlsitemap']['process callback']) && !empty($info['entity keys'])) {
$info['xmlsitemap']['process callback'] = 'xmlsitemap_xmlsitemap_process_entity_links';
}
if (!isset($info['xmlsitemap']['rebuild callback']) && !empty($info['entity keys']['id'])) {
$info['xmlsitemap']['rebuild callback'] = 'xmlsitemap_rebuild_batch_fetch';
}
foreach ($info['bundles'] as $bundle => &$bundle_info) { foreach ($info['bundles'] as $bundle => &$bundle_info) {
$bundle_info += [ $bundle_info += [
'xmlsitemap' => [], 'xmlsitemap' => [],
...@@ -837,18 +835,13 @@ function xmlsitemap_get_link_type_indexed_status($entity_type_id, $bundle = '') ...@@ -837,18 +835,13 @@ function xmlsitemap_get_link_type_indexed_status($entity_type_id, $bundle = '')
try { try {
$query = \Drupal::entityQuery($entity_type_id); $query = \Drupal::entityQuery($entity_type_id);
if ($bundle && $entity_type->hasKey('bundle')) {
if ($bundle) { $query->condition($entity_type->getKey('bundle'), $bundle);
$query->condition($entity_type->getKey('bundle'), $bundle, '=');
} }
// We are only using this for totals, so we can skip the access check.
$query->addTag('xmlsitemap_link_bundle_access'); $query->accessCheck(FALSE);
$query->addTag('xmlsitemap_link_indexed_status'); $query->addTag('xmlsitemap_link_indexed_status');
$query->addMetaData('entity_type_id', $entity_type_id); $status['total'] = $query->count()->execute();
$query->addMetaData('bundle', $bundle);
$query->addMetaData('entity_info', $info);
$query->count();
$status['total'] = $query->execute();
return $status; return $status;
} }
catch (\Exception $e) { catch (\Exception $e) {
...@@ -1628,9 +1621,9 @@ function xmlsitemap_xmlsitemap_index_links($limit) { ...@@ -1628,9 +1621,9 @@ function xmlsitemap_xmlsitemap_index_links($limit) {
$entity_type_manager = \Drupal::entityTypeManager(); $entity_type_manager = \Drupal::entityTypeManager();
$entity_types = $entity_type_manager->getDefinitions(); $entity_types = $entity_type_manager->getDefinitions();
foreach ($entity_types as $entity_type_id => $entity_type) { foreach ($entity_types as $entity_type_id => $entity_type) {
// Don't try to loop over non-content entity types or with no bundle // If an entity type is not supported it will not have link info.
// defined. $info = xmlsitemap_get_link_info($entity_type_id);
if (!xmlsitemap_is_entity_type_supported($entity_type)) { if (empty($info)) {
continue; continue;
} }
...@@ -1642,16 +1635,41 @@ function xmlsitemap_xmlsitemap_index_links($limit) { ...@@ -1642,16 +1635,41 @@ function xmlsitemap_xmlsitemap_index_links($limit) {
try { try {
$query = $entity_type_manager->getStorage($entity_type_id)->getQuery(); $query = $entity_type_manager->getStorage($entity_type_id)->getQuery();
$query->range(0, $limit); $query->range(0, $limit);
if ($entity_type->hasKey('bundle')) { if (!empty($info['entity keys']['bundle'])) {
$bundle_key = $entity_type->getKey('bundle'); $query->condition($info['entity keys']['bundle'], $bundles, 'IN');
$query->condition($bundle_key, $bundles, 'IN'); }
// Perform a subquery against the xmlsitemap table to ensure that we are
// only looking for items that we have not already indexed.
$subquery = \Drupal::database()->select('xmlsitemap', 'x');
$subquery->addField('x', 'id');
$subquery->condition('type', $entity_type_id);
// If the storage for this entity type is against a SQL backend, perform
// a direct subquery condition to avoid needing to load all the IDs.
if ($query instanceof \Drupal\Core\Entity\Query\Sql\Query) {
$query->condition($info['entity keys']['id'], $subquery, 'NOT IN');
} }
$ids = $query->execute(); else {
$query->condition($info['entity keys']['id'], $subquery->execute()->fetchCol(), 'NOT IN');
}
// Access for entities is checked individually for the anonymous user
// when each item is processed. We can skip the access check for the
// query.
$query->accessCheck(FALSE);
$query->addTag('xmlsitemap_index_links');
if ($ids = $query->execute()) {
// Chunk the array into batch sizes.
$chunks = array_chunk($ids, \Drupal::config('xmlsitemap.settings')->get('batch_limit'));
foreach ($chunks as $chunk) {
$info['xmlsitemap']['process callback']($entity_type_id, $chunk);
}
// Chunk the array into batch sizes. \Drupal::logger('xmlsitemap')->info('Indexed @count new @type items.', [
$chunks = array_chunk($ids, \Drupal::config('xmlsitemap.settings')->get('batch_limit')); '@count' => count($ids),
foreach ($chunks as $chunk) { '@type' => $entity_type_id
xmlsitemap_xmlsitemap_process_entity_links($entity_type_id, $chunk); ]);
} }
} }
catch (\Exception $e) { catch (\Exception $e) {
...@@ -2011,24 +2029,6 @@ function xmlsitemap_xmlsitemap_sitemap_operations() { ...@@ -2011,24 +2029,6 @@ function xmlsitemap_xmlsitemap_sitemap_operations() {
return $operations; return $operations;
} }
/**
* Implements hook_query_TAG_alter().
*
* @todo Check if this query alter is still needed.
*/
function xmlsitemap_query_xmlsitemap_link_bundle_access_alter(AlterableInterface $query) {
if ($query instanceof QueryInterface && $entity_type_id = $query->getMetaData('entity_type_id')) {
$entity_type = \Drupal::entityTypeManager()->getDefinition($entity_type_id);
if ($entity_type->hasKey('bundle')) {
$bundle = $query->getMetaData('bundle');
if (empty($bundle)) {
$bundle = xmlsitemap_get_link_type_enabled_bundles($entity_type_id);
}
$query->condition($entity_type->getKey('bundle'), $bundle, is_array($bundle) ? 'IN' : '=');
}
}
}
/** /**
* XML sitemap link type settings callback for frontpage link entity. * XML sitemap link type settings callback for frontpage link entity.
* *
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment