Skip to content
Snippets Groups Projects
Commit 1b95b568 authored by Alex Pott's avatar Alex Pott
Browse files

Issue #3339640: Add Drush command to add existing content to the VG Wort queue

parent 2a860199
No related branches found
No related tags found
1 merge request!15Issue #3339640: Add Drush command to add existing content to the VG Wort queue
......@@ -4,6 +4,15 @@ Integrates the [VG Wort service](https://tom.vgwort.de/portal/index) with Drupal
Install this module to add VG Wort's tracking 1x1 pixel to content entity types
of your choice.
## Adding sending existing content entities to VG Wort
The module provides a [Drush command](https://www.drush.org/latest/) to send
existing content entities to VG Wort. Once you've configured the module, you can
execute
```shell
vendor/bin/drush vgwort:queue node --batch-size=100
```
to add all the existing nodes with counter IDs to the queue.
## Excluding an entity from VG Wort
If you've configured entities to have the VG Wort counter ID you can implement
`hook_vgwort_enable_for_entity()` to exclude specific entities from VG Wort.
......
......@@ -8,5 +8,12 @@
},
"require-dev": {
"drupal/graphql": "^4.4"
},
"extra": {
"drush": {
"services": {
"drush.services.yml": "^11"
}
}
}
}
services:
vgwort.commands:
class: \Drupal\vgwort\Commands\VgwortCommands
arguments: ['@entity_type.manager', '@vgwort.entity_queuer', '@datetime.time', '@entity.memory_cache']
tags:
- { name: drush.command }
<?php
namespace Drupal\vgwort\Commands;
use Consolidation\AnnotatedCommand\CommandData;
use Consolidation\AnnotatedCommand\CommandError;
use Drupal\Component\Datetime\TimeInterface;
use Drupal\Core\Batch\BatchBuilder;
use Drupal\Core\Cache\MemoryCache\MemoryCacheInterface;
use Drupal\Core\DependencyInjection\DependencySerializationTrait;
use Drupal\Core\Entity\EntityChangedInterface;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\vgwort\EntityQueuer;
use Drush\Commands\DrushCommands;
/**
* VGWort Drush commands.
*/
class VgwortCommands extends DrushCommands {
use DependencySerializationTrait;
/**
* @var \Drupal\Core\Entity\EntityTypeManagerInterface
*/
protected $entityTypeManager;
/**
* @var \Drupal\vgwort\EntityQueuer
*/
protected $entityQueuer;
/**
* @var \Drupal\Component\Datetime\TimeInterface
*/
protected $time;
/**
* @var \Drupal\Core\Cache\MemoryCache\MemoryCacheInterface
*/
protected $entityMemoryCache;
/**
* @param \Drupal\Core\Entity\EntityTypeManagerInterface $entityTypeManager
* The entity type manager.
* @param \Drupal\vgwort\EntityQueuer $entityQueuer
* The VG Wort entity queuer.
* @param \Drupal\Component\Datetime\TimeInterface $time
* The time service.
* @param \Drupal\Core\Cache\MemoryCache\MemoryCacheInterface $entityMemoryCache
* The entity memory cache.
*/
public function __construct(EntityTypeManagerInterface $entityTypeManager, EntityQueuer $entityQueuer, TimeInterface $time, MemoryCacheInterface $entityMemoryCache) {
$this->entityTypeManager = $entityTypeManager;
$this->entityQueuer = $entityQueuer;
$this->time = $time;
$this->entityMemoryCache = $entityMemoryCache;
}
/**
* Adds existing content to VGWort queue.
*
* @param string $entityType
* Entity type to process
* @param array $options
* An associative array of options whose values come from cli, aliases, config, etc.
*
* @option batch-size
* How many entities to process in each batch invocation. Defaults to 50.
*
* @usage vgwort-addExistingContentToQueue node
* Adds nodes to queue.
*
* @command vgwort:queue
* @aliases vgwq
* @validate-vgwort-entity-type
* @validate-vgwort-batch-size
*
* NOTE: this command will appear in the _global namespace until there is more
* than one command available.
*/
public function addExistingContentToQueue(string $entityType, $options = ['batch-size' => 50]) {
$batch = new BatchBuilder();
$batch->addOperation([$this, 'batchAddExistingContentToQueue'], [$entityType, $options['batch-size']]);
batch_set($batch->toArray());
$result = drush_backend_batch_process();
$success = FALSE;
if (!is_array($result)) {
$this->logger()->error(dt('Batch process did not return a result array. Returned: !type', ['!type' => gettype($result)]));
}
elseif (!empty($result[0]['#abort'])) {
// Whenever an error occurs the batch process does not continue, so
// this array should only contain a single item, but we still output
// all available data for completeness.
$this->logger()->error(dt('Update aborted by: !process', [
'!process' => implode(', ', $result[0]['#abort']),
]));
}
else {
$success = TRUE;
}
return $success;
}
/**
* Batch callback.
*
* @param $entity_type
* The entity type to process.
* @param int $batch_size
* How many entities to process at a time.
* @param $context
* The batch context.
*/
public function batchAddExistingContentToQueue($entity_type, int $batch_size, &$context) {
$entity_storage = $this->entityTypeManager->getStorage($entity_type);
$entity_type = $this->entityTypeManager->getDefinition($entity_type);
$id_key = $entity_type->getKey('id');
if (!isset($context['sandbox']['@processed'])) {
$context['sandbox']['@processed'] = $context['sandbox']['@queued'] = 0;
$count_query = $entity_storage
->getQuery()
->accessCheck(FALSE)
->count();
$context['sandbox']['@total'] = (int) $count_query->execute();
if ($context['sandbox']['@total'] === 0) {
// If there are no entities to process, then stop immediately.
$context['finished'] = 1;
return;
}
}
$query = $entity_storage->getQuery();
if (isset($context['sandbox']['last_id'])) {
$query->condition($id_key, $context['sandbox']['last_id'], '>');
}
$entities = $entity_storage->loadMultiple($query->accessCheck(FALSE)->sort($id_key)->range(0, $batch_size)->execute());
if (empty($entities)) {
// If there are no entities to process, then stop immediately.
$context['finished'] = 1;
return;
}
foreach ($entities as $entity) {
// @todo Should we add some jitter so that all the existing content is
// available for claiming immediately? Or should be avoid the queue
// completely and just send straight to VG Wort?
if ($entity instanceof EntityChangedInterface) {
// Work out the delay based on the last changed time. Entities older
// than the default delay will have a delay of 0.
$delay = $entity->getChangedTime() + $this->entityQueuer->getDefaultDelay() - $this->time->getCurrentTime();
$queued = $this->entityQueuer->queueEntity($entity, max($delay, 0));
}
else {
$queued = $this->entityQueuer->queueEntity($entity);
}
if ($queued) {
$context['sandbox']['@queued']++;
}
$context['sandbox']['@processed']++;
}
// Make sure to clear entity memory cache so it does not build up
// resulting in a constant increase of memory.
if ($entity_type->isStaticallyCacheable()) {
$this->entityMemoryCache->deleteAll();
}
$context['sandbox']['last_id'] = $entity->id();
$context['finished'] = min(1, $context['sandbox']['@processed'] / $context['sandbox']['@total']);
$context['results']['total'] = $context['sandbox']['@total'];
// Optional message displayed under the progressbar.
$context['message'] = dt('Processed @processed out of @total entities, added @queued to the queue', $context['sandbox']);
}
/**
* Validates an entity type has VG Wort enabled.
*
* @hook validate @validate-vgwort-entity-type
*
* @param \Consolidation\AnnotatedCommand\CommandData $commandData
* The command data to validate.
*
* @return \Consolidation\AnnotatedCommand\CommandError|null
*/
public function validateEntityType(CommandData $commandData): ?CommandError {
$entityType = $commandData->input()->getArgument('entityType');
if (!_vgwort_entity_type_has_counter_id($entityType)) {
return new CommandError(dt('Entity type "@entityType" is not configured for VG Wort', ['@entityType' => $entityType]));
}
return NULL;
}
/**
* Validates the batch size is a positive integer.
*
* @hook validate @validate-vgwort-batch-size
*
* @param \Consolidation\AnnotatedCommand\CommandData $commandData
* The command data to validate.
*
* @return \Consolidation\AnnotatedCommand\CommandError|null
*/
public function validateBatchSize(CommandData $commandData): ?CommandError {
$options = $commandData->options();
if (isset($options['batch-size'])) {
$validatedValue = filter_var($options['batch-size'], FILTER_VALIDATE_INT);
if (!$validatedValue || $validatedValue < 1) {
return new CommandError(dt('The batch-size "@batchSize" is not valid. It must be an integer and greater than 0.', ['@batchSize' => $options['batch-size']]));
}
}
return NULL;
}
}
......@@ -43,29 +43,34 @@ class EntityQueuer {
*
* @param \Drupal\Core\Entity\EntityInterface $entity
* The entity to notify VG Wort about.
* @param int|null $delay
* (optional) Override the default in processing the created queue item.
*
* @return bool
* TRUE if the entity is in the queue, FALSE if not.
*/
public function queueEntity(EntityInterface $entity): void {
public function queueEntity(EntityInterface $entity, int $delay = NULL): bool {
// This service only supports entity types that can be configured in the UI.
// @see \Drupal\vgwort\Form\SettingsForm::buildForm()
if (!$entity instanceof EntityPublishedInterface || !$entity instanceof FieldableEntityInterface) {
return;
return FALSE;
}
// Only published entities can be added to the queue.
if (!$entity->isPublished()) {
return;
return FALSE;
}
// Only entities with a counter ID can be queued.
if ($entity->vgwort_counter_id->isEmpty()) {
return;
return FALSE;
}
$queue = Queue::load('vgwort');
// If there is no queue fail silently. This ensures content can be inserted
// or updated prior to vgwort_post_update_create_queue() running.
if (!$queue instanceof Queue) {
return;
return FALSE;
}
// Failed jobs should be requeued if the entity is published.
......@@ -76,17 +81,29 @@ class EntityQueuer {
// to start again.
$job->setNumRetries(-1);
$queue_backend->retryJob($job, $this->config->get('queue_retry_time'));
return;
return TRUE;
}
elseif ($job && in_array($job->getState(), [Job::STATE_SUCCESS, Job::STATE_PROCESSING, Job::STATE_QUEUED], TRUE)) {
// Nothing to do.
return;
return TRUE;
}
$delay = $this->config->get('registration_wait_days') * 24 * 60 * 60;
$job = RegistrationNotification::createJob($entity);
$queue->enqueueJob($job, $delay);
$queue->enqueueJob($job, $delay ?? $this->getDefaultDelay());
$this->entityJobMapper->merge($job);
return TRUE;
}
/**
* Gets the default delay before processing a queue item.
*
* @return int
* The default delay in seconds.
*
* @see vgwort.settings:registration_wait_days
*/
public function getDefaultDelay(): int {
return $this->config->get('registration_wait_days') * 24 * 60 * 60;
}
}
<?php
namespace Drupal\Tests\vgwort\Kernel;
use Drupal\advancedqueue\Entity\Queue;
use Drupal\node\Entity\Node;
use Drupal\Tests\BrowserTestBase;
use Drush\TestTraits\DrushTestTrait;
/**
* Tests the vgwort drush command.
*
* @group vgwort
*
* @covers \Drupal\vgwort\Commands\VgwortCommands
*/
class DrushTest extends BrowserTestBase {
use DrushTestTrait;
/**
* {@inheritdoc}
*/
protected static $modules = ['vgwort', 'node', 'vgwort_test'];
/**
* {@inheritdoc}
*/
protected $defaultTheme = 'stark';
/**
* {@inheritdoc}
*/
protected function setUp(): void {
parent::setUp();
$this->config('vgwort.settings')
->set('publisher_id', 123456)
->set('image_domain', 'http://example.com')
// Disable VG Wort on all entity types.
->set('entity_types', [])
->set('legal_rights', [
'distribution' => TRUE,
'public_access' => TRUE,
'reproduction' => TRUE,
'declaration_of_granting' => TRUE,
'other_public_communication' => FALSE,
])
->save();
}
public function testInvalidEntityType() {
$this->expectException(\Exception::class);
$this->expectErrorMessageMatches('/Entity type "node" is not configured for VG Wort/');
$this->drush('vgwq', ['node']);
}
public function testInvalidBatchSize() {
$this->config('vgwort.settings')->set('entity_types', ['node' => []])->save();
$this->expectException(\Exception::class);
$this->expectErrorMessageMatches('/The batch-size "-1" is not valid. It must be an integer and greater than 0./');
$this->drush('vgwq', ['node'], ['batch-size' => -1]);
}
public function testDrushCommand() {
$this->createContentType(['type' => 'page']);
// Create 80 nodes.
for ($i = 1; $i <= 80; $i++) {
$node = Node::create([
'type' => 'page',
'title' => 'Page ' . $i,
]);
// Set each node to be created a number of days ago, starting at 80.
$node->setChangedTime(time() - ((80 - $i) * 60 * 60 * 24));
$node->save();
}
\Drupal::state()->set('vgwort_test_vgwort_enable_for_entity', ['70']);
$this->config('vgwort.settings')->set('entity_types', ['node' => []])->save();
$this->drush('vgwort:queue', ['node'], ['batch-size' => 20]);
$this->assertMatchesRegularExpression('/Processed 80 out of 80 entities, added 79 to the queue/', $this->getErrorOutput());
/** @var \Drupal\advancedqueue\Entity\Queue $queue */
$queue = Queue::load('vgwort');
/** @var \Drupal\advancedqueue\Plugin\AdvancedQueue\Backend\Database $queue_backend */
$queue_backend = $queue->getBackend();
$this->assertSame(79, (int) $queue_backend->countJobs()['queued']);
$job = $queue_backend->loadJob(1);
$this->assertLessThanOrEqual(time(), $job->getAvailableTime());
$job = $queue_backend->loadJob(66);
$this->assertLessThanOrEqual(time(), $job->getAvailableTime());
// The last 13 nodes should be available in the future.
$job = $queue_backend->loadJob(67);
$this->assertGreaterThan(time(), $job->getAvailableTime());
$job = $queue_backend->loadJob(79);
$this->assertGreaterThan(time(), $job->getAvailableTime());
$this->assertSame('80', $job->getPayload()['entity_id']);
// Ensure no jobs are actually processed.
$queue_backend->deleteQueue();
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment