From ad38a0b5e0fe69f1b67070955247682f9c6a8e6c Mon Sep 17 00:00:00 2001
From: Mohit Aghera <28092-mohit_rocks@users.noreply.drupalcode.org>
Date: Fri, 21 Jun 2024 02:09:35 +0000
Subject: [PATCH] Issue #3448925 by mohit_aghera, mstrelan, jibran, pghaemim:
Add getFileUrl method to FileDestinationBase and provide cache buster
---
composer.json | 1 +
.../FileDestinationBase.php | 22 +++++++
.../Destination/DatasetDestinationUrlTest.php | 65 +++++++++++++++++++
tests/src/fixtures/test-pipeline-2.json | 17 +++++
4 files changed, 105 insertions(+)
create mode 100644 tests/src/Kernel/Destination/DatasetDestinationUrlTest.php
create mode 100644 tests/src/fixtures/test-pipeline-2.json
diff --git a/composer.json b/composer.json
index a93d492..ec8504e 100644
--- a/composer.json
+++ b/composer.json
@@ -1,5 +1,6 @@
{
"name": "drupal/data_pipelines",
+ "description": "Provides the ability to ingest, validate, transform and index (ElasticSearch) arbitrary datasets",
"type": "drupal-module",
"license": "GPL-2.0-or-later",
"minimum-stability": "dev",
diff --git a/src/Plugin/DatasetDestination/FileDestinationBase.php b/src/Plugin/DatasetDestination/FileDestinationBase.php
index 0dd5037..bd73cd4 100644
--- a/src/Plugin/DatasetDestination/FileDestinationBase.php
+++ b/src/Plugin/DatasetDestination/FileDestinationBase.php
@@ -12,6 +12,7 @@ use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\Core\StreamWrapper\StreamWrapperInterface;
use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface;
use Drupal\Core\StringTranslation\TranslatableMarkup;
+use Drupal\Core\Url;
use Drupal\data_pipelines\Destination\DatasetDestinationPluginBase;
use Drupal\data_pipelines\Entity\DatasetInterface;
use Drupal\data_pipelines\Entity\DestinationInterface;
@@ -186,6 +187,27 @@ abstract class FileDestinationBase extends DatasetDestinationPluginBase implemen
return $this->getDirPath() . $this->getFilename($dataset);
}
+ /**
+ * Get the Url object for the file of the dataset.
+ *
+ * @param \Drupal\data_pipelines\Entity\DatasetInterface $dataset
+ * The dataset.
+ *
+ * @return \Drupal\Core\Url
+ * File Url.
+ */
+ public function getFileUrl(DatasetInterface $dataset): Url {
+ $fileUri = $this->getFilePath($dataset);
+ $filePath = $this->fileSystem->realpath($fileUri);
+ // Add the timestamp as a cache-bursting parameter.
+ $timestamp = file_exists($filePath) ? filemtime($filePath) : time();
+ return Url::fromUri($fileUri, [
+ 'query' => [
+ 't' => $timestamp,
+ ],
+ ]);
+ }
+
/**
* Get the dataset filename.
*
diff --git a/tests/src/Kernel/Destination/DatasetDestinationUrlTest.php b/tests/src/Kernel/Destination/DatasetDestinationUrlTest.php
new file mode 100644
index 0000000..2a5130f
--- /dev/null
+++ b/tests/src/Kernel/Destination/DatasetDestinationUrlTest.php
@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\Tests\data_pipelines\Kernel\Destination;
+
+use Drupal\Core\Url;
+use Drupal\data_pipelines\Plugin\DatasetDestination\JsonDestination;
+use Drupal\Tests\data_pipelines\Kernel\DatasetKernelTestBase;
+
+/**
+ * Tests the getFileUrl of the FileDestinationBase class.
+ *
+ * @group data_pipelines
+ *
+ * @see \Drupal\data_pipelines\Plugin\DatasetDestination\FileDestinationBase
+ */
+class DatasetDestinationUrlTest extends DatasetKernelTestBase {
+
+ /**
+ * Tests that filemtime is added to getUrl method.
+ */
+ public function testFileUrlForDestination(): void {
+ $destinationId = $this->randomMachineName();
+ $destination = $this->createTestMemoryDestination(['id' => $destinationId]);
+ $dataset = $this->createTestDataset(['destinations' => $destination]);
+
+ $fileSystem = \Drupal::service('file_system');
+ $streamWrapperManager = \Drupal::service('stream_wrapper_manager');
+ $configuration = [
+ 'scheme' => 'public',
+ 'dir' => '',
+ ];
+ $plugin_id = 'foo';
+ $plugin_definition = [];
+ $destinationPlugin = new JsonDestination($configuration, $plugin_id, $plugin_definition, $fileSystem, $streamWrapperManager);
+ $file_path = $destinationPlugin->getFilePath($dataset);
+
+ // Create.
+ $this->assertTrue($destinationPlugin->beginProcessing($dataset));
+ $this->assertTrue($destinationPlugin->processChunk($dataset, iterator_to_array($dataset->getDataIterator())));
+ $this->assertEquals('[{"should_we":true,"full_name":"bloggs, joe"},{"should_we":false,"full_name":"bloggs, betty"}]', file_get_contents($file_path));
+
+ $url = $destinationPlugin->getFileUrl($dataset);
+ assert($url instanceof Url);
+ $this->assertArrayHasKey('t', $url->getOption('query'));
+ $timestamp1 = $url->getOption('query')['t'];
+ $this->assertEquals($timestamp1, filemtime(\Drupal::service('file_system')->realpath($destinationPlugin->getFilePath($dataset))));
+
+ sleep(2);
+
+ // Update and test the timestamp again.
+ $dataset->csv_text[0]->value .= "\nY,robin,scherbatsky";
+ $dataset->save();
+ $this->assertTrue($destinationPlugin->beginProcessing($dataset));
+ $this->assertTrue($destinationPlugin->processChunk($dataset, iterator_to_array($dataset->getDataIterator())));
+ $this->assertEquals('[{"should_we":true,"full_name":"bloggs, joe"},{"should_we":false,"full_name":"bloggs, betty"},{"should_we":true,"full_name":"scherbatsky, robin"}]', file_get_contents($file_path));
+
+ $url = $destinationPlugin->getFileUrl($dataset);
+ $timestamp2 = $url->getOption('query')['t'];
+ $this->assertEquals($timestamp2, filemtime(\Drupal::service('file_system')->realpath($destinationPlugin->getFilePath($dataset))));
+ $this->assertNotEquals($timestamp2, $timestamp1);
+ }
+
+}
diff --git a/tests/src/fixtures/test-pipeline-2.json b/tests/src/fixtures/test-pipeline-2.json
new file mode 100644
index 0000000..d0db581
--- /dev/null
+++ b/tests/src/fixtures/test-pipeline-2.json
@@ -0,0 +1,17 @@
+[
+ {
+ "should_we": "Y",
+ "firstname": "joe",
+ "lastname": "bloggs"
+ },
+ {
+ "should_we": "N",
+ "firstname": "betty",
+ "lastname": "bloggs"
+ },
+ {
+ "should_we": "N",
+ "firstname": "john",
+ "lastname": "Doe"
+ }
+]
--
GitLab