diff --git a/composer.json b/composer.json index a93d492797bd7f17c87c6609034563fd87e1a18c..ec8504eaf06697588e6985d2b25921988ee563c2 100644 --- a/composer.json +++ b/composer.json @@ -1,5 +1,6 @@ { "name": "drupal/data_pipelines", + "description": "Provides the ability to ingest, validate, transform and index (ElasticSearch) arbitrary datasets", "type": "drupal-module", "license": "GPL-2.0-or-later", "minimum-stability": "dev", diff --git a/src/Plugin/DatasetDestination/FileDestinationBase.php b/src/Plugin/DatasetDestination/FileDestinationBase.php index 0dd5037fa517f0103d4874463b3859ce77af21b2..bd73cd4e394878605a97871b9a42a76726d46ceb 100644 --- a/src/Plugin/DatasetDestination/FileDestinationBase.php +++ b/src/Plugin/DatasetDestination/FileDestinationBase.php @@ -12,6 +12,7 @@ use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\StreamWrapper\StreamWrapperInterface; use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface; use Drupal\Core\StringTranslation\TranslatableMarkup; +use Drupal\Core\Url; use Drupal\data_pipelines\Destination\DatasetDestinationPluginBase; use Drupal\data_pipelines\Entity\DatasetInterface; use Drupal\data_pipelines\Entity\DestinationInterface; @@ -186,6 +187,27 @@ abstract class FileDestinationBase extends DatasetDestinationPluginBase implemen return $this->getDirPath() . $this->getFilename($dataset); } + /** + * Get the Url object for the file of the dataset. + * + * @param \Drupal\data_pipelines\Entity\DatasetInterface $dataset + * The dataset. + * + * @return \Drupal\Core\Url + * File Url. + */ + public function getFileUrl(DatasetInterface $dataset): Url { + $fileUri = $this->getFilePath($dataset); + $filePath = $this->fileSystem->realpath($fileUri); + // Add the timestamp as a cache-bursting parameter. + $timestamp = file_exists($filePath) ? filemtime($filePath) : time(); + return Url::fromUri($fileUri, [ + 'query' => [ + 't' => $timestamp, + ], + ]); + } + /** * Get the dataset filename. * diff --git a/tests/src/Kernel/Destination/DatasetDestinationUrlTest.php b/tests/src/Kernel/Destination/DatasetDestinationUrlTest.php new file mode 100644 index 0000000000000000000000000000000000000000..2a5130f5effaafaefd5deacde4be231f31c301fe --- /dev/null +++ b/tests/src/Kernel/Destination/DatasetDestinationUrlTest.php @@ -0,0 +1,65 @@ +<?php + +declare(strict_types=1); + +namespace Drupal\Tests\data_pipelines\Kernel\Destination; + +use Drupal\Core\Url; +use Drupal\data_pipelines\Plugin\DatasetDestination\JsonDestination; +use Drupal\Tests\data_pipelines\Kernel\DatasetKernelTestBase; + +/** + * Tests the getFileUrl of the FileDestinationBase class. + * + * @group data_pipelines + * + * @see \Drupal\data_pipelines\Plugin\DatasetDestination\FileDestinationBase + */ +class DatasetDestinationUrlTest extends DatasetKernelTestBase { + + /** + * Tests that filemtime is added to getUrl method. + */ + public function testFileUrlForDestination(): void { + $destinationId = $this->randomMachineName(); + $destination = $this->createTestMemoryDestination(['id' => $destinationId]); + $dataset = $this->createTestDataset(['destinations' => $destination]); + + $fileSystem = \Drupal::service('file_system'); + $streamWrapperManager = \Drupal::service('stream_wrapper_manager'); + $configuration = [ + 'scheme' => 'public', + 'dir' => '', + ]; + $plugin_id = 'foo'; + $plugin_definition = []; + $destinationPlugin = new JsonDestination($configuration, $plugin_id, $plugin_definition, $fileSystem, $streamWrapperManager); + $file_path = $destinationPlugin->getFilePath($dataset); + + // Create. + $this->assertTrue($destinationPlugin->beginProcessing($dataset)); + $this->assertTrue($destinationPlugin->processChunk($dataset, iterator_to_array($dataset->getDataIterator()))); + $this->assertEquals('[{"should_we":true,"full_name":"bloggs, joe"},{"should_we":false,"full_name":"bloggs, betty"}]', file_get_contents($file_path)); + + $url = $destinationPlugin->getFileUrl($dataset); + assert($url instanceof Url); + $this->assertArrayHasKey('t', $url->getOption('query')); + $timestamp1 = $url->getOption('query')['t']; + $this->assertEquals($timestamp1, filemtime(\Drupal::service('file_system')->realpath($destinationPlugin->getFilePath($dataset)))); + + sleep(2); + + // Update and test the timestamp again. + $dataset->csv_text[0]->value .= "\nY,robin,scherbatsky"; + $dataset->save(); + $this->assertTrue($destinationPlugin->beginProcessing($dataset)); + $this->assertTrue($destinationPlugin->processChunk($dataset, iterator_to_array($dataset->getDataIterator()))); + $this->assertEquals('[{"should_we":true,"full_name":"bloggs, joe"},{"should_we":false,"full_name":"bloggs, betty"},{"should_we":true,"full_name":"scherbatsky, robin"}]', file_get_contents($file_path)); + + $url = $destinationPlugin->getFileUrl($dataset); + $timestamp2 = $url->getOption('query')['t']; + $this->assertEquals($timestamp2, filemtime(\Drupal::service('file_system')->realpath($destinationPlugin->getFilePath($dataset)))); + $this->assertNotEquals($timestamp2, $timestamp1); + } + +} diff --git a/tests/src/fixtures/test-pipeline-2.json b/tests/src/fixtures/test-pipeline-2.json new file mode 100644 index 0000000000000000000000000000000000000000..d0db581f7e9838484f4e9ed3c0d998c4d2de6f74 --- /dev/null +++ b/tests/src/fixtures/test-pipeline-2.json @@ -0,0 +1,17 @@ +[ + { + "should_we": "Y", + "firstname": "joe", + "lastname": "bloggs" + }, + { + "should_we": "N", + "firstname": "betty", + "lastname": "bloggs" + }, + { + "should_we": "N", + "firstname": "john", + "lastname": "Doe" + } +]