Issue #3236171 by kim.pepper, jibran, mortim07: Add pluggable outputs (6a5c2fb3) · Commits · project / data_pipelines

composer.json

+7 −2

Original line number	Diff line number	Diff line
		@@ -15,11 +15,16 @@
		}
		},
		"require": {
		"drupal/core": "^8.8 \|\| ^9",
		"drupal/core": "^9",
		"drupal/entity": "^1.0",
		"drupal/elasticsearch_connector": "^7.0",
		"softcreatr/jsonpath": "^0.7"
		},
		"require-dev": {
		"elasticsearch/elasticsearch": "^7"
		},
		"suggest": {
		"elasticsearch/elasticsearch": "Required for the data_pipelines_elasticsearch module"
		},
		"extra": {
		"drush": {
		"services": {

config/install/data_pipelines.settings.yml

deleted100644 → 0

+0 −1

Original line number	Diff line number	Diff line
		index_name_prefix: ''
		No newline at end of file

config/schema/data_pipelines.schema.yml

+24 −4

Original line number	Diff line number	Diff line
		data_pipelines.settings:
		data_pipelines.dataset_destination.*:
		type: config_entity
		label: 'Data pipelines settings'
		label: 'Dataset Destination'
		mapping:
		index_name_prefix:
		id:
		type: string
		label: 'Index name prefix'
		label: 'ID'
		label:
		type: label
		label: 'Label'
		destination:
		type: label
		label: 'Destination'
		destinationSettings:
		type: 'data_pipelines.dataset_destination.destinationSettings.[%parent.destination]'
		label: 'Destination Settings'

		data_pipelines.dataset_destination.destinationSettings.file_json:
		type: config_object
		label: 'JSON File destination settings'
		mapping:
		scheme:
		type: string
		label: 'File scheme'
		dir:
		type: string
		label: 'Sub-directory'

data_pipelines.info.yml

+2 −3

Original line number	Diff line number	Diff line
		name: 'DCS dataset'
		name: 'Data Pipelines'
		package: Data Pipelines
		type: module
		description: 'Provides the ability to manage datasets.'
		core_version_requirement: '^9'
		@@ -7,5 +8,3 @@ dependencies:
		- drupal:file
		- entity:entity
		- drupal:options
		- elasticsearch_connector:elasticsearch_connector
		configure: data_pipelines.settings

data_pipelines.install

+115 −3

Original line number	Diff line number	Diff line
		@@ -5,9 +5,17 @@
		* The data pipelines install file.
		*/

		use Drupal\Core\Config\Entity\ConfigEntityType;
		use Drupal\Core\Entity\EntityDeleteForm;
		use Drupal\Core\Entity\FieldableEntityInterface;
		use Drupal\Core\Entity\Routing\AdminHtmlRouteProvider;
		use Drupal\Core\Field\BaseFieldDefinition;
		use Drupal\Core\Field\FieldStorageDefinitionInterface;
		use Drupal\Core\StringTranslation\TranslatableMarkup;
		use Drupal\data_pipelines\Entity\DatasetInterface;
		use Drupal\data_pipelines\Entity\Destination;
		use Drupal\data_pipelines\EntityHandlers\DatasetDestinationListBuilder;
		use Drupal\data_pipelines\Form\DestinationForm;

		/**
		* Implements hook_update_N().
		@@ -109,9 +117,7 @@ function data_pipelines_update_8004() {
		* Implements hook_update_N().
		*/
		function data_pipelines_update_8005() {
		$data_pipelines_settings = \Drupal::configFactory()->getEditable('data_pipelines.settings');
		$data_pipelines_settings->set('index_name_prefix', '');
		$data_pipelines_settings->save();
		// Deprecated.
		}

		/**
		@@ -154,3 +160,109 @@ function data_pipelines_update_8007() {
		->setRequired(TRUE);
		$entity_definition_update_manager->installFieldStorageDefinition('csv_delimiter', 'data_pipelines', 'data_pipelines', $field_storage_definition);
		}

		/**
		* Update 'allowed_values' for data_pipelines 'status' field.
		*/
		function data_pipelines_update_8008() {
		$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
		$field_storage_definition = $entity_definition_update_manager->getFieldStorageDefinition('status', 'data_pipelines');
		$field_storage_definition->setSetting('allowed_values', [
		DatasetInterface::STATUS_PENDING_VALIDATION => new TranslatableMarkup('Pending validation'),
		DatasetInterface::STATUS_PENDING_PROCESSING => new TranslatableMarkup('Pending processing'),
		DatasetInterface::STATUS_PROCESSED => new TranslatableMarkup('Processed'),
		]);
		$entity_definition_update_manager->updateFieldStorageDefinition($field_storage_definition);
		}

		/**
		* Install 'Data Pipeline Destination' entity.
		*/
		function data_pipelines_update_8009() {
		$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
		$dataset_destination = new ConfigEntityType([
		'id' => 'dataset_destination',
		'label' => new TranslatableMarkup('Data Pipeline Destination'),
		'label_collection' => new TranslatableMarkup('Data Pipeline Destinations'),
		'label_singular' => new TranslatableMarkup('destination'),
		'label_plural' => new TranslatableMarkup('destinations'),
		'label_count' => [
		'singular' => '@count destination',
		'plural' => '@count destinations',
		],
		'entity_keys' => [
		'id' => 'id',
		'label' => 'label',
		],

		'handlers' => [
		'list_builder' => DatasetDestinationListBuilder::class,
		'route_provider' => [
		'html' => AdminHtmlRouteProvider::class,
		],
		'form' => [
		'default' => DestinationForm::class,
		'add' => DestinationForm::class,
		'edit' => DestinationForm::class,
		'delete' => EntityDeleteForm::class,
		],
		],
		'admin_permission' => 'administer dataset destinations',
		'links' => [
		'add-form' => '/admin/config/content/dataset_destinations/add',
		'edit-form' => '/admin/config/content/dataset_destinations/manage/{dataset_destination}',
		'delete-form' => '/admin/config/content/dataset_destinations/manage/{dataset_destination}/delete',
		'collection' => '/admin/config/content/dataset_destinations',
		],
		'config_export' => [
		'id',
		'label',
		'destination',
		'destinationSettings',
		],
		'class' => Destination::class,
		'provider' => 'data_pipelines',
		]);
		$entity_definition_update_manager->installEntityType($dataset_destination);
		}

		/**
		* Update the database field schema to add destination.
		*/
		function data_pipelines_update_8010() {
		$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
		$field_storage_definition = BaseFieldDefinition::create('entity_reference')
		->setLabel(t('Destination'))
		->setDescription(t('The output destination.'))
		->setRequired(TRUE)
		->setCardinality(FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED)
		->setSetting('target_type', 'dataset_destination')
		->setDisplayOptions('form', [
		'type' => 'options_select',
		]);
		$entity_definition_update_manager->installFieldStorageDefinition('destinations', 'data_pipelines', 'data_pipelines', $field_storage_definition);
		}

		/**
		* Update to using the data_pipelines_elasticsearch.
		*/
		function data_pipelines_update_8011() {
		// Update to using the elasticsearch submodule.
		\Drupal::service('module_installer')->install(['data_pipelines_elasticsearch']);
		}

		/**
		* Update the queue names.
		*/
		function data_pipelines_update_8012() {
		$database = \Drupal::database();
		$database->query("UPDATE queue SET name = REPLACE(name, 'data_pipelines_index:', 'data_pipelines_process:') WHERE name LIKE 'data_pipelines_index:%'")->execute();
		}

		/**
		* Remove the deprecated config.
		*/
		function data_pipelines_update_8013() {
		$data_pipelines_settings = \Drupal::configFactory()->getEditable('data_pipelines.settings');
		$data_pipelines_settings->delete();
		}