Issue #2198429 by alexpott: Make deleted fields work with config synch.

core/lib/Drupal/Core/Config/ConfigImporter.php

@@ -499,11 +499,11 @@ public function import() {
    *   Exception thrown if the $sync_step can not be called.
    */
   public function doSyncStep($sync_step, &$context) {
-    if (method_exists($this, $sync_step)) {
+    if (!is_array($sync_step) && method_exists($this, $sync_step)) {
       $this->$sync_step($context);
     }
     elseif (is_callable($sync_step)) {
-      call_user_func_array($sync_step, array(&$context));
+      call_user_func_array($sync_step, array(&$context, $this));
     }
     else {
       throw new \InvalidArgumentException('Invalid configuration synchronization step');
@@ -549,7 +549,7 @@ public function initialize() {
     $sync_steps[] = 'processConfigurations';
 
     // Allow modules to add new steps to configuration synchronization.
-    $this->moduleHandler->alter('config_import_steps', $sync_steps);
+    $this->moduleHandler->alter('config_import_steps', $sync_steps, $this);
     $sync_steps[] = 'finish';
     return $sync_steps;
   }

core/modules/field/config/install/field.settings.yml

-purge_batch_size: 10
+purge_batch_size: 50

core/modules/field/field.module

@@ -6,6 +6,7 @@
 
 use Drupal\Component\Utility\Html;
 use Drupal\Component\Utility\Xss;
+use Drupal\Core\Config\ConfigImporter;
 use Drupal\Core\Entity\EntityTypeInterface;
 use Drupal\Core\Extension\Extension;
 use Drupal\field\Field;
@@ -362,3 +363,49 @@ function field_hook_info() {
 
   return $hooks;
 }
+
+/**
+ * Implements hook_config_import_steps_alter().
+ */
+function field_config_import_steps_alter(&$sync_steps, ConfigImporter $config_importer) {
+  $fields = \Drupal\field\ConfigImporterFieldPurger::getFieldsToPurge(
+    $config_importer->getStorageComparer()->getSourceStorage()->read('core.extension'),
+    $config_importer->getStorageComparer()->getChangelist('delete')
+  );
+  if ($fields) {
+    // Add a step to the beginning of the configuration synchronization process
+    // to purge field data where the module that provides the field is being
+    // uninstalled.
+    array_unshift($sync_steps, array('\Drupal\field\ConfigImporterFieldPurger', 'process'));
+  };
+}
+
+/**
+ * Implements hook_form_FORM_ID_alter().
+ *
+ * Adds a warning if field data will be permanently removed by the configuration
+ * synchronization.
+ *
+ * @see \Drupal\field\ConfigImporterFieldPurger
+ */
+function field_form_config_admin_import_form_alter(&$form, &$form_state) {
+  // Only display the message when there is a storage comparer available and the
+  // form is not submitted.
+  if (isset($form_state['storage_comparer']) && empty($form_state['input'])) {
+    $fields = \Drupal\field\ConfigImporterFieldPurger::getFieldsToPurge(
+      $form_state['storage_comparer']->getSourceStorage()->read('core.extension'),
+      $form_state['storage_comparer']->getChangelist('delete')
+    );
+    if ($fields) {
+      foreach ($fields as $field) {
+        $field_labels[] = $field->label();
+      }
+      drupal_set_message(\Drupal::translation()->formatPlural(
+        count($fields),
+        'This synchronization will delete data from the field %fields.',
+        'This synchronization will delete data from the fields: %fields.',
+        array('%fields' => implode(', ', $field_labels))
+      ), 'warning');
+    }
+  }
+}

core/modules/field/field.purge.inc

@@ -68,11 +68,18 @@
  *
  * @param $batch_size
  *   The maximum number of field data records to purge before returning.
+ * @param string $field_uuid
+ *   (optional) Limit the purge to a specific field.
  */
-function field_purge_batch($batch_size) {
+function field_purge_batch($batch_size, $field_uuid = NULL) {
   // Retrieve all deleted field instances. We cannot use field_info_instances()
   // because that function does not return deleted instances.
-  $instances = entity_load_multiple_by_properties('field_instance_config', array('deleted' => TRUE, 'include_deleted' => TRUE));
+  if ($field_uuid) {
+    $instances = entity_load_multiple_by_properties('field_instance_config', array('deleted' => TRUE, 'include_deleted' => TRUE, 'field_uuid' => $field_uuid));
+  }
+  else {
+    $instances = entity_load_multiple_by_properties('field_instance_config', array('deleted' => TRUE, 'include_deleted' => TRUE));
+  }
   $factory = \Drupal::service('entity.query');
   $info = \Drupal::entityManager()->getDefinitions();
   foreach ($instances as $instance) {
@@ -104,6 +111,11 @@ function field_purge_batch($batch_size) {
         $ids->entity_id = $entity_id;
         $entity = _field_create_entity_from_ids($ids);
         \Drupal::entityManager()->getStorage($entity_type)->onFieldItemsPurge($entity, $instance);
+        $batch_size--;
+      }
+      // Only delete up to the maximum number of records.
+      if ($batch_size == 0) {
+        break;
       }
     }
     else {
@@ -116,6 +128,10 @@ function field_purge_batch($batch_size) {
   $deleted_fields = \Drupal::state()->get('field.field.deleted') ?: array();
   foreach ($deleted_fields as $field) {
     $field = new FieldConfig($field);
+    if ($field_uuid && $field->uuid() != $field_uuid) {
+      // If a specific UUID is provided, only purge the corresponding field.
+      continue;
+    }
 
     // We cannot purge anything if the entity type is unknown (e.g. the
     // providing module was uninstalled).

core/modules/field/lib/Drupal/field/ConfigImporterFieldPurger.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\field\ConfigImporterFieldPurger.
+ */
+
+namespace Drupal\field;
+
+use Drupal\Core\Config\ConfigImporter;
+use Drupal\Core\Config\Entity\ConfigEntityStorage;
+
+/**
+ * Processes field purges before a configuration synchronization.
+ */
+class ConfigImporterFieldPurger {
+
+  /**
+   * Processes fields targeted for purge as part of a configuration sync.
+   *
+   * This takes care of deleting the field if necessary, and purging the data on
+   * the fly.
+   *
+   * @param array $context
+   *   The batch context.
+   * @param \Drupal\Core\Config\ConfigImporter $config_importer
+   *   The config importer.
+   */
+  public static function process(array &$context, ConfigImporter $config_importer) {
+    if (!isset($context['sandbox']['field'])) {
+      static::initializeSandbox($context, $config_importer);
+    }
+
+    // Get the list of fields to purge.
+    $fields = static::getFieldsToPurge($context['sandbox']['field']['extensions'], $config_importer->getUnprocessedConfiguration('delete'));
+    // Get the first field to process.
+    $field = reset($fields);
+    if (!isset($context['sandbox']['field']['current_field_id']) || $context['sandbox']['field']['current_field_id'] != $field->id()) {
+      $context['sandbox']['field']['current_field_id'] = $field->id();
+      // If the field has not been deleted yet we need to do that. This is the
+      // case when the field deletion is staged.
+      if (!$field->deleted) {
+        $field->delete();
+      }
+    }
+    field_purge_batch($context['sandbox']['field']['purge_batch_size'], $field->uuid());
+    $context['sandbox']['field']['current_progress']++;
+    $fields_to_delete_count = count(static::getFieldsToPurge($context['sandbox']['field']['extensions'], $config_importer->getUnprocessedConfiguration('delete')));
+    if ($fields_to_delete_count == 0) {
+      $context['finished'] = 1;
+    }
+    else {
+      $context['finished'] = $context['sandbox']['field']['current_progress'] / $context['sandbox']['field']['steps_to_delete'];
+      $context['message'] = \Drupal::translation()->translate('Purging field @field_label', array('@field_label' => $field->label()));
+    }
+  }
+
+  /**
+   * Initializes the batch context sandbox for processing field deletions.
+   *
+   * This calculates the number of steps necessary to purge all the field data
+   * and saves data for later use.
+   *
+   * @param array $context
+   *   The batch context.
+   * @param \Drupal\Core\Config\ConfigImporter $config_importer
+   *   The config importer.
+   */
+  protected static function initializeSandbox(array &$context, ConfigImporter $config_importer) {
+    $context['sandbox']['field']['purge_batch_size'] = \Drupal::config('field.settings')->get('purge_batch_size');
+    // Save the future list of installed extensions to limit the amount of times
+    // the configuration is read from disk.
+    $context['sandbox']['field']['extensions'] = $config_importer->getStorageComparer()->getSourceStorage()->read('core.extension');
+
+    $context['sandbox']['field']['steps_to_delete'] = 0;
+    $fields = static::getFieldsToPurge($context['sandbox']['field']['extensions'], $config_importer->getUnprocessedConfiguration('delete'));
+    foreach ($fields as $field) {
+      $row_count = $field->entityCount();
+      if ($row_count > 0) {
+        // The number of steps to delete each field is determined by the
+        // purge_batch_size setting. For example if the field has 9 rows and the
+        // batch size is 10 then this will add 1 step to $number_of_steps.
+        $how_many_steps = ceil($row_count / $context['sandbox']['field']['purge_batch_size']);
+        $context['sandbox']['field']['steps_to_delete'] += $how_many_steps;
+      }
+    }
+    // Each field needs one last field_purge_batch() call to remove the last
+    // instance and the field itself.
+    $context['sandbox']['field']['steps_to_delete'] += count($fields);
+
+    $context['sandbox']['field']['current_progress'] = 0;
+  }
+
+  /**
+   * Gets the list of fields to purge before configuration synchronization.
+   *
+   * If, during a configuration synchronization, a field is being deleted and
+   * the module that provides the field type is being uninstalled then the field
+   * data must be purged before the module is uninstalled. Also, if deleted
+   * fields exist whose field types are provided by modules that are being
+   * uninstalled their data need to be purged too.
+   *
+   * @param array $extensions
+   *   The list of extensions that will be enabled after the configuration
+   *   synchronization has finished.
+   * @param array $deletes
+   *   The configuration that will be deleted by the configuration
+   *   synchronization.
+   *
+   * @return \Drupal\field\Entity\FieldConfig[]
+   *   An array of fields that need purging before configuration can be
+   *   synchronized.
+   */
+  public static function getFieldsToPurge(array $extensions, array $deletes) {
+    $providers = array_keys($extensions['module']);
+    $providers[] = 'Core';
+    $fields_to_delete = array();
+
+    // Gather fields that will be deleted during configuration synchronization
+    // where the module that provides the field type is also being uninstalled.
+    $field_ids = array();
+    foreach ($deletes as $config_name) {
+      if (strpos($config_name, 'field.field.') === 0) {
+        $field_ids[] = ConfigEntityStorage::getIDFromConfigName($config_name, 'field.field');
+      }
+    }
+    if (!empty($field_ids)) {
+      $fields = \Drupal::entityQuery('field_config')
+        ->condition('id', $field_ids, 'IN')
+        ->condition('module', $providers, 'NOT IN')
+        ->execute();
+      if (!empty($fields)) {
+        $fields_to_delete = entity_load_multiple('field_config', $fields);
+      }
+    }
+
+    // Gather deleted fields from modules that are being uninstalled.
+    $fields = entity_load_multiple_by_properties('field_config', array('deleted' => TRUE, 'include_deleted' => TRUE));
+    foreach ($fields as $field) {
+      if (!in_array($field->module, $providers)) {
+        $fields_to_delete[$field->id()] = $field;
+      }
+    }
+    return $fields_to_delete;
+  }
+
+}

core/modules/field/lib/Drupal/field/Tests/FieldImportDeleteUninstallTest.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\field\Tests\FieldImportDeleteUninstallTest.
+ */
+
+namespace Drupal\field\Tests;
+
+/**
+ * Tests config sync of deleting fields and instances and uninstalling modules.
+ *
+ * @see \Drupal\field\ConfigImporterFieldPurger
+ * @see field_config_import_steps_alter()
+ */
+class FieldImportDeleteUninstallTest extends FieldUnitTestBase {
+
+  /**
+   * Modules to enable.
+   *
+   * @var array
+   */
+  public static $modules = array('telephone');
+
+  public static function getInfo() {
+    return array(
+      'name' => 'Field config delete and uninstall tests',
+      'description' => 'Delete field and instances during config synchronization and uninstall module that provides the field type.',
+      'group' => 'Field API',
+    );
+  }
+
+  public function setUp() {
+    parent::setUp();
+    // Module uninstall requires the router and users_data tables.
+    // @see drupal_flush_all_caches()
+    // @see user_modules_uninstalled()
+    $this->installSchema('system', array('router'));
+    $this->installSchema('user', array('users_data'));
+  }
+
+  /**
+   * Tests deleting fields and instances as part of config import.
+   */
+  public function testImportDeleteUninstall() {
+    // Create a field to delete to prove that
+    // \Drupal\field\ConfigImporterFieldPurger does not purge fields that are
+    // not related to the configuration synchronization.
+    $unrelated_field = entity_create('field_config', array(
+      'name' => 'field_int',
+      'entity_type' => 'entity_test',
+      'type' => 'integer',
+    ));
+    $unrelated_field->save();
+    $unrelated_field_uuid = $unrelated_field->uuid();
+    entity_create('field_instance_config', array(
+      'entity_type' => 'entity_test',
+      'field_name' => 'field_int',
+      'bundle' => 'entity_test',
+    ))->save();
+
+    // Create a telephone field and instance for validation.
+    $field = entity_create('field_config', array(
+      'name' => 'field_test',
+      'entity_type' => 'entity_test',
+      'type' => 'telephone',
+    ));
+    $field->save();
+    $field_uuid = $field->uuid();
+    entity_create('field_instance_config', array(
+      'entity_type' => 'entity_test',
+      'field_name' => 'field_test',
+      'bundle' => 'entity_test',
+    ))->save();
+
+    $entity = entity_create('entity_test');
+    $value = '+0123456789';
+    $entity->field_test = $value;
+    $entity->field_int = '99';
+    $entity->name->value = $this->randomName();
+    $entity->save();
+
+    // Verify entity has been created properly.
+    $id = $entity->id();
+    $entity = entity_load('entity_test', $id);
+    $this->assertEqual($entity->field_test->value, $value);
+    $this->assertEqual($entity->field_test[0]->value, $value);
+    $this->assertEqual($entity->field_int->value, '99');
+
+    // Delete unrelated field before copying configuration and running the
+    // synchronization.
+    $unrelated_field->delete();
+
+    $active = $this->container->get('config.storage');
+    $staging = $this->container->get('config.storage.staging');
+    $this->copyConfig($active, $staging);
+
+    // Stage uninstall of the Telephone module.
+    $core_extension = \Drupal::config('core.extension')->get();
+    unset($core_extension['module']['telephone']);
+    $staging->write('core.extension', $core_extension);
+
+    // Stage the field deletion
+    $staging->delete('field.field.entity_test.field_test');
+    $staging->delete('field.instance.entity_test.entity_test.field_test');
+
+    $steps = $this->configImporter()->initialize();
+    $this->assertIdentical($steps[0], array('\Drupal\field\ConfigImporterFieldPurger', 'process'), 'The additional process configuration synchronization step has been added.');
+
+    $this->configImporter()->import();
+
+    // This will purge all the data, delete the field and uninstall the
+    // Telephone module.
+    $this->rebuildContainer();
+    $this->assertFalse(\Drupal::moduleHandler()->moduleExists('telephone'));
+    $this->assertFalse(entity_load_by_uuid('field_config', $field_uuid), 'The test field has been deleted by the configuration synchronization');
+    $deleted_fields = \Drupal::state()->get('field.field.deleted') ?: array();
+    $this->assertFalse(isset($deleted_fields[$field_uuid]), 'Telephone field has been completed removed from the system.');
+    $this->assertTrue(isset($deleted_fields[$unrelated_field_uuid]), 'Unrelated field not purged by configuration synchronization.');
+  }
+
+  /**
+   * Tests purging already deleted fields and instances during a config import.
+   */
+  public function testImportAlreadyDeletedUninstall() {
+    // Create a telephone field and instance for validation.
+    $field = entity_create('field_config', array(
+      'name' => 'field_test',
+      'entity_type' => 'entity_test',
+      'type' => 'telephone',
+    ));
+    $field->save();
+    $field_uuid = $field->uuid();
+    entity_create('field_instance_config', array(
+      'entity_type' => 'entity_test',
+      'field_name' => 'field_test',
+      'bundle' => 'entity_test',
+    ))->save();
+
+    // Create 12 entities to ensure that the purging works as expected.
+    for ($i=0; $i < 12; $i++) {
+      $entity = entity_create('entity_test');
+      $value = '+0123456789';
+      $entity->field_test = $value;
+      $entity->name->value = $this->randomName();
+      $entity->save();
+
+      // Verify entity has been created properly.
+      $id = $entity->id();
+      $entity = entity_load('entity_test', $id);
+      $this->assertEqual($entity->field_test->value, $value);
+    }
+
+    // Delete the field.
+    $field->delete();
+
+    $active = $this->container->get('config.storage');
+    $staging = $this->container->get('config.storage.staging');
+    $this->copyConfig($active, $staging);
+
+    // Stage uninstall of the Telephone module.
+    $core_extension = \Drupal::config('core.extension')->get();
+    unset($core_extension['module']['telephone']);
+    $staging->write('core.extension', $core_extension);
+
+    $deleted_fields = \Drupal::state()->get('field.field.deleted') ?: array();
+    $this->assertTrue(isset($deleted_fields[$field_uuid]), 'Field has been deleted and needs purging before configuration synchronization.');
+
+    $steps = $this->configImporter()->initialize();
+    $this->assertIdentical($steps[0], array('\Drupal\field\ConfigImporterFieldPurger', 'process'), 'The additional process configuration synchronization step has been added.');
+
+    $this->configImporter()->import();
+
+    // This will purge all the data, delete the field and uninstall the
+    // Telephone module.
+    $this->rebuildContainer();
+    $this->assertFalse(\Drupal::moduleHandler()->moduleExists('telephone'));
+    $deleted_fields = \Drupal::state()->get('field.field.deleted') ?: array();
+    $this->assertFalse(isset($deleted_fields[$field_uuid]), 'Field has been completed removed from the system.');
+  }
+
+}

core/modules/field/lib/Drupal/field/Tests/FieldImportDeleteUninstallUiTest.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\field\Tests\FieldImportDeleteUninstallUiTest.
+ */
+
+namespace Drupal\field\Tests;
+
+/**
+ * Tests config sync of deleting fields and instances and uninstalling modules.
+ *
+ * @see \Drupal\field\ConfigImporterFieldPurger
+ * @see field_config_import_steps_alter()
+ * @see field_form_config_admin_import_form_alter()
+ */
+class FieldImportDeleteUninstallUiTest extends FieldTestBase {
+
+  /**
+   * Modules to enable.
+   *
+   * @var array
+   */
+  public static $modules = array('entity_test', 'telephone', 'config', 'filter', 'text');
+
+  public static function getInfo() {
+    return array(
+      'name' => 'Field config delete and uninstall UI tests',
+      'description' => 'Delete field and instances during config synchronization and uninstall module that provides the field type through the UI.',
+      'group' => 'Field API',
+    );
+  }
+
+  function setUp() {
+    parent::setUp();
+
+    $this->web_user = $this->drupalCreateUser(array('synchronize configuration'));
+    $this->drupalLogin($this->web_user);
+  }
+
+  /**
+   * Tests deleting fields and instances as part of config import.
+   */
+  public function testImportDeleteUninstall() {
+    // Create a telephone field and instance.
+    $field = entity_create('field_config', array(
+      'name' => 'field_tel',
+      'entity_type' => 'entity_test',
+      'type' => 'telephone',
+    ));
+    $field->save();
+    $tel_field_uuid = $field->uuid();
+    entity_create('field_instance_config', array(
+      'entity_type' => 'entity_test',
+      'field_name' => 'field_tel',
+      'bundle' => 'entity_test',
+    ))->save();
+
+    // Create a text field and instance.
+    $text_field = entity_create('field_config', array(
+      'name' => 'field_text',
+      'entity_type' => 'entity_test',
+      'type' => 'text',
+    ));
+    $text_field->save();
+    $text_field_uuid = $field->uuid();
+    entity_create('field_instance_config', array(
+      'entity_type' => 'entity_test',
+      'field_name' => 'field_text',
+      'bundle' => 'entity_test',
+    ))->save();
+
+    // Create an entity which has values for the telephone and text field.
+    $entity = entity_create('entity_test');
+    $value = '+0123456789';
+    $entity->field_tel = $value;
+    $entity->field_text = $this->randomName(20);
+    $entity->name->value = $this->randomName();
+    $entity->save();
+
+    // Delete the text field before exporting configuration so that we can test
+    // that deleted fields that are provided by modules that will be uninstalled
+    // are also purged and that the UI message includes such fields.
+    $text_field->delete();
+
+    // Verify entity has been created properly.
+    $id = $entity->id();
+    $entity = entity_load('entity_test', $id);
+    $this->assertEqual($entity->field_tel->value, $value);
+    $this->assertEqual($entity->field_tel[0]->value, $value);
+
+    $active = $this->container->get('config.storage');
+    $staging = $this->container->get('config.storage.staging');
+    $this->copyConfig($active, $staging);
+
+    // Stage uninstall of the Telephone module.
+    $core_extension = \Drupal::config('core.extension')->get();
+    unset($core_extension['module']['telephone']);
+    $staging->write('core.extension', $core_extension);
+
+    // Stage the field deletion
+    $staging->delete('field.field.entity_test.field_tel');
+    $staging->delete('field.instance.entity_test.entity_test.field_tel');
+    $this->drupalGet('admin/config/development/configuration');
+    // Test that the message for one field being purged during a configuration
+    // synchronization is correct.
+    $this->assertText('This synchronization will delete data from the field entity_test.field_tel.');
+
+    // Stage an uninstall of the text module to test the message for multiple
+    // fields.
+    unset($core_extension['module']['text']);
+    $staging->write('core.extension', $core_extension);
+    $this->drupalGet('admin/config/development/configuration');
+    $this->assertText('This synchronization will delete data from the fields: entity_test.field_tel, entity_test.field_text.');
+
+    // This will purge all the data, delete the field and uninstall the
+    // Telephone and Text modules.
+    $this->drupalPostForm(NULL, array(), t('Import all'));
+    $this->assertNoText('Field data will be deleted by this synchronization.');
+    $this->rebuildContainer();
+    $this->assertFalse(\Drupal::moduleHandler()->moduleExists('telephone'));
+    $this->assertFalse(entity_load_by_uuid('field_config', $tel_field_uuid), 'The telephone field has been deleted by the configuration synchronization');
+    $deleted_fields = \Drupal::state()->get('field.field.deleted') ?: array();
+    $this->assertFalse(isset($deleted_fields[$tel_field_uuid]), 'Telephone field has been completed removed from the system.');
+    $this->assertFalse(isset($deleted_fields[$text_field_uuid]), 'Text field has been completed removed from the system.');
+  }
+