Issue #2550291 by neclimdul, phenaproxima: Improve and generalize database dump tools

core/lib/Drupal/Core/Command/DbCommandBase.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Command\DbCommandBase.
+ */
+
+namespace Drupal\Core\Command;
+
+use Drupal\Core\Database\Database;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+
+/**
+ * Base command that abstracts handling of database connection arguments.
+ */
+class DbCommandBase extends Command {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function configure() {
+    $this->addOption('database', NULL, InputOption::VALUE_OPTIONAL, 'The database connection name to use.', 'default')
+      ->addOption('database-url', 'db-url', InputOption::VALUE_OPTIONAL, 'A database url to parse and use as the database connection.')
+      ->addOption('prefix', NULL, InputOption::VALUE_OPTIONAL, 'Override or set the table prefix used in the database connection.');
+  }
+
+  /**
+   * Parse input options decide on a database.
+   *
+   * @param \Symfony\Component\Console\Input\InputInterface $input
+   *   Input object.
+   * @return \Drupal\Core\Database\Connection
+   */
+  protected function getDatabaseConnection(InputInterface $input) {
+    // Load connection from a url.
+    if ($input->getOption('database-url')) {
+      // @todo this could probably be refactored to not use a global connection.
+      // Ensure database connection isn't set.
+      if (Database::getConnectionInfo('db-tools')) {
+        throw new \RuntimeException('Database "db-tools" is already defined. Cannot define database provided.');
+      }
+      $info = Database::convertDbUrlToConnectionInfo($input->getOption('database-url'), \Drupal::root());
+      Database::addConnectionInfo('db-tools', 'default', $info);
+      $key = 'db-tools';
+    }
+    else {
+      $key = $input->getOption('database');
+    }
+
+    // If they supplied a prefix, replace it in the connection information.
+    $prefix = $input->getOption('prefix');
+    if ($prefix) {
+      $info = Database::getConnectionInfo($key)['default'];
+      $info['prefix']['default'] = $prefix;
+
+      Database::removeConnection($key);
+      Database::addConnectionInfo($key, 'default', $info);
+    }
+
+    return Database::getConnection('default', $key);
+  }
+
+}

core/lib/Drupal/Core/Command/DbDumpApplication.php

@@ -7,8 +7,6 @@
 
 namespace Drupal\Core\Command;
 
-use Drupal\Core\Database\Connection;
-use Drupal\Core\Extension\ModuleHandlerInterface;
 use Symfony\Component\Console\Application;
 use Symfony\Component\Console\Input\InputInterface;
 
@@ -17,34 +15,6 @@
  */
 class DbDumpApplication extends Application {
 
-  /**
-   * The database connection.
-   *
-   * @var \Drupal\Core\Database\Connection
-   */
-  protected $connection;
-
-  /**
-   * The module handler.
-   *
-   * @var \Drupal\Core\Extension\ModuleHandlerInterface
-   */
-  protected $moduleHandler;
-
-  /**
-   * Construct the application.
-   *
-   * @param \Drupal\Core\Database\Connection $connection
-   *   The database connection.
-   * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
-   *   The module handler.
-   */
-  function __construct(Connection $connection, ModuleHandlerInterface $module_handler) {
-    $this->connection = $connection;
-    $this->moduleHandler = $module_handler;
-    parent::__construct();
-  }
-
   /**
    * {@inheritdoc}
    */
@@ -58,7 +28,7 @@ protected function getCommandName(InputInterface $input) {
   protected function getDefaultCommands() {
     // Even though this is a single command, keep the HelpCommand (--help).
     $default_commands = parent::getDefaultCommands();
-    $default_commands[] = new DbDumpCommand($this->connection, $this->moduleHandler);
+    $default_commands[] = new DbDumpCommand();
     return $default_commands;
   }
 

core/lib/Drupal/Core/Command/DbDumpCommand.php

@@ -9,9 +9,8 @@
 
 use Drupal\Component\Utility\Variable;
 use Drupal\Core\Database\Connection;
-use Drupal\Core\Extension\ModuleHandlerInterface;
-use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 /**
@@ -30,21 +29,7 @@
  *
  * @see \Drupal\Core\Command\DbDumpApplication
  */
-class DbDumpCommand extends Command {
-
-  /**
-   * The database connection.
-   *
-   * @var \Drupal\Core\Database\Connection $connection
-   */
-  protected $connection;
-
-  /**
-   * The module handler.
-   *
-   * @var \Drupal\Core\Extension\ModuleHandlerInterface
-   */
-  protected $moduleHandler;
+class DbDumpCommand extends DbCommandBase {
 
   /**
    * An array of table patterns to exclude completely.
@@ -55,81 +40,78 @@ class DbDumpCommand extends Command {
    */
   protected $excludeTables = ['simpletest.+'];
 
-  /**
-   * Table patterns for which to only dump the schema, no data.
-   *
-   * @var array
-   */
-  protected $schemaOnly = ['cache.*', 'sessions', 'watchdog'];
-
-  /**
-   * Construct the database dump command.
-   *
-   * @param \Drupal\Core\Database\Connection $connection
-   *   The database connection to use.
-   * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
-   *   The module handler to use.
-   */
-  function __construct(Connection $connection, ModuleHandlerInterface $module_handler) {
-    // Check this is MySQL.
-    if ($connection->databaseType() !== 'mysql') {
-      throw new \RuntimeException('This script can only be used with MySQL database backends.');
-    }
-
-    $this->connection = $connection;
-    $this->moduleHandler = $module_handler;
-    parent::__construct();
-  }
-
   /**
    * {@inheritdoc}
    */
   protected function configure() {
     $this->setName('dump-database-d8-mysql')
-      ->setDescription('Dump the current database to a generation script');
+      ->setDescription('Dump the current database to a generation script')
+      ->addOption('schema-only', NULL, InputOption::VALUE_OPTIONAL, 'A comma separated list of tables to only export the schema without data.', 'cache.*,sessions,watchdog');
+    parent::configure();
   }
 
   /**
    * {@inheritdoc}
    */
   protected function execute(InputInterface $input, OutputInterface $output) {
+    $connection = $this->getDatabaseConnection($input);
+
     // If not explicitly set, disable ANSI which will break generated php.
     if ($input->hasParameterOption(['--ansi']) !== TRUE) {
       $output->setDecorated(FALSE);
     }
 
-    $output->writeln($this->generateScript(), OutputInterface::OUTPUT_RAW);
+    $schema_tables = $input->getOption('schema-only');
+    $schema_tables = explode(',', $schema_tables);
+
+    $output->writeln($this->generateScript($connection, $schema_tables), OutputInterface::OUTPUT_RAW);
   }
 
   /**
    * Generates the database script.
    *
-   * @return string
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
+   * @param array $schema_only
+   *   Table patterns for which to only dump the schema, no data.
+   * @return string The PHP script.
    *   The PHP script.
    */
-  protected function generateScript() {
+  protected function generateScript(Connection $connection, array $schema_only = []) {
     $tables = '';
-    foreach ($this->getTables() as $table) {
-      $schema = $this->getTableSchema($table);
-      $data = $this->getTableData($table);
+
+    $schema_only_patterns = [];
+    foreach ($schema_only as $match) {
+      $schema_only_patterns[] = '/^' . $match . '$/';
+    }
+
+    foreach ($this->getTables($connection) as $table) {
+      $schema = $this->getTableSchema($connection, $table);
+      // Check for schema only.
+      if (empty($schema_only_patterns) || preg_replace($schema_only_patterns, '', $table)) {
+        $data = $this->getTableData($connection, $table);
+      }
+      else {
+        $data = [];
+      }
       $tables .= $this->getTableScript($table, $schema, $data);
     }
     $script = $this->getTemplate();
     // Substitute in the tables.
     $script = str_replace('{{TABLES}}', trim($tables), $script);
-    // Modules.
-    $script = str_replace('{{MODULES}}', $this->getModulesScript(), $script);
     return trim($script);
   }
 
   /**
    * Returns a list of tables, not including those set to be excluded.
    *
-   * @return array
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
+   * @return array An array of table names.
    *   An array of table names.
    */
-  protected function getTables() {
-    $tables = array_values($this->connection->schema()->findTables('%'));
+  protected function getTables(Connection $connection) {
+    $tables = array_values($connection->schema()->findTables('%'));
 
     foreach ($tables as $key => $table) {
       // Remove any explicitly excluded tables.
@@ -146,6 +128,8 @@ protected function getTables() {
   /**
    * Returns a schema array for a given table.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $table
    *   The table name.
    *
@@ -154,14 +138,19 @@ protected function getTables() {
    *
    * @todo This implementation is hard-coded for MySQL.
    */
-  protected function getTableSchema($table) {
-    $query = $this->connection->query("SHOW FULL COLUMNS FROM {" . $table . "}");
+  protected function getTableSchema(Connection $connection, $table) {
+    // Check this is MySQL.
+    if ($connection->databaseType() !== 'mysql') {
+      throw new \RuntimeException('This script can only be used with MySQL database backends.');
+    }
+
+    $query = $connection->query("SHOW FULL COLUMNS FROM {" . $table . "}");
     $definition = [];
     while (($row = $query->fetchAssoc()) !== FALSE) {
       $name = $row['Field'];
       // Parse out the field type and meta information.
       preg_match('@([a-z]+)(?:\((\d+)(?:,(\d+))?\))?\s*(unsigned)?@', $row['Type'], $matches);
-      $type  = $this->fieldTypeMap($matches[1]);
+      $type  = $this->fieldTypeMap($connection, $matches[1]);
       if ($row['Extra'] === 'auto_increment') {
         // If this is an auto increment, then the type is 'serial'.
         $type = 'serial';
@@ -170,7 +159,7 @@ protected function getTableSchema($table) {
         'type' => $type,
         'not null' => $row['Null'] === 'NO',
       ];
-      if ($size = $this->fieldSizeMap($matches[1])) {
+      if ($size = $this->fieldSizeMap($connection, $matches[1])) {
         $definition['fields'][$name]['size'] = $size;
       }
       if (isset($matches[2]) && $type === 'numeric') {
@@ -216,10 +205,10 @@ protected function getTableSchema($table) {
     }
 
     // Set primary key, unique keys, and indexes.
-    $this->getTableIndexes($table, $definition);
+    $this->getTableIndexes($connection, $table, $definition);
 
     // Set table collation.
-    $this->getTableCollation($table, $definition);
+    $this->getTableCollation($connection, $table, $definition);
 
     return $definition;
   }
@@ -227,15 +216,17 @@ protected function getTableSchema($table) {
   /**
    * Adds primary key, unique keys, and index information to the schema.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $table
    *   The table to find indexes for.
    * @param array &$definition
    *   The schema definition to modify.
    */
-  protected function getTableIndexes($table, &$definition) {
+  protected function getTableIndexes(Connection $connection, $table, &$definition) {
     // Note, this query doesn't support ordering, so that is worked around
     // below by keying the array on Seq_in_index.
-    $query = $this->connection->query("SHOW INDEX FROM {" . $table . "}");
+    $query = $connection->query("SHOW INDEX FROM {" . $table . "}");
     while (($row = $query->fetchAssoc()) !== FALSE) {
       $index_name = $row['Key_name'];
       $column = $row['Column_name'];
@@ -262,13 +253,15 @@ protected function getTableIndexes($table, &$definition) {
   /**
    * Set the table collation.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $table
    *   The table to find indexes for.
    * @param array &$definition
    *   The schema definition to modify.
    */
-  protected function getTableCollation($table, &$definition) {
-    $query = $this->connection->query("SHOW TABLE STATUS LIKE '{" . $table . "}'");
+  protected function getTableCollation(Connection $connection, $table, &$definition) {
+    $query = $connection->query("SHOW TABLE STATUS LIKE '{" . $table . "}'");
     $data = $query->fetchAssoc();
 
     // Set `mysql_character_set`. This will be ignored by other backends.
@@ -280,21 +273,17 @@ protected function getTableCollation($table, &$definition) {
    *
    * If a table is set to be schema only, and empty array is returned.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $table
    *   The table to query.
    *
    * @return array
    *   The data from the table as an array.
    */
-  protected function getTableData($table) {
-    // Check for schema only.
-    foreach ($this->schemaOnly as $schema_only) {
-      if (preg_match('/^' . $schema_only . '$/', $table)) {
-        return [];
-      }
-    }
-    $order = $this->getFieldOrder($table);
-    $query = $this->connection->query("SELECT * FROM {" . $table . "} " . $order );
+  protected function getTableData(Connection $connection, $table) {
+    $order = $this->getFieldOrder($connection, $table);
+    $query = $connection->query("SELECT * FROM {" . $table . "} " . $order);
     $results = [];
     while (($row = $query->fetchAssoc()) !== FALSE) {
       $results[] = $row;
@@ -305,6 +294,8 @@ protected function getTableData($table) {
   /**
    * Given a database field type, return a Drupal type.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $type
    *   The MySQL field type.
    *
@@ -312,9 +303,9 @@ protected function getTableData($table) {
    *   The Drupal schema field type. If there is no mapping, the original field
    *   type is returned.
    */
-  protected function fieldTypeMap($type) {
+  protected function fieldTypeMap(Connection $connection, $type) {
     // Convert everything to lowercase.
-    $map = array_map('strtolower', $this->connection->schema()->getFieldTypeMap());
+    $map = array_map('strtolower', $connection->schema()->getFieldTypeMap());
     $map = array_flip($map);
 
     // The MySql map contains type:size. Remove the size part.
@@ -324,15 +315,17 @@ protected function fieldTypeMap($type) {
   /**
    * Given a database field type, return a Drupal size.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $type
    *   The MySQL field type.
    *
    * @return string
    *   The Drupal schema field size.
    */
-  protected function fieldSizeMap($type) {
+  protected function fieldSizeMap(Connection $connection, $type) {
     // Convert everything to lowercase.
-    $map = array_map('strtolower', $this->connection->schema()->getFieldTypeMap());
+    $map = array_map('strtolower', $connection->schema()->getFieldTypeMap());
     $map = array_flip($map);
 
     $schema_type = explode(':', $map[$type])[0];
@@ -346,24 +339,26 @@ protected function fieldSizeMap($type) {
   /**
    * Gets field ordering for a given table.
    *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   The database connection to use.
    * @param string $table
    *   The table name.
    *
    * @return string
    *   The order string to append to the query.
    */
-  protected function getFieldOrder($table) {
+  protected function getFieldOrder(Connection $connection, $table) {
     // @todo this is MySQL only since there are no Database API functions for
     // table column data.
     // @todo this code is duplicated in `core/scripts/migrate-db.sh`.
-    $connection_info = $this->connection->getConnectionOptions();
+    $connection_info = $connection->getConnectionOptions();
     // Order by primary keys.
     $order = '';
     $query = "SELECT `COLUMN_NAME` FROM `information_schema`.`COLUMNS`
     WHERE (`TABLE_SCHEMA` = '" . $connection_info['database'] . "')
     AND (`TABLE_NAME` = '{" . $table . "}') AND (`COLUMN_KEY` = 'PRI')
     ORDER BY COLUMN_NAME";
-    $results = $this->connection->query($query);
+    $results = $connection->query($query);
     while (($row = $results->fetchAssoc()) !== FALSE) {
       $order .= $row['COLUMN_NAME'] . ', ';
     }
@@ -384,12 +379,9 @@ protected function getTemplate() {
 <?php
 /**
  * @file
- * Filled installation of Drupal 8.0, for test purposes.
- *
- * This file was generated by the dump-database-d8.php script, from an
- * installation of Drupal 8. It has the following modules installed:
+ * A database agnostic dump for testing purposes.
  *
-{{MODULES}}
+ * This file was generated by the Drupal 8.0 db-tools.php script.
  */
 
 use Drupal\Core\Database\Database;
@@ -431,20 +423,4 @@ protected function getTableScript($table, array $schema, array $data) {
     return $output;
   }
 
-  /**
-   * List of modules enabled for insertion into the script docblock.
-   *
-   * @return string
-   *   The formatted list of enabled modules.
-   */
-  protected function getModulesScript() {
-    $output = '';
-    $modules = $this->moduleHandler->getModuleList();
-    ksort($modules);
-    foreach ($modules as $module => $filename) {
-      $output .= " *  - $module\n";
-    }
-    return rtrim($output, "\n");
-  }
-
 }

core/lib/Drupal/Core/Command/DbImportCommand.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Command\DbDumpCommand.
+ */
+
+namespace Drupal\Core\Command;
+
+use Drupal\Core\Database\Connection;
+use Drupal\Core\Database\Database;
+use Drupal\Core\Database\SchemaObjectExistsException;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Provides a command to import the current database from a script.
+ *
+ * This script runs on databases exported using using one of the database dump
+ * commands and imports it into the current database connection.
+ *
+ * @see \Drupal\Core\Command\DbImportApplication
+ */
+class DbImportCommand extends DbCommandBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function configure() {
+    parent::configure();
+    $this->setName('import')
+      ->setDescription('Import database from a generation script.')
+      ->addArgument('script', InputOption::VALUE_REQUIRED, 'Import script');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function execute(InputInterface $input, OutputInterface $output) {
+    $script = $input->getArgument('script');
+    if (!is_file($script)) {
+      $output->writeln('File must exist.');
+      return;
+    }
+
+    $connection = $this->getDatabaseConnection($input);
+    $this->runScript($connection, $script);
+    $output->writeln('Import completed successfully.');
+  }
+
+  /**
+   * Run the database script.
+   *
+   * @param \Drupal\Core\Database\Connection $connection
+   *   Connection used by the script when included.
+   * @param string $script
+   *   Path to dump script.
+   */
+  protected function runScript(Connection $connection, $script) {
+    if (substr($script, -3) == '.gz') {
+      $script = "compress.zlib://$script";
+    }
+    try {
+      require $script;
+    }
+    catch (SchemaObjectExistsException $e) {
+      throw new \RuntimeException('An existing Drupal installation exists at this location. Try removing all tables or changing the database prefix in your settings.php file.');
+    }
+  }
+
+}

core/lib/Drupal/Core/Command/DbToolsApplication.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Command\DbToolsApplication.
+ */
+
+namespace Drupal\Core\Command;
+
+use Symfony\Component\Console\Application;
+
+/**
+ * Provides a command to import a database generation script.
+ */
+class DbToolsApplication extends Application {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function __construct() {
+    parent::__construct('Database Tools', '8.0.x');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function getDefaultCommands() {
+    $default_commands = parent::getDefaultCommands();
+    $default_commands[] = new DbDumpCommand();
+    $default_commands[] = new DbImportCommand();
+    return $default_commands;
+  }
+
+}

core/modules/system/src/Tests/Update/DbDumpTest.php

@@ -151,21 +151,11 @@ public function testDbDumpCommand() {
       return;
     }
 
-    $application = new DbDumpApplication(Database::getConnection(), $this->container->get('module_handler'));
+    $application = new DbDumpApplication();
     $command = $application->find('dump-database-d8-mysql');
     $command_tester = new CommandTester($command);
     $command_tester->execute([]);
 
-    // The enabled modules should be present in the docblock.
-    $modules = static::$modules;
-    asort($modules);
-    $pattern = preg_quote(implode("\n *  - ", $modules));
-    $this->assertTrue(preg_match('/' . $pattern . '/', $command_tester->getDisplay()), 'Module list is contained in the docblock of the script.');