Issue #1314214 by stefan.r, phayes, ergophobe, YesCT, damienwhaley, kbasarab,...

Issue #1314214 by stefan.r, phayes, ergophobe, YesCT, damienwhaley, kbasarab, Tor Arne Thune, basic, pfrenssen, yannickoo, simolokid, fietserwin, bzrudi71: MySQL driver does not support full UTF-8 (emojis, asian symbols, mathematical symbols)

core/lib/Drupal/Core/Database/Driver/mysql/Connection.php

@@ -64,7 +64,7 @@ public static function open(array &$connection_options = array()) {
     // Character set is added to dsn to ensure PDO uses the proper character
     // set when escaping. This has security implications. See
     // https://www.drupal.org/node/1201452 for further discussion.
-    $dsn .= ';charset=utf8';
+    $dsn .= ';charset=utf8mb4';
     if (!empty($connection_options['database'])) {
       $dsn .= ';dbname=' . $connection_options['database'];
     }
@@ -92,13 +92,13 @@ public static function open(array &$connection_options = array()) {
     $pdo = new \PDO($dsn, $connection_options['username'], $connection_options['password'], $connection_options['pdo']);
 
     // Force MySQL to use the UTF-8 character set. Also set the collation, if a
-    // certain one has been set; otherwise, MySQL defaults to 'utf8_general_ci'
-    // for UTF-8.
+    // certain one has been set; otherwise, MySQL defaults to
+    // 'utf8mb4_general_ci' for utf8mb4.
     if (!empty($connection_options['collation'])) {
-      $pdo->exec('SET NAMES utf8 COLLATE ' . $connection_options['collation']);
+      $pdo->exec('SET NAMES utf8mb4 COLLATE ' . $connection_options['collation']);
     }
     else {
-      $pdo->exec('SET NAMES utf8');
+      $pdo->exec('SET NAMES utf8mb4');
     }
 
     // Set MySQL init_commands if not already defined.  Default Drupal's MySQL

core/lib/Drupal/Core/Database/Driver/mysql/Schema.php

@@ -31,6 +31,19 @@ class Schema extends DatabaseSchema {
    */
   const COMMENT_MAX_COLUMN = 255;
 
+  /**
+   * @var array
+   *   List of MySQL string types.
+   */
+  protected $mysqlStringTypes = array(
+    'VARCHAR',
+    'CHAR',
+    'TINYTEXT',
+    'MEDIUMTEXT',
+    'LONGTEXT',
+    'TEXT',
+  );
+
   /**
    * Get information about the table and database name from the prefix.
    *
@@ -87,7 +100,7 @@ protected function createTableSql($name, $table) {
     // Provide defaults if needed.
     $table += array(
       'mysql_engine' => 'InnoDB',
-      'mysql_character_set' => 'utf8',
+      'mysql_character_set' => 'utf8mb4',
     );
 
     $sql = "CREATE TABLE {" . $name . "} (\n";
@@ -108,8 +121,8 @@ protected function createTableSql($name, $table) {
 
     $sql .= 'ENGINE = ' . $table['mysql_engine'] . ' DEFAULT CHARACTER SET ' . $table['mysql_character_set'];
     // By default, MySQL uses the default collation for new tables, which is
-    // 'utf8_general_ci' for utf8. If an alternate collation has been set, it
-    // needs to be explicitly specified.
+    // 'utf8mb4_general_ci' for utf8mb4. If an alternate collation has been
+    // set, it needs to be explicitly specified.
     // @see DatabaseConnection_mysql
     if (!empty($info['collation'])) {
       $sql .= ' COLLATE ' . $info['collation'];
@@ -129,15 +142,15 @@ protected function createTableSql($name, $table) {
    * Before passing a field out of a schema definition into this function it has
    * to be processed by _db_process_field().
    *
-   * @param $name
+   * @param string $name
    *   Name of the field.
-   * @param $spec
+   * @param array $spec
    *   The field specification, as per the schema data structure format.
    */
   protected function createFieldSql($name, $spec) {
     $sql = "`" . $name . "` " . $spec['mysql_type'];
 
-    if (in_array($spec['mysql_type'], array('VARCHAR', 'CHAR', 'TINYTEXT', 'MEDIUMTEXT', 'LONGTEXT', 'TEXT'))) {
+    if (in_array($spec['mysql_type'], $this->mysqlStringTypes)) {
       if (isset($spec['length'])) {
         $sql .= '(' . $spec['length'] . ')';
       }
@@ -271,7 +284,8 @@ protected function createKeysSql($spec) {
       }
     }
     if (!empty($spec['indexes'])) {
-      foreach ($spec['indexes'] as $index => $fields) {
+      $indexes = $this->getNormalizedIndexes($spec);
+      foreach ($indexes as $index => $fields) {
         $keys[] = 'INDEX `' . $index . '` (' . $this->createKeySql($fields) . ')';
       }
     }
@@ -279,6 +293,63 @@ protected function createKeysSql($spec) {
     return $keys;
   }
 
+  /**
+   * Gets normalized indexes from a table specification.
+   *
+   * Shortens indexes to 191 characters if they apply to utf8mb4-encoded
+   * fields, in order to comply with the InnoDB index limitation of 756 bytes.
+   *
+   * @param $spec
+   *   The table specification.
+   *
+   * @return array
+   *   List of shortened indexes.
+   */
+  protected function getNormalizedIndexes($spec) {
+    $indexes = $spec['indexes'];
+    foreach ($indexes as $index_name => $index_fields) {
+      foreach ($index_fields as $index_key => $index_field) {
+        // Get the name of the field from the index specification.
+        $field_name = is_array($index_field) ? $index_field[0] : $index_field;
+        // Check whether the field is defined in the table specification.
+        if (isset($spec['fields'][$field_name])) {
+          // Get the MySQL type from the processed field.
+          $mysql_field = $this->processField($spec['fields'][$field_name]);
+          if (in_array($mysql_field['mysql_type'], $this->mysqlStringTypes)) {
+            // Check whether we need to shorten the index.
+            if ((!isset($mysql_field['type']) || $mysql_field['type'] != 'varchar_ascii') && (!isset($mysql_field['length']) || $mysql_field['length'] > 191)) {
+              // Limit the index length to 191 characters.
+              $this->shortenIndex($indexes[$index_name][$index_key]);
+            }
+          }
+        }
+      }
+    }
+    return $indexes;
+  }
+
+  /**
+   * Helper function for normalizeIndexes().
+   *
+   * Shortens an index to 191 characters.
+   *
+   * @param array $index
+   *   The index array to be used in createKeySql.
+   *
+   * @see Drupal\Core\Database\Driver\mysql\Schema::createKeySql()
+   * @see Drupal\Core\Database\Driver\mysql\Schema::normalizeIndexes()
+   */
+  protected function shortenIndex(&$index) {
+    if (is_array($index)) {
+      if ($index[1] > 191) {
+        $index[1] = 191;
+      }
+    }
+    else {
+      $index = array($index, 191);
+    }
+  }
+
   protected function createKeySql($fields) {
     $return = array();
     foreach ($fields as $field) {

core/modules/aggregator/src/FeedStorageSchema.php

@@ -33,7 +33,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st
           break;
 
         case 'title':
-          $this->addSharedTableFieldUniqueKey($storage_definition, $schema);
+          $this->addSharedTableFieldIndex($storage_definition, $schema, TRUE);
           break;
       }
     }

core/modules/block_content/src/BlockContentStorageSchema.php

-<?php
-
-/**
- * @file
- * Contains \Drupal\block_content\BlockContentStorageSchema.
- */
-
-namespace Drupal\block_content;
-
-use Drupal\Core\Entity\ContentEntityTypeInterface;
-use Drupal\Core\Entity\Sql\SqlContentEntityStorageSchema;
-use Drupal\Core\Field\FieldStorageDefinitionInterface;
-
-/**
- * Defines the block content schema handler.
- */
-class BlockContentStorageSchema extends SqlContentEntityStorageSchema {
-
-  /**
-   * {@inheritdoc}
-   */
-  protected function getEntitySchema(ContentEntityTypeInterface $entity_type, $reset = FALSE) {
-    $schema = parent::getEntitySchema($entity_type, $reset);
-
-    $schema['block_content_field_data']['unique keys'] += array(
-      'block_content__info' => array('info', 'langcode'),
-    );
-
-    return $schema;
-  }
-
-  /**
-   * {@inheritdoc}
-   */
-  protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $storage_definition, $table_name, array $column_mapping) {
-    $schema = parent::getSharedTableFieldSchema($storage_definition, $table_name, $column_mapping);
-    $field_name = $storage_definition->getName();
-
-    if ($table_name == 'block_content_field_data') {
-      switch ($field_name) {
-        case 'info':
-          // Improves the performance of the block_content__info index defined
-          // in getEntitySchema().
-          $schema['fields'][$field_name]['not null'] = TRUE;
-          break;
-      }
-    }
-
-    return $schema;
-  }
-
-}

core/modules/block_content/src/Entity/BlockContent.php

@@ -23,7 +23,6 @@
  *   bundle_label = @Translation("Custom block type"),
  *   handlers = {
  *     "storage" = "Drupal\Core\Entity\Sql\SqlContentEntityStorage",
- *     "storage_schema" = "Drupal\block_content\BlockContentStorageSchema",
  *     "access" = "Drupal\block_content\BlockContentAccessControlHandler",
  *     "list_builder" = "Drupal\block_content\BlockContentListBuilder",
  *     "view_builder" = "Drupal\block_content\BlockContentViewBuilder",

core/modules/file/src/Entity/File.php

@@ -252,7 +252,8 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
       ->setLabel(t('URI'))
       ->setDescription(t('The URI to access the file (either local or remote).'))
       ->setSetting('max_length', 255)
-      ->setSetting('case_sensitive', TRUE);
+      ->setSetting('case_sensitive', TRUE)
+      ->addConstraint('FileUriUnique');
 
     $fields['filemime'] = BaseFieldDefinition::create('string')
       ->setLabel(t('File MIME type'))

core/modules/file/src/FileStorageSchema.php

@@ -30,7 +30,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st
           break;
 
         case 'uri':
-          $this->addSharedTableFieldUniqueKey($storage_definition, $schema, TRUE);
+          $this->addSharedTableFieldIndex($storage_definition, $schema, TRUE);
           break;
       }
     }

core/modules/file/src/Plugin/Validation/Constraint/FileUriUnique.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\file\Plugin\Validation\Constraint\FileUriUnique.
+ */
+
+namespace Drupal\file\Plugin\Validation\Constraint;
+
+use Symfony\Component\Validator\Constraint;
+
+/**
+ * Supports validating file URIs.
+ *
+ * @Constraint(
+ *   id = "FileUriUnique",
+ *   label = @Translation("File URI", context = "Validation")
+ * )
+ */
+class FileUriUnique extends Constraint {
+
+  public $message = 'The file %value already exists. Enter a unique file URI.';
+
+  /**
+   * {@inheritdoc}
+   */
+  public function validatedBy() {
+    return '\Drupal\Core\Validation\Plugin\Validation\Constraint\UniqueFieldValueValidator';
+  }
+
+}

core/modules/file/src/Tests/SaveTest.php

@@ -8,6 +8,9 @@
 namespace Drupal\file\Tests;
 
 use Drupal\file\Entity\File;
+use Drupal\Core\Entity\EntityStorageException;
+use Drupal\Core\Entity\EntityStorageInterface;
+use Drupal\Core\Entity\Sql\SqlEntityStorageInterface;
 
 /**
  * File saving tests.
@@ -57,16 +60,27 @@ function testFileSave() {
 
     // Try to insert a second file with the same name apart from case insensitivity
     // to ensure the 'uri' index allows for filenames with different cases.
-    $uppercase_file = File::create(array(
+    $uppercase_values = array(
       'uid' => 1,
       'filename' => 'DRUPLICON.txt',
       'uri' => 'public://DRUPLICON.txt',
       'filemime' => 'text/plain',
       'status' => FILE_STATUS_PERMANENT,
-    ));
+    );
+    $uppercase_file = File::create($uppercase_values);
     file_put_contents($uppercase_file->getFileUri(), 'hello world');
+    $violations = $uppercase_file->validate();
+    $this->assertEqual(count($violations), 0, 'No violations when adding an URI with an existing filename in upper case.');
     $uppercase_file->save();
 
+    // Ensure the database URI uniqueness constraint is triggered.
+    $uppercase_file_duplicate = File::create($uppercase_values);
+    file_put_contents($uppercase_file_duplicate->getFileUri(), 'hello world');
+    $violations = $uppercase_file_duplicate->validate();
+    $this->assertEqual(count($violations), 1);
+    $this->assertEqual($violations[0]->getMessage(), t('The file %value already exists. Enter a unique file URI.', [
+      '%value' => $uppercase_file_duplicate->getFileUri(),
+    ]));
     // Ensure that file URI entity queries are case sensitive.
     $fids = \Drupal::entityQuery('file')
       ->condition('uri', $uppercase_file->getFileUri())

core/modules/migrate/src/Plugin/migrate/id_map/Sql.php

@@ -261,6 +261,12 @@ protected function ensureTables() {
       foreach ($this->migration->getSourcePlugin()->getIds() as $id_definition) {
         $mapkey = 'sourceid' . $count++;
         $source_id_schema[$mapkey] = $this->getFieldSchema($id_definition);
+
+        // With InnoDB, utf8mb4-based primary keys can't be over 191 characters.
+        // Use ASCII-based primary keys instead.
+        if (isset($source_id_schema[$mapkey]['type']) && $source_id_schema[$mapkey]['type'] == 'varchar') {
+          $source_id_schema[$mapkey]['type'] = 'varchar_ascii';
+        }
         $pks[] = $mapkey;
       }
 

core/modules/migrate_drupal/src/Tests/Table/d6/System.php

@@ -26,7 +26,7 @@ public function load() {
       ),
       'fields' => array(
         'filename' => array(
-          'type' => 'varchar',
+          'type' => 'varchar_ascii',
           'not null' => TRUE,
           'length' => '255',
           'default' => '',
@@ -916,4 +916,4 @@ public function load() {
   }
 
 }
-#8867fc0eccc6c8439bff0a269ec597ae
+#e15f00f5d9b1c571ee015c40f8fc7b00

core/modules/node/src/Tests/NodeViewTest.php

@@ -33,4 +33,16 @@ public function testHtmlHeadLinks() {
     $this->assertEqual($result[0]['href'], $node->url());
   }
 
+  /**
+   * Tests that we store and retrieve multi-byte UTF-8 characters correctly.
+   */
+  public function testMultiByteUtf8() {
+    $title = '🐝';
+    $this->assertTrue(mb_strlen($title, 'utf-8') < strlen($title), 'Title has multi-byte characters.');
+    $node = $this->drupalCreateNode(array('title' => $title));
+    $this->drupalGet($node->urlInfo());
+    $result = $this->xpath('//span[contains(@class, "field-name-title")]');
+    $this->assertEqual((string) $result[0], $title, 'The passed title was returned.');
+  }
+
 }

core/modules/system/src/Tests/Database/RegressionTest.php

@@ -26,16 +26,16 @@ class RegressionTest extends DatabaseTestBase {
    */
   function testRegression_310447() {
     // That's a 255 character UTF-8 string.
-    $name = str_repeat("é", 255);
+    $job = str_repeat("é", 255);
     db_insert('test')
       ->fields(array(
-        'name' => $name,
+        'name' => $this->randomMachineName(),
         'age' => 20,
-        'job' => 'Dancer',
+        'job' => $job,
       ))->execute();
 
-    $from_database = db_query('SELECT name FROM {test} WHERE name = :name', array(':name' => $name))->fetchField();
-    $this->assertIdentical($name, $from_database, 'The database handles UTF-8 characters cleanly.');
+    $from_database = db_query('SELECT job FROM {test} WHERE job = :job', array(':job' => $job))->fetchField();
+    $this->assertIdentical($job, $from_database, 'The database handles UTF-8 characters cleanly.');
   }
 
   /**

core/modules/system/src/Tests/Database/SchemaTest.php

@@ -72,7 +72,7 @@ function testSchema() {
       $columns = db_query('SHOW FULL COLUMNS FROM {test_table}');
       foreach ($columns as $column) {
         if ($column->Field == 'test_field_string') {
-          $string_check = ($column->Collation == 'utf8_general_ci');
+          $string_check = ($column->Collation == 'utf8mb4_general_ci');
         }
         if ($column->Field == 'test_field_string_ascii') {
           $string_ascii_check = ($column->Collation == 'ascii_general_ci');
@@ -238,6 +238,101 @@ function testSchema() {
     $this->assertTrue(db_table_exists('test_timestamp'), 'Table with database specific datatype was created.');
   }
 
+  /**
+   * Tests that indexes on string fields are limited to 191 characters on MySQL.
+   *
+   * @see \Drupal\Core\Database\Driver\mysql\Schema::getNormalizedIndexes()
+   */
+  function testIndexLength() {
+    if (Database::getConnection()->databaseType() != 'mysql') {
+      return;
+    }
+    $table_specification = array(
+      'fields' => array(
+        'id'  => array(
+          'type' => 'int',
+          'default' => NULL,
+        ),
+        'test_field_text'  => array(
+          'type' => 'text',
+          'not null' => TRUE,
+        ),
+        'test_field_string_long'  => array(
+          'type' => 'varchar',
+          'length' => 255,
+          'not null' => TRUE,
+        ),
+        'test_field_string_ascii_long'  => array(
+          'type' => 'varchar_ascii',
+          'length' => 255,
+        ),
+        'test_field_string_short'  => array(
+          'type' => 'varchar',
+          'length' => 128,
+          'not null' => TRUE,
+        ),
+      ),
+      'indexes' => array(
+        'test_regular' => array(
+          'test_field_text',
+          'test_field_string_long',
+          'test_field_string_ascii_long',
+          'test_field_string_short',
+        ),
+        'test_length' => array(
+          array('test_field_text', 128),
+          array('test_field_string_long', 128),
+          array('test_field_string_ascii_long', 128),
+          array('test_field_string_short', 128),
+        ),
+        'test_mixed' => array(
+          array('test_field_text', 200),
+          'test_field_string_long',
+          array('test_field_string_ascii_long', 200),
+          'test_field_string_short',
+        ),
+      ),
+    );
+    db_create_table('test_table_index_length', $table_specification);
+
+    // Get index information.
+    $results = db_query('SHOW INDEX FROM {test_table_index_length}');
+    $expected_lengths = array(
+      'test_regular' => array(
+        'test_field_text' => 191,
+        'test_field_string_long' => 191,
+        'test_field_string_ascii_long' => NULL,
+        'test_field_string_short' => NULL,
+      ),
+      'test_length' => array(
+        'test_field_text' => 128,
+        'test_field_string_long' => 128,
+        'test_field_string_ascii_long' => 128,
+        'test_field_string_short' => NULL,
+      ),
+      'test_mixed' => array(
+        'test_field_text' => 191,
+        'test_field_string_long' => 191,
+        'test_field_string_ascii_long' => 200,
+        'test_field_string_short' => NULL,
+      ),
+    );
+
+    // Count the number of columns defined in the indexes.
+    $column_count = 0;
+    foreach ($table_specification['indexes'] as $index) {
+      foreach ($index as $field) {
+        $column_count++;
+      }
+    }
+    $test_count = 0;
+    foreach ($results as $result) {
+      $this->assertEqual($result->Sub_part, $expected_lengths[$result->Key_name][$result->Column_name], 'Index length matches expected value.');
+      $test_count++;
+    }
+    $this->assertEqual($test_count, $column_count, 'Number of tests matches expected value.');
+  }
+
   /**
    * Tests inserting data into an existing table.
    *

core/modules/system/src/Tests/Update/UpdatePathTestBaseTest.php

@@ -25,7 +25,7 @@ class UpdatePathTestBaseTest extends UpdatePathTestBase {
    * {@inheritdoc}
    */
   protected function setUp() {
-    $this->databaseDumpFiles = [__DIR__ . '/../../../tests/fixtures/update/drupal-8.beta11.bare.standard.php.gz'];
+    $this->databaseDumpFiles = [__DIR__ . '/../../../tests/fixtures/update/drupal-8.bare.standard.php.gz'];
     parent::setUp();
   }
 

core/modules/system/tests/modules/database_test/database_test.install

@@ -24,7 +24,7 @@ function database_test_schema() {
       ),
       'name' => array(
         'description' => "A person's name",
-        'type' => 'varchar',
+        'type' => 'varchar_ascii',
         'length' => 255,
         'not null' => TRUE,
         'default' => '',
@@ -75,7 +75,7 @@ function database_test_schema() {
       ),
       'job' => array(
         'description' => "The person's job",
-        'type' => 'varchar',
+        'type' => 'varchar_ascii',
         'length' => 255,
         'not null' => TRUE,
         'default' => '',
@@ -106,7 +106,7 @@ function database_test_schema() {
       ),
       'job' => array(
         'description' => "The person's job",
-        'type' => 'varchar',
+        'type' => 'varchar_ascii',
         'length' => 255,
         'not null' => TRUE,
         'default' => '',
@@ -197,7 +197,7 @@ function database_test_schema() {
       ),
       'name' => array(
         'description' => "A person's name.",
-        'type' => 'varchar',
+        'type' => 'varchar_ascii',
         'length' => 255,
         'not null' => FALSE,
         'default' => '',
@@ -228,7 +228,7 @@ function database_test_schema() {
       ),
       'name' => array(
         'description' => "A person's name.",
-        'type' => 'varchar',
+        'type' => 'varchar_ascii',
         'length' => 255,
         'not null' => FALSE,
         'default' => '',

core/modules/system/tests/modules/entity_test/src/Entity/EntityTestStringId.php

@@ -45,7 +45,10 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
     $fields['id'] = BaseFieldDefinition::create('string')
       ->setLabel(t('ID'))
       ->setDescription(t('The ID of the test entity.'))
-      ->setReadOnly(TRUE);
+      ->setReadOnly(TRUE)
+      // In order to work around the InnoDB 191 character limit on utf8mb4
+      // primary keys, we set the character set for the field to ASCII.
+      ->setSetting('is_ascii', TRUE);
     return $fields;
   }
 

core/modules/user/src/UserStorageSchema.php

@@ -52,6 +52,9 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st
           // Improves the performance of the user__name index defined
           // in getEntitySchema().
           $schema['fields'][$field_name]['not null'] = TRUE;
+          // Make sure the field is no longer than 191 characters so we can
+          // add a unique constraint in MySQL.
+          $schema['fields'][$field_name]['length'] = USERNAME_MAX_LENGTH;
           break;
 
         case 'mail':