Commit d579f513 authored by catch's avatar catch

Issue #1314214 by stefan.r, phayes, ergophobe, YesCT, damienwhaley, kbasarab,...

Issue #1314214 by stefan.r, phayes, ergophobe, YesCT, damienwhaley, kbasarab, Tor Arne Thune, basic, pfrenssen, yannickoo, simolokid, fietserwin, bzrudi71: MySQL driver does not support full UTF-8 (emojis, asian symbols, mathematical symbols)
parent 0df55ad1
...@@ -64,7 +64,7 @@ public static function open(array &$connection_options = array()) { ...@@ -64,7 +64,7 @@ public static function open(array &$connection_options = array()) {
// Character set is added to dsn to ensure PDO uses the proper character // Character set is added to dsn to ensure PDO uses the proper character
// set when escaping. This has security implications. See // set when escaping. This has security implications. See
// https://www.drupal.org/node/1201452 for further discussion. // https://www.drupal.org/node/1201452 for further discussion.
$dsn .= ';charset=utf8'; $dsn .= ';charset=utf8mb4';
if (!empty($connection_options['database'])) { if (!empty($connection_options['database'])) {
$dsn .= ';dbname=' . $connection_options['database']; $dsn .= ';dbname=' . $connection_options['database'];
} }
...@@ -92,13 +92,13 @@ public static function open(array &$connection_options = array()) { ...@@ -92,13 +92,13 @@ public static function open(array &$connection_options = array()) {
$pdo = new \PDO($dsn, $connection_options['username'], $connection_options['password'], $connection_options['pdo']); $pdo = new \PDO($dsn, $connection_options['username'], $connection_options['password'], $connection_options['pdo']);
// Force MySQL to use the UTF-8 character set. Also set the collation, if a // Force MySQL to use the UTF-8 character set. Also set the collation, if a
// certain one has been set; otherwise, MySQL defaults to 'utf8_general_ci' // certain one has been set; otherwise, MySQL defaults to
// for UTF-8. // 'utf8mb4_general_ci' for utf8mb4.
if (!empty($connection_options['collation'])) { if (!empty($connection_options['collation'])) {
$pdo->exec('SET NAMES utf8 COLLATE ' . $connection_options['collation']); $pdo->exec('SET NAMES utf8mb4 COLLATE ' . $connection_options['collation']);
} }
else { else {
$pdo->exec('SET NAMES utf8'); $pdo->exec('SET NAMES utf8mb4');
} }
// Set MySQL init_commands if not already defined. Default Drupal's MySQL // Set MySQL init_commands if not already defined. Default Drupal's MySQL
......
...@@ -31,6 +31,19 @@ class Schema extends DatabaseSchema { ...@@ -31,6 +31,19 @@ class Schema extends DatabaseSchema {
*/ */
const COMMENT_MAX_COLUMN = 255; const COMMENT_MAX_COLUMN = 255;
/**
* @var array
* List of MySQL string types.
*/
protected $mysqlStringTypes = array(
'VARCHAR',
'CHAR',
'TINYTEXT',
'MEDIUMTEXT',
'LONGTEXT',
'TEXT',
);
/** /**
* Get information about the table and database name from the prefix. * Get information about the table and database name from the prefix.
* *
...@@ -87,7 +100,7 @@ protected function createTableSql($name, $table) { ...@@ -87,7 +100,7 @@ protected function createTableSql($name, $table) {
// Provide defaults if needed. // Provide defaults if needed.
$table += array( $table += array(
'mysql_engine' => 'InnoDB', 'mysql_engine' => 'InnoDB',
'mysql_character_set' => 'utf8', 'mysql_character_set' => 'utf8mb4',
); );
$sql = "CREATE TABLE {" . $name . "} (\n"; $sql = "CREATE TABLE {" . $name . "} (\n";
...@@ -108,8 +121,8 @@ protected function createTableSql($name, $table) { ...@@ -108,8 +121,8 @@ protected function createTableSql($name, $table) {
$sql .= 'ENGINE = ' . $table['mysql_engine'] . ' DEFAULT CHARACTER SET ' . $table['mysql_character_set']; $sql .= 'ENGINE = ' . $table['mysql_engine'] . ' DEFAULT CHARACTER SET ' . $table['mysql_character_set'];
// By default, MySQL uses the default collation for new tables, which is // By default, MySQL uses the default collation for new tables, which is
// 'utf8_general_ci' for utf8. If an alternate collation has been set, it // 'utf8mb4_general_ci' for utf8mb4. If an alternate collation has been
// needs to be explicitly specified. // set, it needs to be explicitly specified.
// @see DatabaseConnection_mysql // @see DatabaseConnection_mysql
if (!empty($info['collation'])) { if (!empty($info['collation'])) {
$sql .= ' COLLATE ' . $info['collation']; $sql .= ' COLLATE ' . $info['collation'];
...@@ -129,15 +142,15 @@ protected function createTableSql($name, $table) { ...@@ -129,15 +142,15 @@ protected function createTableSql($name, $table) {
* Before passing a field out of a schema definition into this function it has * Before passing a field out of a schema definition into this function it has
* to be processed by _db_process_field(). * to be processed by _db_process_field().
* *
* @param $name * @param string $name
* Name of the field. * Name of the field.
* @param $spec * @param array $spec
* The field specification, as per the schema data structure format. * The field specification, as per the schema data structure format.
*/ */
protected function createFieldSql($name, $spec) { protected function createFieldSql($name, $spec) {
$sql = "`" . $name . "` " . $spec['mysql_type']; $sql = "`" . $name . "` " . $spec['mysql_type'];
if (in_array($spec['mysql_type'], array('VARCHAR', 'CHAR', 'TINYTEXT', 'MEDIUMTEXT', 'LONGTEXT', 'TEXT'))) { if (in_array($spec['mysql_type'], $this->mysqlStringTypes)) {
if (isset($spec['length'])) { if (isset($spec['length'])) {
$sql .= '(' . $spec['length'] . ')'; $sql .= '(' . $spec['length'] . ')';
} }
...@@ -271,7 +284,8 @@ protected function createKeysSql($spec) { ...@@ -271,7 +284,8 @@ protected function createKeysSql($spec) {
} }
} }
if (!empty($spec['indexes'])) { if (!empty($spec['indexes'])) {
foreach ($spec['indexes'] as $index => $fields) { $indexes = $this->getNormalizedIndexes($spec);
foreach ($indexes as $index => $fields) {
$keys[] = 'INDEX `' . $index . '` (' . $this->createKeySql($fields) . ')'; $keys[] = 'INDEX `' . $index . '` (' . $this->createKeySql($fields) . ')';
} }
} }
...@@ -279,6 +293,63 @@ protected function createKeysSql($spec) { ...@@ -279,6 +293,63 @@ protected function createKeysSql($spec) {
return $keys; return $keys;
} }
/**
* Gets normalized indexes from a table specification.
*
* Shortens indexes to 191 characters if they apply to utf8mb4-encoded
* fields, in order to comply with the InnoDB index limitation of 756 bytes.
*
* @param $spec
* The table specification.
*
* @return array
* List of shortened indexes.
*/
protected function getNormalizedIndexes($spec) {
$indexes = $spec['indexes'];
foreach ($indexes as $index_name => $index_fields) {
foreach ($index_fields as $index_key => $index_field) {
// Get the name of the field from the index specification.
$field_name = is_array($index_field) ? $index_field[0] : $index_field;
// Check whether the field is defined in the table specification.
if (isset($spec['fields'][$field_name])) {
// Get the MySQL type from the processed field.
$mysql_field = $this->processField($spec['fields'][$field_name]);
if (in_array($mysql_field['mysql_type'], $this->mysqlStringTypes)) {
// Check whether we need to shorten the index.
if ((!isset($mysql_field['type']) || $mysql_field['type'] != 'varchar_ascii') && (!isset($mysql_field['length']) || $mysql_field['length'] > 191)) {
// Limit the index length to 191 characters.
$this->shortenIndex($indexes[$index_name][$index_key]);
}
}
}
}
}
return $indexes;
}
/**
* Helper function for normalizeIndexes().
*
* Shortens an index to 191 characters.
*
* @param array $index
* The index array to be used in createKeySql.
*
* @see Drupal\Core\Database\Driver\mysql\Schema::createKeySql()
* @see Drupal\Core\Database\Driver\mysql\Schema::normalizeIndexes()
*/
protected function shortenIndex(&$index) {
if (is_array($index)) {
if ($index[1] > 191) {
$index[1] = 191;
}
}
else {
$index = array($index, 191);
}
}
protected function createKeySql($fields) { protected function createKeySql($fields) {
$return = array(); $return = array();
foreach ($fields as $field) { foreach ($fields as $field) {
......
...@@ -33,7 +33,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st ...@@ -33,7 +33,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st
break; break;
case 'title': case 'title':
$this->addSharedTableFieldUniqueKey($storage_definition, $schema); $this->addSharedTableFieldIndex($storage_definition, $schema, TRUE);
break; break;
} }
} }
......
<?php
/**
* @file
* Contains \Drupal\block_content\BlockContentStorageSchema.
*/
namespace Drupal\block_content;
use Drupal\Core\Entity\ContentEntityTypeInterface;
use Drupal\Core\Entity\Sql\SqlContentEntityStorageSchema;
use Drupal\Core\Field\FieldStorageDefinitionInterface;
/**
* Defines the block content schema handler.
*/
class BlockContentStorageSchema extends SqlContentEntityStorageSchema {
/**
* {@inheritdoc}
*/
protected function getEntitySchema(ContentEntityTypeInterface $entity_type, $reset = FALSE) {
$schema = parent::getEntitySchema($entity_type, $reset);
$schema['block_content_field_data']['unique keys'] += array(
'block_content__info' => array('info', 'langcode'),
);
return $schema;
}
/**
* {@inheritdoc}
*/
protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $storage_definition, $table_name, array $column_mapping) {
$schema = parent::getSharedTableFieldSchema($storage_definition, $table_name, $column_mapping);
$field_name = $storage_definition->getName();
if ($table_name == 'block_content_field_data') {
switch ($field_name) {
case 'info':
// Improves the performance of the block_content__info index defined
// in getEntitySchema().
$schema['fields'][$field_name]['not null'] = TRUE;
break;
}
}
return $schema;
}
}
...@@ -23,7 +23,6 @@ ...@@ -23,7 +23,6 @@
* bundle_label = @Translation("Custom block type"), * bundle_label = @Translation("Custom block type"),
* handlers = { * handlers = {
* "storage" = "Drupal\Core\Entity\Sql\SqlContentEntityStorage", * "storage" = "Drupal\Core\Entity\Sql\SqlContentEntityStorage",
* "storage_schema" = "Drupal\block_content\BlockContentStorageSchema",
* "access" = "Drupal\block_content\BlockContentAccessControlHandler", * "access" = "Drupal\block_content\BlockContentAccessControlHandler",
* "list_builder" = "Drupal\block_content\BlockContentListBuilder", * "list_builder" = "Drupal\block_content\BlockContentListBuilder",
* "view_builder" = "Drupal\block_content\BlockContentViewBuilder", * "view_builder" = "Drupal\block_content\BlockContentViewBuilder",
......
...@@ -252,7 +252,8 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ...@@ -252,7 +252,8 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
->setLabel(t('URI')) ->setLabel(t('URI'))
->setDescription(t('The URI to access the file (either local or remote).')) ->setDescription(t('The URI to access the file (either local or remote).'))
->setSetting('max_length', 255) ->setSetting('max_length', 255)
->setSetting('case_sensitive', TRUE); ->setSetting('case_sensitive', TRUE)
->addConstraint('FileUriUnique');
$fields['filemime'] = BaseFieldDefinition::create('string') $fields['filemime'] = BaseFieldDefinition::create('string')
->setLabel(t('File MIME type')) ->setLabel(t('File MIME type'))
......
...@@ -30,7 +30,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st ...@@ -30,7 +30,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st
break; break;
case 'uri': case 'uri':
$this->addSharedTableFieldUniqueKey($storage_definition, $schema, TRUE); $this->addSharedTableFieldIndex($storage_definition, $schema, TRUE);
break; break;
} }
} }
......
<?php
/**
* @file
* Contains \Drupal\file\Plugin\Validation\Constraint\FileUriUnique.
*/
namespace Drupal\file\Plugin\Validation\Constraint;
use Symfony\Component\Validator\Constraint;
/**
* Supports validating file URIs.
*
* @Constraint(
* id = "FileUriUnique",
* label = @Translation("File URI", context = "Validation")
* )
*/
class FileUriUnique extends Constraint {
public $message = 'The file %value already exists. Enter a unique file URI.';
/**
* {@inheritdoc}
*/
public function validatedBy() {
return '\Drupal\Core\Validation\Plugin\Validation\Constraint\UniqueFieldValueValidator';
}
}
...@@ -8,6 +8,9 @@ ...@@ -8,6 +8,9 @@
namespace Drupal\file\Tests; namespace Drupal\file\Tests;
use Drupal\file\Entity\File; use Drupal\file\Entity\File;
use Drupal\Core\Entity\EntityStorageException;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Entity\Sql\SqlEntityStorageInterface;
/** /**
* File saving tests. * File saving tests.
...@@ -57,16 +60,27 @@ function testFileSave() { ...@@ -57,16 +60,27 @@ function testFileSave() {
// Try to insert a second file with the same name apart from case insensitivity // Try to insert a second file with the same name apart from case insensitivity
// to ensure the 'uri' index allows for filenames with different cases. // to ensure the 'uri' index allows for filenames with different cases.
$uppercase_file = File::create(array( $uppercase_values = array(
'uid' => 1, 'uid' => 1,
'filename' => 'DRUPLICON.txt', 'filename' => 'DRUPLICON.txt',
'uri' => 'public://DRUPLICON.txt', 'uri' => 'public://DRUPLICON.txt',
'filemime' => 'text/plain', 'filemime' => 'text/plain',
'status' => FILE_STATUS_PERMANENT, 'status' => FILE_STATUS_PERMANENT,
)); );
$uppercase_file = File::create($uppercase_values);
file_put_contents($uppercase_file->getFileUri(), 'hello world'); file_put_contents($uppercase_file->getFileUri(), 'hello world');
$violations = $uppercase_file->validate();
$this->assertEqual(count($violations), 0, 'No violations when adding an URI with an existing filename in upper case.');
$uppercase_file->save(); $uppercase_file->save();
// Ensure the database URI uniqueness constraint is triggered.
$uppercase_file_duplicate = File::create($uppercase_values);
file_put_contents($uppercase_file_duplicate->getFileUri(), 'hello world');
$violations = $uppercase_file_duplicate->validate();
$this->assertEqual(count($violations), 1);
$this->assertEqual($violations[0]->getMessage(), t('The file %value already exists. Enter a unique file URI.', [
'%value' => $uppercase_file_duplicate->getFileUri(),
]));
// Ensure that file URI entity queries are case sensitive. // Ensure that file URI entity queries are case sensitive.
$fids = \Drupal::entityQuery('file') $fids = \Drupal::entityQuery('file')
->condition('uri', $uppercase_file->getFileUri()) ->condition('uri', $uppercase_file->getFileUri())
......
...@@ -261,6 +261,12 @@ protected function ensureTables() { ...@@ -261,6 +261,12 @@ protected function ensureTables() {
foreach ($this->migration->getSourcePlugin()->getIds() as $id_definition) { foreach ($this->migration->getSourcePlugin()->getIds() as $id_definition) {
$mapkey = 'sourceid' . $count++; $mapkey = 'sourceid' . $count++;
$source_id_schema[$mapkey] = $this->getFieldSchema($id_definition); $source_id_schema[$mapkey] = $this->getFieldSchema($id_definition);
// With InnoDB, utf8mb4-based primary keys can't be over 191 characters.
// Use ASCII-based primary keys instead.
if (isset($source_id_schema[$mapkey]['type']) && $source_id_schema[$mapkey]['type'] == 'varchar') {
$source_id_schema[$mapkey]['type'] = 'varchar_ascii';
}
$pks[] = $mapkey; $pks[] = $mapkey;
} }
......
...@@ -26,7 +26,7 @@ public function load() { ...@@ -26,7 +26,7 @@ public function load() {
), ),
'fields' => array( 'fields' => array(
'filename' => array( 'filename' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'not null' => TRUE, 'not null' => TRUE,
'length' => '255', 'length' => '255',
'default' => '', 'default' => '',
...@@ -916,4 +916,4 @@ public function load() { ...@@ -916,4 +916,4 @@ public function load() {
} }
} }
#8867fc0eccc6c8439bff0a269ec597ae #e15f00f5d9b1c571ee015c40f8fc7b00
...@@ -33,4 +33,16 @@ public function testHtmlHeadLinks() { ...@@ -33,4 +33,16 @@ public function testHtmlHeadLinks() {
$this->assertEqual($result[0]['href'], $node->url()); $this->assertEqual($result[0]['href'], $node->url());
} }
/**
* Tests that we store and retrieve multi-byte UTF-8 characters correctly.
*/
public function testMultiByteUtf8() {
$title = '🐝';
$this->assertTrue(mb_strlen($title, 'utf-8') < strlen($title), 'Title has multi-byte characters.');
$node = $this->drupalCreateNode(array('title' => $title));
$this->drupalGet($node->urlInfo());
$result = $this->xpath('//span[contains(@class, "field-name-title")]');
$this->assertEqual((string) $result[0], $title, 'The passed title was returned.');
}
} }
...@@ -26,16 +26,16 @@ class RegressionTest extends DatabaseTestBase { ...@@ -26,16 +26,16 @@ class RegressionTest extends DatabaseTestBase {
*/ */
function testRegression_310447() { function testRegression_310447() {
// That's a 255 character UTF-8 string. // That's a 255 character UTF-8 string.
$name = str_repeat("é", 255); $job = str_repeat("é", 255);
db_insert('test') db_insert('test')
->fields(array( ->fields(array(
'name' => $name, 'name' => $this->randomMachineName(),
'age' => 20, 'age' => 20,
'job' => 'Dancer', 'job' => $job,
))->execute(); ))->execute();
$from_database = db_query('SELECT name FROM {test} WHERE name = :name', array(':name' => $name))->fetchField(); $from_database = db_query('SELECT job FROM {test} WHERE job = :job', array(':job' => $job))->fetchField();
$this->assertIdentical($name, $from_database, 'The database handles UTF-8 characters cleanly.'); $this->assertIdentical($job, $from_database, 'The database handles UTF-8 characters cleanly.');
} }
/** /**
......
...@@ -72,7 +72,7 @@ function testSchema() { ...@@ -72,7 +72,7 @@ function testSchema() {
$columns = db_query('SHOW FULL COLUMNS FROM {test_table}'); $columns = db_query('SHOW FULL COLUMNS FROM {test_table}');
foreach ($columns as $column) { foreach ($columns as $column) {
if ($column->Field == 'test_field_string') { if ($column->Field == 'test_field_string') {
$string_check = ($column->Collation == 'utf8_general_ci'); $string_check = ($column->Collation == 'utf8mb4_general_ci');
} }
if ($column->Field == 'test_field_string_ascii') { if ($column->Field == 'test_field_string_ascii') {
$string_ascii_check = ($column->Collation == 'ascii_general_ci'); $string_ascii_check = ($column->Collation == 'ascii_general_ci');
...@@ -238,6 +238,101 @@ function testSchema() { ...@@ -238,6 +238,101 @@ function testSchema() {
$this->assertTrue(db_table_exists('test_timestamp'), 'Table with database specific datatype was created.'); $this->assertTrue(db_table_exists('test_timestamp'), 'Table with database specific datatype was created.');
} }
/**
* Tests that indexes on string fields are limited to 191 characters on MySQL.
*
* @see \Drupal\Core\Database\Driver\mysql\Schema::getNormalizedIndexes()
*/
function testIndexLength() {
if (Database::getConnection()->databaseType() != 'mysql') {
return;
}
$table_specification = array(
'fields' => array(
'id' => array(
'type' => 'int',
'default' => NULL,
),
'test_field_text' => array(
'type' => 'text',
'not null' => TRUE,
),
'test_field_string_long' => array(
'type' => 'varchar',
'length' => 255,
'not null' => TRUE,
),
'test_field_string_ascii_long' => array(
'type' => 'varchar_ascii',
'length' => 255,
),
'test_field_string_short' => array(
'type' => 'varchar',
'length' => 128,
'not null' => TRUE,
),
),
'indexes' => array(
'test_regular' => array(
'test_field_text',
'test_field_string_long',
'test_field_string_ascii_long',
'test_field_string_short',
),
'test_length' => array(
array('test_field_text', 128),
array('test_field_string_long', 128),
array('test_field_string_ascii_long', 128),
array('test_field_string_short', 128),
),
'test_mixed' => array(
array('test_field_text', 200),
'test_field_string_long',
array('test_field_string_ascii_long', 200),
'test_field_string_short',
),
),
);
db_create_table('test_table_index_length', $table_specification);
// Get index information.
$results = db_query('SHOW INDEX FROM {test_table_index_length}');
$expected_lengths = array(
'test_regular' => array(
'test_field_text' => 191,
'test_field_string_long' => 191,
'test_field_string_ascii_long' => NULL,
'test_field_string_short' => NULL,
),
'test_length' => array(
'test_field_text' => 128,
'test_field_string_long' => 128,
'test_field_string_ascii_long' => 128,
'test_field_string_short' => NULL,
),
'test_mixed' => array(
'test_field_text' => 191,
'test_field_string_long' => 191,
'test_field_string_ascii_long' => 200,
'test_field_string_short' => NULL,
),
);
// Count the number of columns defined in the indexes.
$column_count = 0;
foreach ($table_specification['indexes'] as $index) {
foreach ($index as $field) {
$column_count++;
}
}
$test_count = 0;
foreach ($results as $result) {
$this->assertEqual($result->Sub_part, $expected_lengths[$result->Key_name][$result->Column_name], 'Index length matches expected value.');
$test_count++;
}
$this->assertEqual($test_count, $column_count, 'Number of tests matches expected value.');
}
/** /**
* Tests inserting data into an existing table. * Tests inserting data into an existing table.
* *
......
...@@ -25,7 +25,7 @@ class UpdatePathTestBaseTest extends UpdatePathTestBase { ...@@ -25,7 +25,7 @@ class UpdatePathTestBaseTest extends UpdatePathTestBase {
* {@inheritdoc} * {@inheritdoc}
*/ */
protected function setUp() { protected function setUp() {
$this->databaseDumpFiles = [__DIR__ . '/../../../tests/fixtures/update/drupal-8.beta11.bare.standard.php.gz']; $this->databaseDumpFiles = [__DIR__ . '/../../../tests/fixtures/update/drupal-8.bare.standard.php.gz'];
parent::setUp(); parent::setUp();
} }
......
...@@ -24,7 +24,7 @@ function database_test_schema() { ...@@ -24,7 +24,7 @@ function database_test_schema() {
), ),
'name' => array( 'name' => array(
'description' => "A person's name", 'description' => "A person's name",
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -75,7 +75,7 @@ function database_test_schema() { ...@@ -75,7 +75,7 @@ function database_test_schema() {
), ),
'job' => array( 'job' => array(
'description' => "The person's job", 'description' => "The person's job",
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '',