From d57ee5f9a3b7a1fc296d785816664e8038d4d2c7 Mon Sep 17 00:00:00 2001
From: Alex Pott <alex.a.pott@googlemail.com>
Date: Tue, 5 May 2015 09:42:09 -0700
Subject: [PATCH] Issue #1923406 by stefan.r, yannickoo, catch, Crell,
amateescu, pwolanin, morgantocker, Damien Tournoud, sun: Use ASCII character
set on alphanumeric fields so we can index all 255 characters
---
core/config/schema/core.data_types.schema.yml | 3 ++
.../lib/Drupal/Core/Cache/DatabaseBackend.php | 16 +++++++----
.../Core/Cache/DatabaseCacheTagsChecksum.php | 2 +-
.../Drupal/Core/Config/DatabaseStorage.php | 4 +--
.../Core/Database/Driver/mysql/Schema.php | 6 ++++
.../Core/Database/Driver/pgsql/Schema.php | 2 ++
.../Core/Database/Driver/sqlite/Schema.php | 2 ++
.../Sql/SqlContentEntityStorageSchema.php | 6 ++--
.../Field/FieldType/EntityReferenceItem.php | 2 +-
.../Plugin/Field/FieldType/LanguageItem.php | 2 ++
.../Plugin/Field/FieldType/StringItem.php | 3 +-
.../Field/Plugin/Field/FieldType/UuidItem.php | 1 +
core/lib/Drupal/Core/Menu/MenuTreeStorage.php | 10 +++----
core/modules/aggregator/src/Entity/Feed.php | 1 +
core/modules/aggregator/src/FeedInterface.php | 3 +-
core/modules/ban/ban.install | 2 +-
core/modules/comment/comment.install | 4 +--
core/modules/comment/src/Entity/Comment.php | 2 ++
core/modules/dblog/dblog.install | 4 +--
core/modules/file/file.install | 6 ++--
core/modules/file/src/Entity/File.php | 1 +
core/modules/locale/locale.install | 14 +++++-----
.../src/Entity/MenuLinkContent.php | 4 ++-
core/modules/node/node.install | 4 +--
core/modules/search/search.install | 8 +++---
core/modules/shortcut/shortcut.install | 2 +-
core/modules/simpletest/simpletest.install | 6 ++--
core/modules/system/database.api.php | 2 ++
.../Tests/Cache/DatabaseBackendUnitTest.php | 20 +++++++++++++
.../system/src/Tests/Database/SchemaTest.php | 20 +++++++++++++
core/modules/system/system.install | 28 +++++++++----------
.../Plugin/Field/FieldType/TextLongItem.php | 2 +-
.../Field/FieldType/TextWithSummaryItem.php | 2 +-
core/modules/user/user.install | 4 +--
core/modules/views/src/EntityViewsData.php | 1 +
.../Sql/SqlContentEntityStorageSchemaTest.php | 12 ++++----
36 files changed, 141 insertions(+), 70 deletions(-)
diff --git a/core/config/schema/core.data_types.schema.yml b/core/config/schema/core.data_types.schema.yml
index cc89fa97a6cf..496808761d84 100644
--- a/core/config/schema/core.data_types.schema.yml
+++ b/core/config/schema/core.data_types.schema.yml
@@ -445,6 +445,9 @@ field.storage_settings.string:
case_sensitive:
type: boolean
label: 'Case sensitive'
+ is_ascii:
+ type: boolean
+ label: 'Contains US ASCII characters only'
field.field_settings.string:
type: mapping
diff --git a/core/lib/Drupal/Core/Cache/DatabaseBackend.php b/core/lib/Drupal/Core/Cache/DatabaseBackend.php
index 81e83bb0a11b..ff12845a78a1 100644
--- a/core/lib/Drupal/Core/Cache/DatabaseBackend.php
+++ b/core/lib/Drupal/Core/Cache/DatabaseBackend.php
@@ -422,22 +422,26 @@ protected function catchException(\Exception $e, $table_name = NULL) {
}
/**
- * Ensures that cache IDs have a maximum length of 255 characters.
+ * Normalizes a cache ID in order to comply with database limitations.
*
* @param string $cid
* The passed in cache ID.
*
* @return string
- * A cache ID that is at most 255 characters long.
+ * An ASCII-encoded cache ID that is at most 255 characters long.
*/
protected function normalizeCid($cid) {
- // Nothing to do if the ID length is 255 characters or less.
- if (strlen($cid) <= 255) {
+ // Nothing to do if the ID is a US ASCII string of 255 characters or less.
+ $cid_is_ascii = mb_check_encoding($cid, 'ASCII');
+ if (strlen($cid) <= 255 && $cid_is_ascii) {
return $cid;
}
// Return a string that uses as much as possible of the original cache ID
// with the hash appended.
$hash = Crypt::hashBase64($cid);
+ if (!$cid_is_ascii) {
+ return $hash;
+ }
return substr($cid, 0, 255 - strlen($hash)) . $hash;
}
@@ -450,7 +454,7 @@ public function schemaDefinition() {
'fields' => array(
'cid' => array(
'description' => 'Primary Key: Unique cache ID.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
@@ -491,7 +495,7 @@ public function schemaDefinition() {
),
'checksum' => array(
'description' => 'The tag invalidation checksum when this entry was saved.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
),
diff --git a/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php b/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php
index cc882eead164..14b34ec0add9 100644
--- a/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php
+++ b/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php
@@ -175,7 +175,7 @@ public function schemaDefinition() {
'fields' => array(
'tag' => array(
'description' => 'Namespace-prefixed tag string.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
diff --git a/core/lib/Drupal/Core/Config/DatabaseStorage.php b/core/lib/Drupal/Core/Config/DatabaseStorage.php
index 6c332731948c..646a1d54a2dd 100644
--- a/core/lib/Drupal/Core/Config/DatabaseStorage.php
+++ b/core/lib/Drupal/Core/Config/DatabaseStorage.php
@@ -192,14 +192,14 @@ protected static function schemaDefinition() {
'fields' => array(
'collection' => array(
'description' => 'Primary Key: Config object collection.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
),
'name' => array(
'description' => 'Primary Key: Config object name.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
diff --git a/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php b/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php
index d0aba49bac8c..8237c79112c8 100644
--- a/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php
+++ b/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php
@@ -144,6 +144,10 @@ protected function createFieldSql($name, $spec) {
if (!empty($spec['binary'])) {
$sql .= ' BINARY';
}
+ // Note we check for the "type" key here. "mysql_type" is VARCHAR:
+ if (isset($spec['type']) && $spec['type'] == 'varchar_ascii') {
+ $sql .= ' CHARACTER SET ascii COLLATE ascii_general_ci';
+ }
}
elseif (isset($spec['precision']) && isset($spec['scale'])) {
$sql .= '(' . $spec['precision'] . ', ' . $spec['scale'] . ')';
@@ -218,6 +222,8 @@ public function getFieldTypeMap() {
// database types back into schema types.
// $map does not use drupal_static as its value never changes.
static $map = array(
+ 'varchar_ascii:normal' => 'VARCHAR',
+
'varchar:normal' => 'VARCHAR',
'char:normal' => 'CHAR',
diff --git a/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php b/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php
index 506baa8e1f62..040b32a7bd9b 100644
--- a/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php
+++ b/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php
@@ -363,6 +363,8 @@ function getFieldTypeMap() {
// database types back into schema types.
// $map does not use drupal_static as its value never changes.
static $map = array(
+ 'varchar_ascii:normal' => 'varchar',
+
'varchar:normal' => 'varchar',
'char:normal' => 'character',
diff --git a/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php b/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php
index fe3f877289a2..154470dfec28 100644
--- a/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php
+++ b/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php
@@ -212,6 +212,8 @@ public function getFieldTypeMap() {
// database types back into schema types.
// $map does not use drupal_static as its value never changes.
static $map = array(
+ 'varchar_ascii:normal' => 'VARCHAR',
+
'varchar:normal' => 'VARCHAR',
'char:normal' => 'CHAR',
diff --git a/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php b/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php
index 6fb4ddfd1a7c..b0669697aa10 100644
--- a/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php
+++ b/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php
@@ -1568,7 +1568,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor
}
else {
$id_schema = array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'description' => 'The entity id this data is attached to',
@@ -1601,7 +1601,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor
'description' => $description_current,
'fields' => array(
'bundle' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
@@ -1617,7 +1617,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor
'entity_id' => $id_schema,
'revision_id' => $revision_id_schema,
'langcode' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 32,
'not null' => TRUE,
'default' => '',
diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php
index ca54aa85cdd7..21d70906b21d 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php
@@ -133,7 +133,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
$columns = array(
'target_id' => array(
'description' => 'The ID of the target entity.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
// If the target entities act as bundles for another entity type,
// their IDs should not exceed the maximum length for bundles.
'length' => $target_type_info->getBundleOf() ? EntityTypeInterface::BUNDLE_MAX_LENGTH : 255,
diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php
index 9ab60a4d4856..b7ce7fb790a0 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php
@@ -44,6 +44,7 @@ class LanguageItem extends FieldItemBase {
public static function propertyDefinitions(FieldStorageDefinitionInterface $field_definition) {
$properties['value'] = DataDefinition::create('string')
->setLabel(t('Language code'))
+ ->setSetting('is_ascii', TRUE)
->setRequired(TRUE);
$properties['language'] = DataReferenceDefinition::create('language')
@@ -75,6 +76,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
'value' => array(
'type' => 'varchar',
'length' => 12,
+ 'is_ascii' => TRUE,
),
),
);
diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php
index 8817c876823e..2329c86ca6be 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php
@@ -32,6 +32,7 @@ class StringItem extends StringItemBase {
public static function defaultStorageSettings() {
return array(
'max_length' => 255,
+ 'is_ascii' => FALSE,
) + parent::defaultStorageSettings();
}
@@ -42,7 +43,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
return array(
'columns' => array(
'value' => array(
- 'type' => 'varchar',
+ 'type' => $field_definition->getSetting('is_ascii') === TRUE ? 'varchar_ascii' : 'varchar',
'length' => (int) $field_definition->getSetting('max_length'),
'binary' => $field_definition->getSetting('case_sensitive'),
),
diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php
index 84d1ec582b7d..1bdd10e76abc 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php
@@ -30,6 +30,7 @@ class UuidItem extends StringItem {
public static function defaultStorageSettings() {
return array(
'max_length' => 128,
+ 'is_ascii' => TRUE,
) + parent::defaultStorageSettings();
}
diff --git a/core/lib/Drupal/Core/Menu/MenuTreeStorage.php b/core/lib/Drupal/Core/Menu/MenuTreeStorage.php
index 2ce68f778922..659d6be3926c 100644
--- a/core/lib/Drupal/Core/Menu/MenuTreeStorage.php
+++ b/core/lib/Drupal/Core/Menu/MenuTreeStorage.php
@@ -1193,7 +1193,7 @@ protected static function schemaDefinition() {
'fields' => array(
'menu_name' => array(
'description' => "The menu name. All links with the same menu name (such as 'tools') are part of the same menu.",
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 32,
'not null' => TRUE,
'default' => '',
@@ -1206,20 +1206,20 @@ protected static function schemaDefinition() {
),
'id' => array(
'description' => 'Unique machine name: the plugin ID.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
),
'parent' => array(
'description' => 'The plugin ID for the parent of this link.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
),
'route_name' => array(
'description' => 'The machine name of a defined Symfony Route this menu item represents.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
),
'route_param_key' => array(
@@ -1281,7 +1281,7 @@ protected static function schemaDefinition() {
),
'provider' => array(
'description' => 'The name of the module that generated this link.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH,
'not null' => TRUE,
'default' => 'system',
diff --git a/core/modules/aggregator/src/Entity/Feed.php b/core/modules/aggregator/src/Entity/Feed.php
index 6f3d2e2dbe85..a5ed08166f78 100644
--- a/core/modules/aggregator/src/Entity/Feed.php
+++ b/core/modules/aggregator/src/Entity/Feed.php
@@ -226,6 +226,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['hash'] = BaseFieldDefinition::create('string')
->setLabel(t('Hash'))
+ ->setSetting('is_ascii', TRUE)
->setDescription(t('Calculated hash of the feed data, used for validating cache.'));
$fields['etag'] = BaseFieldDefinition::create('string')
diff --git a/core/modules/aggregator/src/FeedInterface.php b/core/modules/aggregator/src/FeedInterface.php
index 29c6782e37ad..e678c90e4c88 100644
--- a/core/modules/aggregator/src/FeedInterface.php
+++ b/core/modules/aggregator/src/FeedInterface.php
@@ -170,7 +170,8 @@ public function getHash();
* Sets the calculated hash of the feed data, used for validating cache.
*
* @param string $hash
- * A string containing the calculated hash of the feed.
+ * A string containing the calculated hash of the feed. Must contain
+ * US ASCII characters only.
*
* @return \Drupal\aggregator\FeedInterface
* The class instance that this method is called on.
diff --git a/core/modules/ban/ban.install b/core/modules/ban/ban.install
index 7a5494fa7810..b2ea1fe3d337 100644
--- a/core/modules/ban/ban.install
+++ b/core/modules/ban/ban.install
@@ -20,7 +20,7 @@ function ban_schema() {
),
'ip' => array(
'description' => 'IP address',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 40,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/comment/comment.install b/core/modules/comment/comment.install
index 64aa6bd78f23..327feff349af 100644
--- a/core/modules/comment/comment.install
+++ b/core/modules/comment/comment.install
@@ -46,14 +46,14 @@ function comment_schema() {
'description' => 'The entity_id of the entity for which the statistics are compiled.',
),
'entity_type' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'not null' => TRUE,
'default' => 'node',
'length' => EntityTypeInterface::ID_MAX_LENGTH,
'description' => 'The entity_type of the entity to which this comment is a reply.',
),
'field_name' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'not null' => TRUE,
'default' => '',
'length' => FieldStorageConfig::NAME_MAX_LENGTH,
diff --git a/core/modules/comment/src/Entity/Comment.php b/core/modules/comment/src/Entity/Comment.php
index 798a69f1cf41..f83f54b58c09 100644
--- a/core/modules/comment/src/Entity/Comment.php
+++ b/core/modules/comment/src/Entity/Comment.php
@@ -303,6 +303,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['entity_type'] = BaseFieldDefinition::create('string')
->setLabel(t('Entity type'))
->setDescription(t('The entity type to which this comment is attached.'))
+ ->setSetting('is_ascii', TRUE)
->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH);
$fields['comment_type'] = BaseFieldDefinition::create('entity_reference')
@@ -313,6 +314,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['field_name'] = BaseFieldDefinition::create('string')
->setLabel(t('Comment field name'))
->setDescription(t('The field name through which this comment was added.'))
+ ->setSetting('is_ascii', TRUE)
->setSetting('max_length', FieldStorageConfig::NAME_MAX_LENGTH);
return $fields;
diff --git a/core/modules/dblog/dblog.install b/core/modules/dblog/dblog.install
index b7454c133415..1119c47dbfa6 100644
--- a/core/modules/dblog/dblog.install
+++ b/core/modules/dblog/dblog.install
@@ -25,7 +25,7 @@ function dblog_schema() {
'description' => 'The {users}.uid of the user who triggered the event.',
),
'type' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'default' => '',
@@ -69,7 +69,7 @@ function dblog_schema() {
'description' => 'URL of referring page.',
),
'hostname' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/file/file.install b/core/modules/file/file.install
index 6e188c6a5624..fd3503c2834d 100644
--- a/core/modules/file/file.install
+++ b/core/modules/file/file.install
@@ -20,21 +20,21 @@ function file_schema() {
),
'module' => array(
'description' => 'The name of the module that is using the file.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH,
'not null' => TRUE,
'default' => '',
),
'type' => array(
'description' => 'The name of the object type in which the file is used.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'default' => '',
),
'id' => array(
'description' => 'The primary key of the object using the file.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'default' => 0,
diff --git a/core/modules/file/src/Entity/File.php b/core/modules/file/src/Entity/File.php
index 42de0ae94e84..ff38ab320ae8 100644
--- a/core/modules/file/src/Entity/File.php
+++ b/core/modules/file/src/Entity/File.php
@@ -254,6 +254,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['filemime'] = BaseFieldDefinition::create('string')
->setLabel(t('File MIME type'))
+ ->setSetting('is_ascii', TRUE)
->setDescription(t("The file's MIME type."));
$fields['filesize'] = BaseFieldDefinition::create('integer')
diff --git a/core/modules/locale/locale.install b/core/modules/locale/locale.install
index 81aadf534b4e..025876706274 100644
--- a/core/modules/locale/locale.install
+++ b/core/modules/locale/locale.install
@@ -67,14 +67,14 @@ function locale_schema() {
'description' => 'The original string in English.',
),
'context' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
'description' => 'The context this string applies to.',
),
'version' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 20,
'not null' => TRUE,
'default' => 'none',
@@ -103,7 +103,7 @@ function locale_schema() {
'description' => 'Translation string value in this language.',
),
'language' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 12,
'not null' => TRUE,
'default' => '',
@@ -142,7 +142,7 @@ function locale_schema() {
'description' => 'Unique identifier of this string.',
),
'type' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 50,
'not null' => TRUE,
'default' => '',
@@ -156,7 +156,7 @@ function locale_schema() {
'description' => 'Type dependent location information (file name, path, etc).',
),
'version' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 20,
'not null' => TRUE,
'default' => 'none',
@@ -180,14 +180,14 @@ function locale_schema() {
'description' => 'File import status information for interface translation files.',
'fields' => array(
'project' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => '255',
'not null' => TRUE,
'default' => '',
'description' => 'A unique short name to identify the project the file belongs to.',
),
'langcode' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => '12',
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/menu_link_content/src/Entity/MenuLinkContent.php b/core/modules/menu_link_content/src/Entity/MenuLinkContent.php
index 5570a2a50ed7..d6610a0e15d7 100644
--- a/core/modules/menu_link_content/src/Entity/MenuLinkContent.php
+++ b/core/modules/menu_link_content/src/Entity/MenuLinkContent.php
@@ -250,6 +250,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
->setLabel(t('Bundle'))
->setDescription(t('The content menu link bundle.'))
->setSetting('max_length', EntityTypeInterface::BUNDLE_MAX_LENGTH)
+ ->setSetting('is_ascii', TRUE)
->setReadOnly(TRUE);
$fields['title'] = BaseFieldDefinition::create('string')
@@ -291,7 +292,8 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['menu_name'] = BaseFieldDefinition::create('string')
->setLabel(t('Menu name'))
->setDescription(t('The menu name. All links with the same menu name (such as "tools") are part of the same menu.'))
- ->setDefaultValue('tools');
+ ->setDefaultValue('tools')
+ ->setSetting('is_ascii', TRUE);
$fields['link'] = BaseFieldDefinition::create('link')
->setLabel(t('Link'))
diff --git a/core/modules/node/node.install b/core/modules/node/node.install
index fa0d245c3635..a7cb55e0c018 100644
--- a/core/modules/node/node.install
+++ b/core/modules/node/node.install
@@ -54,7 +54,7 @@ function node_schema() {
),
'langcode' => array(
'description' => 'The {language}.langcode of this node.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 12,
'not null' => TRUE,
'default' => '',
@@ -75,7 +75,7 @@ function node_schema() {
),
'realm' => array(
'description' => 'The realm in which the user must possess the grant ID. Each node access node can define one or more realms.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/search/search.install b/core/modules/search/search.install
index a298f2f93164..f920aa5ae99f 100644
--- a/core/modules/search/search.install
+++ b/core/modules/search/search.install
@@ -20,14 +20,14 @@ function search_schema() {
'description' => 'Search item ID, e.g. node ID for nodes.',
),
'langcode' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => '12',
'not null' => TRUE,
'description' => 'The {languages}.langcode of the item variant.',
'default' => '',
),
'type' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'description' => 'Type of item, e.g. node.',
@@ -67,14 +67,14 @@ function search_schema() {
'description' => 'The {search_dataset}.sid of the searchable item to which the word belongs.',
),
'langcode' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => '12',
'not null' => TRUE,
'description' => 'The {languages}.langcode of the item variant.',
'default' => '',
),
'type' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'description' => 'The {search_dataset}.type of the searchable item to which the word belongs.',
diff --git a/core/modules/shortcut/shortcut.install b/core/modules/shortcut/shortcut.install
index b046142043b8..73986b817829 100644
--- a/core/modules/shortcut/shortcut.install
+++ b/core/modules/shortcut/shortcut.install
@@ -20,7 +20,7 @@ function shortcut_schema() {
'description' => 'The {users}.uid for this set.',
),
'set_name' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 32,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/simpletest/simpletest.install b/core/modules/simpletest/simpletest.install
index 47c0ee2bd404..b4792ea1d92d 100644
--- a/core/modules/simpletest/simpletest.install
+++ b/core/modules/simpletest/simpletest.install
@@ -108,7 +108,7 @@ function simpletest_schema() {
'description' => 'Test ID, messages belonging to the same ID are reported together',
),
'test_class' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
@@ -127,14 +127,14 @@ function simpletest_schema() {
'description' => 'The message itself.',
),
'message_group' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
'description' => 'The message group this message belongs to. For example: warning, browser, user.',
),
'function' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/system/database.api.php b/core/modules/system/database.api.php
index a01ff5992f1d..f3d7fe2120ff 100644
--- a/core/modules/system/database.api.php
+++ b/core/modules/system/database.api.php
@@ -256,6 +256,8 @@
* 'float', 'numeric', or 'serial'. Most types just map to the according
* database engine specific datatypes. Use 'serial' for auto incrementing
* fields. This will expand to 'INT auto_increment' on MySQL.
+ * A special 'varchar_ascii' type is also available for limiting machine
+ * name field to US ASCII characters.
* - 'mysql_type', 'pgsql_type', 'sqlite_type', etc.: If you need to
* use a record type not included in the officially supported list
* of types above, you can specify a type for each database
diff --git a/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php b/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php
index e54198ba5cce..e182cc0b104b 100644
--- a/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php
+++ b/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php
@@ -33,4 +33,24 @@ protected function createCacheBackend($bin) {
return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin);
}
+ /**
+ * {@inheritdoc}
+ */
+ public function testSetGet() {
+ parent::testSetGet();
+ $backend = $this->getCacheBackend();
+
+ // Set up a cache ID that is not ASCII and longer than 255 characters so we
+ // can test cache ID normalization.
+ $cid_long = str_repeat('愛€', 500);
+ $cached_value_long = $this->randomMachineName();
+ $backend->set($cid_long, $cached_value_long);
+ $this->assertIdentical($cached_value_long, $backend->get($cid_long)->data, "Backend contains the correct value for long, non-ASCII cache id.");
+
+ $cid_short = '愛1€';
+ $cached_value_short = $this->randomMachineName();
+ $backend->set($cid_short, $cached_value_short);
+ $this->assertIdentical($cached_value_short, $backend->get($cid_short)->data, "Backend contains the correct value for short, non-ASCII cache id.");
+ }
+
}
diff --git a/core/modules/system/src/Tests/Database/SchemaTest.php b/core/modules/system/src/Tests/Database/SchemaTest.php
index 7811b505b270..9a234edd5b2d 100644
--- a/core/modules/system/src/Tests/Database/SchemaTest.php
+++ b/core/modules/system/src/Tests/Database/SchemaTest.php
@@ -49,6 +49,11 @@ function testSchema() {
'default' => "'\"funky default'\"",
'description' => 'Schema column description for string.',
),
+ 'test_field_string_ascii' => array(
+ 'type' => 'varchar_ascii',
+ 'length' => 255,
+ 'description' => 'Schema column description for ASCII string.',
+ ),
),
);
db_create_table('test_table', $table_specification);
@@ -62,6 +67,21 @@ function testSchema() {
// Assert that the column comment has been set.
$this->checkSchemaComment($table_specification['fields']['test_field']['description'], 'test_table', 'test_field');
+ if (Database::getConnection()->databaseType() == 'mysql') {
+ // Make sure that varchar fields have the correct collation.
+ $columns = db_query('SHOW FULL COLUMNS FROM {test_table}');
+ foreach ($columns as $column) {
+ if ($column->Field == 'test_field_string') {
+ $string_check = ($column->Collation == 'utf8_general_ci');
+ }
+ if ($column->Field == 'test_field_string_ascii') {
+ $string_ascii_check = ($column->Collation == 'ascii_general_ci');
+ }
+ }
+ $this->assertTrue(!empty($string_check), 'string field has the right collation.');
+ $this->assertTrue(!empty($string_ascii_check), 'ASCII string field has the right collation.');
+ }
+
// An insert without a value for the column 'test_table' should fail.
$this->assertFalse($this->tryInsert(), 'Insert without a default failed.');
diff --git a/core/modules/system/system.install b/core/modules/system/system.install
index 082bbc679387..6c3db4608c93 100644
--- a/core/modules/system/system.install
+++ b/core/modules/system/system.install
@@ -685,7 +685,7 @@ function system_schema() {
),
'token' => array(
'description' => "A string token generated against the current user's session id and the batch id, used to ensure that only the user who submitted the batch can effectively access it.",
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
),
@@ -717,14 +717,14 @@ function system_schema() {
),
'event' => array(
'description' => 'Name of event (e.g. contact).',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'default' => '',
),
'identifier' => array(
'description' => 'Identifier of the visitor, such as an IP address or hostname.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
@@ -754,14 +754,14 @@ function system_schema() {
'fields' => array(
'collection' => array(
'description' => 'A named collection of key and value pairs.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
),
'name' => array(
'description' => 'The key of the key-value pair. As KEY is a SQL reserved keyword, name was chosen instead.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
@@ -781,7 +781,7 @@ function system_schema() {
'fields' => array(
'collection' => array(
'description' => 'A named collection of key and value pairs.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
@@ -789,7 +789,7 @@ function system_schema() {
'name' => array(
// KEY is an SQL reserved word, so use 'name' as the key's field name.
'description' => 'The key of the key/value pair.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
@@ -824,7 +824,7 @@ function system_schema() {
'description' => 'Primary Key: Unique item ID.',
),
'name' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
@@ -862,7 +862,7 @@ function system_schema() {
'fields' => array(
'name' => array(
'description' => 'Primary Key: Machine name of this route',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => '',
@@ -911,14 +911,14 @@ function system_schema() {
'fields' => array(
'name' => array(
'description' => 'Primary Key: Unique name.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => ''
),
'value' => array(
'description' => 'A value for the semaphore.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
'not null' => TRUE,
'default' => ''
@@ -961,13 +961,13 @@ function system_schema() {
),
'sid' => array(
'description' => "A session ID (hashed). The value is generated by Drupal's session handlers.",
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
),
'hostname' => array(
'description' => 'The IP address that last used this session ID (sid).',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
@@ -1025,7 +1025,7 @@ function system_schema() {
),
'langcode' => array(
'description' => "The language code this alias is for; if 'und', the alias will be used for unknown languages. Each Drupal path can have an alias for each supported language.",
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 12,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php b/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php
index c55953c68752..9a2419497f14 100644
--- a/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php
+++ b/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php
@@ -34,7 +34,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
'size' => 'big',
),
'format' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
),
),
diff --git a/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php b/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php
index 48d4dacf8240..413ad4797e3a 100644
--- a/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php
+++ b/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php
@@ -68,7 +68,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
'size' => 'big',
),
'format' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 255,
),
),
diff --git a/core/modules/user/user.install b/core/modules/user/user.install
index 7b3c3d3a3936..dd55b49398c8 100644
--- a/core/modules/user/user.install
+++ b/core/modules/user/user.install
@@ -21,14 +21,14 @@ function user_schema() {
),
'module' => array(
'description' => 'The name of the module declaring the variable.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH,
'not null' => TRUE,
'default' => '',
),
'name' => array(
'description' => 'The identifier of the data.',
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'default' => '',
diff --git a/core/modules/views/src/EntityViewsData.php b/core/modules/views/src/EntityViewsData.php
index fa1cf3efc2a5..5cd321531c3c 100644
--- a/core/modules/views/src/EntityViewsData.php
+++ b/core/modules/views/src/EntityViewsData.php
@@ -411,6 +411,7 @@ protected function mapSingleFieldViewsData($table, $field_name, $field_type, $co
case 'char':
case 'string':
case 'varchar':
+ case 'varchar_ascii':
case 'tinytext':
case 'text':
case 'mediumtext':
diff --git a/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php b/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php
index 08f73cb4d08f..e67b8e6a8759 100644
--- a/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php
+++ b/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php
@@ -802,7 +802,7 @@ public function testDedicatedTableSchema() {
'description' => "Data storage for $entity_type_id field $field_name.",
'fields' => array(
'bundle' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => true,
'default' => '',
@@ -828,7 +828,7 @@ public function testDedicatedTableSchema() {
'description' => 'The entity revision id this data is attached to, which for an unversioned entity type is the same as the entity id',
),
'langcode' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 32,
'not null' => true,
'default' => '',
@@ -947,7 +947,7 @@ public function testDedicatedTableSchemaForEntityWithStringIdentifier() {
'description' => "Data storage for $entity_type_id field $field_name.",
'fields' => array(
'bundle' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => true,
'default' => '',
@@ -961,19 +961,19 @@ public function testDedicatedTableSchemaForEntityWithStringIdentifier() {
'description' => 'A boolean indicating whether this data item has been deleted',
),
'entity_id' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => true,
'description' => 'The entity id this data is attached to',
),
'revision_id' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 128,
'not null' => true,
'description' => 'The entity revision id this data is attached to, which for an unversioned entity type is the same as the entity id',
),
'langcode' => array(
- 'type' => 'varchar',
+ 'type' => 'varchar_ascii',
'length' => 32,
'not null' => true,
'default' => '',
--
GitLab