diff --git a/core/config/schema/core.data_types.schema.yml b/core/config/schema/core.data_types.schema.yml index cc89fa97a6cf52a62f0420f971cac57c40d655c1..496808761d84638a3fba96c08816d627a7e9f9a0 100644 --- a/core/config/schema/core.data_types.schema.yml +++ b/core/config/schema/core.data_types.schema.yml @@ -445,6 +445,9 @@ field.storage_settings.string: case_sensitive: type: boolean label: 'Case sensitive' + is_ascii: + type: boolean + label: 'Contains US ASCII characters only' field.field_settings.string: type: mapping diff --git a/core/lib/Drupal/Core/Cache/DatabaseBackend.php b/core/lib/Drupal/Core/Cache/DatabaseBackend.php index 81e83bb0a11b3a429d9e5dad609c6867edb6fdc7..ff12845a78a106a2f45f898aacd53fd4ef44c542 100644 --- a/core/lib/Drupal/Core/Cache/DatabaseBackend.php +++ b/core/lib/Drupal/Core/Cache/DatabaseBackend.php @@ -422,22 +422,26 @@ protected function catchException(\Exception $e, $table_name = NULL) { } /** - * Ensures that cache IDs have a maximum length of 255 characters. + * Normalizes a cache ID in order to comply with database limitations. * * @param string $cid * The passed in cache ID. * * @return string - * A cache ID that is at most 255 characters long. + * An ASCII-encoded cache ID that is at most 255 characters long. */ protected function normalizeCid($cid) { - // Nothing to do if the ID length is 255 characters or less. - if (strlen($cid) <= 255) { + // Nothing to do if the ID is a US ASCII string of 255 characters or less. + $cid_is_ascii = mb_check_encoding($cid, 'ASCII'); + if (strlen($cid) <= 255 && $cid_is_ascii) { return $cid; } // Return a string that uses as much as possible of the original cache ID // with the hash appended. $hash = Crypt::hashBase64($cid); + if (!$cid_is_ascii) { + return $hash; + } return substr($cid, 0, 255 - strlen($hash)) . $hash; } @@ -450,7 +454,7 @@ public function schemaDefinition() { 'fields' => array( 'cid' => array( 'description' => 'Primary Key: Unique cache ID.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -491,7 +495,7 @@ public function schemaDefinition() { ), 'checksum' => array( 'description' => 'The tag invalidation checksum when this entry was saved.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, ), diff --git a/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php b/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php index cc882eead164d83061e9e2df9f3386cfd8fdf333..14b34ec0add9d3aa6ea7142e142ac1fe6e3b2799 100644 --- a/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php +++ b/core/lib/Drupal/Core/Cache/DatabaseCacheTagsChecksum.php @@ -175,7 +175,7 @@ public function schemaDefinition() { 'fields' => array( 'tag' => array( 'description' => 'Namespace-prefixed tag string.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', diff --git a/core/lib/Drupal/Core/Config/DatabaseStorage.php b/core/lib/Drupal/Core/Config/DatabaseStorage.php index 6c332731948c2b5e17cd1960b681eeb7eace7f1e..646a1d54a2ddb06b11b4b8a1a6338de945a63821 100644 --- a/core/lib/Drupal/Core/Config/DatabaseStorage.php +++ b/core/lib/Drupal/Core/Config/DatabaseStorage.php @@ -192,14 +192,14 @@ protected static function schemaDefinition() { 'fields' => array( 'collection' => array( 'description' => 'Primary Key: Config object collection.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', ), 'name' => array( 'description' => 'Primary Key: Config object name.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', diff --git a/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php b/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php index d0aba49bac8cdf95e5e2f2d64a37fd5184536e38..8237c79112c81f925a0820754cef2d3a73bd12ee 100644 --- a/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php +++ b/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php @@ -144,6 +144,10 @@ protected function createFieldSql($name, $spec) { if (!empty($spec['binary'])) { $sql .= ' BINARY'; } + // Note we check for the "type" key here. "mysql_type" is VARCHAR: + if (isset($spec['type']) && $spec['type'] == 'varchar_ascii') { + $sql .= ' CHARACTER SET ascii COLLATE ascii_general_ci'; + } } elseif (isset($spec['precision']) && isset($spec['scale'])) { $sql .= '(' . $spec['precision'] . ', ' . $spec['scale'] . ')'; @@ -218,6 +222,8 @@ public function getFieldTypeMap() { // database types back into schema types. // $map does not use drupal_static as its value never changes. static $map = array( + 'varchar_ascii:normal' => 'VARCHAR', + 'varchar:normal' => 'VARCHAR', 'char:normal' => 'CHAR', diff --git a/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php b/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php index 506baa8e1f62b3e52e3b63a971bef7d5729e39a1..040b32a7bd9b5d4ab91b7351e8c14465c64bb853 100644 --- a/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php +++ b/core/lib/Drupal/Core/Database/Driver/pgsql/Schema.php @@ -363,6 +363,8 @@ function getFieldTypeMap() { // database types back into schema types. // $map does not use drupal_static as its value never changes. static $map = array( + 'varchar_ascii:normal' => 'varchar', + 'varchar:normal' => 'varchar', 'char:normal' => 'character', diff --git a/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php b/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php index fe3f877289a25736aed11528c6fecf1f5ca2dbff..154470dfec28024c07abbe6224a23fe2c9d3fb69 100644 --- a/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php +++ b/core/lib/Drupal/Core/Database/Driver/sqlite/Schema.php @@ -212,6 +212,8 @@ public function getFieldTypeMap() { // database types back into schema types. // $map does not use drupal_static as its value never changes. static $map = array( + 'varchar_ascii:normal' => 'VARCHAR', + 'varchar:normal' => 'VARCHAR', 'char:normal' => 'CHAR', diff --git a/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php b/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php index 6fb4ddfd1a7c944af44b5696003d90bc207ae0f4..b0669697aa1038c2013e0c5954f1a2922d0f730d 100644 --- a/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php +++ b/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorageSchema.php @@ -1568,7 +1568,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor } else { $id_schema = array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'description' => 'The entity id this data is attached to', @@ -1601,7 +1601,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor 'description' => $description_current, 'fields' => array( 'bundle' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', @@ -1617,7 +1617,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor 'entity_id' => $id_schema, 'revision_id' => $revision_id_schema, 'langcode' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 32, 'not null' => TRUE, 'default' => '', diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php index ca54aa85cdd703e6b5cd260e5f921e3c3a1edc08..21d70906b21dea410013789dbe7a7115b44f4645 100644 --- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php +++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/EntityReferenceItem.php @@ -133,7 +133,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) $columns = array( 'target_id' => array( 'description' => 'The ID of the target entity.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', // If the target entities act as bundles for another entity type, // their IDs should not exceed the maximum length for bundles. 'length' => $target_type_info->getBundleOf() ? EntityTypeInterface::BUNDLE_MAX_LENGTH : 255, diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php index 9ab60a4d4856f6a38722293ff229d005b1fb6cbf..b7ce7fb790a0f898f59d75ff96dfcff795fcbdbb 100644 --- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php +++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php @@ -44,6 +44,7 @@ class LanguageItem extends FieldItemBase { public static function propertyDefinitions(FieldStorageDefinitionInterface $field_definition) { $properties['value'] = DataDefinition::create('string') ->setLabel(t('Language code')) + ->setSetting('is_ascii', TRUE) ->setRequired(TRUE); $properties['language'] = DataReferenceDefinition::create('language') @@ -75,6 +76,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) 'value' => array( 'type' => 'varchar', 'length' => 12, + 'is_ascii' => TRUE, ), ), ); diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php index 8817c876823ef5642207fb85c02ad93268d30b4b..2329c86ca6befd2c96a2ed43a9ba5c157dece829 100644 --- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php +++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/StringItem.php @@ -32,6 +32,7 @@ class StringItem extends StringItemBase { public static function defaultStorageSettings() { return array( 'max_length' => 255, + 'is_ascii' => FALSE, ) + parent::defaultStorageSettings(); } @@ -42,7 +43,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) return array( 'columns' => array( 'value' => array( - 'type' => 'varchar', + 'type' => $field_definition->getSetting('is_ascii') === TRUE ? 'varchar_ascii' : 'varchar', 'length' => (int) $field_definition->getSetting('max_length'), 'binary' => $field_definition->getSetting('case_sensitive'), ), diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php index 84d1ec582b7dd929b158d174517209afeaef0682..1bdd10e76abc3a42f63434a2dd4ce0c6a18a0d49 100644 --- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php +++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldType/UuidItem.php @@ -30,6 +30,7 @@ class UuidItem extends StringItem { public static function defaultStorageSettings() { return array( 'max_length' => 128, + 'is_ascii' => TRUE, ) + parent::defaultStorageSettings(); } diff --git a/core/lib/Drupal/Core/Menu/MenuTreeStorage.php b/core/lib/Drupal/Core/Menu/MenuTreeStorage.php index 2ce68f778922bfde184283ee7f7018549728e345..659d6be3926c40a8a9420d1c2497e0f8f762deea 100644 --- a/core/lib/Drupal/Core/Menu/MenuTreeStorage.php +++ b/core/lib/Drupal/Core/Menu/MenuTreeStorage.php @@ -1193,7 +1193,7 @@ protected static function schemaDefinition() { 'fields' => array( 'menu_name' => array( 'description' => "The menu name. All links with the same menu name (such as 'tools') are part of the same menu.", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 32, 'not null' => TRUE, 'default' => '', @@ -1206,20 +1206,20 @@ protected static function schemaDefinition() { ), 'id' => array( 'description' => 'Unique machine name: the plugin ID.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, ), 'parent' => array( 'description' => 'The plugin ID for the parent of this link.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', ), 'route_name' => array( 'description' => 'The machine name of a defined Symfony Route this menu item represents.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, ), 'route_param_key' => array( @@ -1281,7 +1281,7 @@ protected static function schemaDefinition() { ), 'provider' => array( 'description' => 'The name of the module that generated this link.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH, 'not null' => TRUE, 'default' => 'system', diff --git a/core/modules/aggregator/src/Entity/Feed.php b/core/modules/aggregator/src/Entity/Feed.php index 6f3d2e2dbe85b2cf7e2e71d9a1117c84b95b58d1..a5ed08166f78e4208cd3924dad8446541605651d 100644 --- a/core/modules/aggregator/src/Entity/Feed.php +++ b/core/modules/aggregator/src/Entity/Feed.php @@ -226,6 +226,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { $fields['hash'] = BaseFieldDefinition::create('string') ->setLabel(t('Hash')) + ->setSetting('is_ascii', TRUE) ->setDescription(t('Calculated hash of the feed data, used for validating cache.')); $fields['etag'] = BaseFieldDefinition::create('string') diff --git a/core/modules/aggregator/src/FeedInterface.php b/core/modules/aggregator/src/FeedInterface.php index 29c6782e37adc5bc6ff724474149b343db43fdd8..e678c90e4c88a0fdbe50baf145cd66884ca38165 100644 --- a/core/modules/aggregator/src/FeedInterface.php +++ b/core/modules/aggregator/src/FeedInterface.php @@ -170,7 +170,8 @@ public function getHash(); * Sets the calculated hash of the feed data, used for validating cache. * * @param string $hash - * A string containing the calculated hash of the feed. + * A string containing the calculated hash of the feed. Must contain + * US ASCII characters only. * * @return \Drupal\aggregator\FeedInterface * The class instance that this method is called on. diff --git a/core/modules/ban/ban.install b/core/modules/ban/ban.install index 7a5494fa7810a95b34773eaf4a42bbeab7dbbd27..b2ea1fe3d337c72ef402849d25947835c82f1c62 100644 --- a/core/modules/ban/ban.install +++ b/core/modules/ban/ban.install @@ -20,7 +20,7 @@ function ban_schema() { ), 'ip' => array( 'description' => 'IP address', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 40, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/comment/comment.install b/core/modules/comment/comment.install index 64aa6bd78f238955674d1df3bab38ace8ef52c96..327feff349af842a8bc576b41a11d64fe80095b2 100644 --- a/core/modules/comment/comment.install +++ b/core/modules/comment/comment.install @@ -46,14 +46,14 @@ function comment_schema() { 'description' => 'The entity_id of the entity for which the statistics are compiled.', ), 'entity_type' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'not null' => TRUE, 'default' => 'node', 'length' => EntityTypeInterface::ID_MAX_LENGTH, 'description' => 'The entity_type of the entity to which this comment is a reply.', ), 'field_name' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'not null' => TRUE, 'default' => '', 'length' => FieldStorageConfig::NAME_MAX_LENGTH, diff --git a/core/modules/comment/src/Entity/Comment.php b/core/modules/comment/src/Entity/Comment.php index 798a69f1cf41b723266047b267c533de4d67fe46..f83f54b58c092fc768ba9f9ff2afb2da168cb656 100644 --- a/core/modules/comment/src/Entity/Comment.php +++ b/core/modules/comment/src/Entity/Comment.php @@ -303,6 +303,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { $fields['entity_type'] = BaseFieldDefinition::create('string') ->setLabel(t('Entity type')) ->setDescription(t('The entity type to which this comment is attached.')) + ->setSetting('is_ascii', TRUE) ->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH); $fields['comment_type'] = BaseFieldDefinition::create('entity_reference') @@ -313,6 +314,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { $fields['field_name'] = BaseFieldDefinition::create('string') ->setLabel(t('Comment field name')) ->setDescription(t('The field name through which this comment was added.')) + ->setSetting('is_ascii', TRUE) ->setSetting('max_length', FieldStorageConfig::NAME_MAX_LENGTH); return $fields; diff --git a/core/modules/dblog/dblog.install b/core/modules/dblog/dblog.install index b7454c13341599422587aef6fac31080703c70e8..1119c47dbfa62dbdaa37e3480ae6a0a909c95a80 100644 --- a/core/modules/dblog/dblog.install +++ b/core/modules/dblog/dblog.install @@ -25,7 +25,7 @@ function dblog_schema() { 'description' => 'The {users}.uid of the user who triggered the event.', ), 'type' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, 'default' => '', @@ -69,7 +69,7 @@ function dblog_schema() { 'description' => 'URL of referring page.', ), 'hostname' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/file/file.install b/core/modules/file/file.install index 6e188c6a562416b04985d8ad1d1348b6c2379ba0..fd3503c2834d19424f6e453bdf780639fed5b8c6 100644 --- a/core/modules/file/file.install +++ b/core/modules/file/file.install @@ -20,21 +20,21 @@ function file_schema() { ), 'module' => array( 'description' => 'The name of the module that is using the file.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH, 'not null' => TRUE, 'default' => '', ), 'type' => array( 'description' => 'The name of the object type in which the file is used.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, 'default' => '', ), 'id' => array( 'description' => 'The primary key of the object using the file.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, 'default' => 0, diff --git a/core/modules/file/src/Entity/File.php b/core/modules/file/src/Entity/File.php index 42de0ae94e84de65b8171ccdc8a67c9d5aaacfe8..ff38ab320ae85f854f16e538849d37742847928f 100644 --- a/core/modules/file/src/Entity/File.php +++ b/core/modules/file/src/Entity/File.php @@ -254,6 +254,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { $fields['filemime'] = BaseFieldDefinition::create('string') ->setLabel(t('File MIME type')) + ->setSetting('is_ascii', TRUE) ->setDescription(t("The file's MIME type.")); $fields['filesize'] = BaseFieldDefinition::create('integer') diff --git a/core/modules/locale/locale.install b/core/modules/locale/locale.install index 81aadf534b4e4c4af141894d3fbe6b071fe34d72..02587670627414eb5df6878d079566381918eb0d 100644 --- a/core/modules/locale/locale.install +++ b/core/modules/locale/locale.install @@ -67,14 +67,14 @@ function locale_schema() { 'description' => 'The original string in English.', ), 'context' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', 'description' => 'The context this string applies to.', ), 'version' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 20, 'not null' => TRUE, 'default' => 'none', @@ -103,7 +103,7 @@ function locale_schema() { 'description' => 'Translation string value in this language.', ), 'language' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 12, 'not null' => TRUE, 'default' => '', @@ -142,7 +142,7 @@ function locale_schema() { 'description' => 'Unique identifier of this string.', ), 'type' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 50, 'not null' => TRUE, 'default' => '', @@ -156,7 +156,7 @@ function locale_schema() { 'description' => 'Type dependent location information (file name, path, etc).', ), 'version' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 20, 'not null' => TRUE, 'default' => 'none', @@ -180,14 +180,14 @@ function locale_schema() { 'description' => 'File import status information for interface translation files.', 'fields' => array( 'project' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => '255', 'not null' => TRUE, 'default' => '', 'description' => 'A unique short name to identify the project the file belongs to.', ), 'langcode' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => '12', 'not null' => TRUE, 'default' => '', diff --git a/core/modules/menu_link_content/src/Entity/MenuLinkContent.php b/core/modules/menu_link_content/src/Entity/MenuLinkContent.php index 5570a2a50ed730699c83b6621c014829261ccc9d..d6610a0e15d7e93592702a008900f64e110a88b2 100644 --- a/core/modules/menu_link_content/src/Entity/MenuLinkContent.php +++ b/core/modules/menu_link_content/src/Entity/MenuLinkContent.php @@ -250,6 +250,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ->setLabel(t('Bundle')) ->setDescription(t('The content menu link bundle.')) ->setSetting('max_length', EntityTypeInterface::BUNDLE_MAX_LENGTH) + ->setSetting('is_ascii', TRUE) ->setReadOnly(TRUE); $fields['title'] = BaseFieldDefinition::create('string') @@ -291,7 +292,8 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { $fields['menu_name'] = BaseFieldDefinition::create('string') ->setLabel(t('Menu name')) ->setDescription(t('The menu name. All links with the same menu name (such as "tools") are part of the same menu.')) - ->setDefaultValue('tools'); + ->setDefaultValue('tools') + ->setSetting('is_ascii', TRUE); $fields['link'] = BaseFieldDefinition::create('link') ->setLabel(t('Link')) diff --git a/core/modules/node/node.install b/core/modules/node/node.install index fa0d245c3635b124dd55d66f372d7437b336ae9b..a7cb55e0c018ad6243630bbfbc7cd59d9645f27c 100644 --- a/core/modules/node/node.install +++ b/core/modules/node/node.install @@ -54,7 +54,7 @@ function node_schema() { ), 'langcode' => array( 'description' => 'The {language}.langcode of this node.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 12, 'not null' => TRUE, 'default' => '', @@ -75,7 +75,7 @@ function node_schema() { ), 'realm' => array( 'description' => 'The realm in which the user must possess the grant ID. Each node access node can define one or more realms.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/search/search.install b/core/modules/search/search.install index a298f2f93164347a9cef5c389de14a23106e562b..f920aa5ae99fb78507993967d0aada5ab5059941 100644 --- a/core/modules/search/search.install +++ b/core/modules/search/search.install @@ -20,14 +20,14 @@ function search_schema() { 'description' => 'Search item ID, e.g. node ID for nodes.', ), 'langcode' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => '12', 'not null' => TRUE, 'description' => 'The {languages}.langcode of the item variant.', 'default' => '', ), 'type' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, 'description' => 'Type of item, e.g. node.', @@ -67,14 +67,14 @@ function search_schema() { 'description' => 'The {search_dataset}.sid of the searchable item to which the word belongs.', ), 'langcode' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => '12', 'not null' => TRUE, 'description' => 'The {languages}.langcode of the item variant.', 'default' => '', ), 'type' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, 'description' => 'The {search_dataset}.type of the searchable item to which the word belongs.', diff --git a/core/modules/shortcut/shortcut.install b/core/modules/shortcut/shortcut.install index b046142043b843e21a9ea598447c3bc23a8f22d2..73986b817829b44323ee901c105db11c537e3c52 100644 --- a/core/modules/shortcut/shortcut.install +++ b/core/modules/shortcut/shortcut.install @@ -20,7 +20,7 @@ function shortcut_schema() { 'description' => 'The {users}.uid for this set.', ), 'set_name' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 32, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/simpletest/simpletest.install b/core/modules/simpletest/simpletest.install index 47c0ee2bd4048d5b8781e8a7e81ae35fe741502d..b4792ea1d92d1bfe419abbd8ce7985cc88ce4c1a 100644 --- a/core/modules/simpletest/simpletest.install +++ b/core/modules/simpletest/simpletest.install @@ -108,7 +108,7 @@ function simpletest_schema() { 'description' => 'Test ID, messages belonging to the same ID are reported together', ), 'test_class' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -127,14 +127,14 @@ function simpletest_schema() { 'description' => 'The message itself.', ), 'message_group' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', 'description' => 'The message group this message belongs to. For example: warning, browser, user.', ), 'function' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/system/database.api.php b/core/modules/system/database.api.php index a01ff5992f1dc4d045629a9eb0fba0e72c6852b9..f3d7fe2120ff90c91c1865245dffcc2b8beb0e20 100644 --- a/core/modules/system/database.api.php +++ b/core/modules/system/database.api.php @@ -256,6 +256,8 @@ * 'float', 'numeric', or 'serial'. Most types just map to the according * database engine specific datatypes. Use 'serial' for auto incrementing * fields. This will expand to 'INT auto_increment' on MySQL. + * A special 'varchar_ascii' type is also available for limiting machine + * name field to US ASCII characters. * - 'mysql_type', 'pgsql_type', 'sqlite_type', etc.: If you need to * use a record type not included in the officially supported list * of types above, you can specify a type for each database diff --git a/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php b/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php index e54198ba5cce5477d9a61ebec76c9434d798f463..e182cc0b104b41354214ab5e937a1db28d3a8439 100644 --- a/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php +++ b/core/modules/system/src/Tests/Cache/DatabaseBackendUnitTest.php @@ -33,4 +33,24 @@ protected function createCacheBackend($bin) { return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin); } + /** + * {@inheritdoc} + */ + public function testSetGet() { + parent::testSetGet(); + $backend = $this->getCacheBackend(); + + // Set up a cache ID that is not ASCII and longer than 255 characters so we + // can test cache ID normalization. + $cid_long = str_repeat('愛€', 500); + $cached_value_long = $this->randomMachineName(); + $backend->set($cid_long, $cached_value_long); + $this->assertIdentical($cached_value_long, $backend->get($cid_long)->data, "Backend contains the correct value for long, non-ASCII cache id."); + + $cid_short = '愛1€'; + $cached_value_short = $this->randomMachineName(); + $backend->set($cid_short, $cached_value_short); + $this->assertIdentical($cached_value_short, $backend->get($cid_short)->data, "Backend contains the correct value for short, non-ASCII cache id."); + } + } diff --git a/core/modules/system/src/Tests/Database/SchemaTest.php b/core/modules/system/src/Tests/Database/SchemaTest.php index 7811b505b2707742631b5cc427fc01cadbd5579b..9a234edd5b2d40950998e2c7989e8f60a6257950 100644 --- a/core/modules/system/src/Tests/Database/SchemaTest.php +++ b/core/modules/system/src/Tests/Database/SchemaTest.php @@ -49,6 +49,11 @@ function testSchema() { 'default' => "'\"funky default'\"", 'description' => 'Schema column description for string.', ), + 'test_field_string_ascii' => array( + 'type' => 'varchar_ascii', + 'length' => 255, + 'description' => 'Schema column description for ASCII string.', + ), ), ); db_create_table('test_table', $table_specification); @@ -62,6 +67,21 @@ function testSchema() { // Assert that the column comment has been set. $this->checkSchemaComment($table_specification['fields']['test_field']['description'], 'test_table', 'test_field'); + if (Database::getConnection()->databaseType() == 'mysql') { + // Make sure that varchar fields have the correct collation. + $columns = db_query('SHOW FULL COLUMNS FROM {test_table}'); + foreach ($columns as $column) { + if ($column->Field == 'test_field_string') { + $string_check = ($column->Collation == 'utf8_general_ci'); + } + if ($column->Field == 'test_field_string_ascii') { + $string_ascii_check = ($column->Collation == 'ascii_general_ci'); + } + } + $this->assertTrue(!empty($string_check), 'string field has the right collation.'); + $this->assertTrue(!empty($string_ascii_check), 'ASCII string field has the right collation.'); + } + // An insert without a value for the column 'test_table' should fail. $this->assertFalse($this->tryInsert(), 'Insert without a default failed.'); diff --git a/core/modules/system/system.install b/core/modules/system/system.install index 082bbc6793875c6b310e34708457be03d3df063d..6c3db4608c9354472485c5e5deabc72cfc576596 100644 --- a/core/modules/system/system.install +++ b/core/modules/system/system.install @@ -685,7 +685,7 @@ function system_schema() { ), 'token' => array( 'description' => "A string token generated against the current user's session id and the batch id, used to ensure that only the user who submitted the batch can effectively access it.", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, ), @@ -717,14 +717,14 @@ function system_schema() { ), 'event' => array( 'description' => 'Name of event (e.g. contact).', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 64, 'not null' => TRUE, 'default' => '', ), 'identifier' => array( 'description' => 'Identifier of the visitor, such as an IP address or hostname.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', @@ -754,14 +754,14 @@ function system_schema() { 'fields' => array( 'collection' => array( 'description' => 'A named collection of key and value pairs.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', ), 'name' => array( 'description' => 'The key of the key-value pair. As KEY is a SQL reserved keyword, name was chosen instead.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', @@ -781,7 +781,7 @@ function system_schema() { 'fields' => array( 'collection' => array( 'description' => 'A named collection of key and value pairs.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', @@ -789,7 +789,7 @@ function system_schema() { 'name' => array( // KEY is an SQL reserved word, so use 'name' as the key's field name. 'description' => 'The key of the key/value pair.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', @@ -824,7 +824,7 @@ function system_schema() { 'description' => 'Primary Key: Unique item ID.', ), 'name' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -862,7 +862,7 @@ function system_schema() { 'fields' => array( 'name' => array( 'description' => 'Primary Key: Machine name of this route', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -911,14 +911,14 @@ function system_schema() { 'fields' => array( 'name' => array( 'description' => 'Primary Key: Unique name.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '' ), 'value' => array( 'description' => 'A value for the semaphore.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '' @@ -961,13 +961,13 @@ function system_schema() { ), 'sid' => array( 'description' => "A session ID (hashed). The value is generated by Drupal's session handlers.", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, ), 'hostname' => array( 'description' => 'The IP address that last used this session ID (sid).', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', @@ -1025,7 +1025,7 @@ function system_schema() { ), 'langcode' => array( 'description' => "The language code this alias is for; if 'und', the alias will be used for unknown languages. Each Drupal path can have an alias for each supported language.", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 12, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php b/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php index c55953c6875263146f307cc568cc0884a5320fc8..9a2419497f148ae44a5230651033f0a5faa5189f 100644 --- a/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php +++ b/core/modules/text/src/Plugin/Field/FieldType/TextLongItem.php @@ -34,7 +34,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) 'size' => 'big', ), 'format' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, ), ), diff --git a/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php b/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php index 48d4dacf82408557db32529fd93f1e519c62c4c7..413ad4797e3acfcf490ed3bb1e7a9a19a7b9daa4 100644 --- a/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php +++ b/core/modules/text/src/Plugin/Field/FieldType/TextWithSummaryItem.php @@ -68,7 +68,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) 'size' => 'big', ), 'format' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, ), ), diff --git a/core/modules/user/user.install b/core/modules/user/user.install index 7b3c3d3a3936ae010570cec97c8f959022a1f2d5..dd55b49398c8eb04fe333865dba0f8259c0d3aa2 100644 --- a/core/modules/user/user.install +++ b/core/modules/user/user.install @@ -21,14 +21,14 @@ function user_schema() { ), 'module' => array( 'description' => 'The name of the module declaring the variable.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH, 'not null' => TRUE, 'default' => '', ), 'name' => array( 'description' => 'The identifier of the data.', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => TRUE, 'default' => '', diff --git a/core/modules/views/src/EntityViewsData.php b/core/modules/views/src/EntityViewsData.php index fa1cf3efc2a5043a5e48990045cd244ed9cf9e08..5cd321531c3c6c469999f3ec256fffe180ac1691 100644 --- a/core/modules/views/src/EntityViewsData.php +++ b/core/modules/views/src/EntityViewsData.php @@ -411,6 +411,7 @@ protected function mapSingleFieldViewsData($table, $field_name, $field_type, $co case 'char': case 'string': case 'varchar': + case 'varchar_ascii': case 'tinytext': case 'text': case 'mediumtext': diff --git a/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php b/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php index 08f73cb4d08f0cab19e3f4e4b444e754b4bdc915..e67b8e6a8759588af92164c0775c22916e8e28de 100644 --- a/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php +++ b/core/tests/Drupal/Tests/Core/Entity/Sql/SqlContentEntityStorageSchemaTest.php @@ -802,7 +802,7 @@ public function testDedicatedTableSchema() { 'description' => "Data storage for $entity_type_id field $field_name.", 'fields' => array( 'bundle' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => true, 'default' => '', @@ -828,7 +828,7 @@ public function testDedicatedTableSchema() { 'description' => 'The entity revision id this data is attached to, which for an unversioned entity type is the same as the entity id', ), 'langcode' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 32, 'not null' => true, 'default' => '', @@ -947,7 +947,7 @@ public function testDedicatedTableSchemaForEntityWithStringIdentifier() { 'description' => "Data storage for $entity_type_id field $field_name.", 'fields' => array( 'bundle' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => true, 'default' => '', @@ -961,19 +961,19 @@ public function testDedicatedTableSchemaForEntityWithStringIdentifier() { 'description' => 'A boolean indicating whether this data item has been deleted', ), 'entity_id' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => true, 'description' => 'The entity id this data is attached to', ), 'revision_id' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 128, 'not null' => true, 'description' => 'The entity revision id this data is attached to, which for an unversioned entity type is the same as the entity id', ), 'langcode' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 32, 'not null' => true, 'default' => '',