Commit d57ee5f9 authored by alexpott's avatar alexpott

Issue #1923406 by stefan.r, yannickoo, catch, Crell, amateescu, pwolanin,...

Issue #1923406 by stefan.r, yannickoo, catch, Crell, amateescu, pwolanin, morgantocker, Damien Tournoud, sun: Use ASCII character set on alphanumeric fields so we can index all 255 characters
parent c6890085
...@@ -445,6 +445,9 @@ field.storage_settings.string: ...@@ -445,6 +445,9 @@ field.storage_settings.string:
case_sensitive: case_sensitive:
type: boolean type: boolean
label: 'Case sensitive' label: 'Case sensitive'
is_ascii:
type: boolean
label: 'Contains US ASCII characters only'
field.field_settings.string: field.field_settings.string:
type: mapping type: mapping
......
...@@ -422,22 +422,26 @@ protected function catchException(\Exception $e, $table_name = NULL) { ...@@ -422,22 +422,26 @@ protected function catchException(\Exception $e, $table_name = NULL) {
} }
/** /**
* Ensures that cache IDs have a maximum length of 255 characters. * Normalizes a cache ID in order to comply with database limitations.
* *
* @param string $cid * @param string $cid
* The passed in cache ID. * The passed in cache ID.
* *
* @return string * @return string
* A cache ID that is at most 255 characters long. * An ASCII-encoded cache ID that is at most 255 characters long.
*/ */
protected function normalizeCid($cid) { protected function normalizeCid($cid) {
// Nothing to do if the ID length is 255 characters or less. // Nothing to do if the ID is a US ASCII string of 255 characters or less.
if (strlen($cid) <= 255) { $cid_is_ascii = mb_check_encoding($cid, 'ASCII');
if (strlen($cid) <= 255 && $cid_is_ascii) {
return $cid; return $cid;
} }
// Return a string that uses as much as possible of the original cache ID // Return a string that uses as much as possible of the original cache ID
// with the hash appended. // with the hash appended.
$hash = Crypt::hashBase64($cid); $hash = Crypt::hashBase64($cid);
if (!$cid_is_ascii) {
return $hash;
}
return substr($cid, 0, 255 - strlen($hash)) . $hash; return substr($cid, 0, 255 - strlen($hash)) . $hash;
} }
...@@ -450,7 +454,7 @@ public function schemaDefinition() { ...@@ -450,7 +454,7 @@ public function schemaDefinition() {
'fields' => array( 'fields' => array(
'cid' => array( 'cid' => array(
'description' => 'Primary Key: Unique cache ID.', 'description' => 'Primary Key: Unique cache ID.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -491,7 +495,7 @@ public function schemaDefinition() { ...@@ -491,7 +495,7 @@ public function schemaDefinition() {
), ),
'checksum' => array( 'checksum' => array(
'description' => 'The tag invalidation checksum when this entry was saved.', 'description' => 'The tag invalidation checksum when this entry was saved.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
), ),
......
...@@ -175,7 +175,7 @@ public function schemaDefinition() { ...@@ -175,7 +175,7 @@ public function schemaDefinition() {
'fields' => array( 'fields' => array(
'tag' => array( 'tag' => array(
'description' => 'Namespace-prefixed tag string.', 'description' => 'Namespace-prefixed tag string.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
......
...@@ -192,14 +192,14 @@ protected static function schemaDefinition() { ...@@ -192,14 +192,14 @@ protected static function schemaDefinition() {
'fields' => array( 'fields' => array(
'collection' => array( 'collection' => array(
'description' => 'Primary Key: Config object collection.', 'description' => 'Primary Key: Config object collection.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
), ),
'name' => array( 'name' => array(
'description' => 'Primary Key: Config object name.', 'description' => 'Primary Key: Config object name.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
......
...@@ -144,6 +144,10 @@ protected function createFieldSql($name, $spec) { ...@@ -144,6 +144,10 @@ protected function createFieldSql($name, $spec) {
if (!empty($spec['binary'])) { if (!empty($spec['binary'])) {
$sql .= ' BINARY'; $sql .= ' BINARY';
} }
// Note we check for the "type" key here. "mysql_type" is VARCHAR:
if (isset($spec['type']) && $spec['type'] == 'varchar_ascii') {
$sql .= ' CHARACTER SET ascii COLLATE ascii_general_ci';
}
} }
elseif (isset($spec['precision']) && isset($spec['scale'])) { elseif (isset($spec['precision']) && isset($spec['scale'])) {
$sql .= '(' . $spec['precision'] . ', ' . $spec['scale'] . ')'; $sql .= '(' . $spec['precision'] . ', ' . $spec['scale'] . ')';
...@@ -218,6 +222,8 @@ public function getFieldTypeMap() { ...@@ -218,6 +222,8 @@ public function getFieldTypeMap() {
// database types back into schema types. // database types back into schema types.
// $map does not use drupal_static as its value never changes. // $map does not use drupal_static as its value never changes.
static $map = array( static $map = array(
'varchar_ascii:normal' => 'VARCHAR',
'varchar:normal' => 'VARCHAR', 'varchar:normal' => 'VARCHAR',
'char:normal' => 'CHAR', 'char:normal' => 'CHAR',
......
...@@ -363,6 +363,8 @@ function getFieldTypeMap() { ...@@ -363,6 +363,8 @@ function getFieldTypeMap() {
// database types back into schema types. // database types back into schema types.
// $map does not use drupal_static as its value never changes. // $map does not use drupal_static as its value never changes.
static $map = array( static $map = array(
'varchar_ascii:normal' => 'varchar',
'varchar:normal' => 'varchar', 'varchar:normal' => 'varchar',
'char:normal' => 'character', 'char:normal' => 'character',
......
...@@ -212,6 +212,8 @@ public function getFieldTypeMap() { ...@@ -212,6 +212,8 @@ public function getFieldTypeMap() {
// database types back into schema types. // database types back into schema types.
// $map does not use drupal_static as its value never changes. // $map does not use drupal_static as its value never changes.
static $map = array( static $map = array(
'varchar_ascii:normal' => 'VARCHAR',
'varchar:normal' => 'VARCHAR', 'varchar:normal' => 'VARCHAR',
'char:normal' => 'CHAR', 'char:normal' => 'CHAR',
......
...@@ -1568,7 +1568,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor ...@@ -1568,7 +1568,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor
} }
else { else {
$id_schema = array( $id_schema = array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 128, 'length' => 128,
'not null' => TRUE, 'not null' => TRUE,
'description' => 'The entity id this data is attached to', 'description' => 'The entity id this data is attached to',
...@@ -1601,7 +1601,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor ...@@ -1601,7 +1601,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor
'description' => $description_current, 'description' => $description_current,
'fields' => array( 'fields' => array(
'bundle' => array( 'bundle' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 128, 'length' => 128,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -1617,7 +1617,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor ...@@ -1617,7 +1617,7 @@ protected function getDedicatedTableSchema(FieldStorageDefinitionInterface $stor
'entity_id' => $id_schema, 'entity_id' => $id_schema,
'revision_id' => $revision_id_schema, 'revision_id' => $revision_id_schema,
'langcode' => array( 'langcode' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 32, 'length' => 32,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
......
...@@ -133,7 +133,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) ...@@ -133,7 +133,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
$columns = array( $columns = array(
'target_id' => array( 'target_id' => array(
'description' => 'The ID of the target entity.', 'description' => 'The ID of the target entity.',
'type' => 'varchar', 'type' => 'varchar_ascii',
// If the target entities act as bundles for another entity type, // If the target entities act as bundles for another entity type,
// their IDs should not exceed the maximum length for bundles. // their IDs should not exceed the maximum length for bundles.
'length' => $target_type_info->getBundleOf() ? EntityTypeInterface::BUNDLE_MAX_LENGTH : 255, 'length' => $target_type_info->getBundleOf() ? EntityTypeInterface::BUNDLE_MAX_LENGTH : 255,
......
...@@ -44,6 +44,7 @@ class LanguageItem extends FieldItemBase { ...@@ -44,6 +44,7 @@ class LanguageItem extends FieldItemBase {
public static function propertyDefinitions(FieldStorageDefinitionInterface $field_definition) { public static function propertyDefinitions(FieldStorageDefinitionInterface $field_definition) {
$properties['value'] = DataDefinition::create('string') $properties['value'] = DataDefinition::create('string')
->setLabel(t('Language code')) ->setLabel(t('Language code'))
->setSetting('is_ascii', TRUE)
->setRequired(TRUE); ->setRequired(TRUE);
$properties['language'] = DataReferenceDefinition::create('language') $properties['language'] = DataReferenceDefinition::create('language')
...@@ -75,6 +76,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) ...@@ -75,6 +76,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
'value' => array( 'value' => array(
'type' => 'varchar', 'type' => 'varchar',
'length' => 12, 'length' => 12,
'is_ascii' => TRUE,
), ),
), ),
); );
......
...@@ -32,6 +32,7 @@ class StringItem extends StringItemBase { ...@@ -32,6 +32,7 @@ class StringItem extends StringItemBase {
public static function defaultStorageSettings() { public static function defaultStorageSettings() {
return array( return array(
'max_length' => 255, 'max_length' => 255,
'is_ascii' => FALSE,
) + parent::defaultStorageSettings(); ) + parent::defaultStorageSettings();
} }
...@@ -42,7 +43,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition) ...@@ -42,7 +43,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
return array( return array(
'columns' => array( 'columns' => array(
'value' => array( 'value' => array(
'type' => 'varchar', 'type' => $field_definition->getSetting('is_ascii') === TRUE ? 'varchar_ascii' : 'varchar',
'length' => (int) $field_definition->getSetting('max_length'), 'length' => (int) $field_definition->getSetting('max_length'),
'binary' => $field_definition->getSetting('case_sensitive'), 'binary' => $field_definition->getSetting('case_sensitive'),
), ),
......
...@@ -30,6 +30,7 @@ class UuidItem extends StringItem { ...@@ -30,6 +30,7 @@ class UuidItem extends StringItem {
public static function defaultStorageSettings() { public static function defaultStorageSettings() {
return array( return array(
'max_length' => 128, 'max_length' => 128,
'is_ascii' => TRUE,
) + parent::defaultStorageSettings(); ) + parent::defaultStorageSettings();
} }
......
...@@ -1193,7 +1193,7 @@ protected static function schemaDefinition() { ...@@ -1193,7 +1193,7 @@ protected static function schemaDefinition() {
'fields' => array( 'fields' => array(
'menu_name' => array( 'menu_name' => array(
'description' => "The menu name. All links with the same menu name (such as 'tools') are part of the same menu.", 'description' => "The menu name. All links with the same menu name (such as 'tools') are part of the same menu.",
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 32, 'length' => 32,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -1206,20 +1206,20 @@ protected static function schemaDefinition() { ...@@ -1206,20 +1206,20 @@ protected static function schemaDefinition() {
), ),
'id' => array( 'id' => array(
'description' => 'Unique machine name: the plugin ID.', 'description' => 'Unique machine name: the plugin ID.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
), ),
'parent' => array( 'parent' => array(
'description' => 'The plugin ID for the parent of this link.', 'description' => 'The plugin ID for the parent of this link.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
), ),
'route_name' => array( 'route_name' => array(
'description' => 'The machine name of a defined Symfony Route this menu item represents.', 'description' => 'The machine name of a defined Symfony Route this menu item represents.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
), ),
'route_param_key' => array( 'route_param_key' => array(
...@@ -1281,7 +1281,7 @@ protected static function schemaDefinition() { ...@@ -1281,7 +1281,7 @@ protected static function schemaDefinition() {
), ),
'provider' => array( 'provider' => array(
'description' => 'The name of the module that generated this link.', 'description' => 'The name of the module that generated this link.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH, 'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH,
'not null' => TRUE, 'not null' => TRUE,
'default' => 'system', 'default' => 'system',
......
...@@ -226,6 +226,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ...@@ -226,6 +226,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['hash'] = BaseFieldDefinition::create('string') $fields['hash'] = BaseFieldDefinition::create('string')
->setLabel(t('Hash')) ->setLabel(t('Hash'))
->setSetting('is_ascii', TRUE)
->setDescription(t('Calculated hash of the feed data, used for validating cache.')); ->setDescription(t('Calculated hash of the feed data, used for validating cache.'));
$fields['etag'] = BaseFieldDefinition::create('string') $fields['etag'] = BaseFieldDefinition::create('string')
......
...@@ -170,7 +170,8 @@ public function getHash(); ...@@ -170,7 +170,8 @@ public function getHash();
* Sets the calculated hash of the feed data, used for validating cache. * Sets the calculated hash of the feed data, used for validating cache.
* *
* @param string $hash * @param string $hash
* A string containing the calculated hash of the feed. * A string containing the calculated hash of the feed. Must contain
* US ASCII characters only.
* *
* @return \Drupal\aggregator\FeedInterface * @return \Drupal\aggregator\FeedInterface
* The class instance that this method is called on. * The class instance that this method is called on.
......
...@@ -20,7 +20,7 @@ function ban_schema() { ...@@ -20,7 +20,7 @@ function ban_schema() {
), ),
'ip' => array( 'ip' => array(
'description' => 'IP address', 'description' => 'IP address',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 40, 'length' => 40,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
......
...@@ -46,14 +46,14 @@ function comment_schema() { ...@@ -46,14 +46,14 @@ function comment_schema() {
'description' => 'The entity_id of the entity for which the statistics are compiled.', 'description' => 'The entity_id of the entity for which the statistics are compiled.',
), ),
'entity_type' => array( 'entity_type' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'not null' => TRUE, 'not null' => TRUE,
'default' => 'node', 'default' => 'node',
'length' => EntityTypeInterface::ID_MAX_LENGTH, 'length' => EntityTypeInterface::ID_MAX_LENGTH,
'description' => 'The entity_type of the entity to which this comment is a reply.', 'description' => 'The entity_type of the entity to which this comment is a reply.',
), ),
'field_name' => array( 'field_name' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
'length' => FieldStorageConfig::NAME_MAX_LENGTH, 'length' => FieldStorageConfig::NAME_MAX_LENGTH,
......
...@@ -303,6 +303,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ...@@ -303,6 +303,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['entity_type'] = BaseFieldDefinition::create('string') $fields['entity_type'] = BaseFieldDefinition::create('string')
->setLabel(t('Entity type')) ->setLabel(t('Entity type'))
->setDescription(t('The entity type to which this comment is attached.')) ->setDescription(t('The entity type to which this comment is attached.'))
->setSetting('is_ascii', TRUE)
->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH); ->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH);
$fields['comment_type'] = BaseFieldDefinition::create('entity_reference') $fields['comment_type'] = BaseFieldDefinition::create('entity_reference')
...@@ -313,6 +314,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ...@@ -313,6 +314,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['field_name'] = BaseFieldDefinition::create('string') $fields['field_name'] = BaseFieldDefinition::create('string')
->setLabel(t('Comment field name')) ->setLabel(t('Comment field name'))
->setDescription(t('The field name through which this comment was added.')) ->setDescription(t('The field name through which this comment was added.'))
->setSetting('is_ascii', TRUE)
->setSetting('max_length', FieldStorageConfig::NAME_MAX_LENGTH); ->setSetting('max_length', FieldStorageConfig::NAME_MAX_LENGTH);
return $fields; return $fields;
......
...@@ -25,7 +25,7 @@ function dblog_schema() { ...@@ -25,7 +25,7 @@ function dblog_schema() {
'description' => 'The {users}.uid of the user who triggered the event.', 'description' => 'The {users}.uid of the user who triggered the event.',
), ),
'type' => array( 'type' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 64, 'length' => 64,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -69,7 +69,7 @@ function dblog_schema() { ...@@ -69,7 +69,7 @@ function dblog_schema() {
'description' => 'URL of referring page.', 'description' => 'URL of referring page.',
), ),
'hostname' => array( 'hostname' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 128, 'length' => 128,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
......
...@@ -20,21 +20,21 @@ function file_schema() { ...@@ -20,21 +20,21 @@ function file_schema() {
), ),
'module' => array( 'module' => array(
'description' => 'The name of the module that is using the file.', 'description' => 'The name of the module that is using the file.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH, 'length' => DRUPAL_EXTENSION_NAME_MAX_LENGTH,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
), ),
'type' => array( 'type' => array(
'description' => 'The name of the object type in which the file is used.', 'description' => 'The name of the object type in which the file is used.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 64, 'length' => 64,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
), ),
'id' => array( 'id' => array(
'description' => 'The primary key of the object using the file.', 'description' => 'The primary key of the object using the file.',
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 64, 'length' => 64,
'not null' => TRUE, 'not null' => TRUE,
'default' => 0, 'default' => 0,
......
...@@ -254,6 +254,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ...@@ -254,6 +254,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
$fields['filemime'] = BaseFieldDefinition::create('string') $fields['filemime'] = BaseFieldDefinition::create('string')
->setLabel(t('File MIME type')) ->setLabel(t('File MIME type'))
->setSetting('is_ascii', TRUE)
->setDescription(t("The file's MIME type.")); ->setDescription(t("The file's MIME type."));
$fields['filesize'] = BaseFieldDefinition::create('integer') $fields['filesize'] = BaseFieldDefinition::create('integer')
......
...@@ -67,14 +67,14 @@ function locale_schema() { ...@@ -67,14 +67,14 @@ function locale_schema() {
'description' => 'The original string in English.', 'description' => 'The original string in English.',
), ),
'context' => array( 'context' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 255, 'length' => 255,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
'description' => 'The context this string applies to.', 'description' => 'The context this string applies to.',
), ),
'version' => array( 'version' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 20, 'length' => 20,
'not null' => TRUE, 'not null' => TRUE,
'default' => 'none', 'default' => 'none',
...@@ -103,7 +103,7 @@ function locale_schema() { ...@@ -103,7 +103,7 @@ function locale_schema() {
'description' => 'Translation string value in this language.', 'description' => 'Translation string value in this language.',
), ),
'language' => array( 'language' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 12, 'length' => 12,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -142,7 +142,7 @@ function locale_schema() { ...@@ -142,7 +142,7 @@ function locale_schema() {
'description' => 'Unique identifier of this string.', 'description' => 'Unique identifier of this string.',
), ),
'type' => array( 'type' => array(
'type' => 'varchar', 'type' => 'varchar_ascii',
'length' => 50, 'length' => 50,
'not null' => TRUE, 'not null' => TRUE,
'default' => '', 'default' => '',
...@@ -156,7 +156,7 @@ function locale_schema() { ...@@ -156,7 +156,7 @@ function locale_schema() {
'description' => 'Type dependent location information (file name, path, etc).', 'description' => 'Type dependent location information (file name, path, etc).',
), ),
'version' => array( 'version' => array(