Issue #2885413 by quietone, smustgrave, eugene.brit, gambry, mikelutz,...

Issue #2885413 by quietone, smustgrave, eugene.brit, gambry, mikelutz, renatog, ankithashetty, Etroid, sarathkm, catch, alexpott, berdir, daffie, ghost of drupal past, ProFire, mpdonadio: Timestamp field items are affected by 2038 bug

core/assets/scaffold/files/default.settings.php

@@ -854,6 +854,27 @@
 # $settings['migrate_file_public_path'] = '';
 # $settings['migrate_file_private_path'] = '';
 
+/**
+ * Manage the update of timestamp fields so they work after the year 2038.
+ *
+ * For sites with hundreds of thousands of nodes and other entities this update
+ * will not be quick. Two settings are provided to help manage this update.
+ *
+ * The first setting is used to enable or disable the running of the update. Set
+ * it to FALSE to skip the update. The default value is TRUE.
+ *
+ * The second settings allows sites to set a limit on the time the update runs.
+ * Set it the number of seconds the update should run. The following example
+ * allows the update to run for 2 minutes. The default value is 0, which allows
+ * the update to run to completion.
+ *
+ * @code
+ * $settings['timestamp_field_update_y2038_timeout'] = 2 * 60;
+ * @endcode
+ */
+$settings['timestamp_field_update_y2038'] = TRUE;
+$settings['timestamp_field_update_y2038_timeout'] = 0;
+
 /**
  * Load local development override configuration, if available.
  *

core/lib/Drupal/Core/Datetime/Plugin/Field/FieldWidget/TimestampDatetimeWidget.php

@@ -30,7 +30,7 @@ public function formElement(FieldItemListInterface $items, $delta, array $elemen
     $element['value'] = $element + [
       '#type' => 'datetime',
       '#default_value' => $default_value,
-      '#date_year_range' => '1902:2037',
+      '#date_year_range' => '-29227700493:292277026596',
     ];
 
     $element['value']['#description'] = $element['#description'] !== ''

core/lib/Drupal/Core/Field/Plugin/Field/FieldType/TimestampItem.php

@@ -27,8 +27,8 @@
     "ComplexData" => [
       "value" => [
         "Range" => [
-          "min" => "-2147483648",
-          "max" => "2147483648",
+          "min" => "-9223372036854775807",
+          "max" => "9223372036854775807",
         ],
       ],
     ],
@@ -54,6 +54,7 @@ public static function schema(FieldStorageDefinitionInterface $field_definition)
       'columns' => [
         'value' => [
           'type' => 'int',
+          'size' => 'big',
         ],
       ],
     ];

core/modules/field/tests/src/Kernel/Timestamp/TimestampItemTest.php

@@ -88,6 +88,35 @@ public function testDateTime(): void {
 
     // Ensure there is sample value a generated for the field.
     $this->assertNotNull($entity->field_timestamp->value);
+
+    // Ensure field type storage allows bigint and so is not affected by the
+    // 2038 bug. First, set the date to -467BC, the possible first siting of
+    // Halley's comet. Then set the date to 2134, an expected return date of
+    // Halley's comet.
+    $new_value = -76904336343;
+    $entity->field_timestamp->value = $new_value;
+    $this->entityValidateAndSave($entity);
+    $entity = EntityTest::load(2);
+    $this->assertEquals($entity->field_timestamp->value, $new_value);
+
+    $new_value = 5182657201;
+    $entity->field_timestamp->value = $new_value;
+    $this->entityValidateAndSave($entity);
+    $entity = EntityTest::load(2);
+    $this->assertEquals($entity->field_timestamp->value, $new_value);
+
+    // Ensure min and max values are accepted.
+    $new_value = -9223372036854775807;
+    $entity->field_timestamp->value = $new_value;
+    $this->entityValidateAndSave($entity);
+    $entity = EntityTest::load(2);
+    $this->assertEquals($entity->field_timestamp->value, $new_value);
+
+    $new_value = 9223372036854775807;
+    $entity->field_timestamp->value = $new_value;
+    $this->entityValidateAndSave($entity);
+    $entity = EntityTest::load(2);
+    $this->assertEquals($entity->field_timestamp->value, $new_value);
   }
 
 }

core/modules/system/system.install

@@ -14,6 +14,9 @@
 use Drupal\Core\Database\Database;
 use Drupal\Core\DrupalKernel;
 use Drupal\Core\Extension\ExtensionLifecycle;
+use Drupal\Core\Entity\Schema\DynamicallyFieldableEntityStorageSchemaInterface;
+use Drupal\Core\Entity\Sql\SqlContentEntityStorage;
+use Drupal\Core\Field\Plugin\Field\FieldType\TimestampItem;
 use Drupal\Core\File\FileSystemInterface;
 use Drupal\Core\Link;
 use Drupal\Core\Utility\PhpRequirements;
@@ -26,6 +29,7 @@
 use Drupal\Core\StringTranslation\TranslatableMarkup;
 use Drupal\Core\Url;
 use Drupal\Core\Utility\Error;
+use Drupal\Core\Utility\UpdateException;
 use Psr\Http\Client\ClientExceptionInterface;
 use Symfony\Component\HttpFoundation\Request;
 
@@ -1711,3 +1715,142 @@ function _system_advisories_requirements(array &$requirements): void {
     }
   }
 }
+
+/**
+ * Update TimestampItem field schema size to support dates greater than 2038.
+ */
+function system_update_11001(&$sandbox) {
+  // Execute only if allowed setting, 'timestamp_field_update_y2038', is true.
+  if (!Settings::get('timestamp_field_update_y2038', FALSE)) {
+    return t("Update '11001 - Update TimestampItem field schema size to support dates greater than 2038' skipped due to setting of 'timestamp_field_update_y2038' in settings.php");
+  }
+  $timeout = Settings::get('timestamp_field_update_y2038_timeout', 0);
+  if (!isset($sandbox['items'])) {
+    $items = _system_update_get_timestamp_fields();
+    $sandbox['items'] = $items;
+    $sandbox['current'] = 0;
+    $sandbox['num_processed'] = 0;
+    $sandbox['max'] = count($items);
+    $sandbox['start_time'] = \Drupal::time()->getCurrentTime();
+  }
+  elseif ($timeout > 0 && \Drupal::time()->getCurrentTime() > ($sandbox['start_time'] + $timeout)) {
+    // On subsequent runs check if the timeout is exceeded. If it is exceeded
+    // then throw an UpdateException with a helpful message. The exception is
+    // used to ensure that the update is not marked complete.
+    throw new UpdateException(sprintf("Update 10202 failed to complete, stopped after %d seconds", Settings::get('timestamp_field_update_y2038_timeout')));
+  }
+
+  [$entity_type_id, $field_name] = $sandbox['items'][$sandbox['current']] ?? [NULL, NULL];
+  if ($entity_type_id && $field_name) {
+    _system_update_process_timestamp_field($entity_type_id, $field_name);
+  }
+  $sandbox['current']++;
+
+  $sandbox['#finished'] = empty($sandbox['max']) ? 1 : ($sandbox['current'] / $sandbox['max']);
+}
+
+/**
+ * Gets a list fields that use the TimeStampItem class.
+ *
+ * @return string[]
+ *   An array with two elements, an entity type ID and a field name.
+ */
+function _system_update_get_timestamp_fields(): array {
+  $items = [];
+
+  // Get all the field definitions.
+  $field_definitions = \Drupal::service('plugin.manager.field.field_type')->getDefinitions();
+
+  // Get all the field types that use the TimestampItem class.
+  $field_types = array_keys(array_filter($field_definitions, function ($definition) {
+    return is_a($definition['class'], TimestampItem::class, TRUE);
+  }));
+
+  /** @var \Drupal\Core\Entity\EntityFieldManagerInterface $entity_field_manager */
+  $entity_field_manager = \Drupal::service('entity_field.manager');
+
+  // Build a list of all the timestamp fields.
+  foreach ($field_types as $field_type) {
+    $entity_field_map = $entity_field_manager->getFieldMapByFieldType($field_type);
+    foreach ($entity_field_map as $entity_type_id => $fields) {
+      $storage = \Drupal::entityTypeManager()->getStorage($entity_type_id);
+      if ($storage instanceof SqlContentEntityStorage) {
+        foreach (array_keys($fields) as $field_name) {
+          $items[] = [$entity_type_id, $field_name];
+        }
+      }
+    }
+  }
+
+  return $items;
+}
+
+/**
+ * Update a timestamp field to remove Y2038 limitation.
+ *
+ * @param string $entity_type_id
+ *   The entity type ID.
+ * @param string $field_name
+ *   The name of the field that needs to be updated.
+ *
+ * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException
+ * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException
+ * @throws \Drupal\Core\Entity\Sql\SqlContentEntityStorageException
+ */
+function _system_update_process_timestamp_field(string $entity_type_id, string $field_name): void {
+  /** @var \Drupal\Core\Logger\LoggerChannel $logger */
+  $logger = \Drupal::logger('update');
+
+  $entity_type_manager = \Drupal::entityTypeManager();
+  $entity_storage = $entity_type_manager->getStorage($entity_type_id);
+
+  // Get the table mappings for this field.
+  /** @var \Drupal\Core\Entity\Sql\DefaultTableMapping $table_mapping */
+  $storage_definitions = \Drupal::service('entity_field.manager')->getFieldStorageDefinitions($entity_type_id);
+  $table_mapping = $entity_storage->getTableMapping($storage_definitions);
+
+  // Field type column names map to real table column names.
+  $columns = $table_mapping->getColumnNames($field_name);
+  $column_name = $columns['value'] ?? NULL;
+
+  // We are only allowed to change the 'value' column. If that does not exist
+  // due contrib or custom code leave everything unchanged.
+  if (!$column_name) {
+    $logger->notice("Timestamp for entity '$entity_type_id' field '$field_name' not updated because database column was not found.");
+    return;
+  }
+
+  // Get the original storage definition for this field.
+  $last_installed_schema_repository = \Drupal::service('entity.last_installed_schema.repository');
+  $original_storage_definitions = $last_installed_schema_repository->getLastInstalledFieldStorageDefinitions($entity_type_id);
+  $original_storage_definition = $original_storage_definitions[$field_name];
+
+  // Get the current storage definition for this field.
+  $storage_definition = $storage_definitions[$field_name];
+  $storage = $entity_type_manager->getStorage($storage_definition->getTargetEntityTypeId());
+
+  if (!($storage instanceof DynamicallyFieldableEntityStorageSchemaInterface
+    && $storage->requiresFieldStorageSchemaChanges($storage_definition, $original_storage_definition))) {
+    $logger->notice("Timestamp for entity '$entity_type_id' field '$field_name' not updated because field size is already 'big'.");
+    return;
+  }
+
+  // Update the table specification for the timestamp field, setting the size to
+  // 'big'.
+  $schema = \Drupal::database()->schema();
+  $field_schema = $original_storage_definitions[$field_name]->getSchema() ?? $storage_definition->getSchema();
+  $specification = $field_schema['columns']['value'];
+  $specification['size'] = 'big';
+  foreach ($table_mapping->getAllFieldTableNames($field_name) as $table) {
+    $schema->changeField($table, $column_name, $column_name, $specification);
+  }
+
+  // Set flag to let EntityDefinitionUpdateManager know the column changes
+  // have been handled.
+  $storage_definition->setSetting('column_changes_handled', TRUE);
+
+  // Update the tracked entity table schema, setting the size to 'big'.
+  \Drupal::service('entity.definition_update_manager')->updateFieldStorageDefinition($storage_definition);
+
+  $logger->notice("Successfully updated entity '$entity_type_id' field '$field_name' to remove year 2038 limitation.");
+}

core/modules/system/tests/src/Functional/Update/Y2038SchemaUpdateTest.php

+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\Tests\system\Functional\Update;
+
+use Drupal\Core\Database\Database;
+use Drupal\FunctionalTests\Update\UpdatePathTestBase;
+use Drupal\Tests\UpdatePathTestTrait;
+
+/**
+ * Tests update of schema for timestamp fields to bigint.
+ *
+ * @group system
+ */
+class Y2038SchemaUpdateTest extends UpdatePathTestBase {
+
+  use UpdatePathTestTrait;
+
+  /**
+   * A user with some relevant administrative permissions.
+   *
+   * @var \Drupal\user\UserInterface
+   */
+  protected $adminUser;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected $defaultTheme = 'stark';
+
+  /**
+   * The entities and time fields.
+   *
+   * @var string[][]
+   */
+  protected $timestampFields = [
+    ["block_content", "changed"],
+    ["block_content", "revision_created"],
+    ["comment", "changed"],
+    ["comment", "created"],
+    ["file", "changed"],
+    ["file", "created"],
+    ["menu_link_content", "changed"],
+    ["menu_link_content", "revision_created"],
+    ["node", "changed"],
+    ["node", "created"],
+    ["node", "revision_timestamp"],
+    ["taxonomy_term", "changed"],
+    ["taxonomy_term", "content_translation_created"],
+    ["taxonomy_term", "revision_created"],
+    ["user", "access"],
+    ["user", "changed"],
+    ["user", "content_translation_created"],
+    ["user", "created"],
+    ["user", "login"],
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setDatabaseDumpFiles() {
+    $this->databaseDumpFiles = [
+      // Start with a filled standard install of Drupal 10.3.0.
+      DRUPAL_ROOT . '/core/modules/system/tests/fixtures/update/drupal-10.3.0.filled.standard.php.gz',
+    ];
+  }
+
+  /**
+   * Tests update of time fields.
+   */
+  public function testUpdate(): void {
+    if (\Drupal::service('database')->databaseType() == 'sqlite') {
+      $this->markTestSkipped("This test does not support the SQLite database driver.");
+    }
+
+    $this->assertBeforeSpecification(['int', 'integer', 'bigint']);
+
+    $this->runUpdates();
+
+    $this->assertAfterSpecifications(['int']);
+  }
+
+  /**
+   * Asserts the field storage specifications before the update.
+   */
+  public function assertBeforeSpecification($expected_values): void {
+    foreach ($this->timestampFields as $field_data) {
+      [
+        $specification_original,
+        $specification_current,
+      ] = $this->getSpecifications($field_data);
+
+      // The original specification is for a small int.
+      $this->assertArrayNotHasKey('size', $specification_original, "Failed for '$field_data[0]' original specification '$field_data[1]'");
+      $this->assertContains($specification_original['type'], $expected_values, "Failed for '$field_data[0]' original specification '$field_data[1]'");
+
+      // The current specification is for a size of 'big'.
+      $this->assertArrayHasKey('size', $specification_current, "Failed for '$field_data[0]' original specification '$field_data[1]'");
+      $this->assertEquals('big', $specification_current['size'], "Failed for '$field_data[0]' original specification '$field_data[1]'");
+      $this->assertContains($specification_current['type'], $expected_values, "Failed for '$field_data[0]' original specification '$field_data[1]'");
+    }
+  }
+
+  /**
+   * Asserts the field storage specifications after the update.
+   */
+  public function assertAfterSpecifications($expected_values): void {
+    // Log in to access the log messages.
+    $this->adminUser = $this->drupalCreateUser([
+      'access site reports',
+    ]);
+    $this->drupalLogin($this->adminUser);
+    $logs = Database::getConnection()->select('watchdog', 'w')
+      ->fields('w', ['message'])
+      ->condition('message', "% 2038 limitation.", "LIKE")
+      ->execute()
+      ->fetchCol();
+
+    foreach ($this->timestampFields as $field_data) {
+      [
+        $specification_original,
+        $specification_current,
+      ] = $this->getSpecifications($field_data);
+
+      // The original is updated to size of 'big'.
+      $this->assertEquals($specification_original['size'], 'big', "Failed for '$field_data[0]' original specification '$field_data[1]'");
+      $this->assertContains($specification_original['type'], $expected_values, "Failed for '$field_data[0]' original specification '$field_data[1]'");
+
+      // The current specification is still a big integer.
+      $this->assertEquals($specification_current['size'], 'big', "Failed for '$field_data[0]' original specification '$field_data[1]'");
+      $this->assertContains($specification_current['type'], $expected_values, "Failed for '$field_data[0]' original specification '$field_data[1]'");
+
+      // Check the log output for the expected success message.
+      $this_message = "Successfully updated entity '$field_data[0]' field '$field_data[1]' to remove year 2038 limitation.";
+      $this->assertContains($this_message, $logs);
+    }
+    // Confirm the number of fields changed.
+    $this->assertCount(count($this->timestampFields), $logs);
+  }
+
+  /**
+   * Gets the specifications for the provided fields.
+   *
+   * @param array $field_data
+   *   An array with two values, the entity type ID and the field name.
+   *
+   * @return array
+   *   An indexed array containing the original specification and the current
+   *   specification.
+   */
+  public function getSpecifications(array $field_data): array {
+    [$field_data[0], $field_data[1]] = $field_data;
+    $last_installed_schema_repository = \Drupal::service('entity.last_installed_schema.repository');
+
+    // Get the original storage definition for this field.
+    $original_storage_definitions = $last_installed_schema_repository->getLastInstalledFieldStorageDefinitions($field_data[0]);
+    $field_schema_original = $original_storage_definitions[$field_data[1]]->getSchema();
+    $specification_original = $field_schema_original['columns']['value'];
+
+    // Get the current storage definition for this field.
+    $storage_definitions = \Drupal::service('entity_field.manager')
+      ->getFieldStorageDefinitions($field_data[0]);
+    $storage_definition = $storage_definitions[$field_data[1]];
+    $field_schema_current = $storage_definition->getSchema();
+    $specification_current = $field_schema_current['columns']['value'];
+    return [
+      $specification_original,
+      $specification_current,
+    ];
+  }
+
+}

sites/default/default.settings.php

@@ -854,6 +854,27 @@
 # $settings['migrate_file_public_path'] = '';
 # $settings['migrate_file_private_path'] = '';
 
+/**
+ * Manage the update of timestamp fields so they work after the year 2038.
+ *
+ * For sites with hundreds of thousands of nodes and other entities this update
+ * will not be quick. Two settings are provided to help manage this update.
+ *
+ * The first setting is used to enable or disable the running of the update. Set
+ * it to FALSE to skip the update. The default value is TRUE.
+ *
+ * The second settings allows sites to set a limit on the time the update runs.
+ * Set it the number of seconds the update should run. The following example
+ * allows the update to run for 2 minutes. The default value is 0, which allows
+ * the update to run to completion.
+ *
+ * @code
+ * $settings['timestamp_field_update_y2038_timeout'] = 2 * 60;
+ * @endcode
+ */
+$settings['timestamp_field_update_y2038'] = TRUE;
+$settings['timestamp_field_update_y2038_timeout'] = 0;
+
 /**
  * Load local development override configuration, if available.
  *