- #40515: Ensure UTF-8 character set on the database side (and include upgrade...

- #40515: Ensure UTF-8 character set on the database side (and include upgrade path for incorrectly set up databases)

INSTALL.pgsql.txt

@@ -41,6 +41,7 @@ INSTALLATION AND CONFIGURATION
      createdb --encoding=UNICODE --owner=username databasename
 
    If everything works correctly, you'll see a "CREATE DATABASE" notice.
+   Note that the database must be created with UTF-8 (Unicode) encoding.
 
 3. LOAD THE DRUPAL DATABASE SCHEMA
 

database/database.pgsql

@@ -646,7 +646,7 @@ CREATE TABLE system (
   status integer NOT NULL default '0',
   throttle smallint NOT NULL default '0',
   bootstrap integer NOT NULL default '0',
-  schema_version smallint NOT NULL default 0,
+  schema_version smallint NOT NULL default -1,
   weight smallint NOT NULL default 0,
   PRIMARY KEY (filename)
 );

database/updates.inc

@@ -1437,3 +1437,145 @@ function system_update_168() {
 
   return $ret;
 }
+
+function system_update_169() {
+  // Warn PGSQL admins if their database is set up incorrectly
+  if ($GLOBALS['db_type'] == 'pgsql') {
+    $encoding = db_result(db_query('SHOW server_encoding'));
+    if (!in_array(strtolower($encoding), array('unicode', 'utf8'))) {
+      $msg = 'Your PostgreSQL database is set up with the wrong character encoding ('. $encoding .'). It is possible it will not work as expected. It is advised to recreate it with UTF-8/Unicode encoding. More information can be found in the <a href="http://www.postgresql.org/docs/7.4/interactive/multibyte.html">PostgreSQL documentation</a>.';
+      watchdog('php', $msg, WATCHDOG_WARNING);
+      drupal_set_message($msg, 'status');
+    }
+  }
+
+  return _system_update_utf8(array(
+    'access', 'accesslog', 'aggregator_category',
+    'aggregator_category_feed', 'aggregator_category_item',
+    'aggregator_feed', 'aggregator_item', 'authmap', 'blocks',
+    'book', 'boxes', 'cache', 'comments', 'contact',
+    'node_comment_statistics', 'client', 'client_system', 'files',
+    'filter_formats', 'filters', 'flood', 'forum', 'history',
+    'locales_meta', 'locales_source', 'locales_target', 'menu',
+    'moderation_filters', 'moderation_roles', 'moderation_votes',
+    'node', 'node_access', 'node_revisions', 'profile_fields',
+    'profile_values', 'url_alias', 'permission', 'poll', 'poll_votes',
+    'poll_choices', 'role', 'search_dataset', 'search_index',
+    'search_total', 'sessions', 'sequences', 'node_counter',
+    'system', 'term_data', 'term_hierarchy', 'term_node',
+    'term_relation', 'term_synonym', 'users', 'users_roles', 'variable',
+    'vocabulary', 'vocabulary_node_types', 'watchdog'
+  ));
+}
+
+/**
+ * Converts tables to UTF-8 encoding.
+ * 
+ * This update is designed to be re-usable by contrib modules and is
+ * used by system_update_169().
+ */
+function _system_update_utf8($tables) {
+  // Are we starting this update for the first time?
+  if (!isset($_SESSION['update_utf8'])) {
+    switch ($GLOBALS['db_type']) {
+      // Only for MySQL 4.1+
+      case 'mysqli':
+        break;
+      case 'mysql':
+        if (version_compare(mysql_get_server_info($GLOBALS['active_db']), '4.1.0', '<')) {
+          return array();
+        }
+        break;
+      case 'pgsql':
+        return array();
+    }
+
+    // See if database uses UTF-8 already
+    $url = parse_url($GLOBALS['db_url']);
+    $db_name = substr($url['path'], 1);
+    $create = array_pop(db_fetch_array(db_query('SHOW CREATE DATABASE `%s`', $db_name)));
+    if (preg_match('/utf8/i', $create)) {
+      return array();
+    }
+
+    // Make list of tables to convert
+    $_SESSION['update_utf8'] = $tables;
+    // Keep track of total for progress bar
+    $_SESSION['update_utf8_total'] = count($tables);
+  }
+
+  // Fetch remaining tables list
+  $list = &$_SESSION['update_utf8'];
+  $ret = array();
+
+  // Convert a table to UTF-8.
+  // We change all text columns to their correspending binary type,
+  // then back to text, but with a UTF-8 character set.
+  // See: http://dev.mysql.com/doc/refman/4.1/en/charset-conversion.html
+  $types = array('char' => 'binary',
+                 'varchar' => 'varbinary',
+                 'tinytext' => 'tinyblob',
+                 'text' => 'blob',
+                 'mediumtext' => 'mediumblob',
+                 'longtext' => 'longblob');
+
+  // Get next table in list
+  $table = array_shift($list);
+  $convert_to_binary = array();
+  $convert_to_utf8 = array();
+
+  // Set table default charset
+  $ret[] = update_sql("ALTER TABLE \{$table} DEFAULT CHARACTER SET utf8");
+
+  // Find out which columns need converting and build SQL statements
+  $result = db_query("SHOW FULL COLUMNS FROM \{$table}");
+  while ($column = db_fetch_array($result)) {
+    list($type) = explode('(', $column['Type']);
+    if (isset($types[$type])) {
+      $names = 'CHANGE `'. $column['Field'] .'` `'. $column['Field'] .'` ';
+      $attributes = ' DEFAULT '. ($column['Default'] == 'NULL' ? 'NULL ' :
+                     "'". db_escape_string($column['Default']) ."' ") .
+                    ($column['Null'] == 'YES' ? 'NULL' : 'NOT NULL');
+
+      $convert_to_binary[] = $names . preg_replace('/'. $type .'/i', $types[$type], $column['Type']) . $attributes;
+      $convert_to_utf8[] = $names . $column['Type'] .' CHARACTER SET utf8'. $attributes;
+    }
+  }
+
+  if (count($convert_to_binary)) {
+    // Convert text columns to binary
+    $ret[] = update_sql("ALTER TABLE \{$table} ". implode(', ', $convert_to_binary));
+    // Convert binary columns to UTF-8
+    $ret[] = update_sql("ALTER TABLE \{$table} ". implode(', ', $convert_to_utf8));
+  }
+
+  // Are we done?
+  if (count($list) == 0) {
+    unset($_SESSION['update_utf8']);
+    unset($_SESSION['update_utf8_total']);
+    return $ret;
+  }
+
+  // Progress percentage
+  $ret['#finished'] = 1 - (count($list) / $_SESSION['update_utf8_total']);
+  return $ret;
+}
+
+function system_update_170() {
+  if (!variable_get('update_170_done', false)) {
+    switch ($GLOBALS['db_type']) {
+      case 'pgsql':
+        $ret = array();
+        db_change_column($ret, 'system', 'schema_version', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => -1));
+        break;
+
+      case 'mysql':
+      case 'mysqli':
+        db_query('ALTER TABLE {system} CHANGE schema_version schema_version smallint(3) not null default -1');
+        break;
+    }
+    // Set schema version -1 (uninstalled) for disabled modules (only affects contrib).
+    db_query('UPDATE {system} SET schema_version = -1 WHERE status = 0 AND schema_version = 0');
+  }
+  return array();
+}

includes/database.mysql.inc

@@ -62,7 +62,7 @@ function db_connect($url) {
   if (!mysql_select_db(substr($url['path'], 1))) {
     drupal_maintenance_theme();
     drupal_set_title('Unable to select database');
-    print theme('maintenance_page', '<p>We were able to connect to the MySQL database server (which means your username and password is okay) but not able to select the database.</p>
+    print theme('maintenance_page', '<p>We were able to connect to the MySQL database server (which means your username and password are okay) but not able to select the database.</p>
 <p>The MySQL error was: '. theme('placeholder', mysql_error($connection)) .'.</p>
 <p>Currently, the database is '. theme('placeholder', substr($url['path'], 1)) .'. The username is '. theme('placeholder', $url['user']) .' and the database server is '. theme('placeholder', $url['host']) .'.</p>
 <ul>
@@ -74,6 +74,16 @@ function db_connect($url) {
     exit;
   }
 
+  /* On MySQL 4.1 and later, force UTF-8 */
+  if (version_compare(mysql_get_server_info(), '4.1.0', '>=')) {
+    mysql_query('SET NAMES "utf8"', $connection);
+    mysql_query('SET collation_connection="utf8_general_ci"', $connection);
+    mysql_query('SET collation_server="utf8_general_ci"', $connection);
+    mysql_query('SET character_set_client="utf8"', $connection);
+    mysql_query('SET character_set_connection="utf8"', $connection);
+    mysql_query('SET character_set_results="utf8"', $connection);
+    mysql_query('SET character_set_server="utf8"', $connection);
+  }
   return $connection;
 }
 

includes/database.mysqli.inc

@@ -71,6 +71,14 @@ function db_connect($url) {
     exit;
   }
 
+  /* Force UTF-8 */
+  mysqli_query($connection, 'SET NAMES "utf8"');
+  mysqli_query($connection, 'SET collation_connection="utf8_general_ci"');
+  mysqli_query($connection, 'SET collation_server="utf8_general_ci"');
+  mysqli_query($connection, 'SET character_set_client="utf8"');
+  mysqli_query($connection, 'SET character_set_connection="utf8"');
+  mysqli_query($connection, 'SET character_set_results="utf8"');
+  mysqli_query($connection, 'SET character_set_server="utf8"');
 
   /**
    * from: http://bugs.php.net/bug.php?id=33772

includes/database.pgsql.inc

@@ -334,6 +334,16 @@ function db_unlock_tables() {
   db_query('COMMIT');
 }
 
+/**
+ * Verify if the database is set up correctly.
+ */
+function db_check_setup() {
+  $encoding = db_result(db_query('SHOW server_encoding'));
+  if (!in_array(strtolower($encoding), array('unicode', 'utf8'))) {
+    drupal_set_message(t('Your PostgreSQL database is set up with the wrong character encoding (%encoding). It is possibile it will not work as expected. It is advised to recreate it with UTF-8/Unicode encoding. More information can be found in the <a href="%url">PostgreSQL documentation</a>.', array('%encoding' => $encoding, '%url' => 'http://www.postgresql.org/docs/7.4/interactive/multibyte.html')), 'status');
+  }
+}
+
 /**
  * @} End of "ingroup database".
  */

includes/install.inc

 <?php
 // $Id$
 
-define('SCHEMA', 0);
-define('SCHEMA_MIN', 1);
+define('SCHEMA_UNINSTALLED', -1);
+define('SCHEMA_INSTALLED', 0);
 
 
 // The system module (Drupal core) is currently a special case

modules/system.module

@@ -472,6 +472,10 @@ function system_view_general() {
   $form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE);
   $form['cron'] = array_merge($form['cron'], system_cron_settings());
 
+  // Check database setup if necessary
+  if (function_exists('db_check_setup') && empty($_POST)) {
+    db_check_setup();
+  }
   return $form;
 }
 
@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) {
   foreach ($new_modules as $module) {
     // Set the installed schema version for newly-enabled modules
     $versions = drupal_get_schema_versions($module);
-    if ($versions !== FALSE && drupal_get_installed_schema_version($module) == 0) {
-      drupal_set_installed_schema_version($module, max($versions));
+    if (drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
+      drupal_set_installed_schema_version($module, $versions ? max($versions) : SCHEMA_INSTALLED);
     }
   }
 

modules/system/system.module

@@ -472,6 +472,10 @@ function system_view_general() {
   $form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE);
   $form['cron'] = array_merge($form['cron'], system_cron_settings());
 
+  // Check database setup if necessary
+  if (function_exists('db_check_setup') && empty($_POST)) {
+    db_check_setup();
+  }
   return $form;
 }
 
@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) {
   foreach ($new_modules as $module) {
     // Set the installed schema version for newly-enabled modules
     $versions = drupal_get_schema_versions($module);
-    if ($versions !== FALSE && drupal_get_installed_schema_version($module) == 0) {
-      drupal_set_installed_schema_version($module, max($versions));
+    if (drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
+      drupal_set_installed_schema_version($module, $versions ? max($versions) : SCHEMA_INSTALLED);
     }
   }
 

update.php

@@ -132,7 +132,7 @@ function db_change_column(&$ret, $table, $column, $column_new, $type, $attribute
 function update_fix_schema_version() {
   if ($update_start = variable_get('update_start', FALSE)) {
     // Some updates were made to the 4.6 branch and 4.7 branch. This sets
-    // temporary variables to provent the updates from being executed twice and
+    // temporary variables to prevent the updates from being executed twice and
     // throwing errors.
     switch ($update_start) {
       case '2005-04-14':
@@ -151,6 +151,9 @@ function update_fix_schema_version() {
         break;
 
     }
+    // The schema_version column (added below) was changed during 4.7beta.
+    // Update_170 is only for those beta users.
+    variable_set('update_170_done', TRUE);
 
     $sql_updates = array(
       '2004-10-31: first update since Drupal 4.5.0 release' => 110,
@@ -174,18 +177,22 @@ function update_fix_schema_version() {
       '2005-11-14' => 154, '2005-11-27' => 155, '2005-12-03' => 156,
     );
 
+    // Add schema version column
     switch ($GLOBALS['db_type']) {
       case 'pgsql':
         $ret = array();
-        db_add_column($ret, 'system', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => 0));
+        db_add_column($ret, 'system', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => -1));
         break;
 
       case 'mysql':
       case 'mysqli':
-        db_query('ALTER TABLE {system} ADD schema_version smallint(2) unsigned not null default 0');
+        db_query('ALTER TABLE {system} ADD schema_version smallint(3) not null default -1');
         break;
     }
+    // Set all enabled (contrib) modules to schema version 0 (installed)
+    db_query('UPDATE {system} SET schema_version = 0 WHERE status = 1');
 
+    // Set schema version for core
     drupal_set_installed_schema_version('system', $sql_updates[$update_start]);
     variable_del('update_start');
   }
@@ -276,7 +283,7 @@ function update_fix_watchdog() {
 function update_data($module, $number) {
   $ret = module_invoke($module, 'update_'. $number);
   // Assume the update finished unless the update results indicate otherwise.
-  $finished = TRUE;
+  $finished = 1;
   if (isset($ret['#finished'])) {
     $finished = $ret['#finished'];
     unset($ret['#finished']);
@@ -294,7 +301,7 @@ function update_data($module, $number) {
   }
   $_SESSION['update_results'][$module][$number] = array_merge($_SESSION['update_results'][$module][$number], $ret);
 
-  if ($finished) {
+  if ($finished == 1) {
     // Update the installed version
     drupal_set_installed_schema_version($module, $number);
   }
@@ -332,11 +339,11 @@ function update_selection_page() {
   $form['has_js'] = array(
     '#type' => 'hidden',
     '#default_value' => FALSE,
-    '#attributes' => array('id' => 'edit-has_js')
+    '#attributes' => array('id' => 'edit-has_js'),
   );
   $form['submit'] = array(
     '#type' => 'submit',
-    '#value' => 'Update'
+    '#value' => 'Update',
   );
 
   drupal_set_title('Drupal database update');
@@ -375,7 +382,7 @@ function update_progress_page() {
 
   drupal_set_title('Updating');
   $output = '<div id="progress"></div>';
-  $output .= '<p>Updating your site will take a few seconds.</p>';
+  $output .= '<p>Please wait while your site is being updated.</p>';
   return $output;
 }
 
@@ -389,9 +396,10 @@ function update_progress_page() {
 function update_do_updates() {
   while (($update = reset($_SESSION['update_remaining']))) {
     $update_finished = update_data($update['module'], $update['version']);
-    if ($update_finished) {
+    if ($update_finished == 1) {
       // Dequeue the completed update.
       unset($_SESSION['update_remaining'][key($_SESSION['update_remaining'])]);
+      $update_finished = 0; // Make sure this step isn't counted double
     }
     if (timer_read('page') > 1000) {
       break;
@@ -399,12 +407,12 @@ function update_do_updates() {
   }
 
   if ($_SESSION['update_total']) {
-    $percent = floor(($_SESSION['update_total'] - count($_SESSION['update_remaining'])) / $_SESSION['update_total'] * 100);
+    $percent = floor(($_SESSION['update_total'] - count($_SESSION['update_remaining']) + $update_finished) / $_SESSION['update_total'] * 100);
   }
   else {
     $percent = 100;
   }
-  return array($percent, 'Updating '. $update['module'] .' module');
+  return array($percent, isset($update['module']) ? 'Updating '. $update['module'] .' module' : 'Updating complete');
 }
 
 function update_do_update_page() {
@@ -438,7 +446,7 @@ function update_progress_page_nojs() {
   else {
     // This is the first page so return some output immediately.
     $percent = 0;
-    $message = 'Starting updates...';
+    $message = 'Starting updates';
   }
 
   drupal_set_html_head('<meta http-equiv="Refresh" content="0; URL=update.php?op='. $new_op .'">');