Commit 7914ad0f authored by Steven Wittens's avatar Steven Wittens

- #40515: Ensure UTF-8 character set on the database side (and include upgrade...

- #40515: Ensure UTF-8 character set on the database side (and include upgrade path for incorrectly set up databases)
parent 0ff0c4be
...@@ -41,6 +41,7 @@ INSTALLATION AND CONFIGURATION ...@@ -41,6 +41,7 @@ INSTALLATION AND CONFIGURATION
createdb --encoding=UNICODE --owner=username databasename createdb --encoding=UNICODE --owner=username databasename
If everything works correctly, you'll see a "CREATE DATABASE" notice. If everything works correctly, you'll see a "CREATE DATABASE" notice.
Note that the database must be created with UTF-8 (Unicode) encoding.
3. LOAD THE DRUPAL DATABASE SCHEMA 3. LOAD THE DRUPAL DATABASE SCHEMA
......
This diff is collapsed.
...@@ -646,7 +646,7 @@ CREATE TABLE system ( ...@@ -646,7 +646,7 @@ CREATE TABLE system (
status integer NOT NULL default '0', status integer NOT NULL default '0',
throttle smallint NOT NULL default '0', throttle smallint NOT NULL default '0',
bootstrap integer NOT NULL default '0', bootstrap integer NOT NULL default '0',
schema_version smallint NOT NULL default 0, schema_version smallint NOT NULL default -1,
weight smallint NOT NULL default 0, weight smallint NOT NULL default 0,
PRIMARY KEY (filename) PRIMARY KEY (filename)
); );
......
...@@ -1437,3 +1437,145 @@ function system_update_168() { ...@@ -1437,3 +1437,145 @@ function system_update_168() {
return $ret; return $ret;
} }
function system_update_169() {
// Warn PGSQL admins if their database is set up incorrectly
if ($GLOBALS['db_type'] == 'pgsql') {
$encoding = db_result(db_query('SHOW server_encoding'));
if (!in_array(strtolower($encoding), array('unicode', 'utf8'))) {
$msg = 'Your PostgreSQL database is set up with the wrong character encoding ('. $encoding .'). It is possible it will not work as expected. It is advised to recreate it with UTF-8/Unicode encoding. More information can be found in the <a href="http://www.postgresql.org/docs/7.4/interactive/multibyte.html">PostgreSQL documentation</a>.';
watchdog('php', $msg, WATCHDOG_WARNING);
drupal_set_message($msg, 'status');
}
}
return _system_update_utf8(array(
'access', 'accesslog', 'aggregator_category',
'aggregator_category_feed', 'aggregator_category_item',
'aggregator_feed', 'aggregator_item', 'authmap', 'blocks',
'book', 'boxes', 'cache', 'comments', 'contact',
'node_comment_statistics', 'client', 'client_system', 'files',
'filter_formats', 'filters', 'flood', 'forum', 'history',
'locales_meta', 'locales_source', 'locales_target', 'menu',
'moderation_filters', 'moderation_roles', 'moderation_votes',
'node', 'node_access', 'node_revisions', 'profile_fields',
'profile_values', 'url_alias', 'permission', 'poll', 'poll_votes',
'poll_choices', 'role', 'search_dataset', 'search_index',
'search_total', 'sessions', 'sequences', 'node_counter',
'system', 'term_data', 'term_hierarchy', 'term_node',
'term_relation', 'term_synonym', 'users', 'users_roles', 'variable',
'vocabulary', 'vocabulary_node_types', 'watchdog'
));
}
/**
* Converts tables to UTF-8 encoding.
*
* This update is designed to be re-usable by contrib modules and is
* used by system_update_169().
*/
function _system_update_utf8($tables) {
// Are we starting this update for the first time?
if (!isset($_SESSION['update_utf8'])) {
switch ($GLOBALS['db_type']) {
// Only for MySQL 4.1+
case 'mysqli':
break;
case 'mysql':
if (version_compare(mysql_get_server_info($GLOBALS['active_db']), '4.1.0', '<')) {
return array();
}
break;
case 'pgsql':
return array();
}
// See if database uses UTF-8 already
$url = parse_url($GLOBALS['db_url']);
$db_name = substr($url['path'], 1);
$create = array_pop(db_fetch_array(db_query('SHOW CREATE DATABASE `%s`', $db_name)));
if (preg_match('/utf8/i', $create)) {
return array();
}
// Make list of tables to convert
$_SESSION['update_utf8'] = $tables;
// Keep track of total for progress bar
$_SESSION['update_utf8_total'] = count($tables);
}
// Fetch remaining tables list
$list = &$_SESSION['update_utf8'];
$ret = array();
// Convert a table to UTF-8.
// We change all text columns to their correspending binary type,
// then back to text, but with a UTF-8 character set.
// See: http://dev.mysql.com/doc/refman/4.1/en/charset-conversion.html
$types = array('char' => 'binary',
'varchar' => 'varbinary',
'tinytext' => 'tinyblob',
'text' => 'blob',
'mediumtext' => 'mediumblob',
'longtext' => 'longblob');
// Get next table in list
$table = array_shift($list);
$convert_to_binary = array();
$convert_to_utf8 = array();
// Set table default charset
$ret[] = update_sql("ALTER TABLE \{$table} DEFAULT CHARACTER SET utf8");
// Find out which columns need converting and build SQL statements
$result = db_query("SHOW FULL COLUMNS FROM \{$table}");
while ($column = db_fetch_array($result)) {
list($type) = explode('(', $column['Type']);
if (isset($types[$type])) {
$names = 'CHANGE `'. $column['Field'] .'` `'. $column['Field'] .'` ';
$attributes = ' DEFAULT '. ($column['Default'] == 'NULL' ? 'NULL ' :
"'". db_escape_string($column['Default']) ."' ") .
($column['Null'] == 'YES' ? 'NULL' : 'NOT NULL');
$convert_to_binary[] = $names . preg_replace('/'. $type .'/i', $types[$type], $column['Type']) . $attributes;
$convert_to_utf8[] = $names . $column['Type'] .' CHARACTER SET utf8'. $attributes;
}
}
if (count($convert_to_binary)) {
// Convert text columns to binary
$ret[] = update_sql("ALTER TABLE \{$table} ". implode(', ', $convert_to_binary));
// Convert binary columns to UTF-8
$ret[] = update_sql("ALTER TABLE \{$table} ". implode(', ', $convert_to_utf8));
}
// Are we done?
if (count($list) == 0) {
unset($_SESSION['update_utf8']);
unset($_SESSION['update_utf8_total']);
return $ret;
}
// Progress percentage
$ret['#finished'] = 1 - (count($list) / $_SESSION['update_utf8_total']);
return $ret;
}
function system_update_170() {
if (!variable_get('update_170_done', false)) {
switch ($GLOBALS['db_type']) {
case 'pgsql':
$ret = array();
db_change_column($ret, 'system', 'schema_version', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => -1));
break;
case 'mysql':
case 'mysqli':
db_query('ALTER TABLE {system} CHANGE schema_version schema_version smallint(3) not null default -1');
break;
}
// Set schema version -1 (uninstalled) for disabled modules (only affects contrib).
db_query('UPDATE {system} SET schema_version = -1 WHERE status = 0 AND schema_version = 0');
}
return array();
}
...@@ -62,7 +62,7 @@ function db_connect($url) { ...@@ -62,7 +62,7 @@ function db_connect($url) {
if (!mysql_select_db(substr($url['path'], 1))) { if (!mysql_select_db(substr($url['path'], 1))) {
drupal_maintenance_theme(); drupal_maintenance_theme();
drupal_set_title('Unable to select database'); drupal_set_title('Unable to select database');
print theme('maintenance_page', '<p>We were able to connect to the MySQL database server (which means your username and password is okay) but not able to select the database.</p> print theme('maintenance_page', '<p>We were able to connect to the MySQL database server (which means your username and password are okay) but not able to select the database.</p>
<p>The MySQL error was: '. theme('placeholder', mysql_error($connection)) .'.</p> <p>The MySQL error was: '. theme('placeholder', mysql_error($connection)) .'.</p>
<p>Currently, the database is '. theme('placeholder', substr($url['path'], 1)) .'. The username is '. theme('placeholder', $url['user']) .' and the database server is '. theme('placeholder', $url['host']) .'.</p> <p>Currently, the database is '. theme('placeholder', substr($url['path'], 1)) .'. The username is '. theme('placeholder', $url['user']) .' and the database server is '. theme('placeholder', $url['host']) .'.</p>
<ul> <ul>
...@@ -74,6 +74,16 @@ function db_connect($url) { ...@@ -74,6 +74,16 @@ function db_connect($url) {
exit; exit;
} }
/* On MySQL 4.1 and later, force UTF-8 */
if (version_compare(mysql_get_server_info(), '4.1.0', '>=')) {
mysql_query('SET NAMES "utf8"', $connection);
mysql_query('SET collation_connection="utf8_general_ci"', $connection);
mysql_query('SET collation_server="utf8_general_ci"', $connection);
mysql_query('SET character_set_client="utf8"', $connection);
mysql_query('SET character_set_connection="utf8"', $connection);
mysql_query('SET character_set_results="utf8"', $connection);
mysql_query('SET character_set_server="utf8"', $connection);
}
return $connection; return $connection;
} }
......
...@@ -71,6 +71,14 @@ function db_connect($url) { ...@@ -71,6 +71,14 @@ function db_connect($url) {
exit; exit;
} }
/* Force UTF-8 */
mysqli_query($connection, 'SET NAMES "utf8"');
mysqli_query($connection, 'SET collation_connection="utf8_general_ci"');
mysqli_query($connection, 'SET collation_server="utf8_general_ci"');
mysqli_query($connection, 'SET character_set_client="utf8"');
mysqli_query($connection, 'SET character_set_connection="utf8"');
mysqli_query($connection, 'SET character_set_results="utf8"');
mysqli_query($connection, 'SET character_set_server="utf8"');
/** /**
* from: http://bugs.php.net/bug.php?id=33772 * from: http://bugs.php.net/bug.php?id=33772
......
...@@ -334,6 +334,16 @@ function db_unlock_tables() { ...@@ -334,6 +334,16 @@ function db_unlock_tables() {
db_query('COMMIT'); db_query('COMMIT');
} }
/**
* Verify if the database is set up correctly.
*/
function db_check_setup() {
$encoding = db_result(db_query('SHOW server_encoding'));
if (!in_array(strtolower($encoding), array('unicode', 'utf8'))) {
drupal_set_message(t('Your PostgreSQL database is set up with the wrong character encoding (%encoding). It is possibile it will not work as expected. It is advised to recreate it with UTF-8/Unicode encoding. More information can be found in the <a href="%url">PostgreSQL documentation</a>.', array('%encoding' => $encoding, '%url' => 'http://www.postgresql.org/docs/7.4/interactive/multibyte.html')), 'status');
}
}
/** /**
* @} End of "ingroup database". * @} End of "ingroup database".
*/ */
......
<?php <?php
// $Id$ // $Id$
define('SCHEMA', 0); define('SCHEMA_UNINSTALLED', -1);
define('SCHEMA_MIN', 1); define('SCHEMA_INSTALLED', 0);
// The system module (Drupal core) is currently a special case // The system module (Drupal core) is currently a special case
......
...@@ -472,6 +472,10 @@ function system_view_general() { ...@@ -472,6 +472,10 @@ function system_view_general() {
$form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE); $form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE);
$form['cron'] = array_merge($form['cron'], system_cron_settings()); $form['cron'] = array_merge($form['cron'], system_cron_settings());
// Check database setup if necessary
if (function_exists('db_check_setup') && empty($_POST)) {
db_check_setup();
}
return $form; return $form;
} }
...@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) { ...@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) {
foreach ($new_modules as $module) { foreach ($new_modules as $module) {
// Set the installed schema version for newly-enabled modules // Set the installed schema version for newly-enabled modules
$versions = drupal_get_schema_versions($module); $versions = drupal_get_schema_versions($module);
if ($versions !== FALSE && drupal_get_installed_schema_version($module) == 0) { if (drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
drupal_set_installed_schema_version($module, max($versions)); drupal_set_installed_schema_version($module, $versions ? max($versions) : SCHEMA_INSTALLED);
} }
} }
......
...@@ -472,6 +472,10 @@ function system_view_general() { ...@@ -472,6 +472,10 @@ function system_view_general() {
$form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE); $form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE);
$form['cron'] = array_merge($form['cron'], system_cron_settings()); $form['cron'] = array_merge($form['cron'], system_cron_settings());
// Check database setup if necessary
if (function_exists('db_check_setup') && empty($_POST)) {
db_check_setup();
}
return $form; return $form;
} }
...@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) { ...@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) {
foreach ($new_modules as $module) { foreach ($new_modules as $module) {
// Set the installed schema version for newly-enabled modules // Set the installed schema version for newly-enabled modules
$versions = drupal_get_schema_versions($module); $versions = drupal_get_schema_versions($module);
if ($versions !== FALSE && drupal_get_installed_schema_version($module) == 0) { if (drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
drupal_set_installed_schema_version($module, max($versions)); drupal_set_installed_schema_version($module, $versions ? max($versions) : SCHEMA_INSTALLED);
} }
} }
......
...@@ -132,7 +132,7 @@ function db_change_column(&$ret, $table, $column, $column_new, $type, $attribute ...@@ -132,7 +132,7 @@ function db_change_column(&$ret, $table, $column, $column_new, $type, $attribute
function update_fix_schema_version() { function update_fix_schema_version() {
if ($update_start = variable_get('update_start', FALSE)) { if ($update_start = variable_get('update_start', FALSE)) {
// Some updates were made to the 4.6 branch and 4.7 branch. This sets // Some updates were made to the 4.6 branch and 4.7 branch. This sets
// temporary variables to provent the updates from being executed twice and // temporary variables to prevent the updates from being executed twice and
// throwing errors. // throwing errors.
switch ($update_start) { switch ($update_start) {
case '2005-04-14': case '2005-04-14':
...@@ -151,6 +151,9 @@ function update_fix_schema_version() { ...@@ -151,6 +151,9 @@ function update_fix_schema_version() {
break; break;
} }
// The schema_version column (added below) was changed during 4.7beta.
// Update_170 is only for those beta users.
variable_set('update_170_done', TRUE);
$sql_updates = array( $sql_updates = array(
'2004-10-31: first update since Drupal 4.5.0 release' => 110, '2004-10-31: first update since Drupal 4.5.0 release' => 110,
...@@ -174,18 +177,22 @@ function update_fix_schema_version() { ...@@ -174,18 +177,22 @@ function update_fix_schema_version() {
'2005-11-14' => 154, '2005-11-27' => 155, '2005-12-03' => 156, '2005-11-14' => 154, '2005-11-27' => 155, '2005-12-03' => 156,
); );
// Add schema version column
switch ($GLOBALS['db_type']) { switch ($GLOBALS['db_type']) {
case 'pgsql': case 'pgsql':
$ret = array(); $ret = array();
db_add_column($ret, 'system', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => 0)); db_add_column($ret, 'system', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => -1));
break; break;
case 'mysql': case 'mysql':
case 'mysqli': case 'mysqli':
db_query('ALTER TABLE {system} ADD schema_version smallint(2) unsigned not null default 0'); db_query('ALTER TABLE {system} ADD schema_version smallint(3) not null default -1');
break; break;
} }
// Set all enabled (contrib) modules to schema version 0 (installed)
db_query('UPDATE {system} SET schema_version = 0 WHERE status = 1');
// Set schema version for core
drupal_set_installed_schema_version('system', $sql_updates[$update_start]); drupal_set_installed_schema_version('system', $sql_updates[$update_start]);
variable_del('update_start'); variable_del('update_start');
} }
...@@ -276,7 +283,7 @@ function update_fix_watchdog() { ...@@ -276,7 +283,7 @@ function update_fix_watchdog() {
function update_data($module, $number) { function update_data($module, $number) {
$ret = module_invoke($module, 'update_'. $number); $ret = module_invoke($module, 'update_'. $number);
// Assume the update finished unless the update results indicate otherwise. // Assume the update finished unless the update results indicate otherwise.
$finished = TRUE; $finished = 1;
if (isset($ret['#finished'])) { if (isset($ret['#finished'])) {
$finished = $ret['#finished']; $finished = $ret['#finished'];
unset($ret['#finished']); unset($ret['#finished']);
...@@ -294,7 +301,7 @@ function update_data($module, $number) { ...@@ -294,7 +301,7 @@ function update_data($module, $number) {
} }
$_SESSION['update_results'][$module][$number] = array_merge($_SESSION['update_results'][$module][$number], $ret); $_SESSION['update_results'][$module][$number] = array_merge($_SESSION['update_results'][$module][$number], $ret);
if ($finished) { if ($finished == 1) {
// Update the installed version // Update the installed version
drupal_set_installed_schema_version($module, $number); drupal_set_installed_schema_version($module, $number);
} }
...@@ -332,11 +339,11 @@ function update_selection_page() { ...@@ -332,11 +339,11 @@ function update_selection_page() {
$form['has_js'] = array( $form['has_js'] = array(
'#type' => 'hidden', '#type' => 'hidden',
'#default_value' => FALSE, '#default_value' => FALSE,
'#attributes' => array('id' => 'edit-has_js') '#attributes' => array('id' => 'edit-has_js'),
); );
$form['submit'] = array( $form['submit'] = array(
'#type' => 'submit', '#type' => 'submit',
'#value' => 'Update' '#value' => 'Update',
); );
drupal_set_title('Drupal database update'); drupal_set_title('Drupal database update');
...@@ -375,7 +382,7 @@ function update_progress_page() { ...@@ -375,7 +382,7 @@ function update_progress_page() {
drupal_set_title('Updating'); drupal_set_title('Updating');
$output = '<div id="progress"></div>'; $output = '<div id="progress"></div>';
$output .= '<p>Updating your site will take a few seconds.</p>'; $output .= '<p>Please wait while your site is being updated.</p>';
return $output; return $output;
} }
...@@ -389,9 +396,10 @@ function update_progress_page() { ...@@ -389,9 +396,10 @@ function update_progress_page() {
function update_do_updates() { function update_do_updates() {
while (($update = reset($_SESSION['update_remaining']))) { while (($update = reset($_SESSION['update_remaining']))) {
$update_finished = update_data($update['module'], $update['version']); $update_finished = update_data($update['module'], $update['version']);
if ($update_finished) { if ($update_finished == 1) {
// Dequeue the completed update. // Dequeue the completed update.
unset($_SESSION['update_remaining'][key($_SESSION['update_remaining'])]); unset($_SESSION['update_remaining'][key($_SESSION['update_remaining'])]);
$update_finished = 0; // Make sure this step isn't counted double
} }
if (timer_read('page') > 1000) { if (timer_read('page') > 1000) {
break; break;
...@@ -399,12 +407,12 @@ function update_do_updates() { ...@@ -399,12 +407,12 @@ function update_do_updates() {
} }
if ($_SESSION['update_total']) { if ($_SESSION['update_total']) {
$percent = floor(($_SESSION['update_total'] - count($_SESSION['update_remaining'])) / $_SESSION['update_total'] * 100); $percent = floor(($_SESSION['update_total'] - count($_SESSION['update_remaining']) + $update_finished) / $_SESSION['update_total'] * 100);
} }
else { else {
$percent = 100; $percent = 100;
} }
return array($percent, 'Updating '. $update['module'] .' module'); return array($percent, isset($update['module']) ? 'Updating '. $update['module'] .' module' : 'Updating complete');
} }
function update_do_update_page() { function update_do_update_page() {
...@@ -438,7 +446,7 @@ function update_progress_page_nojs() { ...@@ -438,7 +446,7 @@ function update_progress_page_nojs() {
else { else {
// This is the first page so return some output immediately. // This is the first page so return some output immediately.
$percent = 0; $percent = 0;
$message = 'Starting updates...'; $message = 'Starting updates';
} }
drupal_set_html_head('<meta http-equiv="Refresh" content="0; URL=update.php?op='. $new_op .'">'); drupal_set_html_head('<meta http-equiv="Refresh" content="0; URL=update.php?op='. $new_op .'">');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment