Commit 7914ad0f authored by Steven Wittens's avatar Steven Wittens

- #40515: Ensure UTF-8 character set on the database side (and include upgrade...

- #40515: Ensure UTF-8 character set on the database side (and include upgrade path for incorrectly set up databases)
parent 0ff0c4be
......@@ -41,6 +41,7 @@ INSTALLATION AND CONFIGURATION
createdb --encoding=UNICODE --owner=username databasename
If everything works correctly, you'll see a "CREATE DATABASE" notice.
Note that the database must be created with UTF-8 (Unicode) encoding.
3. LOAD THE DRUPAL DATABASE SCHEMA
......
This diff is collapsed.
......@@ -646,7 +646,7 @@ CREATE TABLE system (
status integer NOT NULL default '0',
throttle smallint NOT NULL default '0',
bootstrap integer NOT NULL default '0',
schema_version smallint NOT NULL default 0,
schema_version smallint NOT NULL default -1,
weight smallint NOT NULL default 0,
PRIMARY KEY (filename)
);
......
......@@ -1437,3 +1437,145 @@ function system_update_168() {
return $ret;
}
function system_update_169() {
// Warn PGSQL admins if their database is set up incorrectly
if ($GLOBALS['db_type'] == 'pgsql') {
$encoding = db_result(db_query('SHOW server_encoding'));
if (!in_array(strtolower($encoding), array('unicode', 'utf8'))) {
$msg = 'Your PostgreSQL database is set up with the wrong character encoding ('. $encoding .'). It is possible it will not work as expected. It is advised to recreate it with UTF-8/Unicode encoding. More information can be found in the <a href="http://www.postgresql.org/docs/7.4/interactive/multibyte.html">PostgreSQL documentation</a>.';
watchdog('php', $msg, WATCHDOG_WARNING);
drupal_set_message($msg, 'status');
}
}
return _system_update_utf8(array(
'access', 'accesslog', 'aggregator_category',
'aggregator_category_feed', 'aggregator_category_item',
'aggregator_feed', 'aggregator_item', 'authmap', 'blocks',
'book', 'boxes', 'cache', 'comments', 'contact',
'node_comment_statistics', 'client', 'client_system', 'files',
'filter_formats', 'filters', 'flood', 'forum', 'history',
'locales_meta', 'locales_source', 'locales_target', 'menu',
'moderation_filters', 'moderation_roles', 'moderation_votes',
'node', 'node_access', 'node_revisions', 'profile_fields',
'profile_values', 'url_alias', 'permission', 'poll', 'poll_votes',
'poll_choices', 'role', 'search_dataset', 'search_index',
'search_total', 'sessions', 'sequences', 'node_counter',
'system', 'term_data', 'term_hierarchy', 'term_node',
'term_relation', 'term_synonym', 'users', 'users_roles', 'variable',
'vocabulary', 'vocabulary_node_types', 'watchdog'
));
}
/**
* Converts tables to UTF-8 encoding.
*
* This update is designed to be re-usable by contrib modules and is
* used by system_update_169().
*/
function _system_update_utf8($tables) {
// Are we starting this update for the first time?
if (!isset($_SESSION['update_utf8'])) {
switch ($GLOBALS['db_type']) {
// Only for MySQL 4.1+
case 'mysqli':
break;
case 'mysql':
if (version_compare(mysql_get_server_info($GLOBALS['active_db']), '4.1.0', '<')) {
return array();
}
break;
case 'pgsql':
return array();
}
// See if database uses UTF-8 already
$url = parse_url($GLOBALS['db_url']);
$db_name = substr($url['path'], 1);
$create = array_pop(db_fetch_array(db_query('SHOW CREATE DATABASE `%s`', $db_name)));
if (preg_match('/utf8/i', $create)) {
return array();
}
// Make list of tables to convert
$_SESSION['update_utf8'] = $tables;
// Keep track of total for progress bar
$_SESSION['update_utf8_total'] = count($tables);
}
// Fetch remaining tables list
$list = &$_SESSION['update_utf8'];
$ret = array();
// Convert a table to UTF-8.
// We change all text columns to their correspending binary type,
// then back to text, but with a UTF-8 character set.
// See: http://dev.mysql.com/doc/refman/4.1/en/charset-conversion.html
$types = array('char' => 'binary',
'varchar' => 'varbinary',
'tinytext' => 'tinyblob',
'text' => 'blob',
'mediumtext' => 'mediumblob',
'longtext' => 'longblob');
// Get next table in list
$table = array_shift($list);
$convert_to_binary = array();
$convert_to_utf8 = array();
// Set table default charset
$ret[] = update_sql("ALTER TABLE \{$table} DEFAULT CHARACTER SET utf8");
// Find out which columns need converting and build SQL statements
$result = db_query("SHOW FULL COLUMNS FROM \{$table}");
while ($column = db_fetch_array($result)) {
list($type) = explode('(', $column['Type']);
if (isset($types[$type])) {
$names = 'CHANGE `'. $column['Field'] .'` `'. $column['Field'] .'` ';
$attributes = ' DEFAULT '. ($column['Default'] == 'NULL' ? 'NULL ' :
"'". db_escape_string($column['Default']) ."' ") .
($column['Null'] == 'YES' ? 'NULL' : 'NOT NULL');
$convert_to_binary[] = $names . preg_replace('/'. $type .'/i', $types[$type], $column['Type']) . $attributes;
$convert_to_utf8[] = $names . $column['Type'] .' CHARACTER SET utf8'. $attributes;
}
}
if (count($convert_to_binary)) {
// Convert text columns to binary
$ret[] = update_sql("ALTER TABLE \{$table} ". implode(', ', $convert_to_binary));
// Convert binary columns to UTF-8
$ret[] = update_sql("ALTER TABLE \{$table} ". implode(', ', $convert_to_utf8));
}
// Are we done?
if (count($list) == 0) {
unset($_SESSION['update_utf8']);
unset($_SESSION['update_utf8_total']);
return $ret;
}
// Progress percentage
$ret['#finished'] = 1 - (count($list) / $_SESSION['update_utf8_total']);
return $ret;
}
function system_update_170() {
if (!variable_get('update_170_done', false)) {
switch ($GLOBALS['db_type']) {
case 'pgsql':
$ret = array();
db_change_column($ret, 'system', 'schema_version', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => -1));
break;
case 'mysql':
case 'mysqli':
db_query('ALTER TABLE {system} CHANGE schema_version schema_version smallint(3) not null default -1');
break;
}
// Set schema version -1 (uninstalled) for disabled modules (only affects contrib).
db_query('UPDATE {system} SET schema_version = -1 WHERE status = 0 AND schema_version = 0');
}
return array();
}
......@@ -62,7 +62,7 @@ function db_connect($url) {
if (!mysql_select_db(substr($url['path'], 1))) {
drupal_maintenance_theme();
drupal_set_title('Unable to select database');
print theme('maintenance_page', '<p>We were able to connect to the MySQL database server (which means your username and password is okay) but not able to select the database.</p>
print theme('maintenance_page', '<p>We were able to connect to the MySQL database server (which means your username and password are okay) but not able to select the database.</p>
<p>The MySQL error was: '. theme('placeholder', mysql_error($connection)) .'.</p>
<p>Currently, the database is '. theme('placeholder', substr($url['path'], 1)) .'. The username is '. theme('placeholder', $url['user']) .' and the database server is '. theme('placeholder', $url['host']) .'.</p>
<ul>
......@@ -74,6 +74,16 @@ function db_connect($url) {
exit;
}
/* On MySQL 4.1 and later, force UTF-8 */
if (version_compare(mysql_get_server_info(), '4.1.0', '>=')) {
mysql_query('SET NAMES "utf8"', $connection);
mysql_query('SET collation_connection="utf8_general_ci"', $connection);
mysql_query('SET collation_server="utf8_general_ci"', $connection);
mysql_query('SET character_set_client="utf8"', $connection);
mysql_query('SET character_set_connection="utf8"', $connection);
mysql_query('SET character_set_results="utf8"', $connection);
mysql_query('SET character_set_server="utf8"', $connection);
}
return $connection;
}
......
......@@ -71,6 +71,14 @@ function db_connect($url) {
exit;
}
/* Force UTF-8 */
mysqli_query($connection, 'SET NAMES "utf8"');
mysqli_query($connection, 'SET collation_connection="utf8_general_ci"');
mysqli_query($connection, 'SET collation_server="utf8_general_ci"');
mysqli_query($connection, 'SET character_set_client="utf8"');
mysqli_query($connection, 'SET character_set_connection="utf8"');
mysqli_query($connection, 'SET character_set_results="utf8"');
mysqli_query($connection, 'SET character_set_server="utf8"');
/**
* from: http://bugs.php.net/bug.php?id=33772
......
......@@ -334,6 +334,16 @@ function db_unlock_tables() {
db_query('COMMIT');
}
/**
* Verify if the database is set up correctly.
*/
function db_check_setup() {
$encoding = db_result(db_query('SHOW server_encoding'));
if (!in_array(strtolower($encoding), array('unicode', 'utf8'))) {
drupal_set_message(t('Your PostgreSQL database is set up with the wrong character encoding (%encoding). It is possibile it will not work as expected. It is advised to recreate it with UTF-8/Unicode encoding. More information can be found in the <a href="%url">PostgreSQL documentation</a>.', array('%encoding' => $encoding, '%url' => 'http://www.postgresql.org/docs/7.4/interactive/multibyte.html')), 'status');
}
}
/**
* @} End of "ingroup database".
*/
......
<?php
// $Id$
define('SCHEMA', 0);
define('SCHEMA_MIN', 1);
define('SCHEMA_UNINSTALLED', -1);
define('SCHEMA_INSTALLED', 0);
// The system module (Drupal core) is currently a special case
......
......@@ -472,6 +472,10 @@ function system_view_general() {
$form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE);
$form['cron'] = array_merge($form['cron'], system_cron_settings());
// Check database setup if necessary
if (function_exists('db_check_setup') && empty($_POST)) {
db_check_setup();
}
return $form;
}
......@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) {
foreach ($new_modules as $module) {
// Set the installed schema version for newly-enabled modules
$versions = drupal_get_schema_versions($module);
if ($versions !== FALSE && drupal_get_installed_schema_version($module) == 0) {
drupal_set_installed_schema_version($module, max($versions));
if (drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
drupal_set_installed_schema_version($module, $versions ? max($versions) : SCHEMA_INSTALLED);
}
}
......
......@@ -472,6 +472,10 @@ function system_view_general() {
$form['cron'] = array('#type' => 'fieldset', '#title' => t('Cron jobs'), '#collapsible' => TRUE, '#collapsed' => TRUE);
$form['cron'] = array_merge($form['cron'], system_cron_settings());
// Check database setup if necessary
if (function_exists('db_check_setup') && empty($_POST)) {
db_check_setup();
}
return $form;
}
......@@ -1010,8 +1014,8 @@ function system_modules_submit($form_id, $edit) {
foreach ($new_modules as $module) {
// Set the installed schema version for newly-enabled modules
$versions = drupal_get_schema_versions($module);
if ($versions !== FALSE && drupal_get_installed_schema_version($module) == 0) {
drupal_set_installed_schema_version($module, max($versions));
if (drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
drupal_set_installed_schema_version($module, $versions ? max($versions) : SCHEMA_INSTALLED);
}
}
......
......@@ -132,7 +132,7 @@ function db_change_column(&$ret, $table, $column, $column_new, $type, $attribute
function update_fix_schema_version() {
if ($update_start = variable_get('update_start', FALSE)) {
// Some updates were made to the 4.6 branch and 4.7 branch. This sets
// temporary variables to provent the updates from being executed twice and
// temporary variables to prevent the updates from being executed twice and
// throwing errors.
switch ($update_start) {
case '2005-04-14':
......@@ -151,6 +151,9 @@ function update_fix_schema_version() {
break;
}
// The schema_version column (added below) was changed during 4.7beta.
// Update_170 is only for those beta users.
variable_set('update_170_done', TRUE);
$sql_updates = array(
'2004-10-31: first update since Drupal 4.5.0 release' => 110,
......@@ -174,18 +177,22 @@ function update_fix_schema_version() {
'2005-11-14' => 154, '2005-11-27' => 155, '2005-12-03' => 156,
);
// Add schema version column
switch ($GLOBALS['db_type']) {
case 'pgsql':
$ret = array();
db_add_column($ret, 'system', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => 0));
db_add_column($ret, 'system', 'schema_version', 'smallint', array('not null' => TRUE, 'default' => -1));
break;
case 'mysql':
case 'mysqli':
db_query('ALTER TABLE {system} ADD schema_version smallint(2) unsigned not null default 0');
db_query('ALTER TABLE {system} ADD schema_version smallint(3) not null default -1');
break;
}
// Set all enabled (contrib) modules to schema version 0 (installed)
db_query('UPDATE {system} SET schema_version = 0 WHERE status = 1');
// Set schema version for core
drupal_set_installed_schema_version('system', $sql_updates[$update_start]);
variable_del('update_start');
}
......@@ -276,7 +283,7 @@ function update_fix_watchdog() {
function update_data($module, $number) {
$ret = module_invoke($module, 'update_'. $number);
// Assume the update finished unless the update results indicate otherwise.
$finished = TRUE;
$finished = 1;
if (isset($ret['#finished'])) {
$finished = $ret['#finished'];
unset($ret['#finished']);
......@@ -294,7 +301,7 @@ function update_data($module, $number) {
}
$_SESSION['update_results'][$module][$number] = array_merge($_SESSION['update_results'][$module][$number], $ret);
if ($finished) {
if ($finished == 1) {
// Update the installed version
drupal_set_installed_schema_version($module, $number);
}
......@@ -332,11 +339,11 @@ function update_selection_page() {
$form['has_js'] = array(
'#type' => 'hidden',
'#default_value' => FALSE,
'#attributes' => array('id' => 'edit-has_js')
'#attributes' => array('id' => 'edit-has_js'),
);
$form['submit'] = array(
'#type' => 'submit',
'#value' => 'Update'
'#value' => 'Update',
);
drupal_set_title('Drupal database update');
......@@ -375,7 +382,7 @@ function update_progress_page() {
drupal_set_title('Updating');
$output = '<div id="progress"></div>';
$output .= '<p>Updating your site will take a few seconds.</p>';
$output .= '<p>Please wait while your site is being updated.</p>';
return $output;
}
......@@ -389,9 +396,10 @@ function update_progress_page() {
function update_do_updates() {
while (($update = reset($_SESSION['update_remaining']))) {
$update_finished = update_data($update['module'], $update['version']);
if ($update_finished) {
if ($update_finished == 1) {
// Dequeue the completed update.
unset($_SESSION['update_remaining'][key($_SESSION['update_remaining'])]);
$update_finished = 0; // Make sure this step isn't counted double
}
if (timer_read('page') > 1000) {
break;
......@@ -399,12 +407,12 @@ function update_do_updates() {
}
if ($_SESSION['update_total']) {
$percent = floor(($_SESSION['update_total'] - count($_SESSION['update_remaining'])) / $_SESSION['update_total'] * 100);
$percent = floor(($_SESSION['update_total'] - count($_SESSION['update_remaining']) + $update_finished) / $_SESSION['update_total'] * 100);
}
else {
$percent = 100;
}
return array($percent, 'Updating '. $update['module'] .' module');
return array($percent, isset($update['module']) ? 'Updating '. $update['module'] .' module' : 'Updating complete');
}
function update_do_update_page() {
......@@ -438,7 +446,7 @@ function update_progress_page_nojs() {
else {
// This is the first page so return some output immediately.
$percent = 0;
$message = 'Starting updates...';
$message = 'Starting updates';
}
drupal_set_html_head('<meta http-equiv="Refresh" content="0; URL=update.php?op='. $new_op .'">');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment