Adding content migrate module. Still needs lots of work, just getting this...

Adding content migrate module. Still needs lots of work, just getting this much committed so it is not lost.

modules/content_migrate/content_migrate.admin.inc

+<?php
+// $Id$
+/*
+
+/**
+ * @file content_migrate.admin.inc
+ * Contains page callbacks for the module.
+ */
+
+/**
+ * Form generator for the migration selection form.
+ * 
+ * @todo Make this into a nice table where you have 
+ * an option to check all available fields to migrate
+ * them all at once.
+ */
+function content_migrate_select($form, &$form_state) {
+  $form = array();
+  $options = array('available' => array(), 'converted' => array(), 'missing' => array());
+  
+  $field_values = content_migrate_get_field_values();
+  if (empty($field_values)) {
+    drupal_set_message(t('There is no D6 field information in this database.'));
+    return $form;
+  }
+
+  $type_names = node_type_get_names();
+  $new_fields = array_keys(field_info_fields());
+  
+  // Figure out which field and widget modules are available.
+  $available_modules = array_unique(array_merge(module_implements('field_info'), module_implements('widget_info')));
+  
+  foreach ($field_values as $field_name => $field_value) {
+    $bundles = array();
+    $missing_module = !in_array($field_value['module'], $available_modules);
+    $instance_values = content_migrate_get_instance_values(NULL, $field_name);
+    foreach ($instance_values as $bundle => $instance_value) {
+      $bundles[] = $type_names[$bundle];
+      $label = $instance_value['label'];
+      if (!in_array($instance_value['widget']['module'], $available_modules)) {
+        $missing_module = TRUE;
+      }
+    }
+    $data = array(
+      0 => $field_name,
+      1 => $field_value['type'],
+      2 => implode(', ', $bundles),
+    );
+    if (in_array($field_name, $new_fields)) {
+      $options['converted'][$field_name] = $data;
+    }
+    // TODO, do we need to check for more than the mere presence of a module?
+    elseif ($missing_module) {
+      $options['missing'][$field_name] = $data;
+    }
+    else {
+      $options['available'][$field_name] = $data;
+    }
+  }
+
+  $header = array(t('Field'), t('Field type'), t('Content type(s)'));
+  $form['#tree'] = TRUE;
+  $form['available'] = array(
+    '#type' => 'fieldset',
+    '#collapsible' => TRUE,
+    '#collapsed' => count($options['available']) < 1,
+    '#title' => t('Available fields'),
+    '#description' => t('Fields that have not yet been migrated but are available for migration.'),
+  );
+  $form['available']['data'] = array(
+    '#type' => 'tableselect',
+    '#header' => $header,
+    '#options' => $options['available'],
+    '#empty' => t('No fields are available to be migrated.'),
+  );  
+  $form['available']['submit'] = array(
+    '#type' => 'submit', 
+    '#value' => t('Migrate selected fields'),
+    '#submit' => array('content_migrate_select_submit'),
+  );  
+
+  $form['converted'] = array(
+    '#type' => 'fieldset',
+    '#collapsible' => TRUE,
+    '#collapsed' => count($options['converted']) < 1,
+    '#title' => t('Converted fields'),
+    '#description' => '<p>'. t('Fields that have already been converted. You can choose to roll them back if the conversion did not work correctly. Note that rolling fields back will completely destroy the new field tables.') . ' <span class="error"><strong>' . t('This operation cannot be undone!') . '</strong></span>',
+  );    
+  $form['converted']['data'] = array(
+    '#type' => 'tableselect',
+    '#header' => $header,
+    '#options' => $options['converted'],
+    '#empty' => t('No fields are already converted.'),
+  );
+  $form['converted']['submit'] = array(
+    '#type' => 'submit', 
+    '#value' => t('Roll back selected fields'),
+    '#submit' => array('content_migrate_rollback_submit'),
+  );     
+  
+  $form['missing'] = array(
+    '#type' => 'fieldset',
+    '#collapsible' => TRUE,
+    '#collapsed' => count($options['missing']) < 1,
+    '#title' => t('Unavailable fields'),
+    '#description' => t('Fields that cannot be migrated because some modules are missing.'),
+  );  
+  $form['missing']['data'] = array(
+    '#type' => 'tableselect',
+    '#header' => $header,
+    '#options' => $options['missing'],
+    '#empty' => t('No fields have missing modules.'),
+  ); 
+      
+  return $form;
+}
+
+/**
+ * Submit handler.
+ * 
+ * @TODO add a confirmation on the rollback submission.
+ */
+function content_migrate_rollback_submit($form, &$form_state) {
+  $ret = array();
+  $field_names = array_filter($form_state['values']['converted']['data']);
+  foreach ($field_names as $field_name) {
+    $field = field_info_field($field_name);
+    
+    // Deleting the field only marks it for deletion.
+    field_delete_field($field_name);
+    
+    // We are bypassing the field batch processing
+    // and simply deleting all the data.
+    // The assumption is that the migration was
+    // unsuccessful and will be re-attempted
+    // and we need to remove all traces of the 
+    // new field for later migrations to work.
+    $new_table = content_migrate_new_table($field);
+    db_drop_table($ret, $new_table);
+    
+    $instances = field_read_instances(array('field_id' => $field['id']), array('include_deleted' => 1));
+    foreach ($instances as $instance) {
+      field_purge_instance($instance);
+    }
+    field_purge_field($field);
+    drupal_set_message(t('Rolling back @field_name.', array('@field_name' => $field_name)));
+  }
+}
+
+/**
+ * Submit handler.
+ */
+function content_migrate_select_submit($form, &$form_state) {
+  $field_names = array_filter($form_state['values']['available']['data']);
+  _content_migrate_batch($field_names);
+}
+
+/**
+ * Helper function to create a batch.
+ */
+function _content_migrate_batch($field_names) {
+  $batch = array(
+    'title' => t('Migrating data'),
+    'file'  => drupal_get_path('module', 'content_migrate') . '/content_migrate.admin.inc',
+    'operations' => array(
+      array('_content_migrate_batch_process_create_fields', array($field_names)),
+    ),
+    'finished' => "Field migration is finished",
+    'init_message' => t("Fields migration is starting."),
+    'progress_message' => t('Processed @current out of @total.'),
+    'error_message' => t('Field migration has encountered an error.'),
+  );
+  // Migrate field data one field at a time.
+  foreach ($field_names as $field_name) {
+    $batch['operations'][] = array('_content_migrate_batch_process_migrate_data', array($field_name));
+  }
+  batch_set($batch);
+}
+
+/**
+ * Batch operation callback to create fields.
+ */
+function _content_migrate_batch_process_create_fields($field_names) {
+  $type_names = node_type_get_names();
+  foreach ($field_names as $field_name) {
+    $context['message'] = "creating field $field_name";
+    $field_value = content_migrate_get_field_values($field_name);
+    
+    // Create the field and store the new field 
+    // definition in $context so we can retrieve it later. 
+    try {
+      unset($data['field']['columns']);
+      unset($data['field']['db_storage']); 
+      $field = field_create_field($field_value);
+      $context['fields'][$field_name] = $field;
+      drupal_set_message(t("Created field @field_name", array('@field_name' => $field_name))); 
+      
+      // Create each of the new instances and store 
+      // the new instance definitions in $context.
+      $instance_values = content_migrate_get_instance_values(NULL, $field_name);
+      foreach ($instance_values as $bundle => $instance_value) {
+        try {
+          $instance = field_create_instance($instance_value); 
+          $context['instances'][$fieldname][$instance['bundle']] = $instance;
+          drupal_set_message(t("Created instance of @field_name in bundle @bundle.", array(
+            '@field_name' => $field_name, '@bundle' => $type_names[$instance['bundle']])));
+        }
+        catch (Exception $e) {
+          drupal_set_message(t('Error creating instance of @field_name in bundle @bundle.', array(
+            '@field_name' => $field_name, '@bundle' => $type_names[$instance_value['bundle']])), 'error');
+          drupal_set_message($e, 'error');
+        }
+      }
+    }
+    catch (Exception $e) {
+      drupal_set_message(t("Error creating field @field_name", array('@field_name' => $field_name)), 'error'); 
+      drupal_set_message($e, 'error');
+    }
+
+  }
+  //dsm($field_values);
+  $context['finished'] = TRUE;
+} 
+
+/**
+ * Batch operation callback to migrate data.
+ * Copy old table data to new field table.
+ * 
+ * TODO This query still needs work.
+ */
+function _content_migrate_batch_process_migrate_data($field_name) {
+  $field_value = content_migrate_get_field_values($field_name);
+  $field = $context['fields'][$field_name];
+  foreach ($info['instances'] as $instance_value) {
+    
+    $old_table          = content_migrate_old_table($field_value, $instance_value);
+    $old_cols           = content_migrate_old_columns($field_value, $instance_value);
+    $new_table          = content_migrate_new_table($field);
+    $new_revision_table = content_migrate_new_revision($field);
+    $new_columns        = content_migrate_new_columns($field);
+
+    $query = db_select($old_table, 'old');
+    
+    // We need a new columns for bundle name, entity type, and language.
+    $query->addExpression($instance_value['bundle'], 'bundle');
+    $query->addExpression(1, 'etid');
+    $query->addExpression(LANGUAGE_NONE, 'language');
+    
+    // There are new names for what were the nid and vid columns.
+    $query->addField('old', 'nid', 'entity_id');
+    $query->addField('old', 'vid', 'revision_id');
+    
+    // Add the field columns to the select query.
+    // Use the new column names as aliases in case the 
+    // name changed, hopefully none did.
+    foreach ($old_cols as $key => $col) {
+      $query->addField('old', $col, $new_columns[$key]);
+    }
+    
+    // Add delta, or construct it if missing.
+    if (content_migrate_storage_type($field_value, $instance_value) == CONTENT_DB_STORAGE_PER_FIELD) {
+      $query->addField('old', 'delta', 'delta');
+    }
+    else {
+      $query->addExpression(0, 'delta');
+    }
+    
+    // Now copy the data to the new field tables.
+    try {
+      db_insert($new_table)
+        ->from($query)
+        ->execute();
+      db_insert($new_revision_table)
+        ->from($query)
+        ->execute();
+      drupal_set_message(t('Data migrated for field @field_name in bundle @bundle.', array(
+        '@field_name' => $field_name, '@bundle' => $instance_value['bundle'])));
+    }
+    catch (Exception $e) {
+      drupal_set_message(t('Data could not be migrated for field @field_name in bundle @bundle.', array(
+        '@field_name' => $field_name, '@bundle' => $instance_value['bundle'])), 'error');
+      drupal_set_message($e, 'error');
+    }
+  }
+}
+
+function content_migrate_test() {
+  $field_values = content_migrate_get_field_values();
+  dsm($field_values);
+  foreach ($field_values as $field_name => $field_value) {
+    field_delete_field($field_name);
+    $field = field_create_field($field_value);
+    dsm($field);
+    $instance_values = content_migrate_get_instance_values(NULL, $field_name);
+    foreach ($instance_values as $instance_value) {
+    
+      $old_table          = content_migrate_old_table($field_value, $instance_value);
+      $old_cols           = content_migrate_old_columns($field_value, $instance_value);
+      $new_table          = content_migrate_new_table($field);
+      $new_revision_table = content_migrate_new_revision($field);
+      $new_columns        = content_migrate_new_columns($field);
+      dsm($old_table);
+      dsm($old_cols);
+      dsm($new_table);
+      dsm($new_columns);
+    }
+  }
+}
\ No newline at end of file

modules/content_migrate/content_migrate.info

+; $Id$
+name = Content Migrate
+description = Migrate fields and field data from CCK D6 format to the D7 field format. Required to migrate data, can be disabled once all fields have been migrated.
+core = 7.x
+package = CCK
+files[] = content_migrate.module
+files[] = content_migrate.admin.inc

modules/content_migrate/content_migrate.module

+<?php
+// $Id$
+/**
+ * @file
+ * Code For D6 to D7 field update.
+ */
+define('CONTENT_DB_STORAGE_PER_FIELD', 0);
+define('CONTENT_DB_STORAGE_PER_CONTENT_TYPE', 1);
+
+/**
+ * Implementation of hook_menu().
+ */
+function content_migrate_menu() { 
+  // Demo page.
+  $items['admin/structure/content_migrate'] = array(
+    'title' => 'Migrate fields',
+    'description' => 'Migrate field settings and data from the Drupal 6 version to the Drupal 7 version.',
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('content_migrate_select'),
+    'access arguments' => array('administer content types'),
+    'file' => 'content_migrate.admin.inc',
+  );
+
+  return $items;
+}
+
+/**
+ * Create a D7-style field array from data stored
+ * in the D6 content field tables.
+ *
+ * @param $field_name 
+ *   Optionally request only a specific field name.
+ */
+function content_migrate_get_field_values($field_name = NULL) {
+  $field_values = &drupal_static(__FUNCTION__);
+  if (!is_array($field_values)) {
+    $field_values = array();
+  }
+  if (empty($field_values) && db_table_exists('content_node_field')) {
+    $field_values = array();
+    $query = db_select('content_node_field', 'nf', array('fetch' => PDO::FETCH_ASSOC));
+    $result = $query
+      ->fields('nf')
+      ->execute();
+    
+    foreach ($result as $row) {
+      $field_value = $row;
+    
+      // All Drupal 6 fields were attached to nodes.
+      $field_value['entity_type'] = 'node';
+    
+      $field_value['cardinality'] = $field_value['multiple'] != 1 ? $field_value['multiple'] : FIELD_CARDINALITY_UNLIMITED;
+      
+      // We need column information for the old table.
+      $field_value['columns'] = unserialize($field_value['db_columns']);
+    
+      // Field settings.
+      $default_settings = field_info_field_settings($row['type']);
+      $field_value['settings'] = array_merge($default_settings, unserialize($field_value['global_settings']));
+    
+      unset($field_value['multiple'], $field_value['global_settings'], $field_value['required'], $field_value['db_columns']);
+    
+      // TODO Move this to a hook_content_migrate_field_alter on behalf of text module.
+      // The max_length field can no longer be empty or it will create a SQL error.
+      if ($field_value['type'] == 'text' && empty($field_value['settings']['max_length'])) {
+        $field_value['settings']['max_length'] = 255;
+      }
+            
+      // Let modules change these values.
+      drupal_alter('content_migrate_field', $field_value);
+    
+      // We retain $field_value['columns'] and $field_value['db_storage']
+      // even though they are not used or different in D7 
+      // so we can find the old table information.
+      
+      // Add field definiation to $field_values array.
+      $field_values[$field_value['field_name']] = $field_value;
+    }
+  }
+  if (!empty($field_name)) {
+    return $field_values[$field_name];
+  }
+  return $field_values;
+}
+
+/**
+ * Create a D7-style instance array from data stored
+ * in the D6 content field tables.
+ *
+ * @param $bundle
+ *   Optionally request only instances of a specific bundle.
+ * @param $field_name
+ *   Optionally request only instances of a specific field_name.
+ */
+function content_migrate_get_instance_values($bundle = NULL, $field_name = NULL) {
+  $instance_values = &drupal_static(__FUNCTION__);
+  
+  if (empty($instance_values) && db_table_exists('content_node_field_instance')) {
+    $instance_values = array();
+    $query = db_select('content_node_field_instance', 'ni', array('fetch' => PDO::FETCH_ASSOC));
+    $node_field_alias = $query->join('content_node_field', 'nf', 'ni.field_name=nf.field_name');
+    $result = $query
+      ->fields('ni')
+      ->fields($node_field_alias, array('required', 'type'))
+      ->orderBy('label', 'ASC')
+      ->execute();
+    
+    foreach ($result as $row) {
+      $field_type = $row['type'];   
+      $instance_value = $row;
+    
+      // All Drupal 6 instances were attached to nodes.
+      $instance_value['entity_type'] = 'node';
+    
+      // Unserialize arrays.
+      foreach (array('widget_settings', 'display_settings', 'global_settings') as $key) {
+        $instance_value[$key] = (!empty($instance_value[$key])) ? (array) unserialize($instance_value[$key]) : array();
+      }
+    
+      // Build instance values.
+      $instance_value['bundle'] = $instance_value['type_name'];
+      $instance_value['active'] = $instance_value['widget_active'];
+      $instance_value['module'] = $instance_value['widget_module'];
+      $instance_value['type']   = $instance_value['widget_type'];
+      $instance_value['default_value'] = $instance_value['widget_settings']['default_value'];
+      
+      // Core does not support this, but retain it so 
+      // another module can do something with it
+      // in drupal_alter.
+      $instance_value['default_value_php'] = $instance_value['widget_settings']['default_value_php'];
+    
+      // Build widget values.
+      $instance_value['widget'] = array();
+      
+      // TODO Some widget types have been renamed in D7.
+      $instance_value['widget']['type']     = $instance_value['widget_type'];
+      $instance_value['widget']['weight']   = $instance_value['weight'];
+      $instance_value['widget']['module']   = $instance_value['module'];
+    
+      $default_settings = field_info_widget_settings($row['type']);
+      $instance_value['widget']['settings'] = array_merge($default_settings, $instance_value['widget_settings']);
+    
+      // Build display values.
+      $instance_value['display'] = $instance_value['display_settings'];
+      $label = $instance_value['display']['label'];
+      unset($instance_value['display']['label']);
+      foreach ($instance_value['display'] as $context => $settings) {
+        $instance_value['display'][$context]['label'] = $label['format'];
+        
+        // TODO the format used in D6 does not match the formatter in D7.
+        $instance_value['display'][$context]['type'] = $settings['format'];
+        $instance_value['display'][$context]['settings'] = field_info_formatter_settings($settings['format']);
+        unset($instance_value['display'][$context]['exclude'], $instance_value['display'][$context]['format']);
+      }
+    
+      // Unset unneeded values.
+      unset($instance_value['type_name'], $instance_value['global_settings'], $instance_value['widget_settings'], $instance_value['display_settings'], $instance_value['widget_module'], $instance_value['widget_active'], $instance_value['widget_type'], $instance_value['widget']['settings']['default_value'], $instance_value['widget']['settings']['default_value_php']);
+    
+      // Let modules change these values.
+      drupal_alter('content_migrate_instance', $instance_value);
+    
+      // Add instance information to instance array.
+      $instance_values['instances'][$instance_value['bundle']][$instance_value['field_name']] = $instance_value;
+      $instance_values['fields'][$instance_value['field_name']][$instance_value['bundle']] = $instance_value;
+      
+    }
+  }
+  
+  if (!empty($bundle)) {
+    if (!empty($field_name)) {
+      return $instance_values['instances'][$bundle][$field_name];
+    }
+    else {
+      return $instance_values['instances'][$bundle];
+    }
+  }
+  elseif (!empty($field_name)) {
+    return $instance_values['fields'][$field_name];
+  }
+  return $instance_values;
+}
+
+/**
+ * Helper function for finding the table name
+ * used to store the D6 field data.
+ *
+ * @param $field_value
+ * @param $instance_value
+ */
+function content_migrate_old_table($field_value, $instance_value) {
+  
+  $storage = content_migrate_storage_type($field_value, $instance_value);
+  switch ($storage) {
+    case CONTENT_DB_STORAGE_PER_CONTENT_TYPE :
+      $name = $instance_value['bundle'];
+      return "content_type_$name";
+    case CONTENT_DB_STORAGE_PER_FIELD :
+      $name = $field_value['field_name'];
+      return "content_$name";
+  }
+}
+
+/**
+ * Helper function for finding the type of table
+ * used for storing the D6 field data.
+ *
+ * @param $field_value
+ * @param $instance_value
+ */
+function content_migrate_storage_type($field_value, $instance_value) {
+  if (!empty($field_value['db_storage'])) {
+    return $field_value['db_storage'];
+  }
+  elseif ($field_value['cardinality'] > 0) {
+    $storage = CONTENT_DB_STORAGE_PER_FIELD;
+  }
+  else {
+    $instance_values = content_migrate_get_instance_values(NULL, $field_value['field_name']);
+    if (count($instance_values) > 1) {
+      $storage = CONTENT_DB_STORAGE_PER_FIELD;
+    }
+  }
+  $storage = CONTENT_DB_STORAGE_PER_CONTENT_TYPE;
+  return $storage;
+}
+
+/**
+ * Helper function to find the table for a
+ * D7 field array.
+ *
+ * @param $field
+ */
+function content_migrate_new_table($field) {
+  if (empty($field['storage']['details'])) {
+    return 'field_data_'. $field['field_name'];
+  }
+  $data = $field['storage']['details']['sql'][FIELD_LOAD_CURRENT];
+  return key($data);
+}
+
+function content_migrate_new_revision($field) {
+  if (empty($field['storage']['details'])) {
+    return 'field_revision_'. $field['field_name'];
+  }
+  $data = $field['storage']['details']['sql'][FIELD_LOAD_REVISION];
+  return key($data);
+}
+
+/**
+ * Helper function for finding the column names
+ * used for storing the D6 field data.
+ *
+ * @param $field_value
+ * @param $instance_value
+ */
+function content_migrate_old_columns($field_value, $instance_value) {
+  $columns = array();
+  foreach ($field_value['columns'] as $col => $values) {
+    $columns[] = $field_value['field_name'] .'_'. $col;
+  }
+  return $columns;
+}
+/**
+ * Helper function for figuring out column names
+ * to be used when storing D7 field data.
+ *
+ * @param unknown_type $field
+ * @return unknown
+ */
+function content_migrate_new_columns($field) {
+  if (empty($field['storage']['details'])) {
+    $columns = array();
+    foreach ($field['columns'] as $col => $values) {
+      $columns[] = $field['field_name'] .'_'. $col;
+    }
+    return $columns;
+  }
+  else {
+    return $field['storage']['details']['sql'][FIELD_LOAD_CURRENT];
+  }
+}