Initial import of provisioning framework into drupal contrib. This set of...

Initial import of provisioning framework into drupal contrib. This set of modules extends Drush to provide the ability to install and manage Drupal sites. It is the back end component of the hostmaster system.

provision.inc

+<?php
+/**
+ * @file
+ * The provisioning framework API.
+ *
+ * API functions that are used by the provisioning framework to provide structure to the provisioning modules.
+ *
+ * @see errorhandling
+ * @see logging
+ * @see sitedata
+ * @see provisiontokens
+ */
+
+/**
+ * Invoke provision api calls. 
+ * 
+ * Call the correct hook for all the modules that implement it. We can not use Drupal's default module_invoke, because we
+ * can not pass references through it.
+ * Additionally, the ability to rollback when an error has been encountered is also provided.
+ * If at any point during execution, the provision_get_error() function returns anything but 0, provision_invoke will
+ * trigger $hook_rollback for each of the hooks that implement it, in reverse order from how they were executed.
+ *
+ * @param hook
+ *   The hook name to be executed for all the modules.
+ * @param url
+ *   The url of the site being invoked.
+ * @param data
+ *   A reference to the associated array containing the data for the site. This needs to be a reference, 
+ *   because the modules might provide additional information about the site.
+ * @param rollback
+ *   A boolean specifying whether or not the entire action needs to be rolled back. 
+ *   This is used specifically in commands which implement multiple hooks, such as 'install',
+ *   which implements 'pre_install', 'install' and 'post_install' hooks.
+ * @return
+ *   A boolean specifying whether or not any rollback has been performed. 
+ * 
+ */
+function provision_invoke($hook, $url, &$data, $rollback = false) {
+  
+  if (!$rollback) {
+    foreach (module_implements("provision_$hook") as $name) {
+      $completed[] = $name;
+      $func = $name . "_provision_" . $hook;
+      $func($url, $data);
+      if (provision_get_error()) {
+        # As soon as an error occurs, roll back
+        $rollback = TRUE;
+        break;
+      }
+    }
+  }
+  else {
+    $completed = module_implements("provision_$hook");
+  }
+  
+  if ($rollback) {
+    foreach (array_reverse($completed) as $name) {
+      $func = $name . "_" . $hook . '_rollback';
+      if (function_exists($func)) {
+        $func($url, $data);
+        provision_set_log("Rollback", "Changes for $name module have been rolled back.");
+      }
+    }
+    return FALSE;
+  }
+  else {
+    return TRUE;
+  }
+}
+
+/**
+ * Return output to the command line.
+ *
+ * Provides support for the -b/--backend flag to drush, which returns a serialized data structure.
+ * This feature is used for communication with the front end.
+ *
+ * @param url
+ *   The url of the site being invoked.
+ * @param data
+ *    The complete associative array containing all the aggregated site settings.
+ * @param extra
+ *   An associative array containing additional data to be returned from the command. @see provision_stats_stats()
+ */
+function provision_output($url, $data, $extra = null) {
+  $return = $extra;
+  $return['site'] = $data;
+  $return['error_status'] = provision_get_error(); // error code being returned
+  $return['log'] = provision_get_log(); // Append logging information
+  $return['messages'] = drupal_get_messages();
+  if (drush_get_option(array('b', 'backend'), FALSE)) {
+    print serialize($return);
+  }
+  else {
+    foreach (provision_get_log() as $entry) {
+      printf("%10s|%10s|%10%s", $entry['timestamp'], $entry['type'], $entry['severity'], $entry['message']);
+    }
+    if ($data) {
+      /** TODO : return a cleanly formatted display of all the necessary information */ 
+      print_r($return);
+    }
+    
+  }
+  exit(provision_get_error());
+}
+
+
+/**
+ * @defgroup errorhandling Managing errors that occur in the provisioning framework.
+ * @{
+ * Functions that manage the current error status of the provisioning framework.
+ *
+ * These functions operate by maintaining a static variable that is a bitmask of all the errors that have occurred.
+ * This bitmask value is returned at the end of program execution, and provide the hosting front end more information
+ * on how to diagnose any problems that may have occurred.
+ */
+
+/**
+* @name Error status definitions
+* @{
+* Bitmask values used to generate the error code to return.
+* @see provision_set_error(), provision_get_error(), provision_cmp_error()
+*/
+
+/** Database could not be accessed, or configured */
+define('PROVISION_DB_ERROR', 1);
+/** Drupal was unable to complete it's installation */
+define('PROVISION_INSTALL_ERROR', 2);
+/** Could not create files due to permission error - potentially less severe */ 
+define('PROVISION_PERM_ERROR', 4); 
+/** Web server could not be restarted, or other server related issues - less severe */
+define('PROVISION_WEB_ERROR', 8);
+/** To be used while testing if the provision framework is actually working. */
+define('PROVISION_FRAMEWORK_ERROR', 16);
+/** When the site is not available on this platform, or has not been installed. */
+define('PROVISION_SITE_NOT_FOUND', 32);
+/** When the site is already installed, ie: there is a conflict. */
+define('PROVISION_SITE_INSTALLED', 64);
+
+/**
+ * @} End of "name Error status defintions".
+ */
+
+/**
+ * Set an error code for the error handling system.
+ *
+ * @param error_code
+ *   Any of the defined error status definitions. A numerical bitmask value.
+ * @return
+ *   The current aggregate error status
+ */
+function provision_set_error($error_code = 0) {
+  static $error = 0;
+
+  if ($error_code) {
+    $error = $error | (int) $error_code;    
+  }
+
+  return $error;
+}
+
+/**
+ * Return the current error handling status
+ *
+ * @return
+ *   The current aggregate error status
+ */
+function provision_get_error() {
+  return provision_set_error();
+}
+
+/**
+ * Check if a specific error status has been set.
+ *
+ * @param error
+ *   Any of the defined error status definitions. A numerical bitmask value. 
+ * @return
+ *   TRUE if the specified error has been set, FALSE if not 
+ */
+function provision_cmp_error($error) {
+  return provision_get_error() ^ $error;
+}
+
+/**
+ * @} End of "defgroup errorhandling".
+ */
+
+/**
+ * @defgroup logging Logging information to be provided as output.
+ * @{
+ * Functions that allow the provisioning framework to log messages to be provided to the front end.
+ *
+ * These functions are primarily for diagnostic purposes, but also provide an overview of actions that were taken
+ * by the framework during creation of a site.
+ */
+
+/**
+ * Maintain a static array containing all the log messages
+ *
+ * @param entry
+ *   Associative array containing the log message.
+ * @return
+ *   Entire log history, only if $entry is null
+ */
+function _provision_set_log($entry = null) {
+  static $log = array();
+  if ($entry == null) {
+    return $log;
+  }
+  else {
+    $log[] = $entry;
+  }
+}
+
+/**
+ * Add a log message to the log history.
+ *
+ * @param type
+ *   The type of message to be logged. Common types are 'warning', 'error' and 'notice'.
+ * @param message
+ *   String containing the message to be logged.
+ */
+function provision_log($type, $message) {
+  _provision_set_log(array(
+     'type' => $type, 
+     'message' => $message, 
+     'timestamp' => time()
+    ));
+}
+
+/**
+ * Retrieve the log messages from the log history
+ *
+ * @return
+ *   Entire log history
+ */
+function provision_get_log() {
+  return _provision_set_log();
+}
+
+/**
+ * @} End of "defgroup errorhandling".
+ */
+
+/**
+ * @defgroup sitedata Site data management utility functions.
+ * @{
+ * The provision framework maintains a site.php file in the sites directory, to maintain additional
+ * information from the front end, as well as providing a change history of setting changes. 
+ *
+ * These functions load, save and manage changes made to the site data. This data has diagnostic and infrastructure
+ * values, that allow sites to be more easily moved between different provisioned platforms.
+ */
+
+/**
+ * Returns the aggregated site data from both the pre-existing site.php file, and the options passed to Drush
+ *
+ * This function merges the data from the command line parser, and the information already saved by previous invokations
+ * of the api. This provides a single view of all data relating to the site.
+ * This function also provides sensible defaults for some of the settings.
+ *
+ * @param url
+ *   The url of the site being invoked.
+ * @return
+ *   An associated array containing the relevant settings for the site.
+ */
+function provision_get_site_data($url) {
+  global $args;
+  #TODO: Accept serialized string via unix pipe.
+  foreach ($args['options'] as $key => $value) {
+    if (preg_match("/^site-/", $key)) {
+      $site_data[$key] = $value;
+    }
+  }
+  $site_data['site-url'] = $url;
+  $site_data['site-action-type'] = $args['commands'][1];
+  $docroot = variable_get('provision_root', '');
+  $site_data['site-document-root'] =  ($docroot) ? $docroot . '/webroot' : $_SERVER['DOCUMENT_ROOT'];
+  $site_data['site-temporary-url'] = str_replace('.', '-', $url) . '.' . variable_get('provision_tempurl_base', $_SERVER['HTTP_HOST']);
+  $site_data['site-profile'] = ($site_data['site-profile']) ? $site_data['site-profile'] : variable_get('provision_default_profile', 'default');
+  $site_data['site-ip'] =  variable_get('provision_apache_server_ip', '127.0.0.1');
+  $site_data['site-port'] =  variable_get('provision_apache_server_ip', 80);
+  if ($old_data = provision_load_site_data($url)) {
+    # Merge previously saved data with the new data. This way, old parameters overwrite new ones.
+    $site_data = array_merge(provision_load_site_data($url), $site_data);    
+  }
+  return $site_data;
+}
+
+/**
+ * Load site data stored in the site.php file for the specified site.
+ *
+ * @param url
+ *   The url of the site being invoked
+ * @return
+ *   If the file was found, an associative array of the data that was loaded. Otherwise returns FALSE.
+ */
+function provision_load_site_data($url) {
+  # Load the configuration data.
+  $conf_file = "sites/$url/site.php";
+  if (file_exists($conf_file) && is_readable($conf_file)) {
+    require_once($conf_file);
+    return (array) $data;
+  }
+  return false;
+}
+
+/**
+ * Save modified options to the site.php file
+ *
+ * @param url
+ *   The url of the site being invoked
+ * @param data
+ *   The complete data structure that has been created. Only settings that have been changed will be recorded.
+ */
+function provision_save_site_data($url, $data) {
+  global $args;
+    
+  $old_data = provision_load_site_data($url);
+  
+  $conf_file = "sites/$url/site.php";
+  $fp = fopen($conf_file, "w+"); # Append to the end of the config file.
+  if (!$fp) {
+    provision_log("error", "Site config file could not be written");
+    provision_set_error(PROVISION_PERM_ERROR);
+  }
+  else {
+    fwrite($fp, "<?php\n");
+    $action = array('id' => $data['action-id'], 
+                    'timestamp' => mktime(), 
+                    'action' => $data['site-action-type'], 
+                    'success' => $data['success']);
+    $line = "\n\n\$actions[] = " . var_export($action, TRUE) . ';';
+    fwrite($fp, $line);
+
+    foreach ($data as $key => $value) {
+      if ($data[$key] != $old_data[$key]) {
+        $line = "\n\$data['$key'] = " . var_export($value, true) . ';';
+        fwrite($fp, $line);  
+      }
+    }
+    fwrite($fp, "?>");
+    fclose($fp);
+  }
+}
+
+/**
+ * @} End of "defgroup sitedata".
+ */
+
+
+/**
+ * @defgroup provisiontokens Token replacement support for the provisioning framework
+ * @{
+ * The provisioning framwork makes use of the token module for value replacement in configuration file templates.
+ */
+
+
+/**
+ * Implementation of hook_token_values(). 
+ *
+ * @param type
+ *   A flag indicating the class of substitution tokens to return
+ *   information on. 
+ * @return
+ *   A keyed array listing the substitution tokens. Elements should be
+ *   in the form of: $list[$type][$token] = $description
+ */
+function provision_token_values($type, $object = null) {
+  global $args;
+  switch ($type) {
+    case 'site':
+      $values = array_merge(provision_get_site_data($args['commands'][2]), $object) ;
+      break;
+  }
+  return (array) $values;  
+  
+}
+/**
+ * Implementation of hook_token_list(). 
+ *
+ * @param type
+ *   A flag indicating the class of substitution tokens to return
+ *   information on. 
+ * @return
+ *   A keyed array listing the substitution tokens. Elements should be
+ *   in the form of: $list[$type][$token] = $description
+ */
+function provision_token_list($type = 'all') {
+  if ($type == 'site') {
+    /** TODO: Complete the token list to allow the front end to more easily edit the settings. */
+    $tokens['site']['site-url'] = t("The domain name used to access the site.");
+    $tokens['site']['site-db-type'] = t("");
+    $tokens['site']['site-db-username'] = t("");
+    $tokens['site']['site-db-password'] = t("");
+    $tokens['site']['site-db-name'] = t("");
+    $tokens['site']['site-profile'] = t("");
+    $tokens['site']['site-action-type'] = t("");
+  }
+  return $tokens;
+}
+
+/**
+ * @} End of "defgroup provisiontokens".
+ */
+
+/**
+ * Confirm that provision is running through Drush.
+ *
+ * This module requires a 'bootstrap' and 'configtest' feature, which is to be used during installation, to make sure that
+ * all the provisioning modules have the correct permissions / access to be able to do their jobs. This bootstrap process should
+ * be used when deploying new platforms (ie: new releases).
+ */
+
+function _provision_confirm_drush() {
+  #confirm that code is running through the command line.
+}
+

provision.info

+name = Provision
+description = Allows for the automated provisioning of hosted Drupal sites, via the command line
+package = Provision
+dependencies = drush provision_drupal provision_mysql provision_apache token
\ No newline at end of file

provision.module

+<?php
+/**
+* @file
+*   Provisioning Framework
+*
+* This module provides a framework for a Drupal site to manage and install new Drupal sites, using the command line
+* Drush utility. 
+*
+* It allows for pluggable 'provisioning modules' that can extend and modify the actions that are taken during installation.
+*
+* Each site has the following commands that can be run on it.
+*
+* Implemented : 
+*   install - Install a new Drupal site. The install command uses 3 separate hooks to do it's job, 
+*              namely hook_provision_pre_install(), hook_provision_install() and hook_provision_post_install()
+*   synch   - Recreate all configuration files, to be in synch with changes in the front end.
+*   stats   - Return an associated array of site statistics. (implemented in provision_stats module, is thus optional)
+*   import  - Import the details of an already existing site into the provisioning framework.
+*             This command inspects the settings.php and generates the site.php file that the framework uses for configuration.
+*
+* Not implemented yet :
+*   disable - Disable an installed Drupal site. This is done by removing the virtual host file, so that Apache no longer serves it.
+*   enable  - Re-enable a site that has already been disabled. Recreates the virtual host file.
+*   upgrade - Accepts a site package (backup) as argument, and redeploys it, running the upgrade processes on it.
+*             Uses hook_provision_pre_upgrade(), hook_provision_upgrade() and hook_provision_post_upgrade() hooks,
+*             and allows clean roll back if any errors occur. Will include stringent checking of module versions,
+*             and allow unit tests to be run.
+*   backup  - Generates a tarball containing the sites directory, the site data configuration and the database dump.
+*             This allows the tarball to act as a 'site package', which can be redeployed on other installations,
+*             or used for an upgrade.
+*   rollback - Revert to a previous backup of the site.
+*   rename  - Change the url of a site. This requires moving of files, and numerous other issues.
+*   delete -  Generates a back up of the site, and then removes all references to it.
+*/
+
+
+/** Include the provisioning API. */
+include_once('provision.inc');
+
+/**
+ * Implementation of hook_perm().
+ */
+function hook_perm() {
+  return array("administer provisioning");
+}
+
+
+/**
+ * @defgroup provisionui Configure provisioning framework.
+ * @{
+ */
+
+/**
+ * Implementation of hook_menu().
+ */
+function provision_menu($items, $may_cache = true) {
+  if ($may_cache) {
+    $items[] = array(
+      'path' => 'admin/settings/provision',
+      'title' => t('Provisioning'),
+      'description' => t("Configure how new Drupal sites will be provisioned."),
+      'callback' => 'drupal_get_form',
+      'callback arguments' => array('provision_configure'),
+      'access' => user_access('administer provisioning'),
+    );
+  }
+  return $items;
+}
+
+/**
+ * Menu callback.
+ * 
+ * Configuration screen for the provisioning framework.
+ */
+function provision_configure() {
+  
+  $form['provision_root'] = array(
+    '#type' => 'textfield',
+    '#title' => t('Provision root'),
+    '#description' => t('The path where the provision platform is based.'),
+    '#default_value' => variable_get('provision_root', ereg_replace("/webroot$", "", $_SERVER['DOCUMENT_ROOT'])),
+    '#size' => 40,
+    '#maxlength' => 255,
+  );
+
+  $form['provision_tempurl_base'] = array(
+    '#type' => 'textfield',
+    '#required' => TRUE,
+    '#title' => t('Temporary URL base'),
+    '#description' => t('Each new domain that gets created gets created with a temporary url base that allows it to be immediately accessible.'),
+    '#default_value' => variable_get('provision_tempurl_base', $_SERVER['HTTP_HOST']),
+    '#size' => 40,
+    '#maxlength' => 255,
+  );
+  
+  $form['provision_user'] = array(
+    '#type' => 'textfield',
+    '#title' => t('Provision user'),
+    '#description' => t('The owner of the files. must not be the web server user.'),
+    '#default_value' => variable_get('provision_user', 'hosting'),
+    '#size' => 40,
+    '#maxlength' => 255,
+  );
+  
+  $form['provision_group'] = array(
+    '#type' => 'textfield',
+    '#title' => t('Provision group'),
+    '#description' => t('The group owner of the files. should be the group the web server is running as.'),
+    '#default_value' => variable_get('provision_group', 'apache'),
+    '#size' => 40,
+    '#maxlength' => 255,
+  );
+
+  foreach (module_implements('provision_configure') as $module) {
+    $form[$module] = array(
+      '#type' => 'fieldset',
+      '#title' => module_invoke($module, 'provision_service'),
+      '#collapsible' => TRUE,
+      '#collapsed' => FALSE,
+      '#access' => user_access('administer provisioning'),
+    );
+    $form[$module] = array_merge($form[$module], module_invoke($module, "provision_configure"));
+  }
+  return system_settings_form($form);
+}
+
+/**
+ * @} End of "defgroup provisionui"
+ */
+
+
+/**
+ * @defgroup provisiondrush Command line interface for Provision.
+ * @{
+ */
+/**
+ * Implementation of hook_drush_command().
+ */
+function provision_drush_command() {
+  $items['provision install'] = array(
+    'callback' => 'provision_install_cmd',
+    'description' => 'Provision a new site using the provided data.'
+  );
+
+  $items['provision synch'] = array(
+    'callback' => '_provision_synch',
+    'description' => 'Regenerate all the config files for a site.'
+  );  
+  $items['provision import'] = array(
+    'callback' => '_provision_import',
+    'description' => 'Turn an already running site into a provisioned site.'
+  );  
+  /*
+  $items['provision enable'] = array(
+    'callback' => '_provision_enable',
+    'description' => 'Enable a disabled site.'
+  );
+  $items['provision disable'] = array(
+    'callback' => '_provision_disable',
+    'description' => 'Disable a site.'
+  );  
+  // Not implemented yet.
+  $items['provision delete'] = array(
+    'callback' => '_provision_delete',
+    'description' => 'Delete a site.'
+  );
+
+  $items['provision backup'] = array(
+    'callback' => '_provision_backup',
+    'description' => 'Generate a back up for the site.'
+  );
+  $items['provision rollback'] = array(
+    'callback' => '_provision_rollback',
+    'description' => 'Roll back the site to a previous backup.'
+  );
+  $items['provision deploy'] = array(
+    'callback' => '_provision_deploy',
+    'description' => 'Deploy a backup made on another provisioning platform on this one.'
+  );
+  $items['provision rename'] = array(
+    'callback' => '_provision_rename',
+    'description' => 'Change the url of an existing site.'
+  );
+
+  */
+  return $items;
+}
+
+/**
+ * Drush callback function
+ *
+ * Installs a new site at $url.
+ * It does so by calling hook_provision_pre_install(), hook_provision_install() and hook_provision_post_install().
+ *
+ * @param url
+ *    The url of the site being installed.
+ * @return
+ *    Returns provision_output on success or error. 
+ *    Will exit with a PROVISION_SITE_INSTALLED error if the site already exists.
+ *    Will exit with a PROVISION_FRAMEWORK_ERROR if the command is incorrectly used.
+ */
+function provision_install_cmd($url) {
+  global $args;
+  $data = provision_get_site_data($url);
+
+  if (!$args['commands'][2]) {
+    print "Usage: drush.php provision install DOMAIN [OPTIONS]\n";
+    print "Install a new site for the domain DOMAIN.\n";
+    print "Example: drush.php provision install mydomain.com --site-db-host=localhost\n";
+    provision_set_log("error", "Incorrect usage of the provisioning framework");
+    drupal_set_error(PROVISION_FRAMEWORK_ERROR);
+    provision_output($url, $data);
+  }
+  $ops = array('pre_install', 'install', 'post_install'); # these are the provision hooks that will get triggered.
+
+  if ($data['site-installed'] == TRUE) {
+    print "The site $url is already installed.\nIf you want to re-install it, you will need to delete it first.";
+    drupal_set_error(PROVISION_SITE_INSTALLED);
+    provision_output($url, $data);
+  }
+
+  foreach ($ops as $op ) {
+    $func = "_provision_$op";
+    $func($url, $data);
+  }
+
+  provision_save_site_data($url, $data);
+  provision_output($url, $data);
+}
+
+/**
+ * Drush action. 
+ *
+ * Calls hook_provision_pre_install().
+ * Also responsible for calling creating site directory layout, and the drupal settings file.
+ *
+ * @param url
+ *   The url of the site being invoked.
+ * @param data
+ *   A reference to the associated array containing the data for the site. This needs to be a reference, 
+ *   because the modules might provide additional information about the site.
+ * @return
+ *   Boolean denoting whether the provision_invoke rolled back changes made. 
+ */
+function _provision_pre_install($url, &$data) {  
+  $rolled_back = provision_invoke("pre_install", $url, $data);
+
+  if (!provision_get_error()) {
+    // This is the actual drupal provisioning requirements. 
+    _provision_drupal_create_directories($url, $data['profile']);
+    // Requires at least the database settings to complete.
+    _provision_drupal_create_settings_file($url, $data);
+  }
+  return $rolled_back;
+}
+
+/**
+ * Install drupal site
+ *
+ * The magic here is that we need to drive the install api through this code. 
+ * At this point, we no longer have access to the central database, and we need to be able
+ * to drive this code blind, without causing bad stuff to happen.
+ *
+ * Install profile gets triggered at the end of this code.
+ *
+ * @param url
+ *   The url of the site being invoked.
+ * @param data
+ *   A reference to the associated array containing the data for the site. This needs to be a reference, 
+ *   because the modules might provide additional information about the site.
+ * @return
+ *   Boolean denoting whether the provision_invoke rolled back changes made. 
+ */
+function _provision_install($url, &$data) {
+  $rolled_back = provision_invoke("install", $url, $data);
+
+  if (!$rolled_back) {
+    _provision_drupal_switch_active_site($url); # Change headers and db info, also backs up
+    _provision_drupal_force_load_modules($url);
+    _provision_drupal_install_schema($data['site-profile']);
+    _provision_drupal_force_load_modules();
+    _provision_drupal_switch_active_site(); # This *should* bring the site back to where we were before installing
+  }
+  return $rolled_back;
+}