Commit 35c98012 authored by anarcat's avatar anarcat

import hosting_queue_runner as a submodule in 2.x

parent a8ee8cb5
Hosting queue runner
====================
Simple Drupal module intended to make it easy to run the Aegir tasks
queue with near-instant execution times. The daemon is designed to run
standalone, and started through regular services (e.g. there's an
init.d script available).
Note that before the service is setup and the daemon can be started,
it needs to be enabled as a module in the frontend.
Install this module in your main hostmaster (Aegir) site. You can
enable the feature from: `admin/hosting`. This will disable your
hosting tasks queue for you, ready for you to enable the daemon.
The daemon logs some of its activities to the Drupal watchdog.
Installing as a service
-----------------------
Those instructions will setup the daemon to run as a regular service
in /etc/init.d/ - instructions will vary according to platforms, the
following should work in Debian, running as root.
1. Install the init script in place
cp init.d.example /etc/init.d/hosting-queue-runner
2. Setup symlinks and runlevels
update-rc.d hosting-queue-runner defaults
3. Start the daemon
/etc/init.d/hosting-queue-runner
Supervisord configuration instructions
--------------------------------------
You can also use a daemon like supervisor to make sure the daemon is
restarted if it crashes, but this is optional. An example
configuration file (hosting_queue_runner.conf) is included.
These instructions are for Debian based linux distributions, you may need to
adjust settings for other distributions.
1. Install supervisor
sudo apt-get install supervisor
2. Copy the `hosting_queue_runner.sh` script from the module directory to the
root of the Aegir home directory (usually `/var/aegir`). You will want to
ensure that the script is executable and owned by the Aegir user:
chown aegir:aegir hosting_queue_runner.sh
chmod 700 hosting_queue_runner.sh
3. Copy the supervisor example configuration file from the module directory
to the conf.d directory of supervisor.
cp hosting_queue_runner.conf /etc/supervisor/conf.d/
Adjust the settings in that file to match your environment. If you have a
standard Aegir setup, and have followed the README so far, then you
shouldn't need to change anything.
4. Restart supervisor and add a task to your hosting tasks queue in Aegir,
re-verify a site and see if it executes then you're all set up!
Supervisor keeps a log about the execution of the queue runner that may be
useful if you are trying to resolve an issue where your tasks are not being
executed. The output from the queue runner is also logged by supervisor to
(by default) `/var/log/hosting_queue_runner` it may be useful to view this
log occasionally to ensure that there are no errors being logged. 'Duplicate
task' errors, if you get them are nothing to worry about however.
Troubleshooting
---------------
Try to run the bash script from the command line as the Aegir user yourself, if
you can do this, then the issue is with Supervisor, otherwise it might be an
issue with this module/script.
Look into the Drupal watchdog to see when the daemon has been started
or was restarted. The settings page should also tell you the last time
the daemon was started.
<?php
/**
* @file
* The hosting feature definition for the hosting queue runner.
*/
/**
* Register a hosting feature with Aegir.
*
* This will be used to generate the 'admin/hosting' page.
*
* @return
* associative array indexed by feature key.
*/
function hosting_queue_runner_hosting_feature() {
$features['queue_runner'] = array(
// title to display in form
'title' => t('Hosting queue runner'),
// description
'description' => t('Runs the hosting tasks queue as a daemonized process.'),
// initial status ( HOSTING_FEATURE_DISABLED, HOSTING_FEATURE_ENABLED, HOSTING_FEATURE_REQUIRED )
'status' => HOSTING_FEATURE_DISABLED,
// module to enable/disable alongside feature
'module' => 'hosting_queue_runner',
// associate with a specific node type.
// 'node' => 'nodetype',
// which group to display in ( null , experimental )
'group' => 'experimental'
);
return $features;
}
<?php
/**
* Configuration form for backup schedules
*/
function hosting_queue_runner_settings_form() {
$form['description'] = array(
'#type' => 'markup',
'#value' => t('Note that the settings on this form will only apply to the daemon once it has been restarted, which by default happens as least once an hour.'),
'#weight' => -100,
);
$last_seen = variable_get('hosting_queue_runner_process_started', NULL);
$form['hosting_queue_runner_process_started'] = array(
'#type' => 'item',
'#title' => t('Runner status'),
'#value' => !empty($last_seen) ? t('Last started: @interval ago.', array('@interval' => format_interval(time() - $last_seen))) : t('Never started.'),
);
$form['hosting_queue_runner_post_task_delay'] = array(
'#type' => 'select',
'#title' => t('Post task delay'),
'#description' => t('Tasks are executed as fast as possible, so you may wish to add a delay after the execution of each task. After this delay, new tasks will still start executing almost instantly.'),
'#default_value' => variable_get('hosting_queue_runner_post_task_delay', 0),
'#options' => array(
0 => t('No delay'),
) + drupal_map_assoc(range(1, 60), '_hosting_queue_runner_settings_form_delay_callback'),
);
$form['hosting_queue_runner_process_lifetime'] = array(
'#type' => 'select',
'#title' => t('Process lifetime timeout'),
'#description' => t('Because of memory leaks and bugs in PHP, the daemon automatically stops after this delay, and is restarted. If you are running a lot of tasks, you may want to lower this so it gets restarted more often. In empirical tests, it was found that around 100KB are leaked for every task fired.'),
'#default_value' => variable_get('hosting_queue_runner_process_lifetime', 3600),
'#options' => drupal_map_assoc(array(
60,
60 * 5,
60 * 10,
60 * 15,
60 * 20,
60 * 25,
60 * 30,
60 * 35,
60 * 40,
60 * 45,
60 * 50,
60 * 55,
60 * 60,
), 'format_interval'),
);
return system_settings_form($form);
}
function _hosting_queue_runner_settings_form_delay_callback($value) {
return format_plural($value, '1 second', '@count seconds');
}
[program:hosting_queue_runner]
; Adjust the next line to point to where you copied the script.
command=/var/aegir/hosting_queue_runner.sh
user=aegir
numprocs=1
stdout_logfile=/var/log/hosting_queue_runner
autostart=TRUE
autorestart=TRUE
; Tweak the next line to match your environment.
environment=HOME="/var/aegir",USER="aegir",DRUSH_COMMAND="/var/aegir/drush/drush"
<?php
/**
* @file
* Dispatcher daemon
*
* This file is the heart of the dispatcher drush command. It
* implements most of the backend functionality.
*/
// This is necessary for signal handling to work
declare(ticks=1);
/**
* Implementation of hook_drush_command().
*/
function hosting_queue_runner_drush_command() {
$items = array();
$items['hosting-queue-runner'] = array(
'description' => 'Runs the tasks queue',
'bootstrap' => DRUSH_BOOTSTRAP_DRUPAL_FULL,
'drupal dependencies' => array(
'hosting_queue_runner',
),
// We need this get around a drush bug, see: http://drupal.org/node/704848
// But we remove it below for Drush 4.5 and above.
'callback' => 'drush_hosting_queue_runner',
);
// We don't need the callback defined for Drush > 4.4
if (defined('DRUSH_VERSION') && version_compare(DRUSH_VERSION, '4.4', '>')) {
unset($items['hosting-queue-runner']['callback']);
}
return $items;
}
/**
* Drush command to execute hosting tasks.
*/
function drush_hosting_queue_runner() {
if (function_exists('pcntl_signal')) {
// reload the server on SIGHUP
pcntl_signal(SIGHUP, 'hosting_queue_runner_restart');
pcntl_signal(SIGINT, 'hosting_queue_runner_stop');
pcntl_signal(SIGTERM, 'hosting_queue_runner_stop');
}
// Set a nice high time limit, if we can:
if (function_exists('set_time_limit')) {
@set_time_limit(0);
}
// in some environments (e.g. in "productin") ENV is not actually
// set (!) so try to guess from $_SERVER
if (strpos(ini_get('variables_order'), 'E') === FALSE) {
if (strpos(ini_get('variables_order'), 'S') === FALSE) {
drush_log(dt('Neither $_ENV nor $_SERVER are available to set up proper environment inheritance; ensure E and/or S is set in your php.ini\'s "variables_order" setting.'), 'warning');
}
else {
$_ENV = $_SERVER;
}
}
$end_time = variable_get('hosting_queue_runner_process_lifetime', 3600) + time();
// Record the fact that we're running, so we can give some feedback in the
// frontend.
variable_set('hosting_queue_runner_process_started', time());
watchdog('hosting_queue_runner', 'Started Hosting queue runner, waiting for new tasks');
while (TRUE) {
// Sleep for a second before we look for new tasks.
sleep(1);
// Should we terminate.
if (time() > $end_time) {
// Restart the daemon to recycle leaked memory
hosting_queue_runner_restart();
}
// Get some tasks to run
$tasks = _hosting_get_new_tasks();
foreach ($tasks as $task) {
// We sleep for a second just in case others want to run the task first.
// This guards against other processes that want to add a hosting task
// with arguments and run it immediately, they should be able to do this
// without us getting in there first.
// This is a workaround for http://drupal.org/node/1003536
drush_log(dt('Found task to execute. Pausing before execution.'));
sleep(1);
// Execute the task in the backend
drush_backend_invoke("hosting-task", array($task->nid));
drush_log(dt('Finished executing task.'));
// Delay for a configurable amount of time.
$delay = variable_get('hosting_queue_runner_post_task_delay', 0);
if (!empty($delay)) {
drush_log(dt('Going to sleep for @count seconds after completing task.', array('@count' => $delay)));
sleep($delay);
}
// We're done with this task, this unset might help reduce memory usage.
unset($task);
// Should we terminate.
if (time() > $end_time) {
// Restart the daemon to recycle leaked memory
hosting_queue_runner_restart();
}
}
unset($tasks);
}
}
/**
* Handle interruption signals gracefully
*
* We do not want to interrupt children tasks, so we wait for them
* before stopping.
*/
function hosting_queue_runner_stop($signal) {
watchdog('hosting_queue_runner', 'Received signal @signal, waiting for children to die.', array('@signal' => $signal));
$status = NULL;
pcntl_wait($status);
watchdog('hosting_queue_runner', 'Stopped daemon');
exit($status);
}
/**
* Restart the dispatcher to work around memory leaks
*/
function hosting_queue_runner_restart($signal = NULL) {
// If we received a singal, process it.
if (!is_null($signal)) {
watchdog('hosting_queue_runner', 'Received signal @signal, waiting for children to die.', array('@signal' => $signal));
$status = NULL;
pcntl_wait($status);
}
// We need the PCNTL extension to be able to auto restart.
if (function_exists('pcntl_exec')) {
$drush_command = drush_get_command();
// should be hosting-queue-runner unless something is very wrong
$command = $drush_command['command'];
// Get the drush executable and any arguments it needs.
$args = hosting_queue_runner_drush_find_drush();
$drush = array_shift($args);
// Get the original options passed to this command.
$options = drush_get_context('cli');
// The 'php' option, if needed will be set in $args already.
unset($options['php']);
// We need to reimplement _drush_escape_option() because it adds
// extra leading spaces.
foreach ($options as $key => $value) {
if ($value !== TRUE) {
$args[] = "--$key=" . escapeshellarg($value);
}
else {
$args[] = "--$key";
}
}
array_push($args, '@hostmaster', $command);
watchdog('hosting_queue_runner', 'Restarting queue runner with @drush @args.', array('@drush' => $drush, '@args' => implode(" ", $args)));
pcntl_exec($drush, $args, $_ENV);
watchdog('hosting_queue_runner', 'Could not restart the queue runner, aborting.', array(), WATCHDOG_ERROR);
/* NOTREACHED */
}
else {
watchdog('hosting_queue_runner', 'PCNTL not installed, unable to auto-restart.', array(), WATCHDOG_WARNING);
}
// Explicit exit in case we're handling a signal
exit(1);
}
/**
* Get the proper way to call drush again.
*
* Note that unlike drush_find_drush() we return an array of parts, and we trim
* the 'php' option of extra single quotes.
*
* @see drush_find_drush()
*/
function hosting_queue_runner_drush_find_drush() {
$php = drush_get_option('php');
if (isset($php)) {
$php = trim($php, "'");
$drush = array($php, realpath($_SERVER['argv'][0]) , "--php='$php'");
} else {
$drush = array(realpath($_SERVER['argv']['0']));
}
return $drush;
}
name = Hosting queue runner
description = Allows the hosting queue to be run in a daemon.
package = Hosting
dependencies[] = hosting
core = 6.x
<?php
/**
* Implementation of hook_enable().
*/
function hosting_queue_runner_enable() {
// Disable Aegir's dispatch of the tasks queue.
variable_set('hosting_queue_tasks_enabled', FALSE);
}
/**
* Implementation of hook_disable().
*/
function hosting_queue_runner_disable() {
// Enable Aegir's dispatch of the tasks queue.
variable_set('hosting_queue_tasks_enabled', TRUE);
}
<?php
/**
* Implementation of hook_menu
*/
function hosting_queue_runner_menu() {
$items['admin/hosting/queue_runner'] = array(
'title' => 'Queue runner settings',
'description' => 'Configure the daemonized tasks queue runner.',
'page callback' => 'drupal_get_form',
'page arguments' => array('hosting_queue_runner_settings_form'),
'access arguments' => array('administer hosting queues'),
'type' => MENU_LOCAL_TASK,
'file' => 'hosting_queue_runner.admin.inc',
);
return $items;
}
#!/bin/bash
$DRUSH_COMMAND @hostmaster hosting-queue-runner
#!/bin/sh
### BEGIN INIT INFO
# Provides: hosting-queue-runner
# Required-Start: $network $local_fs
# Required-Stop:
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Queue runner daemon for Aegir
# Description: This module allows the Hosting tasks queue for the
# Aegir project to be 'daemonized' so that tasks
# are run as soon as possible instead of waiting
# for a cron run. This makes Aegir appear much more
# responsive.
### END INIT INFO
# Author: Antoine Beaupré <anarcat@koumbit.org>
# PATH should only include /usr/* if it runs after the mountnfs.sh script
PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="Aegir queue runner" # Introduce a short description here
NAME="hosting-queue-runner" # Introduce the short server's name here
DAEMON=/usr/bin/drush # Introduce the server's location here
DAEMON_ARGS="@hostmaster $NAME" # Arguments to run the daemon with
USER="aegir"
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
# Exit if the package is not installed
[ -x $DAEMON ] || exit 0
# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME
# Load the VERBOSE setting and other rcS variables
. /lib/init/vars.sh
# Define LSB log_* functions.
# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
. /lib/lsb/init-functions
#
# Function that starts the daemon/service
#
do_start()
{
# Return
# 0 if daemon has been started
# 1 if daemon was already running
# 2 if daemon could not be started
start-stop-daemon --start --quiet -c $USER --background \
--make-pidfile --pidfile $PIDFILE --startas $DAEMON \
--test > /dev/null \
|| return 1
start-stop-daemon --start --quiet -c $USER --background \
--make-pidfile --pidfile $PIDFILE --startas $DAEMON -- \
$DAEMON_ARGS \
|| return 2
}
#
# Function that stops the daemon/service
#
do_stop()
{
# Return
# 0 if daemon has been stopped
# 1 if daemon was already stopped
# 2 if daemon could not be stopped
# other if a failure occurred
start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE
RETVAL="$?"
[ "$RETVAL" = 2 ] && return 2
# Many daemons don't delete their pidfiles when they exit.
rm -f $PIDFILE
return "$RETVAL"
}
#
# Function that sends a SIGHUP to the daemon/service
#
do_reload() {
#
# If the daemon can reload its configuration without
# restarting (for example, when it is sent a SIGHUP),
# then implement that here.
#
start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME
return 0
}
case "$1" in
start)
[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC " "$NAME"
do_start
case "$?" in
0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
esac
;;
stop)
[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
do_stop
case "$?" in
0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
esac
;;
status)
status_of_proc "php $DAEMON" "$NAME" && exit 0 || exit $?
;;
#reload|force-reload)
#
# If do_reload() is not implemented then leave this commented out
# and leave 'force-reload' as an alias for 'restart'.
#
#log_daemon_msg "Reloading $DESC" "$NAME"
#do_reload
#log_end_msg $?
#;;
restart|force-reload)
#
# If the "reload" option is implemented then remove the
# 'force-reload' alias
#
log_daemon_msg "Restarting $DESC" "$NAME"
do_stop
case "$?" in
0|1)
do_start
case "$?" in
0) log_end_msg 0 ;;
1) log_end_msg 1 ;; # Old process is still running
*) log_end_msg 1 ;; # Failed to start
esac
;;
*)
# Failed to stop
log_end_msg 1
;;
esac
;;
*)
#echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
exit 3
;;
esac
:
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment