diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..e172b6c3f9641f924b21893555fca24d8dc3d8f6 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,182 @@ +# Is performed before the scripts in the stages step +before_script: + - source /etc/profile + +# Defines stages which are to be executed +stages: + - build + - test + - publish + - upgradetest + +# Stage "build" +build:deb: + stage: build + image: aegir/hostmaster:packaging_base + script: + - export commit=$(git log -n 1 --oneline | sed 's/ .*$//') + - export version=$(sed -ne 's/^[^(]*(\([^)]*\)).*/\1/;1p' debian/changelog) + - export new_version="${version}+${CI_BUILD_ID}.${commit}" + - dch -D unstable --newversion "$new_version" "automatic GitLab CI build ${CI_BUILD_ID}" + - dpkg-buildpackage + - mkdir build + - mv -v ../aegir3*.deb build/ + - mv -v ../aegir3*.tar.gz build/ + - mv -v ../aegir3*.dsc build/ + - mv -v ../aegir3*.changes build/ + + # The files which are to be made available in GitLab as artifacts. + artifacts: + paths: + - build/* + + +# Stage "test" +test:debian-jessie-aegir3-apt: + stage: test + image: debian:jessie + dependencies: + - build:deb + + before_script: + - apt-get update + # Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start. + - echo "#!/bin/sh" > /usr/sbin/policy-rc.d + - echo "exit 0" >> /usr/sbin/policy-rc.d + - apt-get install --yes sudo curl + + script: "scripts/ci-aegir-dev-install-apt-debian8.sh" + +test:debian-stretch-aegir3-apt: + stage: test + image: debian:stretch + allow_failure: true + dependencies: + - build:deb + only: + - 7.x-3.x + + before_script: + - apt-get update + # Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start. + - echo "#!/bin/sh" > /usr/sbin/policy-rc.d + - echo "exit 0" >> /usr/sbin/policy-rc.d + - apt-get install --yes sudo curl + + script: "scripts/ci-aegir-dev-install-apt-debian9.sh" + +test:ubuntu-xenial-aegir3-apt: + stage: test + image: ubuntu:xenial + dependencies: + - build:deb + only: + - 7.x-3.x + + before_script: + - apt-get update + # Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start. + - echo "#!/bin/sh" > /usr/sbin/policy-rc.d + - echo "exit 0" >> /usr/sbin/policy-rc.d + - apt-get install --yes sudo curl + + script: "scripts/ci-aegir-dev-install-apt-ubuntu.sh" + +test:behat: + stage: test + image: aegir/hostmaster:packaging_base + dependencies: + - build:deb + allow_failure: true + variables: + AEGIR_TESTS_VERSION: "master" + + before_script: + - sudo apt-get update + - sudo apt-get install --yes apt-transport-https ca-certificates curl gnupg2 software-properties-common + - curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - + - sudo apt-key fingerprint 0EBFCD88 + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get install --yes docker-ce + - docker info + - git clone http://github.com/aegir-project/tests.git tests + - cd tests + - git checkout $AEGIR_TESTS_VERSION + + # Run prepare scripts. + - cd travis + - sudo bash prepare-docker.sh + - bash prepare-testenv.sh + + script: + + # Tests are included in the docker-compose.yml file in the tests repo. + - sudo docker-compose -f docker-compose-provision.yml run hostmaster + + + + +publish:unstable-repo: + stage: publish + dependencies: + - build:deb + allow_failure: true + + only: + - 7.x-3.x + + image: ruby:2.1 + before_script: + # install ssh-agent + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + + # run ssh-agent + - eval $(ssh-agent -s) + + # add ssh key stored in SSH_PRIVATE_KEY variable to the agent store + - ssh-add <(echo "$SSH_PRIVATE_KEY") + + - mkdir -p ~/.ssh + - ls -la /.dockerenv + - echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts + - cat ~/.ssh/known_hosts + + script: + - ls -lah build/* + - scp build/* ci@aegir0.aegir.coop:/var/www/repos/incoming/ + # reprepro should pick this up in seconds and update the repository index. + + +# Upgrade the latest stable Aegir to our unstable repo. +upgradetest:debian-jessie-aegir3-apt-upgrade: + stage: upgradetest + image: debian:jessie + dependencies: + - publish:unstable-repo + + when: manual + + before_script: + - apt-get update + # Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start. + - echo "#!/bin/sh" > /usr/sbin/policy-rc.d + - echo "exit 0" >> /usr/sbin/policy-rc.d + - apt-get install --yes sudo curl cron + + script: + - "scripts/ci-aegir-stable-install-apt-debian8.sh" + # extra step to run the task queue. + - sudo su aegir --login --command 'drush @hostmaster php-eval "echo hosting_task_count();"' + - sudo su aegir --login --command 'drush @hostmaster hosting-tasks --force' + - sudo su aegir --login --command 'drush @hostmaster php-eval "echo hosting_task_count();"' + - sudo su aegir --login --command 'drush @hostmaster hosting-tasks --force' + - sudo su aegir --login --command 'drush @hostmaster php-eval "echo hosting_task_count();"' + - sudo su aegir --login --command 'drush @hostmaster php-eval "echo hosting_task_count_running();"' + - sleep 2m + - sudo su aegir --login --command 'drush @hostmaster php-eval "echo hosting_task_count_running();"' + # upgrade to the latest version from the unstable repo. + - rm -v /etc/apt/sources.list.d/aegir-stable.list + - echo "deb http://debian2.aegirproject.org unstable main" | sudo tee -a /etc/apt/sources.list.d/aegir-unstable.list + - sudo apt-get update + - sudo apt-get --yes upgrade diff --git a/Provision/Config.php b/Provision/Config.php index 763b0037c66315bb9001044dbcbc9d9aa49858a2..ea77e25df7b5af7887645a22f2403a9c94375b23 100644 --- a/Provision/Config.php +++ b/Provision/Config.php @@ -167,6 +167,11 @@ class Provision_Config { * array. */ private function render_template($template, $variables) { + + // Allow modules to alter the variables before writing to the template. + // @see hook_provision_config_variables_alter() + drush_command_invoke_all_ref('provision_config_variables_alter', $variables, $template, $this); + drush_errors_off(); extract($variables, EXTR_SKIP); // Extract the variables to a local namespace ob_start(); // Start output buffering diff --git a/Provision/Config/Drushrc/Alias.php b/Provision/Config/Drushrc/Alias.php index 71fa03b702113dccd1cd1fec060147716e076b89..e6c2a91eabc5ea0076b75758aa2cc544bb5a17ac 100644 --- a/Provision/Config/Drushrc/Alias.php +++ b/Provision/Config/Drushrc/Alias.php @@ -18,6 +18,14 @@ class Provision_Config_Drushrc_Alias extends Provision_Config_Drushrc { */ function __construct($context, $data = array()) { parent::__construct($context, $data); + + if (is_array($data['aliases'])) { + $data['aliases'] = array_unique($data['aliases']); + } + if (is_array($data['drush_aliases'])) { + $data['drush_aliases'] = array_unique($data['drush_aliases']); + } + $this->data = array( 'aliasname' => ltrim($context, '@'), 'options' => $data, diff --git a/Provision/Service.php b/Provision/Service.php index 58b97bfa82888eaaa26b881dfdd4d53b96ba5e05..2d732b14847d0dcf5d97c794336a821d5351634f 100644 --- a/Provision/Service.php +++ b/Provision/Service.php @@ -55,7 +55,13 @@ class Provision_Service extends Provision_ChainedState { } - // All services have the ability to have an associated restart command and listen port. + /** + * All services have the ability to have an associated restart command and listen port. + * + * If services require a fixed path config file, implement the function symlink_service() inside this method. + * + * @See Provision_Service_http_public::init_server() + */ function init_server() { if (!is_null($this->service)) { if ($this->has_port) { @@ -336,4 +342,27 @@ class Provision_Service extends Provision_ChainedState { static function option_documentation() { return array(); } + + /** + * Save symlink for this server from /var/aegir/config/APPLICATION_NAME.conf -> /var/aegir/config/SERVER/APPLICATION_NAME.conf + * + * If service requires a fixed path config file, implement this function in the init_server() method. + * + * @See Provision_Service_http_public::init_server() + */ + function symlink_service() { + $file = $this->application_name . '.conf'; + // We link the app_name.conf file on the remote server to the right version. + $cmd = sprintf('ln -sf %s %s', + escapeshellarg($this->server->config_path . '/' . $file), + escapeshellarg($this->server->aegir_root . '/config/' . $file) + ); + + if ($this->server->shell_exec($cmd)) { + drush_log(dt("Created symlink for %file on %server", array( + '%file' => $file, + '%server' => $this->server->remote_host, + )), 'notice'); + }; + } } diff --git a/README.md b/README.md index 4afd1294abbfebeb76ff5105e29f0f7d3d87c571..20cda0ed1ab62f491108e5ddbdecae4c78992118 100644 --- a/README.md +++ b/README.md @@ -34,4 +34,6 @@ Other documentation for developers is also available at: http://docs.aegirproject.org/en/3.x/extend/ -Build status: [![Build Status](https://travis-ci.org/aegir-project/provision.svg?branch=7.x-3.x)](https://travis-ci.org/aegir-project/provision) +Build status Travis: [![Build Status](https://travis-ci.org/aegir-project/provision.svg?branch=7.x-3.x)](https://travis-ci.org/aegir-project/provision) (in-depth and manual install) + +Build status GitLab CI: [![build status](https://gitlab.com/aegir/provision/badges/7.x-3.x/build.svg)](https://gitlab.com/aegir/provision/commits/7.x-3.x) (debian packages) diff --git a/aegir-release.make b/aegir-release.make index b6999da54c86b247240ad4cd8541343bc16fba51..d59e783ccc6726c23c47a7ad8c0ef53b9d84e6a4 100644 --- a/aegir-release.make +++ b/aegir-release.make @@ -5,6 +5,7 @@ api = 2 projects[drupal][type] = "core" ; The release.sh script updates the version of hostmaster. -projects[hostmaster][version] = "7.x-3.0-dev" projects[hostmaster][type] = "profile" -projects[hostmaster][variant] = "projects" +projects[hostmaster][download][type] = "git" +projects[hostmaster][download][tag] = "7.x-3.0-dev" +projects[hostmaster][download][url] = "http://git.drupal.org/project/hostmaster.git" diff --git a/debian/changelog b/debian/changelog index f2437c782b5c74a2b84dc09ebba27c71b851eb3b..2694fe79980245bb38a56dd640751e99c04dc77a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,30 @@ +aegir3-provision (3.11~beta2) unstable; urgency=medium + + * Preparations for the 3.11 release, testing the GitLab build process. + * Many bugfixes and UI improvements, see http://aegir.readthedocs.org/en/3.x/release-notes/3.11 + + -- Herman van Rink Tue, 23 May 2017 11:13:35 +0200 + +aegir3-provision (3.11~beta1) unstable; urgency=medium + + * Preparations for the 3.11 release, testing the GitLab build process. + * Many bugfixes and UI improvements, see http://aegir.readthedocs.org/en/3.x/release-notes/3.11 + + -- Herman van Rink Thu, 18 May 2017 14:28:24 +0200 + +aegir3-provision (3.10) unstable; urgency=medium + + * Many bugfixes and UI improvements, see http://aegir.readthedocs.org/en/3.x/release-notes/3.10 + + -- Herman van Rink Thu, 16 Mar 2017 11:08:16 +0100 + +aegir3-provision (3.9) unstable; urgency=medium + + * Makefile tweaks to allow for minor releases to update Drupal core + * Many bugfixes and UI improvements, see http://aegir.readthedocs.org/en/3.x/release-notes/3.9 + + -- Herman van Rink Thu, 05 Jan 2017 20:23:06 +0100 + aegir3-provision (3.9~beta1) unstable; urgency=medium * Preparations for the 3.9 release diff --git a/debian/control b/debian/control index a9630474f24e268e617a9ab4767c3036d5fbfa57..499cab5f0588907d732053eebc73341646cdc063 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Vcs-browser: http://drupalcode.org/project/provision.git Package: aegir3-provision Architecture: all -Depends: ${misc:Depends}, php5-cli (>= 5.3) | php7.0-cli, php5-mysql | php7.0-mysql, mysql-client | mariadb-client, sudo, postfix | mail-transport-agent, apache2 | nginx, adduser, ucf, curl +Depends: ${misc:Depends}, php5-cli (>= 5.3) | php7.0-cli, php5 | php7.0-xml, php5-mysql | php7.0-mysql, mysql-client | mariadb-client, sudo, postfix | mail-transport-agent, apache2 | nginx, adduser, ucf, curl Recommends: mysql-server | mariadb-server, rsync Conflicts: aegir-provision, aegir-provision2, aegir2-provision Replaces: aegir-provision, aegir-provision2, aegir2-provision diff --git a/http/Provision/Config/Apache/server.tpl.php b/http/Provision/Config/Apache/server.tpl.php index 621e7ac13ac2eec560d44d0f884f3c11eac24d14..5efa19db88811ad5557a5836a56b4e5402e5b29c 100644 --- a/http/Provision/Config/Apache/server.tpl.php +++ b/http/Provision/Config/Apache/server.tpl.php @@ -17,18 +17,30 @@ NameVirtualHost *: LoadModule rewrite_module modules/mod_rewrite.so + + # other configuration, not touched by aegir # this allows you to override aegir configuration, as it is included before -Include + # virtual hosts -Include + # platforms -Include + # other configuration, not touched by aegir # this allows to have default (for example during migrations) that are eventually overriden by aegir -Include + diff --git a/http/Provision/Config/Http/Server.php b/http/Provision/Config/Http/Server.php index 14459d7517ea7337c64125ffecfc4874b64b045b..1a9fe5296cd73d12dbc5a01e5cd7a08562da2116 100644 --- a/http/Provision/Config/Http/Server.php +++ b/http/Provision/Config/Http/Server.php @@ -16,23 +16,6 @@ class Provision_Config_Http_Server extends Provision_Config_Http { function write() { parent::write(); - - if (isset($this->data['application_name'])) { - $file = $this->data['application_name'] . '.conf'; - // We link the app_name.conf file on the remote server to the right version. - $cmd = sprintf('ln -sf %s %s', - escapeshellarg($this->data['server']->config_path . '/' . $file), - escapeshellarg($this->data['server']->aegir_root . '/config/' . $file) - ); - - if ($this->data['server']->shell_exec($cmd)) { - drush_log(dt("Created symlink for %file on %server", array( - '%file' => $file, - '%server' => $this->data['server']->remote_host, - ))); - - }; - } } function filename() { diff --git a/http/Provision/Config/Http/Site.php b/http/Provision/Config/Http/Site.php index b3292342cd71e2c505ff383b29d2da58d1be1e2b..35c702625b933d0df556637b87f2be42d21dbaca 100644 --- a/http/Provision/Config/Http/Site.php +++ b/http/Provision/Config/Http/Site.php @@ -11,7 +11,12 @@ class Provision_Config_Http_Site extends Provision_Config_Http { function filename() { - return $this->data['http_vhostd_path'] . '/' . $this->uri; + if (drush_get_option('provision_apache_conf_suffix', FALSE)) { + return $this->data['http_vhostd_path'] . '/' . $this->uri . '.conf'; + } + else { + return $this->data['http_vhostd_path'] . '/' . $this->uri; + } } function process() { diff --git a/http/Provision/Config/Http/Ssl/Site.php b/http/Provision/Config/Http/Ssl/Site.php index 3d91ef3429a787f28c1c99bc77c9e4f1540bfb3c..00658db17ba7ebb9b153b10b09bf89905e0f56f0 100644 --- a/http/Provision/Config/Http/Ssl/Site.php +++ b/http/Provision/Config/Http/Ssl/Site.php @@ -60,6 +60,14 @@ class Provision_Config_Http_Ssl_Site extends Provision_Config_Http_Site { if ($this->ssl_enabled) { // XXX: to be tested, not sure the data structure is sound + // + // ACHTUNG! This deletes even perfectly good certificate and key. + // There is no check in place to determine if the cert is "stale". + // Not sure what the idea was behind this cleanup, but it looks like + // an unfinished work, aggressively deleting existing cert/key pair, + // even if there is absolutely no reason to do so -- like when the site + // is simply migrated to another platform, while its name doesn't change. + // Provision_Service_http_ssl::free_certificate_site($this->ssl_key, $this); } } diff --git a/http/Provision/Config/Nginx/Inc/vhost_include.tpl.php b/http/Provision/Config/Nginx/Inc/vhost_include.tpl.php index 73ac20a2344023a2f37ef04b8d83ea5425f53619..5d9e38d36123fa8dfac657a62a3f9c0880003c80 100644 --- a/http/Provision/Config/Nginx/Inc/vhost_include.tpl.php +++ b/http/Provision/Config/Nginx/Inc/vhost_include.tpl.php @@ -1189,7 +1189,17 @@ location @drupal { return 418; } - rewrite ^/(.*)$ /index.php?q=$1 last; + ### + ### For Drupal >= 7 + ### + if ($sent_http_x_generator) { + add_header X-Info-Gen "Modern"; + rewrite ^ /index.php?$query_string last; + } + ### + ### For Drupal <= 6 + ### + rewrite ^/(.*)$ /index.php?q=$1 last; } @@ -1198,11 +1208,11 @@ location @drupal { ### location @nobots { ### - ### Support for Accelerated Mobile Pages (AMP). + ### Support for Accelerated Mobile Pages (AMP) when bots are redirected below ### - if ( $query_string ~ "^amp$" ) { - rewrite ^/(.*)$ /index.php?q=$1 last; - } + # if ( $query_string ~ "^amp$" ) { + # rewrite ^/(.*)$ /index.php?q=$1 last; + # } ### ### Send all known bots to $args free URLs (optional) @@ -1218,7 +1228,18 @@ location @nobots { if ( $args ~* "=PHP[A-Z0-9]{8}-" ) { return 404; } - rewrite ^/(.*)$ /index.php?q=$1 last; + + ### + ### For Drupal >= 7 + ### + if ($sent_http_x_generator) { + add_header X-Info-Gen "Modern"; + rewrite ^ /index.php?$query_string last; + } + ### + ### For Drupal <= 6 + ### + rewrite ^/(.*)$ /index.php?q=$1 last; } ### diff --git a/http/Provision/Config/Nginx/server.tpl.php b/http/Provision/Config/Nginx/server.tpl.php index ec17484c943f96494ca09346f17fc4775e56dc71..781d84cd8d20224ddac0b25e922dc59f0cd402eb 100644 --- a/http/Provision/Config/Nginx/server.tpl.php +++ b/http/Provision/Config/Nginx/server.tpl.php @@ -288,7 +288,7 @@ map $http_user_agent $deny_on_high_load { ### map $args $is_denied { default ''; - ~*delete.+from|insert.+into|select.+from|union.+select|onload|\.php.+src|system\(.+|document\.cookie|\;|\.\. is_denied; + ~*delete.+from|insert.+into|select.+from|union.+select|onload|\.php.+src|system\(.+|document\.cookie|\;|\.\.\/ is_denied; } diff --git a/http/Provision/Config/Nginx/subdir.tpl.php b/http/Provision/Config/Nginx/subdir.tpl.php index c2669bcbe27dd519ed25300149b6f13815368452..a45b88dfa3795716d7e8e761061dce75444978ec 100644 --- a/http/Provision/Config/Nginx/subdir.tpl.php +++ b/http/Provision/Config/Nginx/subdir.tpl.php @@ -1154,6 +1154,16 @@ location @drupal_ { return 418; } + ### + ### For Drupal >= 7 + ### + if ($sent_http_x_generator) { + add_header X-Info-Gen "Modern"; + rewrite ^ //index.php?$query_string last; + } + ### + ### For Drupal <= 6 + ### rewrite ^//(.*)$ //index.php?q=$1 last; } @@ -1163,11 +1173,11 @@ location @drupal_ { ### location @nobots_ { ### - ### Support for Accelerated Mobile Pages (AMP). + ### Support for Accelerated Mobile Pages (AMP) when bots are redirected below ### - if ( $query_string ~ "^amp$" ) { - rewrite ^//(.*)$ //index.php?q=$1 last; - } + # if ( $query_string ~ "^amp$" ) { + # rewrite ^//(.*)$ //index.php?q=$1 last; + # } ### ### Send all known bots to $args free URLs (optional) @@ -1183,6 +1193,17 @@ location @nobots_ { if ( $args ~* "=PHP[A-Z0-9]{8}-" ) { return 404; } + + ### + ### For Drupal >= 7 + ### + if ($sent_http_x_generator) { + add_header X-Info-Gen "Modern"; + rewrite ^ //index.php?$query_string last; + } + ### + ### For Drupal <= 6 + ### rewrite ^//(.*)$ //index.php?q=$1 last; } diff --git a/http/Provision/Service/http/pack.php b/http/Provision/Service/http/pack.php index 1b7daa55be06a8861620a259c82d64e0f98e7be1..ec816de2f94cc0a3994a8c1e183e1bec211f563c 100644 --- a/http/Provision/Service/http/pack.php +++ b/http/Provision/Service/http/pack.php @@ -64,7 +64,7 @@ class Provision_Service_http_pack extends Provision_Service_http { $this->_each_server($this->server->slave_web_servers, __FUNCTION__, array($config)); } - function delete_config($configi, $data = array()) { + function delete_config($config, $data = array()) { $this->_each_server($this->server->master_web_servers, __FUNCTION__, array($config)); $this->_each_server($this->server->slave_web_servers, __FUNCTION__, array($config)); diff --git a/http/Provision/Service/http/public.php b/http/Provision/Service/http/public.php index d35a3618780a38875dd19b482304308e5741d8a5..4d85fdf93cdf992ec46ea44be71747107a52f6e0 100644 --- a/http/Provision/Service/http/public.php +++ b/http/Provision/Service/http/public.php @@ -57,10 +57,11 @@ class Provision_Service_http_public extends Provision_Service_http { $this->server->http_vhostd_path = "{$app_dir}/vhost.d"; $this->server->http_subdird_path = "{$app_dir}/subdir.d"; $this->server->http_platforms_path = "{$this->server->aegir_root}/platforms"; + + $this->symlink_service(); } } - static function option_documentation() { return array( 'web_group' => 'server with http: OS group for permissions; working default will be attempted', diff --git a/http/Provision/Service/http/ssl.php b/http/Provision/Service/http/ssl.php index 581c04db5dfb7bd766fe7f3413bfb3feb5376384..f30c5e8fa397944aed2ac860a60c2f4f187859a4 100644 --- a/http/Provision/Service/http/ssl.php +++ b/http/Provision/Service/http/ssl.php @@ -192,6 +192,11 @@ class Provision_Service_http_ssl extends Provision_Service_http_public { static function free_certificate_site($ssl_key, $site) { if (empty($ssl_key)) return FALSE; $ssl_dir = $site->platform->server->http_ssld_path . "/" . $ssl_key . "/"; + // Respect hosting_le configuration, if detected -- start + $le_ctrl = d('@server_master')->aegir_root . "/tools/le/.ctrl"; + $immutable = $le_ctrl . "/dont-overwrite-" . $site->uri . ".pid"; + if (is_link($ssl_dir) || is_file($immutable)) return FALSE; + // Respect hosting_le configuration, if detected -- fin // Remove the file system reciept we left for this file if (provision_file()->unlink($ssl_dir . $site->uri . ".receipt")-> succeed(dt("Deleted SSL Certificate association receipt for %site on %server", array( diff --git a/platform/drupal/packages_6.inc b/platform/drupal/packages_6.inc index b48b7c39b7e17311f9c07170adb47e1d7d3fff06..49ee3d43ab72938eeb78215a20e6e86e0ce5c161 100644 --- a/platform/drupal/packages_6.inc +++ b/platform/drupal/packages_6.inc @@ -74,6 +74,11 @@ function _provision_drupal_parse_info_file($filename) { } $value = stripslashes(substr($value1, 1, -1)) . stripslashes(substr($value2, 1, -1)) . $value3; + // Remove any invalid UTF-8 sequences to prevent serialization errors. + if (function_exists('mb_convert_encoding')) { + $value = mb_convert_encoding($value, 'UTF-8', 'UTF-8'); + } + // Parse array syntax $keys = preg_split('/\]?\[/', rtrim($key, ']')); $last = array_pop($keys); @@ -188,3 +193,80 @@ function _provision_drupal_system_map() { drush_log(dt("Found !count themes", array('!count' => sizeof($packages['themes'])))); return $packages; } + +/** + * Find available profiles on this platform. + */ +function _provision_find_profiles() { + $profiles = array(); + + include_once('includes/install.inc'); + $profiles_subdirs[] = "./profiles"; + + foreach($profiles_subdirs as $profiles_subdir) { + if (!$dir = opendir($profiles_subdir)) { + drush_log(dt("Cannot find profiles directory"), 'error'); + return FALSE; + } + + while (FALSE !== ($name = readdir($dir))) { + $languages = array(); + if (($name == '..') || ($name == '.') || (!is_dir("$profiles_subdir/$name"))) { + continue; + } + + $profile = new stdClass(); + $profile->name = $name; + $profile->info = array(); + + $info_file = "$profiles_subdir/$name/$name.info"; + if (file_exists($info_file)) { + $profile->info = provision_parse_info_file($info_file); + // Skip hidden profiles + if (isset($profile->info['hidden']) && $profile->info['hidden'] == 1) { + continue; + } + } + $profile->filename = $info_file; + + // Include code from the profile. + if (file_exists($profile_file = "$profiles_subdir/$name/$name.profile")) { + require_once($profile_file); + } + + $func = $profile->name . "_profile_details"; + if (function_exists($func)) { + $profile->info = array_merge($profile->info, $func()); + } + + $profile_path = $profiles_subdir . '/' . $name; + $profile->info['languages'] = _provision_find_profile_languages($profile_path); + + $profiles[$name] = $profile; + drush_log(dt('Found install profile %name', array('%name' => $name))); + } + } + return $profiles; +} + +/** + * Retrieve a list of paths to search in a certain scope + */ +function _provision_drupal_search_paths($scope, $key = '', $type = 'modules') { + $searchpaths = array(); + $drupal_root = drush_get_context('DRUSH_DRUPAL_ROOT'); + switch ($scope) { + case 'base' : + $searchpaths[] = sprintf("%s/%s", $drupal_root, $type); + $searchpaths[] = sprintf("%s/sites/all/%s", $drupal_root, $type); + break; + default : + if ($key) { + $searchpaths[] = sprintf("%s/%s/%s/%s", $drupal_root, $scope, $key, $type); + } + break; + + } + return $searchpaths; +} + diff --git a/platform/drupal/packages_7.inc b/platform/drupal/packages_7.inc index 01cacbfb1f1a32194be6d5d1b28eba1d63caaa43..89517c4ac2cbf7919498474d41fb1e4c6990f2d6 100644 --- a/platform/drupal/packages_7.inc +++ b/platform/drupal/packages_7.inc @@ -39,7 +39,12 @@ function _provision_drupal_find_themes($scope, $key = '') { $files[$name]->template = TRUE; } } - $files[$name]->version = $files[$name]->info['version']; + if (!empty($files[$name]->info['version'])) { + $files[$name]->version = $files[$name]->info['version']; + } + else { + $files[$name]->version = NULL; + } } return $files; } @@ -77,6 +82,11 @@ function _provision_drupal_parse_info_file($filename) { } $value = stripslashes(substr($value1, 1, -1)) . stripslashes(substr($value2, 1, -1)) . $value3; + // Remove any invalid UTF-8 sequences to prevent serialization errors. + if (function_exists('mb_convert_encoding')) { + $value = mb_convert_encoding($value, 'UTF-8', 'UTF-8'); + } + // Parse array syntax $keys = preg_split('/\]?\[/', rtrim($key, ']')); $last = array_pop($keys); @@ -194,3 +204,86 @@ function _provision_drupal_system_map() { drush_log(dt("Found !count themes", array('!count' => sizeof($packages['themes'])))); return $packages; } + +/** + * Find available profiles on this platform. + */ +function _provision_find_profiles() { + $profiles = array(); + + include_once('includes/install.inc'); + $profiles_subdirs[] = "./profiles"; + + foreach($profiles_subdirs as $profiles_subdir) { + if (!$dir = opendir($profiles_subdir)) { + drush_log(dt("Cannot find profiles directory"), 'error'); + return FALSE; + } + + while (FALSE !== ($name = readdir($dir))) { + $languages = array(); + if (($name == '..') || ($name == '.') || (!is_dir("$profiles_subdir/$name"))) { + continue; + } + + $profile = new stdClass(); + $profile->name = $name; + $profile->info = array(); + + $info_file = "$profiles_subdir/$name/$name.info"; + if (file_exists($info_file)) { + $profile->info = provision_parse_info_file($info_file); + // Skip hidden profiles + if (isset($profile->info['hidden']) && $profile->info['hidden'] == 1) { + continue; + } + } + $profile->filename = $info_file; + + // Include code from the profile. + if (file_exists($profile_file = "$profiles_subdir/$name/$name.profile")) { + require_once($profile_file); + } + + $func = $profile->name . "_profile_details"; + if (function_exists($func)) { + $profile->info = array_merge($profile->info, $func()); + } + + $profile_path = $profiles_subdir . '/' . $name; + $profile->info['languages'] = _provision_find_profile_languages($profile_path); + + // Drupal 7 renamed the default install profile to 'standard' + // Aegir now allows projects to specify an "old short name" to provide an upgrade path when projects get renamed. + if ($profile->name == 'standard') { + $profile->info['old_short_name'] = 'default'; + } + + $profiles[$name] = $profile; + drush_log(dt('Found install profile %name', array('%name' => $name))); + } + } + return $profiles; +} + +/** + * Retrieve a list of paths to search in a certain scope + */ +function _provision_drupal_search_paths($scope, $key = '', $type = 'modules') { + $searchpaths = array(); + $drupal_root = drush_get_context('DRUSH_DRUPAL_ROOT'); + switch ($scope) { + case 'base' : + $searchpaths[] = sprintf("%s/%s", $drupal_root, $type); + $searchpaths[] = sprintf("%s/sites/all/%s", $drupal_root, $type); + break; + default : + if ($key) { + $searchpaths[] = sprintf("%s/%s/%s/%s", $drupal_root, $scope, $key, $type); + } + break; + + } + return $searchpaths; +} + diff --git a/platform/drupal/packages_8.inc b/platform/drupal/packages_8.inc index 272c3a61a0b9c10a07c3679022d9108c6f55b90d..edf79696fc0b925c86f2aa5629daefe064b19499 100644 --- a/platform/drupal/packages_8.inc +++ b/platform/drupal/packages_8.inc @@ -91,17 +91,26 @@ function _provision_drupal_system_map() { */ function _provision_system_query($type) { $packages = array(); - foreach (system_get_info($type) as $name => $package) { - $package = (object) $package; + + if ($type == 'theme') { + /** @var \Drupal\Core\Extension\Extension[] $extensions_data */ + $extensions_data = \Drupal::service('theme_handler')->rebuildThemeData(); + } + // Modules. + else { + /** @var \Drupal\Core\Extension\Extension[] $extensions_data */ + $extensions_data = system_rebuild_module_data(); + } + + foreach ($extensions_data as $name => $extension_data) { + $package = (object) $extension_data->info; $package->filename = drupal_get_filename($type, $name); $frags = explode("/", $package->filename); // Flag site-specific packages. if ($frags[0] == 'sites' && $frags[1] != 'all') { $package->platform = -1; } - // In Drupal 8, system_get_info returns enabled modules/themes. - $package->status = 1; - + $package->status = $extension_data->status; $package->filename = realpath($package->filename); if ($type == 'module') { @@ -113,3 +122,64 @@ function _provision_system_query($type) { return $packages; } + +/** + * Find available profiles on this platform. + */ +function _provision_find_profiles() { + $profile_dirs = array('./core/profiles', './profiles'); + + $profiles = array(); + foreach($profile_dirs as $profile_dir) { + if (!is_dir($profile_dir)) { + drush_log(dt("Cannot find profiles directory %dir", array('%dir' => $profile_dir)), 'error'); + return FALSE; + } + + $info_files = drush_scan_directory($profile_dir, "/\.info.yml$/"); + foreach ($info_files as $path => $info_file) { + $path = realpath($path); + $info = Symfony\Component\Yaml\Yaml::parse($path); + if ($info['type'] == 'profile' && (!isset($info['hidden']) || !$info['hidden'])) { + $profile = new stdClass(); + $profile->name = $info['name']; + $profile->info = $info; + $profile->filename = $path; + $profile->path = dirname($path); + $profile->info['languages'] = _provision_find_profile_languages($profile->path); + + $profiles[basename($profile->path)] = $profile; + drush_log(dt('Found install profile %name', array('%name' => $name))); + } + } + } + return $profiles; +} + +/** + * Retrieve a list of paths to search in a certain scope + */ +function _provision_drupal_search_paths($scope, $key = '', $type = 'modules') { + $searchpaths = array(); + $drupal_root = drush_get_context('DRUSH_DRUPAL_ROOT'); + switch ($scope) { + case 'base': + $searchpaths[] = sprintf("%s/%s", $drupal_root, $type); + $searchpaths[] = sprintf("%s/core/%s", $drupal_root, $type); + $searchpaths[] = sprintf("%s/sites/all/%s", $drupal_root, $type); + break; + case 'profiles': + foreach (_provision_find_profiles() as $profile) { + $searchpaths[] = $profile->path; + } + break; + default: + if ($key) { + $searchpaths[] = sprintf("%s/%s/%s/%s", $drupal_root, $scope, $key, $type); + } + break; + + } + return $searchpaths; +} + diff --git a/platform/install.provision.inc b/platform/install.provision.inc index afc645690d4abe492d7b816fd01f737feff06ecf..47913f531391cd66495dc1539436ec5f954c8a78 100644 --- a/platform/install.provision.inc +++ b/platform/install.provision.inc @@ -19,7 +19,35 @@ function drush_provision_drupal_provision_install_validate() { return drush_set_error("PROVISION_URL_REQUIRED", dt("You need to specify a valid url to install a site")); } if (_provision_drupal_site_exists()) { - return drush_set_error('PROVISION_SITE_INSTALLED'); + + // If "force-reinstall" option is set, delete the database and files. + if (drush_get_option('force-reinstall', FALSE)) { + + drush_log(dt('Forcing reinstall...'), 'ok'); + + // Load the current database name from drushrc.php. + // I cannot find another way to find the current db_name! + require_once(d()->site_path . '/drushrc.php'); + $old_db_name = $options['db_name']; + + if (d()->service('db')->database_exists($old_db_name)) { + d()->service('db')->drop_database($old_db_name); + drush_log(dt('Dropped database @database.', array( + '@database' => $old_db_name, + )), 'ok'); + } + + // Destroy site_path. + if (file_exists(d()->site_path)) { + _provision_recursive_delete( d()->site_path ); + drush_log(dt('Deleted @site_path.', array('@site_path' => d()->site_path)), 'ok'); + } + } + + // Check again if site does not exist after the forced reinstall. + if (_provision_drupal_site_exists()) { + return drush_set_error('PROVISION_SITE_INSTALLED'); + } } } diff --git a/platform/provision_drupal.drush.inc b/platform/provision_drupal.drush.inc index 129c45926202df9fbc85b11c4d27871cf2ef8574..233d60f28af34cfa4ed0251b96519f7e0060d244 100644 --- a/platform/provision_drupal.drush.inc +++ b/platform/provision_drupal.drush.inc @@ -63,8 +63,11 @@ function provision_drupal_drush_exit() { } } elseif (d()->type === 'platform') { - // Don't generate the drushrc.php on provision-save/delete commands. - if (!preg_match("/^provision-(save|delete)/", $command[0])) { + // Don't generate the drushrc.php on provision-save/delete/git-clone commands. + // Avoiding running on git-clone is needed in order to let it work with makefiles in the git repo. + // If this function runs, the platform directory is created, and drush make fails because the directory already exists. + // @TODO: Determine what tasks should save platform data and only act on those. + if (!preg_match("/^provision-(save|delete|git-clone)/", $command[0])) { provision_save_platform_data(); } } @@ -369,105 +372,6 @@ function _provision_drupal_rebuild_caches() { } } -/** - * Find available profiles on this platform. - */ -function _provision_find_profiles() { - $profiles = array(); - - if (drush_drupal_major_version() >= 8) { - include_once('core/includes/install.inc'); - $profiles_subdirs[] = "./core/profiles"; - $profiles_subdirs[] = "./profiles"; - } - else { - include_once('includes/install.inc'); - $profiles_subdirs[] = "./profiles"; - } - - foreach($profiles_subdirs as $profiles_subdir) { - if (!$dir = opendir($profiles_subdir)) { - drush_log(dt("Cannot find profiles directory"), 'error'); - return FALSE; - } - - while (FALSE !== ($name = readdir($dir))) { - $languages = array(); - if (($name == '..') || ($name == '.') || (!is_dir("$profiles_subdir/$name"))) { - continue; - } - - $profile = new stdClass(); - $profile->name = $name; - $profile->info = array(); - - if (drush_drupal_major_version() >= 8) { - $yaml_file = "$profiles_subdir/$name/$name.info.yml"; - if(!file_exists($yaml_file)) { - drush_log(dt("@name.info.yml not found.", array("@name" => $name)), 'notice'); - unset($files[$name]); - continue; - } - $profile->info = Symfony\Component\Yaml\Yaml::parse($yaml_file); - - // Skip hidden profiles. - if (isset($profile->info['hidden']) && $profile->info['hidden'] == 1) { - continue; - } - - if (!empty($profile->info['name'])) { - $profile->name = $profile->info['name']; - } - $profile->filename = $yaml_file; - } - else { - $info_file = "$profiles_subdir/$name/$name.info"; - if (file_exists($info_file)) { - $profile->info = provision_parse_info_file($info_file); - // Skip hidden profiles - if (isset($profile->info['hidden']) && $profile->info['hidden'] == 1) { - continue; - } - } - $profile->filename = $info_file; - } - - // Include code from the profile. - if (file_exists($profile_file = "$profiles_subdir/$name/$name.profile")) { - require_once($profile_file); - } - - $func = $profile->name . "_profile_details"; - if (function_exists($func)) { - $profile->info = array_merge($profile->info, $func()); - } - - $languages['en'] = 1; - // Find languages available - $files = array_keys(drush_scan_directory($profiles_subdir . '/' . $name . '/translations', '/\.po$/', array('.', '..', 'CVS'), 0, FALSE, 'filepath')); - $files = array_merge($files, array_keys(drush_scan_directory($profiles_subdir . '/' . $name , '/\.po$/', array('.', '..', 'CVS'), 0, FALSE, 'filepath'))); - if (is_array($files)) { - foreach ($files as $file) { - if (preg_match('!(/|\.)([^\./]+)\.po$!', $file, $langcode)) { - $languages[$langcode[2]] = 1; // use the language name as an index to weed out duplicates - } - } - } - $profile->info['languages'] = array_keys($languages); - - // Drupal 7 renamed the default install profile to 'standard' - // Aegir now allows projects to specify an "old short name" to provide an upgrade path when projects get renamed. - if ($profile->name == 'standard') { - $profile->info['old_short_name'] = 'default'; - } - - $profiles[$name] = $profile; - drush_log(dt('Found install profile %name', array('%name' => $name))); - } - } - return $profiles; -} - function provision_drupal_find_sites() { $sites = array(); if ($dir = opendir("./sites")) { @@ -637,28 +541,6 @@ function provision_drupal_system_map() { return _provision_drupal_system_map(); } -/** - * Retrieve a list of paths to search in a certain scope - */ -function _provision_drupal_search_paths($scope, $key = '', $type = 'modules') { - $searchpaths = array(); - $drupal_root = drush_get_context('DRUSH_DRUPAL_ROOT'); - switch ($scope) { - case 'base' : - $searchpaths[] = sprintf("%s/%s", $drupal_root, $type); - $searchpaths[] = sprintf("%s/core/%s", $drupal_root, $type); - $searchpaths[] = sprintf("%s/sites/all/%s", $drupal_root, $type); - break; - default : - if ($key) { - $searchpaths[] = sprintf("%s/%s/%s/%s", $drupal_root, $scope, $key, $type); - } - break; - - } - return $searchpaths; -} - /** * Find modules in a certain scope. * @@ -844,3 +726,25 @@ function _provision_client_delete_symlink() { ->fail('Failed to delete symlink @path: @reason'); } } + +/** + * Find available languages for a profile. + */ +function _provision_find_profile_languages($profile_path) { + $languages = array(); + + // English is default. + $languages['en'] = 1; + + // Find languages available. + $files = array_keys(drush_scan_directory($profile_path . '/translations', '/\.po$/', array('.', '..', 'CVS'), 0, FALSE, 'filepath')); + $files = array_merge($files, array_keys(drush_scan_directory($profile_path, '/\.po$/', array('.', '..', 'CVS'), 0, FALSE, 'filepath'))); + if (is_array($files)) { + foreach ($files as $file) { + if (preg_match('!(/|\.)([^\./]+)\.po$!', $file, $langcode)) { + $languages[$langcode[2]] = 1; // use the language name as an index to weed out duplicates + } + } + } + return array_keys($languages); +} diff --git a/provision.api.php b/provision.api.php index 3f491ff921cd46797e762dca12b4be21f985d4ca..50fd9e2d35a761472f113a3ee9e9e88d5a840abd 100644 --- a/provision.api.php +++ b/provision.api.php @@ -14,8 +14,15 @@ * $options['provision_backup_suffix'] = '.tar.bz2'; * * provision_verify_platforms_before_migrate - When migrating many sites turning this off can save time, default TRUE. + * * provision_backup_suffix - Method to set the compression used for backups... e.g. '.tar.bz2' or '.tar.', defaults to '.tar.gz'. * + * provision_apache_conf_suffix + * Set to TRUE to generate apache vhost files with a .conf suffix, default FALSE. + * This takes advantage of the IncludeOptional statment introduced in Apache 2.3.6. + * WARNING: After turning this on you need to re-verify all your sites, then then servers, + * and then cleanup the old configfiles (those without the .conf suffix). + * */ /** @@ -215,6 +222,31 @@ function hook_provision_config_load_templates_alter(&$templates, $config) { $templates = array(); } +/** + * Alter the variables used for rendering a config file. + * + * When implementing this hook, the function name should start with your file's name, not "drush_". + * + * @param $variables + * The variables that are about to be injected into the template. + * @param $template + * The template file chosen for use. + * @param $config + * The Provision_config object trying to find its template. + * + * @see hook_provision_config_load_templates() + * @see hook_provision_config_load_templates_alter() + */ +function hook_provision_config_variables_alter(&$variables, $template, $config) { + + // If this is the vhost template and the http service is Docker... + if (is_a($config, 'Provision_Config_Apache_Site') && is_a(d()->platform->service('http'), 'Provision_Service_http_apache_docker')) { + + // Force the listen port to be 80. + $variables['http_port'] = '80'; + } +} + /** * Alter the array of directories to create. * diff --git a/provision.drush.inc b/provision.drush.inc index 8331e41daf4b5321f56f34b93678bcbf5d5a754b..f417a8e269b06b6986e3344443536fd70fb17b61 100644 --- a/provision.drush.inc +++ b/provision.drush.inc @@ -113,6 +113,7 @@ function provision_drush_command() { 'options' => array( 'client_email' => dt('The email address of the client to use.'), 'profile' => dt('The profile to use when installing the site.'), + 'force-reinstall' => dt('Delete the sites database and files, before attempting to install.'), ), 'allow-additional-options' => TRUE, 'bootstrap' => DRUSH_BOOTSTRAP_DRUPAL_ROOT diff --git a/scripts/ci-aegir-dev-install-apt-debian8.sh b/scripts/ci-aegir-dev-install-apt-debian8.sh new file mode 100755 index 0000000000000000000000000000000000000000..b07e8616c2ba1caf99097f8bf11634aeff17482e --- /dev/null +++ b/scripts/ci-aegir-dev-install-apt-debian8.sh @@ -0,0 +1,35 @@ +# +# Install Aegir debian packages located in the 'build/' directory. +# These are provided by the GitLab CI build stage. +# +# This script is tuned for Debian 8 - Jessie. +# + +sudo apt-get update +echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections +#echo "debconf debconf/priority select critical" | debconf-set-selections + + +echo mysql-server-5.5 mysql-server/root_password password PASSWORD | debconf-set-selections +echo mysql-server-5.5 mysql-server/root_password_again password PASSWORD | debconf-set-selections + +debconf-set-selections <