Commit abd6c365 authored by catch's avatar catch
Browse files

Issue #3386076 by fjgarlin, catch, longwave, larowlan, mstrelan, el7cosmos,... 

Issue #3386076 by fjgarlin, catch, longwave, larowlan, mstrelan, el7cosmos, RoSk0, xurizaemon, poker10, alexpott, bbrala, nick_schuch: GitLab CI integration for core

(cherry picked from commit dfa16b8a)
parent 81a81f13

.gitlab-ci.yml

0 → 100644
+114 −0
Original line number Diff line number Diff line 
################

# Drupal GitLabCI template.

#

# Based off GitlabCI templates project: https://git.drupalcode.org/project/gitlab_templates

# Guide: https://www.drupal.org/docs/develop/git/using-gitlab-to-contribute-to-drupal/gitlab-ci

#

# With thanks to:

# - The GitLab Acceleration Initiative participants

# - DrupalSpoons

################



################

# Includes

#

# Additional configuration can be provided through includes.

# One advantage of include files is that if they are updated upstream, the

# changes affect all pipelines using that include.

#

# Includes can be overridden by re-declaring anything provided in an include,

# here in gitlab-ci.yml.

# https://docs.gitlab.com/ee/ci/yaml/includes.html#override-included-configuration-values

################



include:

  - project: $_GITLAB_TEMPLATES_REPO

    ref: $_GITLAB_TEMPLATES_REF

    file:

      - '/includes/include.drupalci.variables.yml'

      - '/includes/include.drupalci.workflows.yml'



################

# Variables

#

# Overriding variables

# - To override one or more of these variables, simply declare your own variables keyword.

# - Keywords declared directly in .gitlab-ci.yml take precedence over include files.

# - Documentation:  https://docs.gitlab.com/ee/ci/variables/

# - Predefined variables: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html

#

################



variables:

  _TARGET_PHP: "8.2"

  COMPOSER: composer.json

  # Let composer know what self.version means.

  COMPOSER_ROOT_VERSION: "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}-dev"

  CONCURRENCY: 32



################

# Stages

#

# Each job is assigned to a stage, defining the order in which the jobs are executed.

# Jobs in the same stage run in parallel.

#

# If all jobs in a stage succeed, the pipeline will proceed to the next stage.

# If any job in the stage fails, the pipeline will exit early.

################



.default-stage: &default-stage

  stage: test

  trigger:

    # Rely on the status of the child pipeline.

    strategy: depend

    include:

      - local: .gitlab-ci/pipeline.yml

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event"



.run-manually: &run-manually

  when: manual



'PHP 8.2 MySQL 5.7':

  <<: [ *default-stage, *run-manually ]

  variables:

    _TARGET_DB_TYPE: "mysql"

    _TARGET_DB_VERSION: "5.7"

    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production



'PHP 8.2 MySQL 8':

  <<: *default-stage

  variables:

    _TARGET_DB_TYPE: "mysql"

    _TARGET_DB_VERSION: "8"

    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event"



'PHP 8.2 PostgreSQL 13.5':

  <<: [ *default-stage, *run-manually ]

  variables:

    _TARGET_DB_TYPE: "pgsql"

    _TARGET_DB_VERSION: "13.5"

    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production



'PHP 8.2 PostgreSQL 14.1':

  <<: [ *default-stage, *run-manually ]

  variables:

    _TARGET_DB_TYPE: "pgsql"

    _TARGET_DB_VERSION: "14.1"

    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production



# 'PHP 8.2 SQLite 3.26.0':

#   <<: [ *default-stage, *run-manually ]

#   variables:

#     _TARGET_DB_TYPE: "sqlite"

#     _TARGET_DB_VERSION: "3.26.0"

#     _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production



# 'PHP 8.2 MariaDB 10.3.22':

#   <<: [ *default-stage, *run-manually ]

#   variables:

#     _TARGET_DB_TYPE: "mariadb"

#     _TARGET_DB_VERSION: "10.3.22"

#     _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production

.gitlab-ci/pipeline.yml

0 → 100644
+364 −0
Original line number Diff line number Diff line 
# cspell:ignore drupaltestbot drupaltestbotpw



stages:

  ################

  # Build

  #

  # Assemble the test environment.

  ################

  - 🏗️ Build



  ################

  # Code quality checks

  #

  # This stage includes any codebase validation before running tests.

  ################

  - 🪄 Lint



  ################

  # Test

  #

  # The test phase actually executes the tests, as well as gathering results

  # and artifacts.

  ################

  - 🗜️ Test



#############

# Templates #

#############



.default-job-settings: &default-job-settings

  interruptible: true

  allow_failure: false

  image:

    name: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production

  rules:

    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"



.composer-cache: &composer-cache

  key:

    files:

      - ./composer.json

      - ./composer.lock

  paths:

    - ./vendor



.yarn-cache: &yarn-cache

  key:

    files:

      - ./core/package.json

      - ./core/yarn.lock

  paths:

    - ./core/node_modules



.pull-composer-cache: &pull-composer-cache

  cache:

    policy: pull

    <<: *composer-cache

  dependencies:

    - '📦️ Composer'



.with-composer-cache: &with-composer-cache

  needs:

    - '📦️ Composer'

  <<: *pull-composer-cache



.with-yarn-cache: &with-yarn-cache

  dependencies:

    - '📦️ Yarn'

  needs:

    - '📦️ Yarn'

  cache:

    policy: pull

    <<: *yarn-cache



.junit-artifacts: &junit-artifacts

  artifacts:

    expose_as: junit

    expire_in: 6 mos

    paths:

      - junit.xml

    reports:

      junit: junit.xml



.with-linting: &with-linting

  needs:

    - '📦️ Composer'

    - '🧹 PHP Static Analysis (phpstan)'

    - '🧹 PHP Coding standards (PHPCS)'

    - '🧹 Compilation check'

    - '📦️ Yarn'

    - '📔 Spell-checking'

    - '🧹 JavaScript linting (eslint)'

    - '🧹 CSS linting (stylelint)'

  <<: *pull-composer-cache



.with-unit-tests: &with-unit-tests

  needs:

    - '⚡️ PHPUnit Unit'

    - '📦️ Composer'

  <<: *pull-composer-cache



.with-composer-yarn-and-unit-tests: &with-composer-yarn-and-unit-tests

  needs:

    - '📦️ Composer'

    - '⚡️ PHPUnit Unit'

    - '📦️ Yarn'

  dependencies:

    - '📦️ Yarn'

    - '📦️ Composer'



.test-variables: &test-variables

  FF_NETWORK_PER_BUILD: 1

  SIMPLETEST_BASE_URL: http://localhost/subdirectory

  DB_DRIVER: mysql

  MYSQL_ROOT_PASSWORD: root

  MYSQL_DATABASE: mysql

  MYSQL_USER: drupaltestbot

  MYSQL_PASSWORD: drupaltestbotpw

  MARIADB_TAG: $_TARGET_DB_VERSION

  POSTGRES_TAG: $_TARGET_DB_VERSION

  POSTGRES_DB: drupaltestbot

  POSTGRES_USER: drupaltestbot

  POSTGRES_PASSWORD: drupaltestbotpw

  MINK_DRIVER_ARGS_WEBDRIVER: '["chrome", {"browserName":"chrome","chromeOptions":{"args":["--disable-dev-shm-usage","--disable-gpu","--headless"]}}, "http://localhost:9515"]'

  CI_PARALLEL_NODE_INDEX: $CI_NODE_INDEX

  CI_PARALLEL_NODE_TOTAL: $CI_NODE_TOTAL



.with-database: &with-database

  name: $_DB_IMAGE

  alias: database



.with-chrome: &with-chrome

  name: $_CONFIG_DOCKERHUB_ROOT/chromedriver:production

  alias: chrome

  entrypoint:

    - chromedriver

    - "--no-sandbox"

    - "--log-path=/tmp/chromedriver.log"

    - "--verbose"

    - "--whitelisted-ips="



.phpunit-artifacts: &phpunit-artifacts

  artifacts:

    when: always

    expire_in: 6 mos

    reports:

      junit: ./sites/default/files/simpletest/phpunit-*.xml

    paths:

      - ./sites/default/files/simpletest/phpunit-*.xml

      - ./sites/simpletest/browser_output



.setup-webroot: &setup-webserver

  before_script:

    - ln -s $CI_PROJECT_DIR /var/www/html/subdirectory

    - sudo service apache2 start



.run-tests: &run-tests

  script:

    # Determine DB driver.

    - |

      [[ $_TARGET_DB_TYPE == "sqlite" ]] && export SIMPLETEST_DB=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite

      [[ $_TARGET_DB_TYPE == "mysql" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql

      [[ $_TARGET_DB_TYPE == "mariadb" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql

      [[ $_TARGET_DB_TYPE == "pgsql" ]] && export SIMPLETEST_DB=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql

    - export

    - mkdir -p ./sites/simpletest ./sites/default/files ./build/logs/junit /var/www/.composer

    - chown -R www-data:www-data ./sites ./build/logs/junit ./vendor /var/www/

    - sudo -u www-data git config --global --add safe.directory $CI_PROJECT_DIR

    # Need to pass this along directly.

    - sudo MINK_DRIVER_ARGS_WEBDRIVER="$MINK_DRIVER_ARGS_WEBDRIVER" -u www-data php ./core/scripts/run-tests.sh --color --keep-results --types "$TESTSUITE" --concurrency "$CONCURRENCY" --repeat "1" --sqlite "./sites/default/files/.sqlite" --dburl $SIMPLETEST_DB --url $SIMPLETEST_BASE_URL --verbose --non-html --all --ci-parallel-node-index $CI_PARALLEL_NODE_INDEX --ci-parallel-node-total $CI_PARALLEL_NODE_TOTAL



################

# Jobs

#

# Jobs define what scripts are actually executed in each stage.

#

# The 'rules' keyword can also be used to define conditions for each job.

#

# Documentation: https://docs.gitlab.com/ee/ci/jobs/

################



################

# Build Jobs

################



'ℹ️ Output build parameters':

  <<: *default-job-settings

  stage: 🏗️ Build

  script:

    - echo "Checking variables"

    - echo $_TARGET_PHP

    - echo $_TARGET_DB_TYPE

    - echo $_TARGET_DB_VERSION



'📦️ Composer':

  <<: *default-job-settings

  stage: 🏗️ Build

  cache:

    <<: *composer-cache

  artifacts:

    expire_in: 1 week

    expose_as: 'web-vendor'

    paths:

      - vendor/

  script:

      - export

      - composer validate

      - composer install



'📦️ Yarn':

  <<: *default-job-settings

  stage: 🏗️ Build

  cache:

    <<: *yarn-cache

  artifacts:

    expire_in: 1 week

    expose_as: 'yarn-vendor'

    paths:

      - core/node_modules/

  script:

    # Installs all core javascript dependencies and adds junit formatter.

    - yarn --cwd ./core add stylelint-junit-formatter



################

# Lint Jobs

################



'🧹 PHP Coding standards (PHPCS)':

  <<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ]

  stage: 🪄 Lint

  script:

    - composer phpcs -- --report-junit=junit.xml --report-full --report-summary



'🧹 PHP Static Analysis (phpstan)':

  <<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ]

  stage: 🪄 Lint

  script:

    # Turn off apc to avoid corrupt composer cache.

    - php -d apc.enabled=0 -d apc.enable_cli=0 vendor/bin/phpstan analyze --configuration=./core/phpstan.neon.dist --error-format=junit > junit.xml



'🧹 CSS linting (stylelint)':

  <<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ]

  stage: 🪄 Lint

  script:

    - yarn run --cwd=./core lint:css --color --custom-formatter node_modules/stylelint-junit-formatter > junit.xml



'🧹 Compilation check':

  <<: [ *with-yarn-cache, *default-job-settings ]

  stage: 🪄 Lint

  script:

    - yarn run --cwd=./core build:css --check

    - cd core && yarn run -s check:ckeditor5



'🧹 JavaScript linting (eslint)':

  <<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ]

  stage: 🪄 Lint

  script:

    - yarn --cwd=./core run -s lint:core-js-passing --format junit > junit.xml



'📔 Spell-checking':

  <<: [ *with-yarn-cache, *default-job-settings ]

  stage: 🪄 Lint

  script:

    - git fetch origin

    - export MODIFIED=`git diff --name-only origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}|while read r;do echo "$CI_PROJECT_DIR/$r";done|tr "\n" " "`

    - echo $MODIFIED | tr ' ' '\n' | yarn --cwd=./core run -s spellcheck:core --no-must-find-files --file-list stdin



################

# Test Jobs

################



'⚡️ PHPUnit Unit':

  <<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]

  stage: 🗜️ Test

  services:

    # There are some unit tests that need a database.

    # @todo Remove after https://www.drupal.org/project/drupal/issues/3386217

    - <<: *with-database

  variables:

    <<: *test-variables

    TESTSUITE: PHPUnit-Unit

    CONCURRENCY: "$CONCURRENCY"



'⚙️️ PHPUnit Kernel':

  <<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]

  stage: 🗜️ Test

  parallel: 3

  variables:

    <<: *test-variables

    TESTSUITE: PHPUnit-Kernel

    CONCURRENCY: "$CONCURRENCY"

  services:

    - <<: *with-database



'🖱️️️ PHPUnit Functional Javascript':

  <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]

  stage: 🗜️ Test

  variables:

    <<: *test-variables

    TESTSUITE: PHPUnit-FunctionalJavascript

    CONCURRENCY: 15

  services:

    - <<: *with-database

    - <<: *with-chrome



'👷️️️ PHPUnit Build':

  <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]

  stage: 🗜️ Test

  variables:

    <<: *test-variables

    TESTSUITE: PHPUnit-Build

    CONCURRENCY: "$CONCURRENCY"

  services:

    - <<: *with-database



'🌐️️ PHPUnit Functional':

  <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]

  stage: 🗜️ Test

  parallel: 8

  variables:

    <<: *test-variables

    TESTSUITE: PHPUnit-Functional

    CONCURRENCY: "$CONCURRENCY"

  services:

    - <<: *with-database



'🦉️️️ Nightwatch':

  <<: [ *with-composer-yarn-and-unit-tests, *setup-webserver, *default-job-settings ]

  stage: 🗜️ Test

  variables:

    <<: *test-variables

  services:

    - <<: *with-database

    - <<: *with-chrome

  script:

    # Determine DB driver.

    - |

      [[ $_TARGET_DB_TYPE == "sqlite" ]] && export DRUPAL_TEST_DB_URL=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite

      [[ $_TARGET_DB_TYPE == "mysql" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql

      [[ $_TARGET_DB_TYPE == "mariadb" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql

      [[ $_TARGET_DB_TYPE == "pgsql" ]] && export DRUPAL_TEST_DB_URL=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql

    - export

    - cp ./core/.env.example ./core/.env

    # dotenv-safe/config does not support environment variables

    # @see https://github.com/rolodato/dotenv-safe/issues/126

    # @todo move this to `variables` when the above is resolved

    - echo "DRUPAL_TEST_BASE_URL='http://localhost/subdirectory'" >> ./core/.env

    - echo "DRUPAL_TEST_CHROMEDRIVER_AUTOSTART=false" >> ./core/.env

    - echo "DRUPAL_TEST_DB_URL='${DRUPAL_TEST_DB_URL}'" >> ./core/.env

    - echo "DRUPAL_TEST_WEBDRIVER_HOSTNAME='localhost'" >> ./core/.env

    - echo "DRUPAL_TEST_WEBDRIVER_CHROME_ARGS='--disable-dev-shm-usage --disable-gpu --headless'" >> ./core/.env

    - echo "DRUPAL_TEST_WEBDRIVER_PORT='9515'" >> ./core/.env

    - echo "DRUPAL_NIGHTWATCH_OUTPUT='"nightwatch_output"'" >> ./core/.env

    - cat ./core/.env

    - mkdir -p ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output

    - chown -R www-data:www-data ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output ./core/.env

    - sudo BABEL_DISABLE_CACHE=1 -u www-data yarn --cwd ./core test:nightwatch

  artifacts:

    when: always

    expire_in: 6 mos

    reports:

      junit: ./nightwatch_output/*.xml

    paths:

      - ./nightwatch_output

core/misc/cspell/dictionary.txt

+3 −0
Original line number Diff line number Diff line
@@ -322,9 +322,11 @@ distro 
ditka

divs

dnumber

dockerhub

docroot

docroots

dolore

dotenv

downcasting

doxygen

dragtable
@@ -547,6 +549,7 @@ insidekeyword 
instaclick

instantiator

interactable

interruptible

introspectable

invalidators

invalididentifier

core/package.json

+1 −1
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ 
    "prettier": "prettier --write \"./**/*.js\"",

    "spellcheck": "cspell -c .cspell.json",

    "spellcheck:make-drupal-dict": "rm -f misc/cspell/dictionary.txt && touch misc/cspell/dictionary.txt && yarn -s spellcheck:core --unique --words-only | perl -Mopen=locale -pe '$_=lc$_' | LC_ALL=en_US.UTF-8 tr -d \\\\\\\\ | LC_ALL=C sort -u -o misc/cspell/dictionary.txt",

    "spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\"",

    "spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\" \".gitlab-ci/*\" \".gitlab-ci.yml\"",

    "vendor-update": "node ./scripts/js/vendor-update.js",

    "watch:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js --watch",

    "build:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js",

core/scripts/run-tests.sh

+8 −0
Original line number Diff line number Diff line
@@ -404,6 +404,8 @@ function simpletest_script_parse_args() { 
    'execute-test' => '',

    'xml' => '',

    'non-html' => FALSE,

    'ci-parallel-node-index' => 1,

    'ci-parallel-node-total' => 1,

  ];



  // Override with set values.
@@ -1024,6 +1026,12 @@ function simpletest_script_get_test_list() { 
    simpletest_script_print_error('No valid tests were specified.');

    exit(SIMPLETEST_SCRIPT_EXIT_FAILURE);

  }



  if ((int) $args['ci-parallel-node-total'] > 1) {

    $tests_per_job = ceil(count($test_list) / $args['ci-parallel-node-total']);

    $test_list = array_slice($test_list, ($args['ci-parallel-node-index'] - 1) * $tests_per_job, $tests_per_job);

  }



  return $test_list;

}