Issue #3386076 by fjgarlin, catch, longwave, larowlan, mstrelan, el7cosmos,...

Issue #3386076 by fjgarlin, catch, longwave, larowlan, mstrelan, el7cosmos, RoSk0, xurizaemon, poker10, alexpott, bbrala, nick_schuch: GitLab CI integration for core

.gitlab-ci.yml

+################
+# Drupal GitLabCI template.
+#
+# Based off GitlabCI templates project: https://git.drupalcode.org/project/gitlab_templates
+# Guide: https://www.drupal.org/docs/develop/git/using-gitlab-to-contribute-to-drupal/gitlab-ci
+#
+# With thanks to:
+# - The GitLab Acceleration Initiative participants
+# - DrupalSpoons
+################
+
+################
+# Includes
+#
+# Additional configuration can be provided through includes.
+# One advantage of include files is that if they are updated upstream, the
+# changes affect all pipelines using that include.
+#
+# Includes can be overridden by re-declaring anything provided in an include,
+# here in gitlab-ci.yml.
+# https://docs.gitlab.com/ee/ci/yaml/includes.html#override-included-configuration-values
+################
+
+include:
+  - project: $_GITLAB_TEMPLATES_REPO
+    ref: $_GITLAB_TEMPLATES_REF
+    file:
+      - '/includes/include.drupalci.variables.yml'
+      - '/includes/include.drupalci.workflows.yml'
+
+################
+# Variables
+#
+# Overriding variables
+# - To override one or more of these variables, simply declare your own variables keyword.
+# - Keywords declared directly in .gitlab-ci.yml take precedence over include files.
+# - Documentation:  https://docs.gitlab.com/ee/ci/variables/
+# - Predefined variables: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+#
+################
+
+variables:
+  _TARGET_PHP: "8.2"
+  COMPOSER: composer.json
+  # Let composer know what self.version means.
+  COMPOSER_ROOT_VERSION: "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}-dev"
+  CONCURRENCY: 32
+
+################
+# Stages
+#
+# Each job is assigned to a stage, defining the order in which the jobs are executed.
+# Jobs in the same stage run in parallel.
+#
+# If all jobs in a stage succeed, the pipeline will proceed to the next stage.
+# If any job in the stage fails, the pipeline will exit early.
+################
+
+.default-stage: &default-stage
+  stage: test
+  trigger:
+    # Rely on the status of the child pipeline.
+    strategy: depend
+    include:
+      - local: .gitlab-ci/pipeline.yml
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+.run-manually: &run-manually
+  when: manual
+
+'PHP 8.2 MySQL 5.7':
+  <<: [ *default-stage, *run-manually ]
+  variables:
+    _TARGET_DB_TYPE: "mysql"
+    _TARGET_DB_VERSION: "5.7"
+    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
+
+'PHP 8.2 MySQL 8':
+  <<: *default-stage
+  variables:
+    _TARGET_DB_TYPE: "mysql"
+    _TARGET_DB_VERSION: "8"
+    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+'PHP 8.2 PostgreSQL 13.5':
+  <<: [ *default-stage, *run-manually ]
+  variables:
+    _TARGET_DB_TYPE: "pgsql"
+    _TARGET_DB_VERSION: "13.5"
+    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
+
+'PHP 8.2 PostgreSQL 14.1':
+  <<: [ *default-stage, *run-manually ]
+  variables:
+    _TARGET_DB_TYPE: "pgsql"
+    _TARGET_DB_VERSION: "14.1"
+    _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
+
+# 'PHP 8.2 SQLite 3.26.0':
+#   <<: [ *default-stage, *run-manually ]
+#   variables:
+#     _TARGET_DB_TYPE: "sqlite"
+#     _TARGET_DB_VERSION: "3.26.0"
+#     _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production
+
+# 'PHP 8.2 MariaDB 10.3.22':
+#   <<: [ *default-stage, *run-manually ]
+#   variables:
+#     _TARGET_DB_TYPE: "mariadb"
+#     _TARGET_DB_VERSION: "10.3.22"
+#     _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production

.gitlab-ci/pipeline.yml

+# cspell:ignore drupaltestbot drupaltestbotpw
+
+stages:
+  ################
+  # Build
+  #
+  # Assemble the test environment.
+  ################
+  - 🏗️ Build
+
+  ################
+  # Code quality checks
+  #
+  # This stage includes any codebase validation before running tests.
+  ################
+  - 🪄 Lint
+
+  ################
+  # Test
+  #
+  # The test phase actually executes the tests, as well as gathering results
+  # and artifacts.
+  ################
+  - 🗜️ Test
+
+#############
+# Templates #
+#############
+
+.default-job-settings: &default-job-settings
+  interruptible: true
+  allow_failure: false
+  image:
+    name: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+
+.composer-cache: &composer-cache
+  key:
+    files:
+      - ./composer.json
+      - ./composer.lock
+  paths:
+    - ./vendor
+
+.yarn-cache: &yarn-cache
+  key:
+    files:
+      - ./core/package.json
+      - ./core/yarn.lock
+  paths:
+    - ./core/node_modules
+
+.pull-composer-cache: &pull-composer-cache
+  cache:
+    policy: pull
+    <<: *composer-cache
+  dependencies:
+    - '📦️ Composer'
+
+.with-composer-cache: &with-composer-cache
+  needs:
+    - '📦️ Composer'
+  <<: *pull-composer-cache
+
+.with-yarn-cache: &with-yarn-cache
+  dependencies:
+    - '📦️ Yarn'
+  needs:
+    - '📦️ Yarn'
+  cache:
+    policy: pull
+    <<: *yarn-cache
+
+.junit-artifacts: &junit-artifacts
+  artifacts:
+    expose_as: junit
+    expire_in: 6 mos
+    paths:
+      - junit.xml
+    reports:
+      junit: junit.xml
+
+.with-linting: &with-linting
+  needs:
+    - '📦️ Composer'
+    - '🧹 PHP Static Analysis (phpstan)'
+    - '🧹 PHP Coding standards (PHPCS)'
+    - '🧹 Compilation check'
+    - '📦️ Yarn'
+    - '📔 Spell-checking'
+    - '🧹 JavaScript linting (eslint)'
+    - '🧹 CSS linting (stylelint)'
+  <<: *pull-composer-cache
+
+.with-unit-tests: &with-unit-tests
+  needs:
+    - '⚡️ PHPUnit Unit'
+    - '📦️ Composer'
+  <<: *pull-composer-cache
+
+.with-composer-yarn-and-unit-tests: &with-composer-yarn-and-unit-tests
+  needs:
+    - '📦️ Composer'
+    - '⚡️ PHPUnit Unit'
+    - '📦️ Yarn'
+  dependencies:
+    - '📦️ Yarn'
+    - '📦️ Composer'
+
+.test-variables: &test-variables
+  FF_NETWORK_PER_BUILD: 1
+  SIMPLETEST_BASE_URL: http://localhost/subdirectory
+  DB_DRIVER: mysql
+  MYSQL_ROOT_PASSWORD: root
+  MYSQL_DATABASE: mysql
+  MYSQL_USER: drupaltestbot
+  MYSQL_PASSWORD: drupaltestbotpw
+  MARIADB_TAG: $_TARGET_DB_VERSION
+  POSTGRES_TAG: $_TARGET_DB_VERSION
+  POSTGRES_DB: drupaltestbot
+  POSTGRES_USER: drupaltestbot
+  POSTGRES_PASSWORD: drupaltestbotpw
+  MINK_DRIVER_ARGS_WEBDRIVER: '["chrome", {"browserName":"chrome","chromeOptions":{"args":["--disable-dev-shm-usage","--disable-gpu","--headless"]}}, "http://localhost:9515"]'
+  CI_PARALLEL_NODE_INDEX: $CI_NODE_INDEX
+  CI_PARALLEL_NODE_TOTAL: $CI_NODE_TOTAL
+
+.with-database: &with-database
+  name: $_DB_IMAGE
+  alias: database
+
+.with-chrome: &with-chrome
+  name: $_CONFIG_DOCKERHUB_ROOT/chromedriver:production
+  alias: chrome
+  entrypoint:
+    - chromedriver
+    - "--no-sandbox"
+    - "--log-path=/tmp/chromedriver.log"
+    - "--verbose"
+    - "--whitelisted-ips="
+
+.phpunit-artifacts: &phpunit-artifacts
+  artifacts:
+    when: always
+    expire_in: 6 mos
+    reports:
+      junit: ./sites/default/files/simpletest/phpunit-*.xml
+    paths:
+      - ./sites/default/files/simpletest/phpunit-*.xml
+      - ./sites/simpletest/browser_output
+
+.setup-webroot: &setup-webserver
+  before_script:
+    - ln -s $CI_PROJECT_DIR /var/www/html/subdirectory
+    - sudo service apache2 start
+
+.run-tests: &run-tests
+  script:
+    # Determine DB driver.
+    - |
+      [[ $_TARGET_DB_TYPE == "sqlite" ]] && export SIMPLETEST_DB=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite
+      [[ $_TARGET_DB_TYPE == "mysql" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
+      [[ $_TARGET_DB_TYPE == "mariadb" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
+      [[ $_TARGET_DB_TYPE == "pgsql" ]] && export SIMPLETEST_DB=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql
+    - export
+    - mkdir -p ./sites/simpletest ./sites/default/files ./build/logs/junit /var/www/.composer
+    - chown -R www-data:www-data ./sites ./build/logs/junit ./vendor /var/www/
+    - sudo -u www-data git config --global --add safe.directory $CI_PROJECT_DIR
+    # Need to pass this along directly.
+    - sudo MINK_DRIVER_ARGS_WEBDRIVER="$MINK_DRIVER_ARGS_WEBDRIVER" -u www-data php ./core/scripts/run-tests.sh --color --keep-results --types "$TESTSUITE" --concurrency "$CONCURRENCY" --repeat "1" --sqlite "./sites/default/files/.sqlite" --dburl $SIMPLETEST_DB --url $SIMPLETEST_BASE_URL --verbose --non-html --all --ci-parallel-node-index $CI_PARALLEL_NODE_INDEX --ci-parallel-node-total $CI_PARALLEL_NODE_TOTAL
+
+################
+# Jobs
+#
+# Jobs define what scripts are actually executed in each stage.
+#
+# The 'rules' keyword can also be used to define conditions for each job.
+#
+# Documentation: https://docs.gitlab.com/ee/ci/jobs/
+################
+
+################
+# Build Jobs
+################
+
+'ℹ️ Output build parameters':
+  <<: *default-job-settings
+  stage: 🏗️ Build
+  script:
+    - echo "Checking variables"
+    - echo $_TARGET_PHP
+    - echo $_TARGET_DB_TYPE
+    - echo $_TARGET_DB_VERSION
+
+'📦️ Composer':
+  <<: *default-job-settings
+  stage: 🏗️ Build
+  cache:
+    <<: *composer-cache
+  artifacts:
+    expire_in: 1 week
+    expose_as: 'web-vendor'
+    paths:
+      - vendor/
+  script:
+      - export
+      - composer validate
+      - composer install
+
+'📦️ Yarn':
+  <<: *default-job-settings
+  stage: 🏗️ Build
+  cache:
+    <<: *yarn-cache
+  artifacts:
+    expire_in: 1 week
+    expose_as: 'yarn-vendor'
+    paths:
+      - core/node_modules/
+  script:
+    # Installs all core javascript dependencies and adds junit formatter.
+    - yarn --cwd ./core add stylelint-junit-formatter
+
+################
+# Lint Jobs
+################
+
+'🧹 PHP Coding standards (PHPCS)':
+  <<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ]
+  stage: 🪄 Lint
+  script:
+    - composer phpcs -- --report-junit=junit.xml --report-full --report-summary
+
+'🧹 PHP Static Analysis (phpstan)':
+  <<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ]
+  stage: 🪄 Lint
+  script:
+    # Turn off apc to avoid corrupt composer cache.
+    - php -d apc.enabled=0 -d apc.enable_cli=0 vendor/bin/phpstan analyze --configuration=./core/phpstan.neon.dist --error-format=junit > junit.xml
+
+'🧹 CSS linting (stylelint)':
+  <<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ]
+  stage: 🪄 Lint
+  script:
+    - yarn run --cwd=./core lint:css --color --custom-formatter node_modules/stylelint-junit-formatter > junit.xml
+
+'🧹 Compilation check':
+  <<: [ *with-yarn-cache, *default-job-settings ]
+  stage: 🪄 Lint
+  script:
+    - yarn run --cwd=./core build:css --check
+    - cd core && yarn run -s check:ckeditor5
+
+'🧹 JavaScript linting (eslint)':
+  <<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ]
+  stage: 🪄 Lint
+  script:
+    - yarn --cwd=./core run -s lint:core-js-passing --format junit > junit.xml
+
+'📔 Spell-checking':
+  <<: [ *with-yarn-cache, *default-job-settings ]
+  stage: 🪄 Lint
+  script:
+    - git fetch origin
+    - export MODIFIED=`git diff --name-only origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}|while read r;do echo "$CI_PROJECT_DIR/$r";done|tr "\n" " "`
+    - echo $MODIFIED | tr ' ' '\n' | yarn --cwd=./core run -s spellcheck:core --no-must-find-files --file-list stdin
+
+################
+# Test Jobs
+################
+
+'⚡️ PHPUnit Unit':
+  <<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
+  stage: 🗜️ Test
+  services:
+    # There are some unit tests that need a database.
+    # @todo Remove after https://www.drupal.org/project/drupal/issues/3386217
+    - <<: *with-database
+  variables:
+    <<: *test-variables
+    TESTSUITE: PHPUnit-Unit
+    CONCURRENCY: "$CONCURRENCY"
+
+'⚙️️ PHPUnit Kernel':
+  <<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
+  stage: 🗜️ Test
+  parallel: 3
+  variables:
+    <<: *test-variables
+    TESTSUITE: PHPUnit-Kernel
+    CONCURRENCY: "$CONCURRENCY"
+  services:
+    - <<: *with-database
+
+'🖱️️️ PHPUnit Functional Javascript':
+  <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
+  stage: 🗜️ Test
+  variables:
+    <<: *test-variables
+    TESTSUITE: PHPUnit-FunctionalJavascript
+    CONCURRENCY: 15
+  services:
+    - <<: *with-database
+    - <<: *with-chrome
+
+'👷️️️ PHPUnit Build':
+  <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
+  stage: 🗜️ Test
+  variables:
+    <<: *test-variables
+    TESTSUITE: PHPUnit-Build
+    CONCURRENCY: "$CONCURRENCY"
+  services:
+    - <<: *with-database
+
+'🌐️️ PHPUnit Functional':
+  <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
+  stage: 🗜️ Test
+  parallel: 8
+  variables:
+    <<: *test-variables
+    TESTSUITE: PHPUnit-Functional
+    CONCURRENCY: "$CONCURRENCY"
+  services:
+    - <<: *with-database
+
+'🦉️️️ Nightwatch':
+  <<: [ *with-composer-yarn-and-unit-tests, *setup-webserver, *default-job-settings ]
+  stage: 🗜️ Test
+  variables:
+    <<: *test-variables
+  services:
+    - <<: *with-database
+    - <<: *with-chrome
+  script:
+    # Determine DB driver.
+    - |
+      [[ $_TARGET_DB_TYPE == "sqlite" ]] && export DRUPAL_TEST_DB_URL=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite
+      [[ $_TARGET_DB_TYPE == "mysql" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
+      [[ $_TARGET_DB_TYPE == "mariadb" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
+      [[ $_TARGET_DB_TYPE == "pgsql" ]] && export DRUPAL_TEST_DB_URL=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql
+    - export
+    - cp ./core/.env.example ./core/.env
+    # dotenv-safe/config does not support environment variables
+    # @see https://github.com/rolodato/dotenv-safe/issues/126
+    # @todo move this to `variables` when the above is resolved
+    - echo "DRUPAL_TEST_BASE_URL='http://localhost/subdirectory'" >> ./core/.env
+    - echo "DRUPAL_TEST_CHROMEDRIVER_AUTOSTART=false" >> ./core/.env
+    - echo "DRUPAL_TEST_DB_URL='${DRUPAL_TEST_DB_URL}'" >> ./core/.env
+    - echo "DRUPAL_TEST_WEBDRIVER_HOSTNAME='localhost'" >> ./core/.env
+    - echo "DRUPAL_TEST_WEBDRIVER_CHROME_ARGS='--disable-dev-shm-usage --disable-gpu --headless'" >> ./core/.env
+    - echo "DRUPAL_TEST_WEBDRIVER_PORT='9515'" >> ./core/.env
+    - echo "DRUPAL_NIGHTWATCH_OUTPUT='"nightwatch_output"'" >> ./core/.env
+    - cat ./core/.env
+    - mkdir -p ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output
+    - chown -R www-data:www-data ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output ./core/.env
+    - sudo BABEL_DISABLE_CACHE=1 -u www-data yarn --cwd ./core test:nightwatch
+  artifacts:
+    when: always
+    expire_in: 6 mos
+    reports:
+      junit: ./nightwatch_output/*.xml
+    paths:
+      - ./nightwatch_output

core/misc/cspell/dictionary.txt

@@ -322,9 +322,11 @@ distro
 ditka
 divs
 dnumber
+dockerhub
 docroot
 docroots
 dolore
+dotenv
 downcasting
 doxygen
 dragtable
@@ -540,6 +542,7 @@ insidekeyword
 instaclick
 instantiator
 interactable
+interruptible
 introspectable
 invalidators
 invalididentifier

core/package.json

@@ -22,7 +22,7 @@
     "prettier": "prettier --write \"./**/*.js\"",
     "spellcheck": "cspell -c .cspell.json",
     "spellcheck:make-drupal-dict": "rm -f misc/cspell/dictionary.txt && touch misc/cspell/dictionary.txt && yarn -s spellcheck:core --unique --words-only | perl -Mopen=locale -pe '$_=lc$_' | LC_ALL=en_US.UTF-8 tr -d \\\\\\\\ | LC_ALL=C sort -u -o misc/cspell/dictionary.txt",
-    "spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\"",
+    "spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\" \".gitlab-ci/*\" \".gitlab-ci.yml\"",
     "vendor-update": "node ./scripts/js/vendor-update.js",
     "watch:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js --watch",
     "build:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js",

core/scripts/run-tests.sh

@@ -404,6 +404,8 @@ function simpletest_script_parse_args() {
     'execute-test' => '',
     'xml' => '',
     'non-html' => FALSE,
+    'ci-parallel-node-index' => 1,
+    'ci-parallel-node-total' => 1,
   ];
 
   // Override with set values.
@@ -1024,6 +1026,12 @@ function simpletest_script_get_test_list() {
     simpletest_script_print_error('No valid tests were specified.');
     exit(SIMPLETEST_SCRIPT_EXIT_FAILURE);
   }
+
+  if ((int) $args['ci-parallel-node-total'] > 1) {
+    $tests_per_job = ceil(count($test_list) / $args['ci-parallel-node-total']);
+    $test_list = array_slice($test_list, ($args['ci-parallel-node-index'] - 1) * $tests_per_job, $tests_per_job);
+  }
+
   return $test_list;
 }