ci: Run tests in parallel (#39988)

Mostly for the coverage tests, let's see if we can speed things up a bit this way. Looks like it cut the time about in half. 👍 To make this work, we had to generate a separate wp-tests-config file for each plugin with a different `DB_NAME` in each one so the tests from different plugins wouldn't deadlock with each other. That then required an environment variable, `WP_TESTS_CONFIG_FILE_PATH`, to tell the plugins' test boostrap which config file to use.
Automattic · Oct 31, 2024 · ac7ffa6 · ac7ffa6
1 parent bf10de7
commit ac7ffa6
Show file tree

Hide file tree

Showing 8 changed files with 146 additions and 80 deletions.
diff --git a/.github/files/setup-wordpress-env.sh b/.github/files/setup-wordpress-env.sh
@@ -25,8 +25,6 @@ password=root
 EOF
 chmod 0600 ~/.my.cnf
 mysql -e "set global wait_timeout = 3600;"
-mysql -e "DROP DATABASE IF EXISTS wordpress_tests;"
-mysql -e "CREATE DATABASE wordpress_tests;"
 echo "::endgroup::"
 
 echo "::group::Preparing WordPress from \"$WP_BRANCH\" branch";
@@ -128,6 +126,22 @@ for PLUGIN in projects/plugins/*/composer.json; do
 	JSON="$(jq --tab --arg dir "$BASE/$DIR" --argjson pkgversions "$PKGVERSIONS" '( .repositories // empty | .[] | select( .options.monorepo ) ) |= ( .url |= "\($dir)/\(.)" | .options.symlink |= false | .options.versions |= $pkgversions )' "/tmp/wordpress-$WP_BRANCH/src/wp-content/plugins/$NAME/composer.json")"
 	echo "$JSON" > "/tmp/wordpress-$WP_BRANCH/src/wp-content/plugins/$NAME/composer.json"
 
+	# Set up a wp test config for each plugin.
+	WP_TEST_CONFIG="/tmp/wordpress-$WP_BRANCH/wp-tests-config.$NAME.php"
+	DBNAME="wptests_${NAME//-/_}"
+
+	mysql -e "DROP DATABASE IF EXISTS $DBNAME;"
+	mysql -e "CREATE DATABASE $DBNAME;"
+
+	cp "/tmp/wordpress-$WP_BRANCH/wp-tests-config-sample.php" "$WP_TEST_CONFIG"
+	sed -i "s/youremptytestdbnamehere/$DBNAME/" "$WP_TEST_CONFIG"
+	sed -i "s/yourusernamehere/root/" "$WP_TEST_CONFIG"
+	sed -i "s/yourpasswordhere/root/" "$WP_TEST_CONFIG"
+	sed -i "s/localhost/127.0.0.1/" "$WP_TEST_CONFIG"
+
+	# If WooCommerce is installed, be sure we get the monorepo versions rather than the versions distributed with that.
+	echo "define( 'JETPACK_AUTOLOAD_DEV', true );" >> "$WP_TEST_CONFIG"
+
 	echo "::endgroup::"
 done
 
@@ -173,15 +187,7 @@ if [[ "$WITH_WPCOMSH" == true ]]; then
 	echo "::endgroup::"
 fi
 
-cd "/tmp/wordpress-$WP_BRANCH"
-
-cp wp-tests-config-sample.php wp-tests-config.php
-sed -i "s/youremptytestdbnamehere/wordpress_tests/" wp-tests-config.php
-sed -i "s/yourusernamehere/root/" wp-tests-config.php
-sed -i "s/yourpasswordhere/root/" wp-tests-config.php
-sed -i "s/localhost/127.0.0.1/" wp-tests-config.php
-
-# If WooCommerce is installed, be sure we get the monorepo versions rather than the versions distributed with that.
-echo "define( 'JETPACK_AUTOLOAD_DEV', true );" >> wp-tests-config.php
+# Catch anything that doesn't use the WP_TESTS_CONFIG_FILE_PATH env variable.
+echo '<?php die( "Use the WP_TESTS_CONFIG_FILE_PATH environment variable to locate a customized config." );' > "/tmp/wordpress-$WP_BRANCH/wp-tests-config.php"
 
 exit $EXIT
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -119,9 +119,23 @@ jobs:
           CHANGED: ${{ steps.changed.outputs.projects }}
         run: |
           EXIT=0
+          declare -A PIDS
+          PIDS=()
+          MAXPIDS=$( nproc )
+          FAILED=()
           mkdir artifacts
           [[ "$TEST_SCRIPT" == "test-coverage" ]] && mkdir coverage
           for P in composer.json projects/*/*/composer.json; do
+            if [[ ${#PIDS[@]} -ge $MAXPIDS ]]; then
+              if ! wait -fn -p PID "${!PIDS[@]}"; then
+                echo "::error::Tests for ${PIDS[$PID]} failed!"
+                FAILED+=( "${PIDS[$PID]}" )
+                EXIT=1
+              fi
+              echo "Finished ${PIDS[$PID]}"
+              unset PIDS[$PID]
+            fi
+
             if [[ "$P" == "composer.json" ]]; then
               DIR="."
               SLUG="monorepo"
@@ -149,69 +163,89 @@ jobs:
                   continue
                 elif [[ $CODE -ne 0 ]]; then
                   echo "::error::Script skip-$TEST_SCRIPT failed to run $CODE!"
+                  FAILED+=( "$SLUG" )
                   EXIT=1
                   continue
                 fi
               fi
 
-              echo "::group::Running tests for $SLUG"
-
-              # Composer install, if appropriate. Note setup-wordpress-env.sh did it already for plugins.
-              NEED_COMPOSER=false
-              if [[ "${SLUG%%/*}" != "plugins" && ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) ]]; then
-                NEED_COMPOSER=true
-                if [[ "$TEST_SCRIPT" == "test-coverage" ]] &&
-                  ! jq -e '.scripts["test-php"]' "$DIR/composer.json" &>/dev/null
-                then
-                  echo "Skipping composer install, assuming test-coverage is only JS because the project has no test-php."
-                  NEED_COMPOSER=false
-                fi
-              fi
-              if $NEED_COMPOSER; then
-                if [[ ! -f "$DIR/composer.lock" ]]; then
-                  echo 'No composer.lock, running `composer update`'
-                  composer --working-dir="$DIR" update
-                elif composer --working-dir="$DIR" check-platform-reqs --lock; then
-                  echo 'Platform reqs pass, running `composer install`'
-                  composer --working-dir="$DIR" install
-                else
-                  echo 'Platform reqs failed, running `composer update`'
-                  composer --working-dir="$DIR" update
+              echo "Running tests for $SLUG"
+              {
+                # Composer install, if appropriate. Note setup-wordpress-env.sh did it already for plugins.
+                if [[ "${SLUG%%/*}" != "plugins" && ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) ]]; then
+                  if [[ "$TEST_SCRIPT" == "test-coverage" ]] &&
+                    ! jq -e '.scripts["test-php"]' "$DIR/composer.json" &>/dev/null
+                  then
+                    echo "Skipping composer install, assuming test-coverage is only JS because the project has no test-php."
+                  else
+                    if [[ ! -f "$DIR/composer.lock" ]]; then
+                      echo 'No composer.lock, running `composer update`'
+                      composer --working-dir="$DIR" update
+                    elif composer --working-dir="$DIR" check-platform-reqs --lock; then
+                      echo 'Platform reqs pass, running `composer install`'
+                      composer --working-dir="$DIR" install
+                    else
+                      echo 'Platform reqs failed, running `composer update`'
+                      composer --working-dir="$DIR" update
+                    fi
+
+                    if [[ "$WP_BRANCH" == 'trunk' || "$WP_BRANCH" == 'previous' ]]; then
+                      VER=$(composer --format=json --working-dir="$DIR" show | jq -r '.installed[] | select( .name == "roots/wordpress" ) | .version')
+                      if [[ -n "$VER" ]]; then
+                        INSVER=$WORDPRESS_TAG
+                        [[ "$WORDPRESS_TAG" == 'trunk' ]] && INSVER="dev-main as $VER"
+                        echo "Supposed to run tests against WordPress $WORDPRESS_TAG, so setting roots/wordpress and roots/wordpress-no-content to \"$INSVER\""
+                        # Composer seems to sometimes have issues with deleting the wordpress dir on its own, so do it manually first.
+                        rm -rf "$DIR/wordpress"
+                        composer --working-dir="$DIR" require --dev roots/wordpress="$INSVER" roots/wordpress-no-content="$INSVER"
+                      fi
+                    fi
+                  fi
                 fi
 
-                if [[ "$WP_BRANCH" == 'trunk' || "$WP_BRANCH" == 'previous' ]]; then
-                  VER=$(composer --format=json --working-dir="$DIR" show | jq -r '.installed[] | select( .name == "roots/wordpress" ) | .version')
-                  if [[ -n "$VER" ]]; then
-                    INSVER=$WORDPRESS_TAG
-                    [[ "$WORDPRESS_TAG" == 'trunk' ]] && INSVER="dev-main as $VER"
-                    echo "Supposed to run tests against WordPress $WORDPRESS_TAG, so setting roots/wordpress and roots/wordpress-no-content to \"$INSVER\""
-                    # Composer seems to sometimes have issues with deleting the wordpress dir on its own, so do it manually first.
-                    rm -rf "$DIR/wordpress"
-                    composer --working-dir="$DIR" require --dev roots/wordpress="$INSVER" roots/wordpress-no-content="$INSVER"
-                  fi
+                if [[ "${SLUG%%/*}" == "plugins" ]]; then
+                  export WP_TESTS_CONFIG_FILE_PATH="$WORDPRESS_DEVELOP_DIR/wp-tests-config.${SLUG##*/}.php"
                 fi
-              fi
 
-              mkdir -p "artifacts/$SLUG"
-              export ARTIFACTS_DIR="$GITHUB_WORKSPACE/artifacts/$SLUG"
-              if [[ "$TEST_SCRIPT" == "test-coverage" ]]; then
-                mkdir -p "coverage/$SLUG"
-                export COVERAGE_DIR="$GITHUB_WORKSPACE/coverage/$SLUG"
-              fi
-              FAIL=false
-              if ! composer run --timeout=0 --working-dir="$DIR" "$TEST_SCRIPT"; then
-                FAIL=true
-                EXIT=1
-              fi
+                mkdir -p "artifacts/$SLUG"
+                export ARTIFACTS_DIR="$GITHUB_WORKSPACE/artifacts/$SLUG"
+                if [[ "$TEST_SCRIPT" == "test-coverage" ]]; then
+                  mkdir -p "coverage/$SLUG"
+                  export COVERAGE_DIR="$GITHUB_WORKSPACE/coverage/$SLUG"
+                fi
+                FAIL=false
+                if ! composer run --timeout=0 --working-dir="$DIR" "$TEST_SCRIPT"; then
+                  FAIL=true
+                fi
 
-              # Actions seems to slow down if there are a lot of files, so clean up Composer stuff after each test.
-              # We don't do it for JS stuff, as that might break things with how JS does package deps.
-              rm -rf "$DIR/vendor" "$DIR/jetpack_vendor" "$DIR/wordpress"
+                # Actions seems to slow down if there are a lot of files, so clean up Composer stuff after each test.
+                # We don't do it for JS stuff, as that might break things with how JS does package deps.
+                rm -rf "$DIR/vendor" "$DIR/jetpack_vendor" "$DIR/wordpress"
 
-              echo "::endgroup::"
-              $FAIL && echo "::error::Tests for $SLUG failed!"
+                if $FAIL; then
+                  echo "Tests for $SLUG failed!"
+                  exit 1
+                fi
+              } 2> >( sed -u 's!^!['"$SLUG"'] !' >&2 ) > >( sed -u 's!^!['"$SLUG"'] !' ) &
+              PIDS[$!]=$SLUG
             fi
           done
+          while [[ ${#PIDS[@]} -gt 0 ]]; do
+            if ! wait -fn -p PID "${!PIDS[@]}"; then
+              echo "::error::Tests for ${PIDS[$PID]} failed!"
+              FAILED+=( "${PIDS[$PID]}" )
+              EXIT=1
+            fi
+            echo "Finished ${PIDS[$PID]}"
+            unset PIDS[$PID]
+          done
+
+          if [[ ${#FAILED[@]} -gt 0 ]]; then
+            echo ''
+            echo 'The following tests failed:'
+            printf " - %s\n" "${FAILED[@]}"
+          fi
+
           exit $EXIT
 
       - name: Process coverage results

diff --git a/docs/examples/bootstrap.php b/docs/examples/bootstrap.php
@@ -65,6 +65,12 @@ function _manually_load_plugin() {
 }
 tests_add_filter( 'muplugins_loaded', '_manually_load_plugin' );
 
+// Override WP_TESTS_CONFIG_FILE_PATH via environment.
+// Important for monorepo CI, if you don't do this then different test runs might collide!
+if ( false !== getenv( 'WP_TESTS_CONFIG_FILE_PATH' ) ) {
+	define( 'WP_TESTS_CONFIG_FILE_PATH', getenv( 'WP_TESTS_CONFIG_FILE_PATH' ) );
+}
+
 // Start up the WP testing environment.
 require $_tests_dir . '/includes/bootstrap.php';
 

diff --git a/projects/plugins/crm/changelog/update-ci-tests-in-parallel b/projects/plugins/crm/changelog/update-ci-tests-in-parallel
@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Use WP_TESTS_CONFIG_FILE_PATH env var in tests.
+
+
diff --git a/projects/plugins/crm/tests/php/bootstrap.php b/projects/plugins/crm/tests/php/bootstrap.php
@@ -89,6 +89,12 @@ function _jpcrm_manually_load_plugin() {
 
 tests_add_filter( 'muplugins_loaded', '_jpcrm_manually_load_plugin' );
 
+// Override WP_TESTS_CONFIG_FILE_PATH via environment.
+// Important for monorepo CI, if you don't do this then different test runs might collide!
+if ( false !== getenv( 'WP_TESTS_CONFIG_FILE_PATH' ) ) {
+	define( 'WP_TESTS_CONFIG_FILE_PATH', getenv( 'WP_TESTS_CONFIG_FILE_PATH' ) );
+}
+
 /**
  * Start up the WP testing environment.
  */

diff --git a/projects/plugins/jetpack/changelog/update-ci-tests-in-parallel b/projects/plugins/jetpack/changelog/update-ci-tests-in-parallel
@@ -0,0 +1,4 @@
+Significance: patch
+Type: other
+
+Run coverage tests in parallel.
diff --git a/projects/plugins/jetpack/tests/action-test-coverage.sh b/projects/plugins/jetpack/tests/action-test-coverage.sh
@@ -2,23 +2,22 @@
 
 set -eo pipefail
 
-echo "::group::Jetpack Admimnpage coverage"
-pnpm run test-client --coverage --collectCoverageFrom='_inc/client/state/**/*.js' --coverageDirectory="$COVERAGE_DIR/client" --coverageReporters=clover
-pnpm run test-gui --coverage --collectCoverageFrom='_inc/client/**/*.js' --coverageDirectory="$COVERAGE_DIR/gui" --coverageReporters=clover
-echo "::endgroup::"
+# To run the PHP tests in parallel, we need to create custom configs for each.
+for test in lfs multi; do
+	P="${WP_TESTS_CONFIG_FILE_PATH%.php}.$test.php"
+	cp "$WP_TESTS_CONFIG_FILE_PATH" "$P"
+	sed -i "s!wptests_jetpack!wptests_jetpack_$test!" "$P"
+	mysql -e "DROP DATABASE IF EXISTS wptests_jetpack_$test;"
+	mysql -e "CREATE DATABASE wptests_jetpack_$test;"
+done
 
-echo "::group::Jetpack Extensions coverage"
-pnpm run test-extensions --coverage --collectCoverageFrom='extensions/**/*.js' --coverageDirectory="$COVERAGE_DIR/extensions" --coverageReporters=clover
-echo "::endgroup::"
+declare -A TESTS
+TESTS=()
+TESTS[php-normal]="php -dpcov.directory=. \"$(command -v phpunit)\" --coverage-clover \"$COVERAGE_DIR/backend/clover.xml\""
+TESTS[php-lfs]="WP_TESTS_CONFIG_FILE_PATH=\"${WP_TESTS_CONFIG_FILE_PATH%.php}.lfs.php\" LEGACY_FULL_SYNC=1 php -dpcov.directory=. \"$(command -v phpunit)\" --group=legacy-full-sync --coverage-clover \"$COVERAGE_DIR/legacy-sync/clover.xml\""
+TESTS[php-multisite]="WP_TESTS_CONFIG_FILE_PATH=\"${WP_TESTS_CONFIG_FILE_PATH%.php}.multi.php\" WP_MULTISITE=1 php -dpcov.directory=. \"$(command -v phpunit)\" -c tests/php.multisite.xml --coverage-clover \"$COVERAGE_DIR/multisite/clover.xml\""
+TESTS[client]="pnpm run test-client --coverage --collectCoverageFrom='_inc/client/state/**/*.js' --coverageDirectory=\"$COVERAGE_DIR/client\" --coverageReporters=clover"
+TESTS[gui]="pnpm run test-gui --coverage --collectCoverageFrom='_inc/client/state/**/*.js' --coverageDirectory=\"$COVERAGE_DIR/client\" --coverageReporters=clover"
+TESTS[extensions]="pnpm run test-extensions --coverage --collectCoverageFrom='_inc/client/state/**/*.js' --coverageDirectory=\"$COVERAGE_DIR/client\" --coverageReporters=clover"
 
-echo "::group::Jetpack Backend coverage"
-php -dpcov.directory=. "$(command -v phpunit)" --coverage-clover "$COVERAGE_DIR/backend/clover.xml"
-echo "::endgroup::"
-
-echo "::group::Jetpack Legacy full sync coverage"
-LEGACY_FULL_SYNC=1 php -dpcov.directory=. "$(command -v phpunit)" --group=legacy-full-sync --coverage-clover "$COVERAGE_DIR/legacy-sync/clover.xml"
-echo "::endgroup::"
-
-echo "::group::Jetpack Multisite coverage"
-WP_MULTISITE=1 php -dpcov.directory=. "$(command -v phpunit)" -c tests/php.multisite.xml --coverage-clover "$COVERAGE_DIR/multisite/clover.xml"
-echo "::endgroup::"
+pnpm exec concurrently --kill-others-on-fail --max-processes '100%' --names "$( IFS=,; echo "${!TESTS[*]}" )" "${TESTS[@]}"
diff --git a/projects/plugins/jetpack/tests/php/bootstrap.php b/projects/plugins/jetpack/tests/php/bootstrap.php
@@ -183,6 +183,12 @@ function jetpack_full_sync_immediately_off( $modules ) {
 	tests_add_filter( 'jetpack_sync_modules', 'jetpack_full_sync_immediately_off' );
 }
 
+// Override WP_TESTS_CONFIG_FILE_PATH via environment.
+// Important for monorepo CI, if you don't do this then different test runs might collide!
+if ( false !== getenv( 'WP_TESTS_CONFIG_FILE_PATH' ) ) {
+	define( 'WP_TESTS_CONFIG_FILE_PATH', getenv( 'WP_TESTS_CONFIG_FILE_PATH' ) );
+}
+
 require $test_root . '/includes/bootstrap.php';
 
 // Load the shortcodes module to test properly.