diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..b2a9c12d2d6a6 --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,277 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. +# +# This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on +# WordPress.org. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' + type: string + default: 'nightly' + plugin-count: + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250.' + type: string + default: '100' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Fetch the most popular plugins + id: plugin-shards + env: + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # The API caps `per_page` at 250 and returns 250 without complaint for anything larger, so counts + # above that have to be paged. + PAGE_SIZE: '250' + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' + run: | + set -euo pipefail + + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.3' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '8.4' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + SLACK_GHA_TIMEOUT_WEBHOOK: ${{ secrets.SLACK_GHA_TIMEOUT_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + needs: [ slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..4ca645f3c7aa9 --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,365 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + os: + description: 'Operating system to run tests on.' + required: false + type: 'string' + default: 'ubuntu-24.04' + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.3' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard.' + required: true + type: 'string' + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '8.4' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Sets up PHP. + # - Downloads the specified version of WordPress. + # - Creates a `wp-config.php` file with debugging and error logging enabled. + # - Installs WordPress. + # - Starts the PHP built-in web server so HTTP requests can be made against the site. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Writes a results table to the workflow summary and fails the job if any plugin fataled. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ${{ inputs.os }} + timeout-minutes: 30 + + services: + database: + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + - name: Download WordPress ${{ inputs.wp-version }} + run: wp core download --version="${WP_VERSION}" + env: + WP_VERSION: ${{ inputs.wp-version }} + + - name: Create wp-config.php file + run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost="127.0.0.1:${DB_PORT}" + env: + DB_PORT: ${{ job.services.database.ports['3306'] }} + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by + # recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that spawned + # it, and the two deadlock until curl gives up. Catching fatals that only happen on a scheduled event is + # worth doing, but it needs to run through WP-CLI rather than a loopback, which is follow up work. + - name: Enable debugging and error logging + run: | + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw + + - name: Install WordPress + run: | + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email + env: + SITE_URL: http://127.0.0.1:8889 + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. + - name: Start the PHP built-in web server + env: + PHP_CLI_SERVER_WORKERS: '4' + run: | + set -uo pipefail + + nohup php -S 127.0.0.1:8889 -t "$( pwd )" > "${RUNNER_TEMP}/php-server.log" 2>&1 & + + # Wait for the server to start answering before any plugin is installed. + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "http://127.0.0.1:8889/wp-login.php"; then + printf 'The PHP built-in server is ready.\n' + exit 0 + fi + sleep 1 + done + + printf 'The PHP built-in server did not start.\n' + cat "${RUNNER_TEMP}/php-server.log" + exit 1 + + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + PHP_VERSION: ${{ inputs.php-version }} + SITE_URL: http://127.0.0.1:8889 + WP_VERSION: ${{ inputs.wp-version }} + run: | + # `set -e` is deliberately not used here: a plugin that fatals must not stop the remaining + # plugins in the shard from being tested. + set -uo pipefail + + RESULTS="${RUNNER_TEMP}/plugin-results.tsv" + RESPONSE_BODY="${RUNNER_TEMP}/response.html" + : > "${RESULTS}" + + # record + # + # Appends one tab separated row to the results file. The reason is flattened so that it cannot + # break the markdown table that is generated from these rows later on. + record() { + SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + } + + # check_url + # + # Requests a path on the test site and prints a reason to stdout when the response looks broken. + # Prints nothing when the request looks healthy. + check_url() { + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense + # status like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors + # back on for itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi + } + + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # cleanup_plugin + # + # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # command here is allowed to fail and the plugin directory is removed directly as a fallback. + # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. + cleanup_plugin() { + wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + + if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${2}" + fi + + rm -rf "wp-content/plugins/${1}" + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true + } + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + printf '::group::%s\n' "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + PLUGIN_DIR="" + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f wp-content/debug.log + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, + # since it generally means a network flake or a plugin that is no longer in the directory. + if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then + record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" + cleanup_plugin "${SLUG}" "" + printf '::endgroup::\n' + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met, most often a + # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means + # every WooCommerce extension lands here. That is core working as designed rather than a + # fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + + # Step 3: boot all of core plus the active plugin in a CLI context. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + STATUS="FAIL" + REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and + # debug log checks below, which see the same code in a real request. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + + # Step 4: request the front page and the login screen through the PHP built-in server. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then + grep 'PHP Fatal' wp-content/debug.log + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" + fi + + # Step 6: record the outcome and put the site back the way it was found. + record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" + cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + printf '::endgroup::\n' + done < <( printf '%s' "${PLUGIN_SLUGS}" | jq -r '.[]' ) + + PASS_COUNT="$( awk -F '\t' '$3 == "PASS" { count++ } END { print count + 0 }' "${RESULTS}" )" + FAIL_COUNT="$( awk -F '\t' '$3 == "FAIL" { count++ } END { print count + 0 }' "${RESULTS}" )" + SKIP_COUNT="$( awk -F '\t' '$3 == "SKIPPED" { count++ } END { print count + 0 }' "${RESULTS}" )" + + { + printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" + printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" + printf '| Plugin | Version | Result | Details |\n' + printf '| --- | --- | --- | --- |\n' + + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + case "${ROW_STATUS}" in + PASS ) ICON=':white_check_mark:' ;; + FAIL ) ICON=':x:' ;; + * ) ICON=':warning:' ;; + esac + + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + done < "${RESULTS}" + + printf '\n' + } >> "${GITHUB_STEP_SUMMARY}" + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + exit 1 + fi + + printf 'No plugins fataled against WordPress %s.\n' "${WP_VERSION}" + + - name: Show the web server log + if: ${{ failure() }} + run: cat "${RUNNER_TEMP}/php-server.log"