tests.yml

name: Tests

on:
  pull_request:
  push:
    branches: [ 'trunk', '*/branch-*' ]
concurrency:
  group: tests-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

env:
  COMPOSER_ROOT_VERSION: "dev-trunk"

jobs:
  create-matrix:
    name: "Determine tests matrix"
    runs-on: ubuntu-latest
    timeout-minutes: 1  # 2021-02-03: Should only take a second.
    outputs:
      matrix: ${{ steps.create-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
      - id: create-matrix
        run: |
          MATRIX="$(.github/files/generate-ci-matrix.php)"
          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"

  run-tests:
    name: ${{ matrix.name }}
    runs-on: ubuntu-latest
    needs: create-matrix
    services:
      database:
        image: mariadb:10.4
        env:
          MARIADB_ROOT_PASSWORD: root
        ports:
          - 3306:3306
        options: --health-cmd="healthcheck.sh --su-mysql --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=5
    continue-on-error: ${{ matrix.experimental }}
    timeout-minutes: ${{ matrix.timeout }}
    env:
      TEST_SCRIPT: ${{ matrix.script }}
      WP_BRANCH: ${{ matrix.wp }}
      PHP_VERSION: ${{ matrix.php }}
      NODE_VERSION: ${{ matrix.node }}
      MONOREPO_BASE: ${{ github.workspace }}
      WITH_WOOCOMMERCE: ${{ matrix.with-woocommerce }}
      WITH_WPCOMSH: ${{ matrix.with-wpcomsh }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson( needs.create-matrix.outputs.matrix ) }}

    steps:
      - uses: actions/checkout@v4
        with:
          # codecov.io requires a fetch depth > 1.
          fetch-depth: 2

      # For pull requests, list-changed-projects.sh needs the merge base.
      # But it doesn't have to be checked out.
      - name: Deepen to merge base
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/deepen-to-merge-base
        with:
          checkout: false

      - name: Setup tools
        uses: ./.github/actions/tool-setup
        with:
          php: ${{ matrix.php }}
          node: ${{ matrix.node }}

      - name: Monorepo install
        run: |
          echo "::group::Pnpm"
          pnpm install
          echo "::endgroup::"

          # If we're going to be making WorDBless use WP "nightlies", remove the relevant package from Composer's cache to get the latest version.
          if [[ "$WP_BRANCH" == 'trunk' && "$TEST_SCRIPT" == "test-php" ]]; then
            echo "::group::Clear composer cache for roots/wordpress"
            DIR=$(composer config cache-files-dir)
            rm -rf "$DIR/roots/wordpress" "$DIR/roots/wordpress-no-content"
            echo "::endgroup::"
          fi

      - name: Detect changed projects
        id: changed
        run: |
          CHANGED="$(EXTRA=test .github/files/list-changed-projects.sh)"

          # Only test certain plugins in combination with WC
          if [[ "$WITH_WOOCOMMERCE" == true ]]; then
            echo "Testing with WooCommerce, filtering for projects that have WooCommerce-specific tests."
            CHANGED=$( jq -c 'with_entries( select( .key == "plugins/jetpack" ) )' <<<"$CHANGED" )
          fi

          ANY_PLUGINS="$(jq --argjson changed "$CHANGED" -n '$changed | with_entries( select( .key | startswith( "plugins/" ) ) ) | any')"
          echo "projects=${CHANGED}" >> "$GITHUB_OUTPUT"
          echo "any-plugins=${ANY_PLUGINS}" >> "$GITHUB_OUTPUT"

      - name: Select WordPress version
        if: steps.changed.outputs.any-plugins != 'true' && matrix.wp != 'none'
        run: .github/files/select-wordpress-tag.sh

      - name: Setup WordPress environment for plugin tests
        env:
          API_TOKEN_GITHUB: ${{ secrets.GITHUB_TOKEN }}
          CHANGED: ${{ steps.changed.outputs.projects }}
        if: steps.changed.outputs.any-plugins == 'true' && matrix.wp != 'none'
        run: .github/files/setup-wordpress-env.sh

      - name: Run project tests
        env:
          FORCE_PACKAGE_TESTS: ${{ matrix.force-package-tests && 'true' || 'false' }}
          CHANGED: ${{ steps.changed.outputs.projects }}
        run: |
          EXIT=0
          mkdir artifacts
          for P in composer.json projects/*/*/composer.json; do
            if [[ "$P" == "composer.json" ]]; then
              DIR="."
              SLUG="monorepo"
            else
              DIR="${P%/composer.json}"
              SLUG="${DIR#projects/}"
            fi

            if [[ "${SLUG%%/*}" == "plugins" ]]; then
              # Plugins run from the WordPress checkout, not the monorepo checkout.
              if [[ -d "/tmp/wordpress-$WP_BRANCH/src/wp-content/$SLUG" ]]; then
                DIR="/tmp/wordpress-$WP_BRANCH/src/wp-content/$SLUG"
              fi
            elif [[ "$WP_BRANCH" != 'latest' && "$WP_BRANCH" != 'none' && "$FORCE_PACKAGE_TESTS" != "true" ]]; then
              echo "Skipping $SLUG, only plugins run for WP_BRANCH = $WP_BRANCH"
              continue
            fi

            if ! jq --argjson changed "$CHANGED" --arg p "$SLUG" -ne '$changed[$p] // false' > /dev/null; then
              echo "Skipping $SLUG, no changes in it or its dependencies"
            elif ! jq --arg script "$TEST_SCRIPT" -e '.scripts[$script] // false' "$P" > /dev/null; then
              echo "Skipping $SLUG, no test script is defined in composer.json"
            elif php -r 'exit( preg_match( "/^>=\\s*(\\d+\\.\\d+)$/", $argv[1], $m ) && version_compare( PHP_VERSION, $m[1], "<" ) ? 0 : 1 );' "$( jq -r '.require.php // ""' "$P" )"; then
              echo "Skipping $SLUG, requires PHP $( jq -r '.require.php // ""' "$P" ) but PHP version is $( php -r 'echo PHP_VERSION;' )"
            else
              if jq --arg script "skip-$TEST_SCRIPT" -e '.scripts[$script] // false' "$P" > /dev/null; then
                { composer --working-dir="$DIR" run "skip-$TEST_SCRIPT"; CODE=$?; } || true
                if [[ $CODE -eq 3 ]]; then
                  echo "Skipping tests for $SLUG due to skip-$TEST_SCRIPT script"
                  continue
                elif [[ $CODE -ne 0 ]]; then
                  echo "::error::Script skip-$TEST_SCRIPT failed to run $CODE!"
                  EXIT=1
                  continue
                fi
              fi

              echo "::group::Running tests for $SLUG"

              # Composer install, if appropriate. Note setup-wordpress-env.sh did it already for plugins.
              if [[ "${SLUG%%/*}" != "plugins" && "$TEST_SCRIPT" == "test-php" ]]; then
                if [[ ! -f "$DIR/composer.lock" ]]; then
                  echo 'No composer.lock, running `composer update`'
                  composer --working-dir="$DIR" update
                elif composer --working-dir="$DIR" check-platform-reqs --lock; then
                  echo 'Platform reqs pass, running `composer install`'
                  composer --working-dir="$DIR" install
                else
                  echo 'Platform reqs failed, running `composer update`'
                  composer --working-dir="$DIR" update
                fi

                if [[ "$WP_BRANCH" == 'trunk' || "$WP_BRANCH" == 'previous' ]]; then
                  VER=$(composer --format=json --working-dir="$DIR" show | jq -r '.installed[] | select( .name == "roots/wordpress" ) | .version')
                  if [[ -n "$VER" ]]; then
                    INSVER=$WORDPRESS_TAG
                    [[ "$WORDPRESS_TAG" == 'trunk' ]] && INSVER="dev-main as $VER"
                    echo "Supposed to run tests against WordPress $WORDPRESS_TAG, so setting roots/wordpress and roots/wordpress-no-content to \"$INSVER\""
                    # Composer seems to sometimes have issues with deleting the wordpress dir on its own, so do it manually first.
                    rm -rf "$DIR/wordpress"
                    composer --working-dir="$DIR" require --dev roots/wordpress="$INSVER" roots/wordpress-no-content="$INSVER"
                  fi
                fi
              fi

              mkdir -p "artifacts/$SLUG"
              export ARTIFACTS_DIR="$GITHUB_WORKSPACE/artifacts/$SLUG"
              FAIL=false
              if ! composer run --timeout=0 --working-dir="$DIR" "$TEST_SCRIPT"; then
                FAIL=true
                EXIT=1
              fi

              # Actions seems to slow down if there are a lot of files, so clean up Composer stuff after each test.
              # We don't do it for JS stuff, as that might break things with how JS does package deps.
              rm -rf "$DIR/vendor" "$DIR/jetpack_vendor" "$DIR/wordpress"

              echo "::endgroup::"
              $FAIL && echo "::error::Tests for $SLUG failed!"
            fi
          done
          exit $EXIT

      - name: Check for artifacts
        id: check-artifacts
        # Default for `if` is `success()`, we want this to run always.
        if: always()
        run: |
          [[ -d artifacts ]] && find artifacts -type d -empty -delete
          if [[ -d artifacts ]]; then
            echo "any=true" >> "$GITHUB_OUTPUT"
          else
            echo "any=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Upload artifacts
        if: always() && steps.check-artifacts.outputs.any == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifact }}
          path: artifacts
          include-hidden-files: true
          retention-days: 7

  storybook-test:
    name: Storybook tests
    runs-on: ubuntu-latest
    timeout-minutes: 20 # 2024-02-23 Wild guess
    continue-on-error: true # Until it passes
    steps:
      - uses: actions/checkout@v4

      # For pull requests, list-changed-projects.sh needs the merge base.
      # But it doesn't have to be checked out.
      - name: Deepen to merge base
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/deepen-to-merge-base
        with:
          checkout: false

      - name: Setup tools
        uses: ./.github/actions/tool-setup

      - name: Monorepo install
        run: |
          echo "::group::Pnpm"
          pnpm install
          echo "::endgroup::"

      - name: Detect changed projects
        id: changed
        run: |
          CHANGED=$( .github/files/list-changed-projects.sh )
          PROJECTS=$( node -e 'const r = { "js-packages/storybook": true }; for ( const p of require( "./projects/js-packages/storybook/storybook/projects.js" ) ) { const m = p.match( /\/projects\/([^/]+\/[^/]+)(?:$|\/)/ ); m && ( r[ m[1] ] = true ); } console.log( JSON.stringify( r ) );' )

          ANY=$( jq --argjson changed "$CHANGED" --argjson projects "$PROJECTS" -n '$changed | with_entries( select( $projects[ .key ] ) ) | any' )
          echo "any=${ANY}" >> "$GITHUB_OUTPUT"

      - name: Build storybook
        if: steps.changed.outputs.any == 'true'
        run: |
          pnpm jetpack build -v js-packages/storybook

      - name: Install playwright
        if: steps.changed.outputs.any == 'true'
        run: |
          cd projects/js-packages/storybook
          pnpm exec playwright install --with-deps chromium

      - name: Test storybook
        if: steps.changed.outputs.any == 'true'
        run: |
          cd projects/js-packages/storybook
          node bin/webserver.mjs
          REFERENCE_URL=https://automattic.github.io/jetpack-storybook/ pnpm exec test-storybook -c storybook --url 'http://127.0.0.1:6006/index.html'

  # Probably this should be a linting test, but we don't run linting on trunk or release branches.
  plugin-deps:
    name: Check plugin monorepo dep versions
    runs-on: ubuntu-latest
    timeout-minutes: 2  # 2022-09-08: Should only take a few seconds.
    steps:
      - uses: actions/checkout@v4
      - name: Setup tools
        uses: ./.github/actions/tool-setup
        with:
          node: false
      - name: Run check
        run: |
          if [[ "$GITHUB_EVENT_NAME" == 'push' ]]; then
            REF="${GITHUB_REF#refs/heads/}"
          elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'pull_request_target' ]]; then
            REF="$GITHUB_BASE_REF"
          else
            echo "::error::Unsupported github event \"$GITHUB_EVENT_NAME\""
            exit 1
          fi
          echo "Detected target ref \"$REF\""

          if [[ "$REF" == trunk ]]; then
            ARGS=( --dev )
          elif [[ "$REF" == */branch-* ]]; then
            ARGS=( --release )
            TMP="$(jq -r --arg P "${REF%%/branch-*}" '.extra["release-branch-prefix"] | if type == "array" then . else [ . ] end | if index( $P ) then input_filename | match( "^projects/plugins/([^/]+)/composer.json$" ).captures[0].string else empty end' projects/plugins/*/composer.json)"
            while IFS= read -r LINE; do
              ARGS+=( "$LINE" )
            done <<<"$TMP"
          else
            echo "Unsupported ref \"$REF\", ignoring"
            exit 0
          fi

          echo "Running tools/check-plugin-monorepo-deps.sh ${ARGS[@]}"
          tools/check-plugin-monorepo-deps.sh "${ARGS[@]}"

  # Probably this should be a linting test too, but we don't run linting on trunk or release branches.
  phan:
    name: Static analysis
    runs-on: ubuntu-latest
    timeout-minutes: 20  # 2024-05-02: Up to about 8 minutes now that we're running against the old WP stubs too.
    steps:
      - uses: actions/checkout@v4
      - name: Setup tools
        uses: ./.github/actions/tool-setup
      - name: Pnpm install
        run: pnpm install
      - name: Run phan
        run: pnpm jetpack phan --all -v --update-baseline --format github
      - name: Run phan for previous WP version too
        env:
          # Don't bother complaining about unused suppressions that may be used with the newer stubs. See .phan/config.base.php for how this gets applied.
          NO_PHAN_UNUSED_SUPPRESSION: 1
        run: |
          composer update --prefer-lowest php-stubs/wordpress-stubs php-stubs/wordpress-tests-stubs
          # Don't re-update baselines here, only check.
          pnpm jetpack phan --all -v --format github
      - name: Check baselines
        run: |
          # Anything changed? (with a side of printing the diff)
          if git diff --exit-code --ignore-matching-lines='^    // ' -- .phan/baseline.php '*/.phan/baseline.php'; then
            exit 0
          fi

          # Collect which projects changed to suggest the right command.
          PROJECTS=()
          for f in $( git -c core.quotepath=off diff --name-only -- .phan/baseline.php '*/.phan/baseline.php' ); do
            # --name-only and --ignore-matching-lines don't combine, so we have to do the check separately.
            if git diff --quiet --exit-code --ignore-matching-lines='^    // ' -- "$f"; then
              continue
            fi

            if [[ "$f" == ".phan/baseline.php" ]]; then
              SLUG=monorepo
            elif [[ "$f" == projects/*/*/.phan/baseline.php ]]; then
              SLUG=${f%/.phan/baseline.php}
              SLUG=${SLUG#projects/}
            elif SLUG=$( grep -v '^[ \t]*\/\/' .phan/monorepo-pseudo-projects.jsonc | jq -re --arg f "${f%.phan/baseline.php}" 'to_entries[] | select( .value == $f ) | .key' ); then
              : # Ok
            else
              SLUG=
            fi
            if grep -q 'This baseline has no suppressions' "$f"; then
              if [[ -n "$SLUG" ]]; then
                echo "::error file=$f::This Phan baseline is now empty (good job!). You may remove it, or if you want to keep it (e.g. if you expect new unfixed issues to be added in the future) you can run \`jetpack phan --update-baseline $SLUG\` to update it."
              else
                echo "::error file=$f::This Phan baseline is now empty (good job!). You may remove it."
              fi
            elif [[ -n "$SLUG" ]]; then
              PROJECTS+=( "$SLUG" )
            else
              echo "::error file=$f::This Phan baseline has changed and should be updated. This Action was unable to determine the command needed to update it; please report this to the Garage team."
            fi
          done
          if [[ ${#PROJECTS[@]} -gt 0 ]]; then
            echo "::error::Phan baselines have changed (good job!). Run \`jetpack phan --update-baseline ${PROJECTS[*]}\` to update them."
          fi
          exit 1