From 01c27cbe376783d851025fb87e55c7bbeb035b5f Mon Sep 17 00:00:00 2001 From: jrfnl Date: Tue, 12 Dec 2023 14:41:30 +0100 Subject: [PATCH] GH Actions: test against different versions of all CS dependencies As things were, when testing against "dev" versions, only PHPCS "dev" was used, while WPCS now has more dependencies, i.e. PHPCSUtils and PHPCSExtra. This commit updates the workflows to take that into account better. It also improves the readability of the workflows by using multi-line command in a few places. Notes: * This renames the `phpcs_version` matrix variable to `dependencies` in all workflows to make it clear this is not just about PHPCS itself, but about all dependencies. * The CS check for the own codebase is always run against "dev" versions of all dependencies as an early detection system for upstream issues. * The "Ruleset" checks are run against low/stable/dev. * The fixer conflict check is run against dev only, as conflicts detected cannot be fixed anymore in already released versions. * The "quick check" now only runs against low/stable. * The "test" run run against both low/stable by default and now has some extra builds against "dev" versions. - These extra builds could be set to "continue on error", but I've chosen not to do so as if any issues are detected, they should be fixed asap as they will impact WPCS sooner rather than later. - Code coverage will now be run against low/high PHP with low/stable dependencies and no longer against dev. * Also note that, while the "quick test"/"test" runs shouldn't need PHPCSExtra, as that is a dependency used only in the ruleset, not in our sniff code, I've still included PHPCSExtra in the upgrade/downgrade block for consistency. These changes do mean that the "required build"/branch protection will need to be updated (again) before the PR can be merged. I will do so once the PR has been approved. --- .github/workflows/basic-qa.yml | 43 ++++++++++++++------ .github/workflows/quicktest.yml | 35 +++++++---------- .github/workflows/unit-tests.yml | 67 +++++++++++++++++++++++--------- 3 files changed, 93 insertions(+), 52 deletions(-) diff --git a/.github/workflows/basic-qa.yml b/.github/workflows/basic-qa.yml index ef45e215a..9d20cb411 100644 --- a/.github/workflows/basic-qa.yml +++ b/.github/workflows/basic-qa.yml @@ -12,6 +12,11 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + PHPCS_DEV: 'dev-master' + UTILS_DEV: 'dev-develop' + EXTRA_DEV: 'dev-develop' + jobs: # Check code style of sniffs, rulesets and XML documentation. # Check that all sniffs are feature complete. @@ -37,9 +42,13 @@ jobs: - name: Validate the composer.json file run: composer validate --no-check-all --strict - # Using PHPCS `master` as an early detection system for bugs upstream. - - name: Set PHPCS version - run: composer require squizlabs/php_codesniffer:"dev-master" --no-update --no-scripts --no-interaction + # Using dev versions of the dependencies as an early detection system for bugs upstream. + - name: "Composer: set PHPCS dependencies (dev)" + run: > + composer require --no-update --no-scripts --no-interaction + squizlabs/php_codesniffer:"${{ env.PHPCS_DEV }}" + phpcsstandards/phpcsutils:"${{ env.UTILS_DEV }}" + phpcsstandards/phpcsextra:"${{ env.EXTRA_DEV }}" - name: Install Composer dependencies uses: ramsey/composer-install@v2 @@ -93,15 +102,15 @@ jobs: # Makes sure the rulesets don't throw unexpected errors or warnings. # This workflow needs to be run against a high PHP version to prevent triggering the syntax error check. - # It also needs to be run against all PHPCS versions WPCS is tested against. + # It also needs to be run against all dependency versions WPCS is tested against. ruleset-tests: runs-on: ubuntu-latest strategy: matrix: php: [ 'latest' ] - phpcs_version: [ 'lowest', 'dev-master' ] + dependencies: [ 'lowest', 'stable', 'dev' ] - name: "Ruleset test: PHP ${{ matrix.php }} on PHPCS ${{ matrix.phpcs_version }}" + name: "Ruleset test: PHP ${{ matrix.php }} on PHPCS ${{ matrix.dependencies }}" steps: - name: Checkout repository @@ -115,9 +124,13 @@ jobs: ini-values: error_reporting = E_ALL & ~E_DEPRECATED coverage: none - - name: "Set PHPCS version (master)" - if: ${{ matrix.phpcs_version != 'lowest' }} - run: composer require squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-update --no-scripts --no-interaction + - name: "Composer: set PHPCS dependencies for tests (dev)" + if: ${{ matrix.dependencies == 'dev' }} + run: > + composer require --no-update --no-scripts --no-interaction + squizlabs/php_codesniffer:"${{ env.PHPCS_DEV }}" + phpcsstandards/phpcsutils:"${{ env.UTILS_DEV }}" + phpcsstandards/phpcsextra:"${{ env.EXTRA_DEV }}" - name: Install Composer dependencies uses: ramsey/composer-install@v2 @@ -126,8 +139,13 @@ jobs: # Bust the cache at least once a month - output format: YYYY-MM. custom-cache-suffix: $(date -u "+%Y-%m") - - name: "Set PHPCS version (lowest)" - run: composer update squizlabs/php_codesniffer --prefer-lowest --no-scripts --no-interaction + - name: "Composer: downgrade PHPCS dependencies for tests (lowest)" + if: ${{ matrix.dependencies == 'lowest' }} + run: > + composer update --prefer-lowest --no-scripts --no-interaction + squizlabs/php_codesniffer + phpcsstandards/phpcsutils + phpcsstandards/phpcsextra - name: Test the WordPress-Core ruleset run: $(pwd)/vendor/bin/phpcs -ps ./Tests/RulesetCheck/class-ruleset-test.inc --standard=WordPress-Core @@ -150,8 +168,9 @@ jobs: # Test for fixer conflicts by running the auto-fixers of the complete WPCS over the test case files. # This is not an exhaustive test, but should give an early indication for typical fixer conflicts. # If only fixable errors are found, the exit code will be 1, which can be interpreted as success. + # This check is only run against "dev" as conflicts can not be fixed for already released versions. - name: Test for fixer conflicts (fixes expected) - if: ${{ matrix.phpcs_version == 'dev-master' }} + if: ${{ matrix.dependencies == 'dev' }} id: phpcbf continue-on-error: true run: | diff --git a/.github/workflows/quicktest.yml b/.github/workflows/quicktest.yml index 82b2a0cda..547b0518d 100644 --- a/.github/workflows/quicktest.yml +++ b/.github/workflows/quicktest.yml @@ -24,48 +24,39 @@ jobs: strategy: matrix: php: [ '5.4', 'latest' ] - phpcs_version: [ 'lowest', 'dev-master' ] + dependencies: [ 'lowest', 'stable' ] - name: QTest - PHP ${{ matrix.php }} on PHPCS ${{ matrix.phpcs_version }} + name: QTest - PHP ${{ matrix.php }} on PHPCS ${{ matrix.dependencies }} steps: - name: Checkout repository uses: actions/checkout@v4 - # On stable PHPCS versions, allow for PHP deprecation notices. - # Unit tests don't need to fail on those for stable releases where those issues won't get fixed anymore. - - name: Setup ini config - id: set_ini - run: | - if [ "${{ matrix.phpcs_version }}" != "dev-master" ]; then - echo 'PHP_INI=error_reporting=E_ALL & ~E_DEPRECATED, display_errors=On' >> $GITHUB_OUTPUT - else - echo 'PHP_INI=error_reporting=-1, display_errors=On' >> $GITHUB_OUTPUT - fi - - name: Set up PHP uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php }} - ini-values: ${{ steps.set_ini.outputs.PHP_INI }} + # With stable PHPCS dependencies, allow for PHP deprecation notices. + # Unit tests don't need to fail on those for stable releases where those issues won't get fixed anymore. + ini-values: error_reporting=-1, display_errors=On coverage: ${{ github.ref_name == 'develop' && 'xdebug' || 'none' }} - - name: "Set PHPCS version (master)" - if: ${{ matrix.phpcs_version != 'lowest' }} - run: composer require squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-update --no-scripts --no-interaction - - name: Install Composer dependencies uses: ramsey/composer-install@v2 with: # Bust the cache at least once a month - output format: YYYY-MM. custom-cache-suffix: $(date -u "+%Y-%m") - - name: "Set PHPCS version (lowest)" - if: ${{ matrix.phpcs_version == 'lowest' }} - run: composer update squizlabs/php_codesniffer --prefer-lowest --ignore-platform-req=php+ --no-scripts --no-interaction + - name: "Composer: downgrade PHPCS dependencies for tests (lowest)" + if: ${{ matrix.dependencies == 'lowest' }} + run: > + composer update --prefer-lowest --no-scripts --no-interaction + squizlabs/php_codesniffer + phpcsstandards/phpcsutils + phpcsstandards/phpcsextra - name: Lint PHP files against parse errors - if: ${{ matrix.phpcs_version == 'dev-master' }} + if: ${{ matrix.dependencies == 'stable' }} run: composer lint -- --checkstyle - name: Run the unit tests without code coverage diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index fdd7760dd..3d7bfd6dd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -14,47 +14,70 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + PHPCS_DEV: 'dev-master' + UTILS_DEV: 'dev-develop' + EXTRA_DEV: 'dev-develop' + jobs: # Runs the test suite against all supported branches and combinations. - # Linting is performed on all jobs run against PHPCS `dev-master`. + # Linting is performed on all jobs run with dependencies `stable`. test: runs-on: ubuntu-latest strategy: matrix: php: [ '5.5', '5.6', '7.0', '7.1', '7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.4' ] - phpcs_version: [ 'lowest', 'dev-master' ] + dependencies: [ 'lowest', 'stable' ] extensions: [ '' ] coverage: [false] include: - php: '7.4' - phpcs_version: 'dev-master' + dependencies: 'stable' extensions: ':mbstring' # = Disable Mbstring. coverage: true # Make sure coverage is recorded for this too. # Run code coverage builds against high/low PHP and high/low PHPCS. - php: '5.4' - phpcs_version: 'dev-master' + dependencies: 'stable' extensions: '' coverage: true - php: '5.4' - phpcs_version: 'lowest' + dependencies: 'lowest' extensions: '' coverage: true - php: '8.3' - phpcs_version: 'dev-master' + dependencies: 'stable' extensions: '' coverage: true - php: '8.3' - phpcs_version: 'lowest' + dependencies: 'lowest' extensions: '' coverage: true + # Test against dev versions of all dependencies with select PHP versions for early detection of issues. + - php: '5.4' + dependencies: 'dev' + extensions: '' + coverage: false + - php: '7.0' + dependencies: 'dev' + extensions: '' + coverage: false + - php: '7.4' + dependencies: 'dev' + extensions: '' + coverage: false + - php: '8.3' + dependencies: 'dev' + extensions: '' + coverage: false + # Add extra build to test against PHPCS 4. #- php: '7.4' - # phpcs_version: '4.0.x-dev as 3.9.99' + # dependencies: '4.0.x-dev as 3.9.99' - name: PHP ${{ matrix.php }} on PHPCS ${{ matrix.phpcs_version }} + name: PHP ${{ matrix.php }} on PHPCS ${{ matrix.dependencies }} continue-on-error: ${{ matrix.php == '8.4' }} @@ -62,12 +85,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - # On stable PHPCS versions, allow for PHP deprecation notices. + # With stable PHPCS dependencies, allow for PHP deprecation notices. # Unit tests don't need to fail on those for stable releases where those issues won't get fixed anymore. - name: Setup ini config id: set_ini run: | - if [ "${{ matrix.phpcs_version }}" != "dev-master" ]; then + if [ "${{ matrix.dependencies }}" != "dev" ]; then echo 'PHP_INI=error_reporting=E_ALL & ~E_DEPRECATED, display_errors=On' >> $GITHUB_OUTPUT else echo 'PHP_INI=error_reporting=-1, display_errors=On' >> $GITHUB_OUTPUT @@ -81,9 +104,13 @@ jobs: coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} tools: cs2pr - - name: "Set PHPCS version (master)" - if: ${{ matrix.phpcs_version != 'lowest' }} - run: composer require squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-update --no-scripts --no-interaction + - name: "Composer: set PHPCS dependencies for tests (dev)" + if: ${{ matrix.dependencies == 'dev' }} + run: > + composer require --no-update --no-scripts --no-interaction + squizlabs/php_codesniffer:"${{ env.PHPCS_DEV }}" + phpcsstandards/phpcsutils:"${{ env.UTILS_DEV }}" + phpcsstandards/phpcsextra:"${{ env.EXTRA_DEV }}" - name: Install Composer dependencies uses: ramsey/composer-install@v2 @@ -91,12 +118,16 @@ jobs: # Bust the cache at least once a month - output format: YYYY-MM. custom-cache-suffix: $(date -u "+%Y-%m") - - name: "Set PHPCS version (lowest)" - if: ${{ matrix.phpcs_version == 'lowest' }} - run: composer update squizlabs/php_codesniffer --prefer-lowest --ignore-platform-req=php+ --no-scripts --no-interaction + - name: "Composer: downgrade PHPCS dependencies for tests (lowest)" + if: ${{ ! startsWith( matrix.php, '8' ) && matrix.dependencies == 'lowest' }} + run: > + composer update --prefer-lowest --no-scripts --no-interaction + squizlabs/php_codesniffer + phpcsstandards/phpcsutils + phpcsstandards/phpcsextra - name: Lint PHP files against parse errors - if: ${{ matrix.phpcs_version == 'dev-master' }} + if: ${{ matrix.dependencies == 'stable' }} run: composer lint -- --checkstyle | cs2pr - name: Run the unit tests without code coverage