-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add actions and workflows for QIT tests
- Loading branch information
Showing
5 changed files
with
382 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
name: Run QIT for all extensions | ||
|
||
# **What it does**: Runs a suite of tests for all Grow extensions. | ||
# **Why we have it**: To be able to run tests for all extensions at once. For example when we want to compatibility test a new version of the platform (WP/WC). | ||
|
||
on: | ||
workflow_dispatch: | ||
inputs: | ||
# Basic params. | ||
version: | ||
description: 'Version to be tested' | ||
required: true | ||
type: choice | ||
options: | ||
- latest | ||
- dev | ||
|
||
# Configure which tests to run. | ||
test-activation: | ||
description: 'Should activation be tested?' | ||
required: true | ||
default: true | ||
type: boolean | ||
test-security: | ||
description: 'Should security be tested?' | ||
required: true | ||
default: true | ||
type: boolean | ||
test-phpstan: | ||
description: 'Should phpstan be tested?' | ||
required: true | ||
default: true | ||
type: boolean | ||
test-api: | ||
description: 'Should API be tested?' | ||
required: true | ||
default: true | ||
type: boolean | ||
test-e2e: | ||
description: 'Should E2E be tested? (takes a lot of time)' | ||
required: true | ||
default: false | ||
type: boolean | ||
|
||
# Advanced customization. | ||
options: | ||
description: 'Additional options for `qit` command, like `--optional_features=hpos`.' | ||
required: false | ||
|
||
# Used to test this PR, to be removed after before merging. | ||
push: | ||
branches: | ||
- add/qit-workflows | ||
|
||
jobs: | ||
qit-tests: | ||
name: Run QIT Tests | ||
uses: ./.github/workflows/run-qit-extension.yml | ||
secrets: inherit | ||
strategy: | ||
# Allow to test extensions even if one of them fails. | ||
fail-fast: false | ||
matrix: | ||
# List of extensions to be tested. | ||
extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration] | ||
# Run both options for PR review purposes. | ||
version: [latest, dev] | ||
with: | ||
# We're running tests for all versions, just for PR review purposes to be replaced before merging. | ||
# version: ${{ inputs.version }} | ||
version: ${{ matrix.version }} | ||
# Conditional statements are here to allow testing on push triggers, without manual input. To be removed before merging. | ||
test-activation: ${{ inputs.test-activation || true }} | ||
test-security: ${{ inputs.test-security || true }} | ||
test-phpstan: ${{ inputs.test-phpstan || true }} | ||
test-api: ${{ inputs.test-api || false }} | ||
test-e2e: ${{ inputs.test-e2e || false }} | ||
extension: ${{ matrix.extension }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,153 @@ | ||
name: Run QIT for a given extensions | ||
|
||
# **What it does**: Runs a set of QIT tests for a given extension. | ||
# **Why we have it**: To reuse across other repos, to make a full test of a single extension. | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
# Basic params. | ||
extension: | ||
description: Extension to test | ||
required: true | ||
type: string | ||
version: | ||
description: | | ||
Version to be tested: `latest` or `dev`. | ||
The `dev` option will make the action look up for an `{extension}.zip` file in the root | ||
of the repository under the `gha-dev-build` tag. | ||
required: true | ||
default: 'latest' | ||
type: string | ||
|
||
# Customize which types to run. | ||
test-activation: | ||
description: 'Should activation be tested?' | ||
default: true | ||
type: boolean | ||
test-security: | ||
description: 'Should security be tested?' | ||
default: true | ||
type: boolean | ||
test-phpstan: | ||
description: 'Should PHPStan be tested?' | ||
default: true | ||
type: boolean | ||
test-api: | ||
description: 'Should API be tested?' | ||
default: true | ||
type: boolean | ||
test-e2e: | ||
description: 'Should E2E be tested?' | ||
default: true | ||
type: boolean | ||
|
||
# Advanced customization. | ||
options: | ||
description: 'Additional options for `qit` command, like `--optional_features=hpos`' | ||
type: string | ||
|
||
outputs: | ||
statuses: | ||
description: "Statuses of all tests. Array of integers." | ||
value: ${{ jobs.qit-tests.outputs.statuses }} | ||
|
||
|
||
jobs: | ||
qit-tests: | ||
name: Run QIT Tests | ||
runs-on: ubuntu-20.04 | ||
env: | ||
QIT_DISABLE_ONBOARDING: yes | ||
dev_build: ${{ inputs.version == 'dev' && format('{0}.zip', inputs.extension) || '' }} | ||
outputs: | ||
statuses: ${{ toJSON( steps.*.outputs.status ) }} | ||
steps: | ||
|
||
# Checkout repository to reuse local action. To be removed once we switch to the published action. | ||
- uses: actions/checkout@v3 | ||
with: | ||
fetch-depth: 2 | ||
path: 'this-repo' | ||
|
||
- name: Download plugins | ||
if: ${{ env.dev_build != '' }} | ||
uses: robinraju/[email protected] | ||
with: | ||
repository: "woocommerce/${{ inputs.extension }}" | ||
tag: 'gha-dev-build' | ||
fileName: ${{ env.dev_build }} | ||
token: ${{ secrets.BOT_GH_TOKEN }} | ||
|
||
- name: Install QIT via composer | ||
run: composer require woocommerce/qit-cli | ||
- name: Add Partner | ||
run: | | ||
./vendor/bin/qit partner:add \ | ||
--user='${{ secrets.QIT_PARTNER_USER }}' \ | ||
--application_password='${{ secrets.QIT_PARTNER_SECRET }}' | ||
- name: Create results dir | ||
run: mkdir -p ./qit-results/${{ inputs.extension }} | ||
|
||
- name: Activation test | ||
id: activation-test | ||
if: ${{ inputs.test-activation == true }} | ||
# Switch to published action once released. | ||
uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate | ||
timeout-minutes: 5 | ||
with: | ||
type: activation | ||
extension: ${{ inputs.extension }} | ||
extension-file: ${{ env.dev_build }} | ||
options: ${{ inputs.options }} | ||
|
||
- name: Security test | ||
id: security-test | ||
if: ${{ inputs.test-security == true }} | ||
uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate | ||
timeout-minutes: 5 | ||
with: | ||
type: security | ||
extension: ${{ inputs.extension }} | ||
extension-file: ${{ env.dev_build }} | ||
options: ${{ inputs.options }} | ||
|
||
- name: PHPStan test | ||
id: phpstan-test | ||
if: ${{ inputs.test-phpstan == true }} | ||
uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate | ||
timeout-minutes: 5 | ||
with: | ||
type: phpstan | ||
extension: ${{ inputs.extension }} | ||
extension-file: ${{ env.dev_build }} | ||
options: ${{ inputs.options }} | ||
|
||
- name: API test | ||
id: api-test | ||
if: ${{ inputs.test-api == true }} | ||
uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate | ||
timeout-minutes: 5 | ||
with: | ||
type: api | ||
extension: ${{ inputs.extension }} | ||
extension-file: ${{ env.dev_build }} | ||
options: ${{ inputs.options }} | ||
|
||
- name: E2E test | ||
id: e2e-test | ||
if: ${{ inputs.test-e2e == true }} | ||
uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate | ||
timeout-minutes: 30 | ||
with: | ||
type: e2e | ||
extension: ${{ inputs.extension }} | ||
extension-file: ${{ env.dev_build }} | ||
options: ${{ inputs.options }} | ||
|
||
- name: Upload results | ||
uses: actions/upload-artifact@v1 | ||
with: | ||
name: ${{ inputs.extension }} | ||
path: ./qit-results/${{ inputs.extension }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
packages/github-actions/actions/run-qit-annotate/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
# Run QIT test | ||
|
||
This action provides the following functionality for GitHub Actions users: | ||
|
||
- Run `qit` test of given type for a given extension | ||
- Annotate the results | ||
- Forward status code, so the consumer can decide how to conclude the results | ||
|
||
|
||
## Usage | ||
|
||
See [action.yml](action.yml) | ||
|
||
### Prerequisites | ||
|
||
- QIT needs to be [installed](https://woocommerce.github.io/qit-documentation/#/cli/getting-started?id=installing-qit) and [authenticated](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli) | ||
- To setup QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`. | ||
- The action assumes there is `qit-results` directory. You can change it's name using `results-folder` input. | ||
|
||
|
||
### Basic: | ||
|
||
```yaml | ||
jobs: | ||
qit-test: | ||
name: Run QIT Tests | ||
runs-on: ubuntu-20.04 | ||
env: | ||
QIT_DISABLE_ONBOARDING: yes | ||
steps: | ||
- name: Install QIT via composer | ||
run: composer require woocommerce/qit-cli | ||
|
||
- name: Add Partner | ||
run: | | ||
./vendor/bin/qit partner:add \ | ||
--user='${{ secrets.QIT_PARTNER_USER }}' \ | ||
--application_password='${{ secrets.QIT_PARTNER_SECRET }}' | ||
- name: Create results dir | ||
run: mkdir -p ./qit-results/automatewoo | ||
|
||
- name: Security test | ||
id: security-test | ||
uses: woocommerce/grow/run-qit-annotate@actions-v1 | ||
timeout-minutes: 5 | ||
with: | ||
type: security | ||
extension: automatewoo | ||
options: '--optional_features=hpos' | ||
|
||
- name: Echo status | ||
shell: bash | ||
run: echo ${{ jobs.security-test.outputs.status }} | ||
``` |
94 changes: 94 additions & 0 deletions
94
packages/github-actions/actions/run-qit-annotate/action.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
name: Run QIT test | ||
description: Runs QIT test and annotates the results. | ||
|
||
# **What it does**: Runs a single QIT test and annotates the results. | ||
# **Why we have it**: To reuse across other workflows to test extensions. | ||
|
||
inputs: | ||
# Basic params. | ||
extension: | ||
description: Extension to test | ||
required: true | ||
extension-file: | ||
description: Custom build of the extension to test. If not given, the latest release will be used. | ||
required: false | ||
type: | ||
description: Type of test to run | ||
required: false | ||
default: activation | ||
# Advanced customization. | ||
options: | ||
description: Options to pass to the qit command | ||
required: false | ||
default: '' | ||
results-filename: | ||
description: Custom file name for results | ||
required: false | ||
results-folder: | ||
description: Folder to store results | ||
required: false | ||
default: qit-results | ||
|
||
outputs: | ||
status: | ||
description: "Exit code of the test. May be used, for example, to fail the workflow if the test fails." | ||
value: ${{ steps.run-test.outputs.status }} | ||
summary: | ||
description: "Short summary of the test" | ||
value: ${{ steps.read-summary.outputs.summary }} | ||
resultsURL: | ||
description: "URL to the results of the test" | ||
value: ${{ steps.read-summary.outputs.resultURL }} | ||
|
||
branding: | ||
icon: 'award' | ||
color: 'purple' | ||
|
||
runs: | ||
using: composite | ||
steps: | ||
- name: Run test | ||
id: run-test | ||
# Do not fail when the `qit` fails, so we can annotate the results. | ||
shell: bash --noprofile --norc {0} | ||
continue-on-error: true | ||
env: | ||
# If the custom build is provided, pass it to the `zip` param. | ||
zip: ${{ inputs.extension-file && format('--zip={0}', inputs.extension-file) || '' }} | ||
report_file: ${{ inputs.results-filename || format( '{0}/{1}/{2}/{3}.txt', github.workspace, inputs.results-folder, inputs.extension, inputs.type ) }} | ||
status: 0 | ||
run: | | ||
./vendor/bin/qit run:${{ inputs.type }} ${{ inputs.extension }} \ | ||
$zip \ | ||
${{ inputs.options }} \ | ||
--wait \ | ||
> $report_file || status=$? | ||
echo "status=$status" >> "$GITHUB_OUTPUT" | ||
echo "report_file=$report_file" >> "$GITHUB_OUTPUT" | ||
cat $report_file | ||
# Parse the report file, to fetch the essential information. | ||
- name: Read summary | ||
id: read-summary | ||
shell: bash | ||
if: '!cancelled()' | ||
run: | | ||
summary=`grep -Po "(?<=Test Summary)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never` | ||
resultURL=`grep -Po "(?<=Result Url)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never` | ||
echo "summary=$summary" >> $GITHUB_OUTPUT | ||
echo "resultURL=$resultURL" >> $GITHUB_OUTPUT | ||
# Annotate the results according to the exit code. | ||
- name: Annotate | ||
if: '!cancelled()' | ||
shell: bash | ||
run: | | ||
summary="${{ inputs.type }}: ${{ steps.read-summary.outputs.summary }} - ${{ steps.read-summary.outputs.resultURL }}"; | ||
case ${{ steps.run-test.outputs.status }} in | ||
0) echo "::notice ::$summary" | ||
;; | ||
2) echo "::warning ::$summary" | ||
;; | ||
*) echo "::error ::$summary" | ||
;; | ||
esac |