diff --git a/.github/workflows/pmm-ui-tests-matrix.yml b/.github/workflows/pmm-ui-tests-matrix.yml index ba7f15f9b..201b3cd28 100644 --- a/.github/workflows/pmm-ui-tests-matrix.yml +++ b/.github/workflows/pmm-ui-tests-matrix.yml @@ -14,26 +14,26 @@ on: default: 'main' required: true pmm_server_version: - description: 'PMM Server docker image' - default: 'perconalab/pmm-server:dev-latest' + description: 'PMM Server version to upgrade (latest|dev-latest|x.xx.x|x.xx.x-rc)' + default: 'dev-latest' required: true pmm_client_version: - description: 'PMM Client version to test (dev-latest|pmm2-latest|pmm2-rc|x.xx.x)' + description: 'PMM Client version to upgrade from (dev-latest|pmm2-latest|pmm2-rc|x.xx.x)' default: 'dev-latest' required: true - jobs: - rbac: - name: RBAC + inventory: + name: Inventory uses: ./.github/workflows/pmm-ui-tests.yml secrets: inherit with: pmm_ui_tests_branch: ${{ github.event.inputs.pmm_ui_tests_branch || github.head_ref || 'main '}} - pmm_test_flag: '@rbac' - pmm_server_version: ${{ github.event.inputs.pmm_server_version || 'perconalab/pmm-server:dev-latest' }} - pmm_client_version: ${{ github.event.inputs.pmm_client_version || 'dev-latest' }} - pmm_clients: '--addclient=ps,1 --addclient=pdpgsql,1' pmm_qa_branch: ${{ github.event.inputs.pmm_qa_branch || 'main' }} + pmm_server_version: ${{ inputs.pmm_server_version || 'dev-latest' }} + client_version: ${{ inputs.pmm_client_version || 'dev-latest' }} + tags_for_playwright: '@inventory' + client_flags: '--addclient=modb,1 --addclient=ps,1 --addclient=pdpgsql,1' + version_string: ${{ inputs.pmm_server_version || 'dev-latest' }} portal: name: Portal @@ -41,20 +41,22 @@ jobs: secrets: inherit with: pmm_ui_tests_branch: ${{ github.event.inputs.pmm_ui_tests_branch || github.head_ref || 'main '}} - pmm_test_flag: '@portal' - pmm_server_version: ${{ github.event.inputs.pmm_server_version || 'perconalab/pmm-server:dev-latest' }} - pmm_client_version: ${{ github.event.inputs.pmm_client_version || 'dev-latest' }} - pmm_clients: '' pmm_qa_branch: ${{ github.event.inputs.pmm_qa_branch || 'main' }} + pmm_server_version: ${{ inputs.pmm_server_version || 'dev-latest' }} + client_version: ${{ inputs.pmm_client_version || 'dev-latest' }} + tags_for_playwright: '@portal' + client_flags: '' + version_string: ${{ inputs.pmm_server_version || 'dev-latest' }} - inventory: - name: Inventory + rbac: + name: RBAC uses: ./.github/workflows/pmm-ui-tests.yml secrets: inherit with: pmm_ui_tests_branch: ${{ github.event.inputs.pmm_ui_tests_branch || github.head_ref || 'main '}} - pmm_test_flag: '@inventory' - pmm_server_version: ${{ github.event.inputs.pmm_server_version || 'perconalab/pmm-server:dev-latest' }} - pmm_client_version: ${{ github.event.inputs.pmm_client_version || 'dev-latest' }} - pmm_clients: '--addclient=modb,1 --addclient=ps,1 --addclient=pdpgsql,1' pmm_qa_branch: ${{ github.event.inputs.pmm_qa_branch || 'main' }} + pmm_server_version: ${{ inputs.pmm_server_version || 'dev-latest' }} + client_version: ${{ inputs.pmm_client_version || 'dev-latest' }} + tags_for_playwright: '@rbac' + client_flags: '--addclient=ps,1 --addclient=pdpgsql,1' + version_string: ${{ inputs.pmm_server_version || 'dev-latest' }} diff --git a/.github/workflows/pmm-ui-tests.yml b/.github/workflows/pmm-ui-tests.yml index 51b374305..4b48e867f 100644 --- a/.github/workflows/pmm-ui-tests.yml +++ b/.github/workflows/pmm-ui-tests.yml @@ -2,206 +2,301 @@ name: ui-tests-pipeline on: - workflow_call: + workflow_dispatch: inputs: pmm_ui_tests_branch: - description: 'Target branch for pmm-ui-tests repository' + description: 'pmm-ui-tests repository branch' type: string default: 'main' required: true - pmm_test_flag: + tests_tag: description: 'Flag to run only specific portion of the tests.' type: string required: false + pmm_qa_branch: + description: 'pmm-qa repository branch(for setup)' + type: string + default: 'main' + required: true pmm_server_version: - description: 'Version of the pmm server used for testing' + description: 'PMM Server version to test (latest|dev-latest|x.xx.x|x.xx.x-rc|PR-xx-xxx)' type: string default: 'dev-latest' required: true - pmm_client_version: - description: 'Version of the pmm client used for testing' + client_version: + description: 'PMM Client version to test (dev-latest|pmm2-latest|pmm2-rc|x.xx.x)' type: string default: 'dev-latest' required: true pmm_clients: - description: 'Clients for pmm-server' + description: 'Services to setup and connect to pmm-server' type: string default: '--addclient=ps,1' required: true - pmm_qa_branch: - description: 'Branch for the pmm-qa repository.' - type: string - default: 'main' - required: true - workflow_dispatch: + workflow_call: inputs: pmm_ui_tests_branch: - description: 'Target branch for pmm-ui-tests repository' + description: "Branch for PMM-UI tests to checkout" + required: false type: string - default: 'main' - required: true - pmm_test_flag: - description: 'Flag to run only specific portion of the tests.' + pmm_qa_branch: + description: 'pmm-qa repository branch(for setup)' + required: false type: string + sha: + description: "SHA (leave empty if running manually, default - 'null')" required: false + type: string pmm_server_version: - description: 'Version of the pmm server used for testing' + description: 'PMM Server version to test (latest|dev-latest|x.xx.x|x.xx.x-rc|PR)' type: string default: 'dev-latest' - required: true - pmm_client_version: - description: 'Version of the pmm client used for testing' + required: false + server_image: + description: "pmm-server docker image, default perconalab/pmm-server:dev-latest" + required: false type: string - default: 'dev-latest' - required: true - pmm_clients: - description: 'Clients for pmm-server' + client_version: + description: "pmm2-client version Tarball or Dev-latest, default is dev-latest" + required: false type: string - default: '--addclient=ps,1' - required: true - pmm_qa_branch: - description: 'Branch for the pmm-qa repository.' + client_image: + description: "pmm2-client docker image, default perconalab/pmm-client:dev-latest" + required: false + type: string + client_flags: + description: "example: --addclient=ps,1 --ps-version=5.7" + required: false + type: string + tags_for_tests: + description: "CodeceptJS tag expression, ex: @settings-fb" + type: string + tags_for_playwright: + description: "Playwright ag expression, ex: @settings-fb" + type: string + version_string: + description: 'x.xx.x version string for josb name' type: string - default: 'main' - required: true jobs: - tests: - name: Tests - runs-on: ubuntu-latest + ui-tests-e2e: + name: ${{ inputs.version_string || inputs.pmm_server_version || 'dev-latest' }} + # runs-on: ubuntu-latest Mongo Replica setup fails in ubuntu-latest for some reason. Additional investigation needed + runs-on: ubuntu-20.04 timeout-minutes: 60 env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SHA: ${{ inputs.sha || 'null' }} - PMM_BASE_URL: https://127.0.0.1 - ADMIN_PASSWORD: admin + PMM_QA_BRANCH: ${{ inputs.pmm_qa_branch || 'main' }} + PMM_QA_GIT_BRANCH: ${{ inputs.pmm_qa_branch || 'main' }} + PMM_UI_BRANCH: ${{ inputs.pmm_ui_tests_branch || 'main' }} + CLIENT_FLAGS: ${{ inputs.client_flags || '' }} + TAGS_FOR_TESTS: ${{ inputs.tags_for_tests || '@settings-fb' }} - UI_TESTS_BRANCH: ${{ inputs.pmm_ui_tests_branch }} - PMM_CLIENTS: ${{ inputs.pmm_clients }} - PMM_TEST_FLAG: ${{ inputs.pmm_test_flag }} + ### Variables also used in tests + SERVER_IP: "192.168.0.1" + PMM_UI_URL: "http://192.168.0.1/" + ADMIN_PASSWORD: 'admin-password' + # PATH_TO_PMM_QA defined in job steps + DOCKER_VERSION: ${{ inputs.server_image }} - PMM_SERVER_VERSION: ${{ inputs.pmm_server_version }} - PMM_CLIENT_VERSION: ${{ inputs.pmm_client_version }} - PMM_QA_BRANCH: ${{ inputs.pmm_qa_branch }} + ########################################## + ### Credentials required for tests ### + ########################################## - OKTA_TOKEN: ${{ secrets.OKTA_TOKEN }} - OAUTH_ISSUER_URL: 'https://id-dev.percona.com/oauth2/aus15pi5rjdtfrcH51d7' - OAUTH_CLIENT_ID: ${{ secrets.OKTA_OAUTH_CLIENT_ID }} - OAUTH_CLIENT_SECRET: ${{ secrets.OKTA_OAUTH_CLIENT_SECRET }} - OAUTH_PMM_CLIENT_ID: ${{ secrets.OKTA_OAUTH_PMM_CLIENT_ID }} - OAUTH_PMM_CLIENT_SECRET: ${{ secrets.OKTA_OAUTH_PMM_CLIENT_SECRET }} - OAUTH_DEV_HOST: ${{ secrets.OAUTH_DEV_HOST }} - OAUTH_SCOPES: percona - - # Variables for E2E tests - MAILOSAUR_API_KEY: ${{ secrets.MAILOSAUR_API_KEY }} - MAILOSAUR_UI_TESTS_SERVER_ID: ${{ secrets.MAILOSAUR_UI_TESTS_SERVER_ID }} - MAILOSAUR_API_TESTS_SERVER_ID: ${{ secrets.MAILOSAUR_API_TESTS_SERVER_ID }} + # Backup tests credentials + BACKUP_LOCATION_ACCESS_KEY: ${{ secrets.BACKUP_LOCATION_ACCESS_KEY }} + BACKUP_LOCATION_SECRET_KEY: ${{ secrets.BACKUP_LOCATION_SECRET_KEY }} + # Portal tests credentials + PORTAL_BASE_URL: 'https://portal-dev.percona.com' + OAUTH_DEV_HOST: ${{ secrets.OAUTH_DEV_HOST }} + OAUTH_ISSUER_URL: 'https://id-dev.percona.com/oauth2/aus15pi5rjdtfrcH51d7' + OKTA_TOKEN: ${{ secrets.OKTA_TOKEN }} SERVICENOW_LOGIN: percona_platform SERVICENOW_PASSWORD: ${{ secrets.SERVICENOW_PASSWORD }} SERVICENOW_DEV_URL: 'https://perconadev.service-now.com/api/x_pellc_percona_pl/platform/settest' steps: - - name: PMM server version ${{ inputs.pmm_server_version }} and UI tests for flag "${{ inputs.pmm_test_flag }}" and pmm-ui-tests branch ${{ inputs.pmm_ui_tests_branch }} + - name: Create status check if: ${{ env.SHA != 'null' }} - uses: percona/gh-action-Sibz-github-status-action@v1 + uses: percona-platform/github-status-action@v1 continue-on-error: true with: authToken: ${{ secrets.GITHUB_TOKEN }} - context: "${{ env.PMM_TEST_FLAG }} UI tests" + context: "${{ env.TAGS_FOR_TESTS || inputs.tags_for_playwright }} UI tests" description: "Test execution ${{ job.status }}" state: "pending" repository: ${{ github.repository }} target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" sha: ${{ env.SHA }} - - name: 'Checkout UI tests: "${{ inputs.pmm_ui_tests_branch }}"' + - name: 'Prevent both Codecept and Playwright tests execution' + if: ${{ !!inputs.tags_for_tests && !!inputs.tags_for_playwright }} + run: | + echo "Both Codecept(tags_for_tests) and Playwright(tags_for_playwright) tests are forbidden in a single test run!" + echo "Please use only one tag input for job call" + exit 1 + + - name: 'Checkout PMM UI tests: "${{ inputs.pmm_ui_tests_branch }}"' uses: actions/checkout@v3 with: - ref: ${{ inputs.pmm_ui_tests_branch }} + ref: ${{ env.PMM_UI_BRANCH }} repository: percona/pmm-ui-tests path: ./pmm-ui-tests - - name: 'Install playwright' - working-directory: ./pmm-ui-tests/playwright-tests - run: | - npm ci - npx playwright install - - name: 'Checkout pmm-qa: "${{ inputs.pmm_qa_branch }}"' uses: actions/checkout@v3 with: - # token: ${{ secrets.ROBOT_TOKEN }} repository: percona/pmm-qa ref: ${{ inputs.pmm_qa_branch }} path: ./pmm-qa - - name: 'Setup : "${{ inputs.pmm_server_version }}"' - working-directory: pmm-qa/pmm-integration - run: | - npm install - sudo npx ts-node ./integration-setup.ts --ci --setup-docker-pmm-server --rbac --pmm-server-docker-tag=${{ env.PMM_SERVER_VERSION }} --pmm-client-version=${{ env.PMM_CLIENT_VERSION }} - timeout 100 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost/ping)" != "200" ]]; do sleep 5; done' || false + - name: 'Export "PATH_TO_PMM_QA" variable' + working-directory: ./pmm-qa + run: echo "PATH_TO_PMM_QA=$(pwd)" >> $GITHUB_ENV - - name: 'Setup ' + - name: Setup tools run: | - sudo bash ./pmm-qa/pmm-tests/pmm2-client-setup.sh --pmm_server_ip 127.0.0.1 --client_version ${{ env.PMM_CLIENT_VERSION }} --admin_password admin --use_metrics_mode no + npm install -g bats + sudo apt-get install -y apt-transport-https ca-certificates dirmngr ansible libaio1 libaio-dev libnuma-dev libncurses5 socat sysbench + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 + echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ + /etc/apt/sources.list.d/clickhouse.list + sudo apt-get update + sudo apt-get install -y clickhouse-client + sudo curl -s https://raw.githubusercontent.com/datacharmer/dbdeployer/master/scripts/dbdeployer-install.sh | bash + ls -la + # use ${{ env.PATH_TO_PMM_QA }} instead of pushd + pushd pmm-qa + wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/get_download_link.sh + chmod +x get_download_link.sh + popd + + - name: 'Select image' + if: ${{ !inputs.server_image && !!inputs.pmm_server_version }} + id: get_pmm_image shell: bash + run: | + tag=${{ inputs.pmm_server_version }} + repo=percona/pmm-server + if [[ "$tag" == "dev-latest" ]]; then + repo=perconalab/pmm-server + fi + if [[ "$tag" =~ "-rc" ]]; then + repo=perconalab/pmm-server + fi + if [[ "$tag" =~ "PR-" ]]; then + repo=perconalab/pmm-server-fb + fi + echo "image=$repo:$tag" >> "$GITHUB_OUTPUT" + echo "DOCKER_VERSION=$repo/pmm-server:$tag" >> $GITHUB_ENV - - name: Setup Databases for the PMM-Server - working-directory: pmm-qa/pmm-integration + - name: 'Setup : "${{ inputs.server_image || steps.get_pmm_image.outputs.image }}"' + working-directory: ./pmm-ui-tests run: | - sudo npx ts-node ./integration-setup.ts --ci \ - ${{ env.PMM_CLIENTS }} - sleep 30 - sudo pmm-admin list - shell: bash + PWD=$(pwd) PMM_SERVER_IMAGE=${{ inputs.server_image || steps.get_pmm_image.outputs.image }} docker-compose up -d + + ### wait for service to be ready + export CONTAINER_NAME="pmm-server" + export LOGS="pmm-managed entered RUNNING state" + attempt=0 + while [ $attempt -le 3 ]; do + attempt=$(( $attempt + 1 )) + echo "Waiting for ${CONTAINER_NAME} to be up (attempt: $attempt)..." + result=$(docker logs ${CONTAINER_NAME} 2>&1) + if grep "${LOGS}" <<< $result ; then + echo "${CONTAINER_NAME} is ready!" + break + fi + sleep 10 + done; + timeout 100 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://admin:admin@${{ env.SERVER_IP }}/ping)" != "200" ]]; do sleep 5; done' || false + + docker exec pmm-server change-admin-password ${{ env.ADMIN_PASSWORD }} + bash -x testdata/db_setup.sh - - name: Execute UI tests - if: ${{ inputs.pmm_test_flag == 'null' }} - id: ui-tests-all - working-directory: pmm-ui-tests/playwright-tests + - name: 'Setup ' + working-directory: ./pmm-qa run: | - npx playwright test --project="chromium" --quiet -# env: -# NODE_TLS_REJECT_UNAUTHORIZED: 0 + sudo bash -x pmm-tests/pmm2-client-setup.sh \ + --pmm_server_ip ${{ env.SERVER_IP }} --client_version ${{ inputs.client_version }} \ + --admin_password ${{ env.ADMIN_PASSWORD }} --use_metrics_mode no + + - name: Setup Services for the PMM Server + working-directory: ./pmm-qa + run: sudo -E bash -x pmm-tests/pmm-framework.sh ${{ env.CLIENT_FLAGS }} --pmm2 + + - name: Install CodeceptJs + if: ${{ !!inputs.tags_for_tests }} + working-directory: ./pmm-ui-tests + run: | + npm ci + envsubst < env.list > env.generated.list + + - name: 'Run Codecept e2e tests: ${{ inputs.tags_for_tests }}' + if: ${{ !!inputs.tags_for_tests }} + working-directory: ./pmm-ui-tests + env: + PMM_URL : "http://admin:${{ env.ADMIN_PASSWORD }}@${{ env.SERVER_IP }}" + run: | + sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js + ./node_modules/.bin/codeceptjs run -c pr.codecept.js --grep "${{ inputs.tags_for_tests }}" + + - name: 'Install playwright' + if: ${{ !!inputs.tags_for_playwright }} + working-directory: ./pmm-ui-tests/playwright-tests + run: | + npm ci + npx playwright install - - name: 'Run UI tests: ${{ env.PMM_TEST_FLAG }}' - if: ${{ inputs.pmm_test_flag != 'null' }} - id: ui-tests-flagged - working-directory: pmm-ui-tests/playwright-tests + - name: 'Run Playwright e2e tests: ${{ inputs.tags_for_playwright }}' + if: ${{ !!inputs.tags_for_playwright }} + working-directory: ./pmm-ui-tests/playwright-tests run: | - project="${{ contains(env.PMM_TEST_FLAG, 'portal') && 'Portal' || 'Chromium' }}" - npx playwright test --project=$project --grep ${{ env.PMM_TEST_FLAG }} --pass-with-no-tests + project="${{ contains(inputs.tags_for_playwright, 'portal') && 'Portal' || 'Chromium' }}" + npx playwright test --project=$project --grep ${{ inputs.tags_for_playwright }} --pass-with-no-tests # env: # NODE_TLS_REJECT_UNAUTHORIZED: 0 - name: 'Create report name' - if: failure() + if: ${{ failure() && !!inputs.tags_for_playwright }} run: | # TODO: add job id for matrix call - job_tag=$(echo "${{ inputs.pmm_test_flag }}" | sed -e "s/-pre-upgrade//" -e "s/@//") - report_name="$job_tag"-report + job_tag=$(echo "${{ inputs.tags_for_playwright }}" | sed -e "s/-pre-upgrade//" -e "s/@//") + report_name=${{ inputs.version_string || inputs.client_version }}-"$job_tag"-report echo $report_name echo "REPORT_NAME=$report_name" >> $GITHUB_ENV - name: Generate and Attach the report - if: failure() + if: ${{ failure() && !!inputs.tags_for_playwright }} uses: actions/upload-artifact@v3 with: name: ${{ env.REPORT_NAME }} path: ./pmm-ui-tests/playwright-tests/playwright-report + - uses: actions/github-script@v6 + if: ${{ always() }} + id: artifact_name + with: + script: | + return `artifacts_for_${process.env.TAGS_FOR_TESTS.replaceAll('|', '')}` + result-encoding: string + - name: Create status check - uses: percona/gh-action-Sibz-github-status-action@v1 - if: ${{ env.SHA != 'null' && always() }} + if: ${{ always() && env.SHA != 'null' }} + uses: percona-platform/github-status-action@v1 continue-on-error: true with: authToken: ${{ secrets.GITHUB_TOKEN }} - context: "${{ env.PMM_TEST_FLAG }} UI tests" + context: "${{ env.TAGS_FOR_TESTS || inputs.tags_for_playwright }} UI tests" description: "Test execution ${{ job.status }}" state: ${{ job.status }} repository: ${{ github.repository }} target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" sha: ${{ env.SHA }} + +# - name: Setup tmate session on failure +# if: ${{ failure() }} +# uses: percona-platform/action-tmate@v2 diff --git a/.github/workflows/portal-tests-matrix.yml b/.github/workflows/portal-tests-matrix.yml index 7d0c25f36..3fa1c0b03 100644 --- a/.github/workflows/portal-tests-matrix.yml +++ b/.github/workflows/portal-tests-matrix.yml @@ -3,7 +3,17 @@ name: "Portal Tests Matrix" on: push: branches: - - "PMM-7-fix-portal-pipeline" + - PMM-7-update-gh + workflow_dispatch: + inputs: + pmm_ui_tests_branch: + description: 'pmm-ui-tests repository branch' + default: 'main' + required: true + pmm_qa_branch: + description: 'pmm-qa repository branch(for setup)' + default: 'main' + required: true jobs: get_versions: @@ -15,19 +25,20 @@ jobs: portal: name: 'Portal / Integration' - uses: ./.github/workflows/portal-ui-tests.yml + uses: ./.github/workflows/pmm-ui-tests.yml secrets: inherit needs: get_versions strategy: fail-fast: false matrix: old_version: ${{ fromJSON(needs.get_versions.outputs.version_matrix) }} + include: + - old_version: dev-latest with: - pmm_ui_tests_branch: ${{ inputs.pmm_ui_tests_branch || 'main' }} - pmm_test_flag: '@portal' - pmm_server_version: ${{ matrix.old_version }} - pmm_client_version: ${{ matrix.old_version }} + pmm_ui_tests_branch: ${{ inputs.pmm_ui_tests_branch || 'PMM-7-update-gh' }} pmm_qa_branch: ${{ inputs.pmm_qa_branch || 'main' }} - pmm_clients: '' - version_string_from: ${{needs.get_versions.outputs.start_version}} - + pmm_server_version: ${{ matrix.old_version }} + client_version: ${{ matrix.old_version }} + tags_for_playwright: '@portal' + client_flags: '' + version_string: ${{needs.get_versions.outputs.start_version}}