Schedule Daily #245
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schedule Daily | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" | |
| workflow_dispatch: | |
| env: | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| CI_COMMIT_SHA: ${{ github.sha }} | |
| CI_JOB_NAME: ${{ github.job }} | |
| CI_JOB_ID: ${{ github.job }} # github does not expose this variable https://github.com/orgs/community/discussions/8945 | |
| CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| CI_PIPELINE_SOURCE: ${{ github.event_name }} | |
| CI_PROJECT_DIR: ${{ github.workspace }} | |
| CI_PULL_REQUEST_TARGET_BRANCH_NAME: ${{ github.ref_name }} # this workflow will always run on the default branch | |
| ROOT_PIPELINE_ID: ${{ github.run_id }} | |
| BUILDEVENT_DATASET: "github-ci-dfinity" | |
| jobs: | |
| cut-release-candidate: | |
| name: Cut RC | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| ref: ${{ github.ref }} | |
| token: ${{ secrets.IDX_PUSH_TO_PR }} | |
| - name: Cut Release Candidate | |
| shell: bash | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "IDX GitLab Automation" | |
| RC_BRANCH_NAME="rc--$(date '+%Y-%m-%d_%H-%M')" | |
| git switch --force-create "$RC_BRANCH_NAME" HEAD | |
| git push --force --set-upstream origin "$RC_BRANCH_NAME" | |
| rust-benchmarks: | |
| name: Bazel Run Rust Benchmarks | |
| runs-on: | |
| # see linux-x86-64 runner group | |
| labels: rust-benchmarks | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:1c0e901df3c7a97fc440c271881400ce6d2e586e2a89cdc39ec939e3dfe5de76 | |
| # running on bare metal machine using ubuntu user | |
| options: --user ubuntu -v /cache:/cache | |
| timeout-minutes: 720 # 12 hours | |
| strategy: | |
| matrix: | |
| targets: | |
| - "//rs/crypto/..." | |
| - "//rs/state_manager/..." | |
| - "//rs/certification/..." | |
| - "//rs/boundary_node/ic_boundary/..." | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Rust Benchmarks | |
| id: rust-benchmarks | |
| shell: bash | |
| run: | | |
| ./ci/scripts/rust-benchmarks.sh | |
| env: | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| BAZEL_COMMAND: "run" | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" | |
| RUST_BACKTRACE: "full" | |
| TARGETS: ${{ matrix.targets }} | |
| bazel-test-bare-metal: | |
| name: Bazel Test Bare Metal | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:1c0e901df3c7a97fc440c271881400ce6d2e586e2a89cdc39ec939e3dfe5de76 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 120 | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Run Bazel Launch Bare Metal | |
| shell: bash | |
| run: | | |
| echo "$ZH2_DLL01_CSV_SECRETS" > file1 | |
| echo "$ZH2_FILE_SHARE_KEY" > file2 && chmod 400 file2 | |
| # Run bare metal installation test | |
| # shellcheck disable=SC2046,SC2086 | |
| bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \ | |
| //ic-os/setupos/envs/dev:launch_bare_metal -- \ | |
| --config_path "$(realpath ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \ | |
| --csv_filename "$(realpath file1)" \ | |
| --file_share_ssh_key "$(realpath file2)" \ | |
| --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \ | |
| --file_share_username ci_interim \ | |
| --ci_mode | |
| # Run bare metal node performance benchmarks | |
| # shellcheck disable=SC2046,SC2086 | |
| bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \ | |
| //ic-os/setupos/envs/dev:launch_bare_metal -- \ | |
| --config_path "$(realpath ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \ | |
| --csv_filename "$(realpath file1)" \ | |
| --file_share_ssh_key "$(realpath file2)" \ | |
| --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \ | |
| --file_share_username ci_interim \ | |
| --ci_mode \ | |
| --benchmark | |
| bazel clean | |
| env: | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| ZH2_DLL01_CSV_SECRETS: "${{ secrets.ZH2_DLL01_CSV_SECRETS }}" | |
| ZH2_FILE_SHARE_KEY: "${{ secrets.ZH2_FILE_SHARE_KEY }}" | |
| fi-tests-nightly: | |
| name: Bazel Test FI Nightly | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:1c0e901df3c7a97fc440c271881400ce6d2e586e2a89cdc39ec939e3dfe5de76 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 720 # 12 hours | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Run FI Tests Nightly | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//rs/rosetta-api/..." | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=fi_tests_nightly --test_env=SSH_AUTH_SOCK --test_timeout=43200" | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Upload bazel-bep | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| nns-tests-nightly: | |
| name: Bazel Test NNS Nightly | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:1c0e901df3c7a97fc440c271881400ce6d2e586e2a89cdc39ec939e3dfe5de76 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Run NNS Tests Nightly | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//rs/nns/..." | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=nns_tests_nightly --test_env=SSH_AUTH_SOCK --test_env=NNS_CANISTER_UPGRADE_SEQUENCE=all" | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Upload bazel-bep | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| system-tests-benchmarks-nightly: | |
| name: Bazel System Test Benchmarks | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:1c0e901df3c7a97fc440c271881400ce6d2e586e2a89cdc39ec939e3dfe5de76 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 480 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Set Benchmark Targets | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| echo "BENCHMARK_TARGETS=$(bazel query 'attr(tags, system_test_benchmark, //rs/...)' | tr '\n' ' ')" >> $GITHUB_ENV | |
| - name: Test System Test Benchmarks | |
| id: bazel-system-test-benchmarks | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: ${{ env.BENCHMARK_TARGETS }} | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| # note: there's just one performance cluster, so the job can't be parallelized | |
| BAZEL_EXTRA_ARGS: "--test_tag_filters=system_test_benchmark --//bazel:enable_upload_perf_systest_results=True --keep_going --jobs 1" | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Upload bazel-bep | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| dependency-scan-nightly: | |
| name: Dependency Scan Nightly | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:1c0e901df3c7a97fc440c271881400ce6d2e586e2a89cdc39ec939e3dfe5de76 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 60 | |
| permissions: | |
| actions: write | |
| env: | |
| SHELL_WRAPPER: "/usr/bin/time" | |
| CARGO_WASMPACK_VERSION: "0.12.1" | |
| DEFAULT_NODE_VERSION: "20" | |
| CI_PROJECT_PATH: ${{ github.repository }} | |
| CI_PIPELINE_ID: ${{ github.run_id }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| SLACK_PSEC_BOT_OAUTH_TOKEN: ${{ secrets.SLACK_PSEC_BOT_OAUTH_TOKEN }} | |
| GITHUB_REF: ${{ github.ref }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Setup environment deps | |
| id: setup-environment-deps | |
| shell: bash | |
| run: | | |
| pip3 install --ignore-installed -r requirements.txt | |
| cargo install wasm-pack --version "${CARGO_WASMPACK_VERSION}" | |
| source "${NVM_DIR}/nvm.sh" | |
| nvm use ${DEFAULT_NODE_VERSION} | |
| node --version | |
| npm --version | |
| - name: Run Dependency Scan Nightly | |
| id: dependency-scan-nightly | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies | |
| cd ci/src/dependencies/ | |
| $SHELL_WRAPPER python3 job/bazel_rust_ic_scanner_periodic_job.py | |
| $SHELL_WRAPPER python3 job/npm_scanner_periodic_job.py | |
| $SHELL_WRAPPER python3 job/bazel_trivy_container_ic_scanner_periodic_job.py |