Build and test NBS on demand #98
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test NBS on demand | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| build_target: | |
| type: string | |
| default: "cloud/blockstore/apps/,cloud/filestore/apps/,cloud/disk_manager/,cloud/tasks/" | |
| description: "limit build to specific target" | |
| test_target: | |
| type: string | |
| default: "cloud/blockstore/,cloud/filestore/,cloud/disk_manager/,cloud/tasks/" | |
| description: "limit test to specific target" | |
| build_preset: | |
| type: choice | |
| default: "relwithdebinfo" | |
| description: "Build preset" | |
| options: | |
| - debug | |
| - relwithdebinfo | |
| - release-asan | |
| - release-tsan | |
| - release-msan | |
| - release-ubsan | |
| cache_update: | |
| type: boolean | |
| default: false | |
| description: "Update build cache for build and cache" | |
| test_size: | |
| type: choice | |
| default: "small" | |
| description: "test size to run" | |
| options: | |
| - small | |
| - small,medium | |
| - small,medium,large | |
| test_type: | |
| type: choice | |
| default: "unittest" | |
| description: "type of tests to run" | |
| options: | |
| - unittest,clang_tidy,gtest | |
| - py3test,py2test,pytest,flake8,black,py2_flake8 | |
| - go_test,gofmt | |
| - unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,go_test,gofmt | |
| sleep_after_tests: | |
| type: string | |
| default: "0" | |
| description: "Amount of seconds to sleep after tests" | |
| run_build: | |
| type: boolean | |
| default: true | |
| description: "run build" | |
| run_tests: | |
| type: boolean | |
| default: true | |
| description: "run tests" | |
| workflow_call: | |
| inputs: | |
| build_target: | |
| type: string | |
| default: "cloud/blockstore/apps/,cloud/filestore/apps/,cloud/disk_manager/,cloud/tasks/" | |
| description: "limit build to specific target" | |
| test_target: | |
| type: string | |
| default: "cloud/blockstore/,cloud/filestore/,cloud/disk_manager/,cloud/tasks/" | |
| description: "limit test to specific target" | |
| build_preset: | |
| type: string | |
| default: "relwithdebinfo" | |
| vm_name_suffix: | |
| type: string | |
| default: "" | |
| test_size: | |
| type: string | |
| default: "small,medium,large" | |
| test_type: | |
| type: string | |
| default: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,go_test,gofmt" | |
| run_build: | |
| type: boolean | |
| default: true | |
| run_tests: | |
| type: boolean | |
| default: true | |
| sleep_after_tests: | |
| type: string | |
| default: "0" | |
| description: "Amount of seconds to sleep after tests" | |
| cache_update_build: | |
| type: boolean | |
| default: false | |
| description: "Update build cache in build stage" | |
| cache_update_tests: | |
| type: boolean | |
| default: false | |
| description: "Update build cache in test stage" | |
| link_threads: | |
| type: string | |
| default: "24" | |
| description: "--link-threads in ya make" | |
| test_threads: | |
| type: string | |
| default: "32" | |
| description: "--test-threads in ya test" | |
| upload_ya_dir: | |
| type: string | |
| default: "no" | |
| description: "Upload ya dir to s3" | |
| clean_ya_dir: | |
| type: string | |
| default: "no" | |
| description: "Clean ya dir from image before building" | |
| use_network_cache: | |
| type: string | |
| default: "yes" | |
| description: "Use network cache" | |
| disk_type: | |
| type: string | |
| default: "network-ssd-nonreplicated" | |
| description: "Disk type for VM" | |
| jobs: | |
| provide-runner: | |
| name: Start self-hosted runner (${{ inputs.build_preset }}) | |
| timeout-minutes: 60 | |
| if: always() | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-runner.outputs.label }} | |
| instance-id: ${{ steps.start-runner.outputs.instance-id }} | |
| runner_ipv4: ${{ steps.start-runner.outputs.external-ipv4 }} | |
| runner_local_ipv4: ${{ steps.start-runner.outputs.local-ipv4 }} | |
| steps: | |
| - name: checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: ${{ !contains(github.event.pull_request.labels.*.name, 'rebase') && 1 || 0 }} | |
| - name: rebase PR | |
| if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }} | |
| shell: bash | |
| run: | | |
| git config user.email "[email protected]" | |
| git config user.name "Rebase Robotovich" | |
| git fetch origin ${{ github.event.pull_request.base.ref }} | |
| git rebase origin/${{ github.event.pull_request.base.ref }} | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Start runner | |
| id: start-runner | |
| if: always() | |
| uses: ./.github/actions/runner_create | |
| timeout-minutes: 60 | |
| with: | |
| repo_owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| service_account_key: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }} | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| vm_folder: bjeuq5o166dq4ukv3eec | |
| vm_name: ${{ github.event.pull_request.number && format('pr-{0}{1}-{2}-{3}', github.event.pull_request.number, inputs.vm_name_suffix, github.run_id, github.run_attempt) || format('run-{0}-{1}', github.run_id, github.run_attempt) }} | |
| vm_zone: eu-north1-c | |
| vm_cpu: 60 | |
| vm_memory: 420 | |
| vm_disk_type: ${{ inputs.disk_type || 'network-ssd-nonreplicated' }} | |
| vm_disk_size: 1023 | |
| vm_subnet: f8uh0ml4rhb45nde9p75 | |
| vm_image: ${{ vars.IMAGE_ID_2204 }} | |
| vm_labels: ${{ github.event.pull_request.number && format('pr={0},run={1}-{2},repo={3},owner={4}', github.event.pull_request.number, github.run_id, github.run_attempt, github.event.repository.name, github.repository_owner) || format('run={0}-{1},repo={2},owner={3}', github.run_id, github.run_attempt, github.event.repository.name, github.repository_owner) }} | |
| vm_user_passwd: ${{ secrets.VM_USER_PASSWD }} | |
| build-and-test: | |
| name: Build and test NBS (${{ inputs.build_preset }}) [id=${{ needs.provide-runner.outputs.instance-id }} local_ip=${{ needs.provide-runner.outputs.runner_local_ipv4 }} ip=${{ needs.provide-runner.outputs.runner_ipv4 }}] | |
| uses: ./.github/workflows/build_and_test_ya.yaml | |
| needs: | |
| - provide-runner | |
| with: | |
| runner_kind: self-hosted | |
| runner_label: ${{ needs.provide-runner.outputs.label }} | |
| runner_instance_id: ${{ needs.provide-runner.outputs.instance-id }} | |
| runner_ipv4: ${{ needs.provide-runner.outputs.runner_ipv4 }} | |
| build_target: ${{ inputs.build_target }} | |
| test_target: ${{ inputs.test_target }} | |
| build_preset: ${{ inputs.build_preset }} | |
| run_build: ${{ inputs.run_build }} | |
| run_tests: ${{ inputs.run_tests }} | |
| test_size: ${{ inputs.test_size }} | |
| test_type: ${{ inputs.test_type }} | |
| link_threads: ${{ inputs.link_threads || '24' }} | |
| test_threads: ${{ inputs.test_threads || '12' }} | |
| sleep_after_tests: ${{ inputs.sleep_after_tests }} | |
| cache_update_build: ${{ fromJson(github.event_name == 'workflow_dispatch' && format('{0}', inputs.cache_update) || format('{0}', inputs.cache_update_build)) }} | |
| cache_update_tests: ${{ fromJson(github.event_name == 'workflow_dispatch' && format('{0}', inputs.cache_update) || format('{0}', inputs.cache_update_tests)) }} | |
| upload_ya_dir: ${{ github.event == 'workflow_dispatch' && 'no' || inputs.upload_ya_dir}} | |
| clean_ya_dir: ${{ github.event == 'workflow_dispatch' && 'no' || inputs.clean_ya_dir }} | |
| use_network_cache: ${{ github.event == 'workflow_dispatch' && 'yes'|| inputs.use_network_cache }} | |
| secrets: inherit | |
| sleep-if-needed: | |
| name: Sleep ${{ needs.build-and-test.outputs.sleep_after_tests || '0' }}s if build failed | |
| if: ${{ !cancelled() }} | |
| needs: | |
| - provide-runner # required to get output from the start-runner job | |
| - build-and-test # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Sleep ${{ needs.build-and-test.outputs.sleep_after_tests || '0' }}s if build failed | |
| shell: bash | |
| if: ${{ needs.build-and-test.outputs.sleep_after_tests && needs.build-and-test.outputs.sleep_after_tests != '0' }} | |
| # we need to use binary because builting doesn't produce process | |
| # which we can kill to stop the vm before it's done | |
| run: | | |
| echo "sleeping ${{ needs.build-and-test.outputs.sleep_after_tests }}s" | |
| /usr/bin/sleep ${{ needs.build-and-test.outputs.sleep_after_tests }} | |
| release-runner: | |
| name: Release self-hosted runner (${{ inputs.build_preset }}) [id=${{ needs.provide-runner.outputs.instance-id }} local_ip=${{ needs.provide-runner.outputs.runner_local_ipv4 }} ip=${{ needs.provide-runner.outputs.runner_ipv4 }}] | |
| needs: | |
| - provide-runner # required to get output from the start-runner job | |
| - build-and-test # required to wait when the main job is done | |
| - sleep-if-needed # required to wait until all debug is done | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: ${{ !contains(github.event.pull_request.labels.*.name, 'rebase') && 1 || 0 }} | |
| - name: rebase PR | |
| if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }} | |
| shell: bash | |
| run: | | |
| git config user.email "[email protected]" | |
| git config user.name "Rebase Robotovich" | |
| git fetch origin ${{ github.event.pull_request.base.ref }} | |
| git rebase origin/${{ github.event.pull_request.base.ref }} | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Stop runner | |
| uses: ./.github/actions/runner_remove | |
| if: always() | |
| timeout-minutes: 60 | |
| with: | |
| service_account_key: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }} | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| repo_owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| vm_id: ${{ needs.provide-runner.outputs.instance-id }} |