autowarefoundation · xmfcx · Aug 20, 2024 · Aug 20, 2024 · Aug 20, 2024 · Aug 20, 2024
diff --git a/.github/README.md b/.github/README.md
@@ -0,0 +1,55 @@
+# Overall CI Infrastructure
+
+## Machine Types
+
+### Standard GitHub-hosted runners
+
+- [Documentation](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories)
+
+These runners are utilized by the majority of the workflows.
+They are free to use for public repositories, with a concurrency limit of 20 jobs per organization.
+
+**Listed specs:**
+
+| vCPU | RAM   | Storage (SSD) |
+| ---- | ----- | ------------- |
+| 4    | 16 GB | 14 GB         |
+
+**Note:** While the official documentation lists 14 GB of storage, the actual available storage is approximately 73 GB.
+
+### AWS CodeBuild runners
+
+- [AWS CodeBuild Documentation](https://docs.aws.amazon.com/codebuild/latest/userguide/action-runner.html)
+
+These runners are employed for workflows that require more resources and are funded by the Autoware Foundation budget.
+
+**Relevant machine types:**
+
+| Instance Type   | Memory | vCPUs | Price per Minute |
+| --------------- | ------ | ----- | ---------------- |
+| arm1.large      | 16 GiB | 8     | $0.015           |
+| general1.medium | 7 GB   | 4     | $0.01            |
+| general1.large  | 15 GB  | 8     | $0.02            |
+| general1.xlarge | 72 GiB | 36    | $0.0798          |
+| gpu1.small      | 15 GB  | 4     | $0.05            |
+
+**Sources:**
+
+- [Compute images supported with the CodeBuild-hosted GitHub Actions runner](https://docs.aws.amazon.com/codebuild/latest/userguide/sample-github-action-runners-update-yaml.images.html)
+- [AWS CodeBuild pricing](https://aws.amazon.com/codebuild/pricing/)
+
+## Key workflows and their runners
+
+| Workflow                           | Trigger               | Runner         |
+| ---------------------------------- | --------------------- | -------------- |
+| build-and-test (cuda)              | merge to main         | general1.large |
+| build-and-test-daily               | daily on main         | github-std     |
+| build-and-test-daily-arm64         | daily on main         | arm1.large     |
+| build-and-test-differential        | PR update             | github-std     |
+| build-and-test-differential (cuda) | PR update             | general1.large |
+| build-and-test-differential-arm64  | PR update (arm label) | arm1.large     |
+
+## Additional notes
+
+- We use [`taskset`](https://manpages.ubuntu.com/manpages/jammy/man1/taskset.1.html) from GNU Coreutils to limit the number of cores utilized by build processes. This is done to prevent overloading the self-hosted runners.
+  - The number of cores is limited to `vCPU count - 1`.
diff --git a/.github/workflows/build-and-test-daily-arm64.yaml b/.github/workflows/build-and-test-daily-arm64.yaml
@@ -5,9 +5,13 @@
     - cron: 0 0 * * *
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   build-and-test-daily-arm64:
-    runs-on: [self-hosted, linux, ARM64]
+    runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large
     container: ${{ matrix.container }}${{ matrix.container-suffix }}
     strategy:
       fail-fast: false
@@ -30,6 +34,9 @@
       - name: Show disk space before the tasks
         run: df -h
 
+      - name: Show machine specs
+        run: lscpu && free -h
+
       - name: Remove exec_depend
         uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
 
@@ -55,6 +62,7 @@
           target-packages: ${{ steps.get-self-packages.outputs.self-packages }}
           build-depends-repos: ${{ matrix.build-depends-repos }}
           cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+          build-pre-command: taskset --cpu-list 0-6
 
       - name: Test
         if: ${{ steps.get-self-packages.outputs.self-packages != '' }}

diff --git a/.github/workflows/build-and-test-differential-arm64.yaml b/.github/workflows/build-and-test-differential-arm64.yaml
@@ -8,6 +8,10 @@
       - reopened
       - labeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   make-sure-label-is-present:
     uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1
@@ -17,7 +21,7 @@
   build-and-test-differential-arm64:
     needs: make-sure-label-is-present
     if: ${{ needs.make-sure-label-is-present.outputs.result == 'true' }}
-    runs-on: [self-hosted, linux, ARM64]
+    runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large
     container: ${{ matrix.container }}${{ matrix.container-suffix }}
     strategy:
       fail-fast: false
@@ -44,6 +48,9 @@
       - name: Show disk space before the tasks
         run: df -h
 
+      - name: Show machine specs
+        run: lscpu && free -h
+
       - name: Remove exec_depend
         uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
 
@@ -69,6 +76,7 @@
           target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
           build-depends-repos: ${{ matrix.build-depends-repos }}
           cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+          build-pre-command: taskset --cpu-list 0-6
 
       - name: Test
         id: test

diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml
@@ -8,6 +8,10 @@
       - reopened
       - labeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 env:
   CC: /usr/lib/ccache/gcc
   CXX: /usr/lib/ccache/g++
@@ -36,9 +40,11 @@
             container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe
             build-depends-repos: build_depends.repos
           - container-suffix: -cuda
-            runner: [self-hosted, linux, X64, cpu]
+            runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large
+            build-pre-command: taskset --cpu-list 0-6
           - container-suffix: ""
             runner: ubuntu-latest
+            build-pre-command: ""
     steps:
       - name: Set PR fetch depth
         run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
@@ -52,6 +58,9 @@
       - name: Show disk space before the tasks
         run: df -h
 
+      - name: Show machine specs
+        run: lscpu && free -h
+
       - name: Remove exec_depend
         uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
 
@@ -74,14 +83,6 @@
           restore-keys: |
             ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-
 
-      # Limit ccache size only for CUDA builds to avoid running out of disk space
-      - name: Limit ccache size
-        if: ${{ matrix.container-suffix == '-cuda' }}
-        run: |
-          rm -f "${CCACHE_DIR}/ccache.conf"
-          echo -e "# Set maximum cache size\nmax_size = 1MB" >> "${CCACHE_DIR}/ccache.conf"
-        shell: bash
-
       - name: Show ccache stats before build
         run: du -sh ${CCACHE_DIR} && ccache -s
         shell: bash
@@ -104,6 +105,7 @@
           target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
           build-depends-repos: ${{ matrix.build-depends-repos }}
           cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+          build-pre-command: ${{ matrix.build-pre-command }}
 
       - name: Show ccache stats after build
         run: du -sh ${CCACHE_DIR} && ccache -s

diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -6,13 +6,17 @@
       - main
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 env:
   CC: /usr/lib/ccache/gcc
   CXX: /usr/lib/ccache/g++
 
 jobs:
   build-and-test:
-    runs-on: [self-hosted, linux, X64, gpu]
+    runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large
     container: ${{ matrix.container }}${{ matrix.container-suffix }}
     strategy:
       fail-fast: false
@@ -34,6 +38,9 @@
       - name: Show disk space before the tasks
         run: df -h
 
+      - name: Show machine specs
+        run: lscpu && free -h
+
       - name: Remove exec_depend
         uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
 
@@ -84,6 +91,7 @@
           target-packages: ${{ steps.get-self-packages.outputs.self-packages }}
           build-depends-repos: ${{ matrix.build-depends-repos }}
           cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+          build-pre-command: taskset --cpu-list 0-6
 
       - name: Show ccache stats after build
         run: du -sh ${CCACHE_DIR} && ccache -s