ci: test aws codebuild

Signed-off-by: M. Fatih Cırıt <[email protected]>
autowarefoundation · Aug 20, 2024 · a37e933 · a37e933
1 parent 12f4d23
commit a37e933
Show file tree

Hide file tree

Showing 4 changed files with 155 additions and 12 deletions.
diff --git a/.github/workflows/build-and-test-codebuild.yaml b/.github/workflows/build-and-test-codebuild.yaml
@@ -0,0 +1,136 @@
+name: build-and-test-codebuild
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CC: /usr/lib/ccache/gcc
+  CXX: /usr/lib/ccache/g++
+
+jobs:
+  build-and-test:
+    runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large
+    container: ${{ matrix.container }}${{ matrix.container-suffix }}
+    strategy:
+      fail-fast: false
+      matrix:
+        rosdistro:
+          - humble
+        container-suffix:
+          - -cuda
+        include:
+          - rosdistro: humble
+            container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe
+            build-depends-repos: build_depends.repos
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Show disk space before the tasks
+        run: df -h
+
+      - name: Show machine specs
+        run: lscpu && free -h
+
+      - name: Remove exec_depend
+        uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
+
+      - name: Get self packages
+        id: get-self-packages
+        uses: autowarefoundation/autoware-github-actions/get-self-packages@v1
+
+      - name: Create ccache directory
+        run: |
+          mkdir -p ${CCACHE_DIR}
+          du -sh ${CCACHE_DIR} && ccache -s
+        shell: bash
+
+      - name: Attempt to restore ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /root/.ccache
+          key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }}
+          restore-keys: |
+            ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-
+
+      - name: Limit ccache size
+        run: |
+          rm -f "${CCACHE_DIR}/ccache.conf"
+          echo -e "# Set maximum cache size\nmax_size = 600MB" >> "${CCACHE_DIR}/ccache.conf"
+        shell: bash
+
+      - name: Show ccache stats before build
+        run: du -sh ${CCACHE_DIR} && ccache -s
+        shell: bash
+
+      - name: Export CUDA state as a variable for adding to cache key
+        run: |
+          build_type_cuda_state=nocuda
+          if [[ "${{ matrix.container-suffix }}" == "-cuda" ]]; then
+            build_type_cuda_state=cuda
+          fi
+          echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}"
+          echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state"
+        shell: bash
+
+      - name: Build
+        if: ${{ steps.get-self-packages.outputs.self-packages != '' }}
+        uses: autowarefoundation/autoware-github-actions/colcon-build@feat/allow-thread-count-control
+        with:
+          rosdistro: ${{ matrix.rosdistro }}
+          target-packages: ${{ steps.get-self-packages.outputs.self-packages }}
+          build-depends-repos: ${{ matrix.build-depends-repos }}
+          cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+          colcon-parallel-workers-flag: --parallel-workers 3
+          makeflags: -j2
+
+      - name: Show ccache stats after build
+        run: du -sh ${CCACHE_DIR} && ccache -s
+        shell: bash
+
+      # Only keep save the -cuda version because cuda packages covers non-cuda packages too
+      - name: Push the ccache cache
+        if: matrix.container-suffix == '-cuda'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            /root/.ccache
+          key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }}
+
+      - name: Test
+        if: ${{ steps.get-self-packages.outputs.self-packages != '' }}
+        id: test
+        uses: autowarefoundation/autoware-github-actions/colcon-test@v1
+        with:
+          rosdistro: ${{ matrix.rosdistro }}
+          target-packages: ${{ steps.get-self-packages.outputs.self-packages }}
+          build-depends-repos: ${{ matrix.build-depends-repos }}
+
+      - name: Upload coverage to CodeCov
+        if: ${{ steps.test.outputs.coverage-report-files != '' }}
+        uses: codecov/codecov-action@v4
+        with:
+          files: ${{ steps.test.outputs.coverage-report-files }}
+          fail_ci_if_error: false
+          verbose: true
+          flags: total
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Show disk space after the tasks
+        run: df -h
diff --git a/.github/workflows/build-and-test-daily-arm64.yaml b/.github/workflows/build-and-test-daily-arm64.yaml
@@ -5,9 +5,13 @@ on:
     - cron: 0 0 * * *
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   build-and-test-daily-arm64:
-    runs-on: [self-hosted, linux, ARM64]
+    runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large
     container: ${{ matrix.container }}${{ matrix.container-suffix }}
     strategy:
       fail-fast: false
@@ -49,7 +53,7 @@ jobs:
 
       - name: Build
         if: ${{ steps.get-self-packages.outputs.self-packages != '' }}
-        uses: autowarefoundation/autoware-github-actions/colcon-build@v1
+        uses: autowarefoundation/autoware-github-actions/colcon-build@feat/allow-thread-count-control
         with:
           rosdistro: ${{ matrix.rosdistro }}
           target-packages: ${{ steps.get-self-packages.outputs.self-packages }}

diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml
@@ -8,6 +8,10 @@ on:
       - reopened
       - labeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 env:
   CC: /usr/lib/ccache/gcc
   CXX: /usr/lib/ccache/g++
@@ -36,7 +40,7 @@ jobs:
             container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe
             build-depends-repos: build_depends.repos
           - container-suffix: -cuda
-            runner: [self-hosted, linux, X64, cpu]
+            runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large
           - container-suffix: ""
             runner: ubuntu-latest
     steps:
@@ -52,6 +56,9 @@ jobs:
       - name: Show disk space before the tasks
         run: df -h
 
+      - name: Show machine specs
+        run: lscpu && free -h
+
       - name: Remove exec_depend
         uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
 
@@ -74,14 +81,6 @@ jobs:
           restore-keys: |
             ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-
 
-      # Limit ccache size only for CUDA builds to avoid running out of disk space
-      - name: Limit ccache size
-        if: ${{ matrix.container-suffix == '-cuda' }}
-        run: |
-          rm -f "${CCACHE_DIR}/ccache.conf"
-          echo -e "# Set maximum cache size\nmax_size = 1MB" >> "${CCACHE_DIR}/ccache.conf"
-        shell: bash
-
       - name: Show ccache stats before build
         run: du -sh ${CCACHE_DIR} && ccache -s
         shell: bash
@@ -98,12 +97,14 @@ jobs:
 
       - name: Build
         if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }}
-        uses: autowarefoundation/autoware-github-actions/colcon-build@v1
+        uses: autowarefoundation/autoware-github-actions/colcon-build@feat/allow-thread-count-control
         with:
           rosdistro: ${{ matrix.rosdistro }}
           target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
           build-depends-repos: ${{ matrix.build-depends-repos }}
           cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+          colcon-parallel-workers-flag: --parallel-workers 3
+          makeflags: -j2
 
       - name: Show ccache stats after build
         run: du -sh ${CCACHE_DIR} && ccache -s

diff --git a/sensing/autoware_pointcloud_preprocessor/include/autoware/pointcloud_preprocessor/filter.hpp b/sensing/autoware_pointcloud_preprocessor/include/autoware/pointcloud_preprocessor/filter.hpp
@@ -85,6 +85,8 @@
 #include <autoware/universe_utils/ros/static_transform_buffer.hpp>
 #include <autoware/universe_utils/system/stop_watch.hpp>
 
+// TODO(xmfcx): remove this line before merging..
+
 namespace autoware::pointcloud_preprocessor
 {
 namespace sync_policies = message_filters::sync_policies;