From a37e933e997be830a541b5593a383f9daf9ce6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Fatih=20C=C4=B1r=C4=B1t?= Date: Tue, 20 Aug 2024 11:12:37 +0300 Subject: [PATCH] ci: test aws codebuild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: M. Fatih Cırıt --- .../workflows/build-and-test-codebuild.yaml | 136 ++++++++++++++++++ .../workflows/build-and-test-daily-arm64.yaml | 8 +- .../build-and-test-differential.yaml | 21 +-- .../pointcloud_preprocessor/filter.hpp | 2 + 4 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/build-and-test-codebuild.yaml diff --git a/.github/workflows/build-and-test-codebuild.yaml b/.github/workflows/build-and-test-codebuild.yaml new file mode 100644 index 0000000000000..96e16a1440dd9 --- /dev/null +++ b/.github/workflows/build-and-test-codebuild.yaml @@ -0,0 +1,136 @@ +name: build-and-test-codebuild + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + push: + branches: + - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +env: + CC: /usr/lib/ccache/gcc + CXX: /usr/lib/ccache/g++ + +jobs: + build-and-test: + runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large + container: ${{ matrix.container }}${{ matrix.container-suffix }} + strategy: + fail-fast: false + matrix: + rosdistro: + - humble + container-suffix: + - -cuda + include: + - rosdistro: humble + container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe + build-depends-repos: build_depends.repos + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Show disk space before the tasks + run: df -h + + - name: Show machine specs + run: lscpu && free -h + + - name: Remove exec_depend + uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 + + - name: Get self packages + id: get-self-packages + uses: autowarefoundation/autoware-github-actions/get-self-packages@v1 + + - name: Create ccache directory + run: | + mkdir -p ${CCACHE_DIR} + du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + - name: Attempt to restore ccache + uses: actions/cache/restore@v4 + with: + path: | + /root/.ccache + key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }} + restore-keys: | + ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}- + + - name: Limit ccache size + run: | + rm -f "${CCACHE_DIR}/ccache.conf" + echo -e "# Set maximum cache size\nmax_size = 600MB" >> "${CCACHE_DIR}/ccache.conf" + shell: bash + + - name: Show ccache stats before build + run: du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + - name: Export CUDA state as a variable for adding to cache key + run: | + build_type_cuda_state=nocuda + if [[ "${{ matrix.container-suffix }}" == "-cuda" ]]; then + build_type_cuda_state=cuda + fi + echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" + echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" + shell: bash + + - name: Build + if: ${{ steps.get-self-packages.outputs.self-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-build@feat/allow-thread-count-control + with: + rosdistro: ${{ matrix.rosdistro }} + target-packages: ${{ steps.get-self-packages.outputs.self-packages }} + build-depends-repos: ${{ matrix.build-depends-repos }} + cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + colcon-parallel-workers-flag: --parallel-workers 3 + makeflags: -j2 + + - name: Show ccache stats after build + run: du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + # Only keep save the -cuda version because cuda packages covers non-cuda packages too + - name: Push the ccache cache + if: matrix.container-suffix == '-cuda' + uses: actions/cache/save@v4 + with: + path: | + /root/.ccache + key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }} + + - name: Test + if: ${{ steps.get-self-packages.outputs.self-packages != '' }} + id: test + uses: autowarefoundation/autoware-github-actions/colcon-test@v1 + with: + rosdistro: ${{ matrix.rosdistro }} + target-packages: ${{ steps.get-self-packages.outputs.self-packages }} + build-depends-repos: ${{ matrix.build-depends-repos }} + + - name: Upload coverage to CodeCov + if: ${{ steps.test.outputs.coverage-report-files != '' }} + uses: codecov/codecov-action@v4 + with: + files: ${{ steps.test.outputs.coverage-report-files }} + fail_ci_if_error: false + verbose: true + flags: total + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Show disk space after the tasks + run: df -h diff --git a/.github/workflows/build-and-test-daily-arm64.yaml b/.github/workflows/build-and-test-daily-arm64.yaml index 0d71e13a9ff58..76ce793bb986f 100644 --- a/.github/workflows/build-and-test-daily-arm64.yaml +++ b/.github/workflows/build-and-test-daily-arm64.yaml @@ -5,9 +5,13 @@ on: - cron: 0 0 * * * workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: build-and-test-daily-arm64: - runs-on: [self-hosted, linux, ARM64] + runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large container: ${{ matrix.container }}${{ matrix.container-suffix }} strategy: fail-fast: false @@ -49,7 +53,7 @@ jobs: - name: Build if: ${{ steps.get-self-packages.outputs.self-packages != '' }} - uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + uses: autowarefoundation/autoware-github-actions/colcon-build@feat/allow-thread-count-control with: rosdistro: ${{ matrix.rosdistro }} target-packages: ${{ steps.get-self-packages.outputs.self-packages }} diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index d41e6cf481b76..232e0a63e2ec5 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -8,6 +8,10 @@ on: - reopened - labeled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + env: CC: /usr/lib/ccache/gcc CXX: /usr/lib/ccache/g++ @@ -36,7 +40,7 @@ jobs: container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe build-depends-repos: build_depends.repos - container-suffix: -cuda - runner: [self-hosted, linux, X64, cpu] + runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large - container-suffix: "" runner: ubuntu-latest steps: @@ -52,6 +56,9 @@ jobs: - name: Show disk space before the tasks run: df -h + - name: Show machine specs + run: lscpu && free -h + - name: Remove exec_depend uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 @@ -74,14 +81,6 @@ jobs: restore-keys: | ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}- - # Limit ccache size only for CUDA builds to avoid running out of disk space - - name: Limit ccache size - if: ${{ matrix.container-suffix == '-cuda' }} - run: | - rm -f "${CCACHE_DIR}/ccache.conf" - echo -e "# Set maximum cache size\nmax_size = 1MB" >> "${CCACHE_DIR}/ccache.conf" - shell: bash - - name: Show ccache stats before build run: du -sh ${CCACHE_DIR} && ccache -s shell: bash @@ -98,12 +97,14 @@ jobs: - name: Build if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} - uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + uses: autowarefoundation/autoware-github-actions/colcon-build@feat/allow-thread-count-control with: rosdistro: ${{ matrix.rosdistro }} target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} build-depends-repos: ${{ matrix.build-depends-repos }} cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + colcon-parallel-workers-flag: --parallel-workers 3 + makeflags: -j2 - name: Show ccache stats after build run: du -sh ${CCACHE_DIR} && ccache -s diff --git a/sensing/autoware_pointcloud_preprocessor/include/autoware/pointcloud_preprocessor/filter.hpp b/sensing/autoware_pointcloud_preprocessor/include/autoware/pointcloud_preprocessor/filter.hpp index 789f377e7ec95..9b703112515cb 100644 --- a/sensing/autoware_pointcloud_preprocessor/include/autoware/pointcloud_preprocessor/filter.hpp +++ b/sensing/autoware_pointcloud_preprocessor/include/autoware/pointcloud_preprocessor/filter.hpp @@ -85,6 +85,8 @@ #include #include +// TODO(xmfcx): remove this line before merging.. + namespace autoware::pointcloud_preprocessor { namespace sync_policies = message_filters::sync_policies;