Build stages #2302
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Requires secret `ACTIONS_WORKFLOW_TOKEN` with `workflow` scope | |
| name: Build stages | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| base_ref: | |
| description: 'Base reference if triggered by a pull request' | |
| required: false | |
| default: "" | |
| jobs: | |
| update-check: | |
| name: Update GitHub check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Update check's target URL | |
| uses: Sibz/[email protected] | |
| with: | |
| authToken: ${{ secrets.ACTIONS_WORKFLOW_TOKEN }} | |
| context: "Build C-PAC images" | |
| state: pending | |
| target_url: ${{ github.server_url }}/${{ github.repository}}/actions/runs/${{ github.run_id }} | |
| Ubuntu: | |
| name: Build C-PAC Ubuntu | |
| needs: | |
| - update-check | |
| strategy: | |
| matrix: | |
| Dockerfile: | |
| - Ubuntu.bionic-non-free | |
| - Ubuntu.xenial-20200114 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out C-PAC (depth 0) | |
| if: github.event.inputs.base_ref != '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out C-PAC (depth 2) | |
| if: github.event.inputs.base_ref == '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Get changed files since fork point | |
| uses: tj-actions/[email protected] | |
| if: github.event.inputs.base_ref != '' | |
| id: changed-files-base | |
| with: | |
| use_fork_point: "true" | |
| files: .github/Dockerfiles/* | |
| - name: Get changed files since last commit | |
| uses: tj-actions/[email protected] | |
| if: github.event.inputs.base_ref == '' | |
| id: changed-files | |
| with: | |
| since_last_remote_commit: "true" | |
| files: .github/Dockerfiles/* | |
| - name: Set tag & see if it exists | |
| continue-on-error: true | |
| run: | | |
| TAG=$(sed 's/\./:/' <(echo ${{ matrix.Dockerfile }})) | |
| DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]') | |
| echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV | |
| docker manifest inspect $DOCKER_TAG >/dev/null || echo "not_yet_exists=1" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) | |
| - name: Build and push Docker image | |
| uses: docker/[email protected] | |
| with: | |
| file: .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.DOCKER_TAG }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) | |
| stages: | |
| name: Build Docker stage images | |
| needs: Ubuntu | |
| strategy: | |
| matrix: | |
| Dockerfile: | |
| - AFNI.16.2.07.neurodocker-xenial | |
| - AFNI.21.1.00-bionic | |
| - ANTs.2.2.0.neurodocker-bionic | |
| - c3d.1.0.0-xenial | |
| - connectome-workbench.1.3.2-1.neurodebian-bionic | |
| - connectome-workbench.1.3.2-2.neurodebian-xenial | |
| - FSL.5.0.9-5.neurodebian-xenial | |
| - FSL.5.0.10-bionic | |
| - ICA-AROMA.0.4.5-xenial | |
| - msm.2.0-bionic | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out C-PAC (depth 0) | |
| if: github.event.inputs.base_ref != '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out C-PAC (depth 2) | |
| if: github.event.inputs.base_ref == '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Get changed files since fork point | |
| uses: tj-actions/[email protected] | |
| if: github.event.inputs.base_ref != '' | |
| id: changed-files-base | |
| with: | |
| use_fork_point: "true" | |
| files: | | |
| dev/docker_data/required_afni_pkgs.txt | |
| .github/Dockerfiles/* | |
| - name: Get changed files since last commit | |
| uses: tj-actions/[email protected] | |
| if: github.event.inputs.base_ref == '' | |
| id: changed-files | |
| with: | |
| since_last_remote_commit: "true" | |
| files: | | |
| dev/docker_data/required_afni_pkgs.txt | |
| .github/Dockerfiles/* | |
| - name: Clear up some space on runner | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Set tag & see if it exists | |
| continue-on-error: true | |
| run: | | |
| TAG=$(sed 's/\./:/' <(echo ${{ matrix.Dockerfile }})) | |
| DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]') | |
| echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV | |
| docker manifest inspect $DOCKER_TAG >/dev/null || echo "not_yet_exists=1" >> $GITHUB_OUTPUT | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Prep Dockerfiles for forked repository | |
| if: github.repository_owner != 'FCP-INDI' && contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| run: | | |
| .github/scripts/local_ghcr .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile ${{ github.repository_owner }} $DOCKER_TAG | |
| cat .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| - name: See Dockerfile | |
| run: cat .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Build and push Docker image | |
| uses: docker/[email protected] | |
| with: | |
| context: . | |
| file: .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.DOCKER_TAG }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| build-base: | |
| needs: stages | |
| runs-on: ubuntu-latest | |
| env: | |
| BUILD_CACHE: /home/runner/.docker/buildkit | |
| strategy: | |
| matrix: | |
| variant: | |
| - standard | |
| - ABCD-HCP | |
| - fMRIPrep-LTS | |
| steps: | |
| - name: Maximize build space | |
| uses: easimon/maximize-build-space@v4 | |
| with: | |
| remove-dotnet: 'true' | |
| remove-android: 'true' | |
| remove-haskell: 'true' | |
| overprovision-lvm: 'true' | |
| - name: Check out C-PAC (depth 0) | |
| if: github.event.inputs.base_ref != '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out C-PAC (depth 2) | |
| if: github.event.inputs.base_ref == '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Prep source files | |
| run: | | |
| sed -i -e 's/^/\.github\/Dockerfiles\//' .github/stage_requirements/${{ matrix.variant }}.txt | |
| echo 'dev/docker_data/required_afni_pkgs.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt | |
| echo '.github/workflows/build_stages.yml' >> .github/stage_requirements/${{ matrix.variant }}.txt | |
| echo '.github/stage_requirements/${{ matrix.variant }}.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt | |
| - name: Get changed files since fork point | |
| uses: tj-actions/[email protected] | |
| if: github.event.inputs.base_ref != '' | |
| id: changed-files-base | |
| with: | |
| use_fork_point: "true" | |
| files_from_source_file: | | |
| .github/stage_requirements/${{ matrix.variant }}.txt | |
| - name: Get changed files since last commit | |
| uses: tj-actions/[email protected] | |
| if: github.event.inputs.base_ref == '' | |
| id: changed-files | |
| with: | |
| since_last_remote_commit: "true" | |
| files_from_source_file: | | |
| .github/stage_requirements/${{ matrix.variant }}.txt | |
| - name: Set tag & see if it exists | |
| continue-on-error: true | |
| run: | | |
| TAG="stage-base:${{ matrix.variant }}-$(cat version)" | |
| DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]') | |
| echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV | |
| docker manifest inspect $DOCKER_TAG >/dev/null || echo "not_yet_exists=1" >> $GITHUB_OUTPUT | |
| id: docker_tag | |
| - name: Clear up some space on runner | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| - name: Prep Dockerfiles for forked repository | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| run: | | |
| .github/scripts/local_ghcr .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile ${{ github.repository_owner }} $DOCKER_TAG | |
| cat .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile | |
| - name: See Dockerfile | |
| run: cat .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| - name: Build and push base image | |
| uses: docker/[email protected] | |
| with: | |
| context: . | |
| file: .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.DOCKER_TAG }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| trigger-next-workflow: | |
| needs: build-base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out C-PAC | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Build and test C-PAC | |
| run: gh workflow run build_and_test.yml --ref ${GITHUB_REF_NAME} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.ACTIONS_WORKFLOW_TOKEN }} |