Skip to content

Improved the current SDA workflow to reach the North American runs with 6400 sites. #5974

Improved the current SDA workflow to reach the North American runs with 6400 sites.

Improved the current SDA workflow to reach the North American runs with 6400 sites. #5974

Workflow file for this run

name: Docker GHA
on:
push:
branches:
- main
- develop
tags:
- "v*.*.*"
pull_request:
merge_group:
workflow_dispatch:
inputs:
r_version:
description: 'R version to use'
required: true
type: choice
default: "4.1"
options:
- 4.1
- 4.2
- 4.3
- 4.4
- devel
# set up the environment, either input or default
env:
R_VERSION: ${{ github.event.inputs.r_version || '4.1' }}
# there are 3 jobs to build different images
jobs:
# ----------------------------------------------------------------------
# depends image has all the dependencies installed
# ----------------------------------------------------------------------
depends:
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- uses: actions/checkout@v4
# create metadata for image
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
# list of Docker images to use as base name for tags
images: |
pecan/depends
ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/depends
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# setup docker build
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Inspect Builder
run: |
echo "Name: ${{ steps.buildx.outputs.name }}"
echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
echo "Status: ${{ steps.buildx.outputs.status }}"
echo "Flags: ${{ steps.buildx.outputs.flags }}"
echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
# login to registries
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# build the docker images
- name: Build and push depends
uses: docker/build-push-action@v6
with:
context: docker/depends
file: docker/depends/Dockerfile
push: true
platforms: "linux/amd64"
cache-from: type=registry,ref=pecan/depends:buildcache
cache-to: type=registry,ref=pecan/depends:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
R_VERSION=${{ env.R_VERSION }}
GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
# ----------------------------------------------------------------------
# base image has PEcAn compiled and installed, and depends on depends
# ----------------------------------------------------------------------
base:
runs-on: ubuntu-latest
needs: depends
permissions:
packages: write
steps:
- uses: actions/checkout@v4
# create metadata for image
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
# list of Docker images to use as base name for tags
images: |
pecan/base
ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/base
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# setup docker build
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Inspect Builder
run: |
echo "Name: ${{ steps.buildx.outputs.name }}"
echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
echo "Status: ${{ steps.buildx.outputs.status }}"
echo "Flags: ${{ steps.buildx.outputs.flags }}"
echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
# login to registries
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# build the docker images
- name: Build and push base
uses: docker/build-push-action@v6
with:
context: .
file: docker/base/Dockerfile
push: true
platforms: "linux/amd64"
cache-from: type=registry,ref=pecan/base:buildcache
cache-to: type=registry,ref=pecan/base:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
R_VERSION=${{ env.R_VERSION }}
FROM_IMAGE=depends
IMAGE_VERSION=${{ steps.meta.outputs.version }}
GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
PECAN_VERSION=${{ steps.meta.outputs.version }}
PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }}
PECAN_GIT_CHECKSUM=${{ github.sha }}
PECAN_GIT_DATE=${{ github.event.repository.updated_at }}
# ----------------------------------------------------------------------
# models image has some python installed to run models, depends on base
# ----------------------------------------------------------------------
models:
runs-on: ubuntu-latest
needs: base
permissions:
packages: write
steps:
- uses: actions/checkout@v4
# create metadata for image
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
# list of Docker images to use as models name for tags
images: |
pecan/models
ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/models
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# setup docker build
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Inspect Builder
run: |
echo "Name: ${{ steps.buildx.outputs.name }}"
echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
echo "Status: ${{ steps.buildx.outputs.status }}"
echo "Flags: ${{ steps.buildx.outputs.flags }}"
echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
# login to registries
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# build the docker images
- name: Build and push models
uses: docker/build-push-action@v6
with:
context: docker/models
file: docker/models/Dockerfile
push: true
platforms: "linux/amd64"
cache-from: type=registry,ref=pecan/models:buildcache
cache-to: type=registry,ref=pecan/models:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
R_VERSION=${{ env.R_VERSION }}
FROM_IMAGE=depends
IMAGE_VERSION=${{ steps.meta.outputs.version }}
GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
PECAN_VERSION=${{ steps.meta.outputs.version }}
PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }}
PECAN_GIT_CHECKSUM=${{ github.sha }}
PECAN_GIT_DATE=${{ github.event.repository.updated_at }}
# ----------------------------------------------------------------------
# Next are images that have models installed
# ----------------------------------------------------------------------
modelsbinary:
runs-on: ubuntu-latest
needs: models
permissions:
packages: write
strategy:
fail-fast: false
matrix:
name:
- basgra
- biocro
# - ed2_2.2.0
- ed2_git
- maespa
- sipnet
include:
- name: basgra
CONTEXT: models/basgra
DOCKERFILE: models/basgra/Dockerfile
PLATFORM: "linux/amd64"
MODEL: basgra
VERSION: BASGRA_N_v1
- name: biocro
CONTEXT: models/biocro
DOCKERFILE: models/biocro/Dockerfile
PLATFORM: "linux/amd64"
MODEL: biocro
VERSION: "0.95"
# - name: ed2_2.2.0
# CONTEXT: models/ed
# DOCKERFILE: models/ed/Dockerfile
# PLATFORM: "linux/amd64"
# MODEL: ed2
# VERSION: "2.2.0"
- name: ed2_git
CONTEXT: models/ed
DOCKERFILE: models/ed/Dockerfile
PLATFORM: "linux/amd64"
MODEL: ed2
VERSION: "git"
- name: maespa
CONTEXT: models/maespa
DOCKERFILE: models/maespa/Dockerfile
PLATFORM: "linux/amd64"
MODEL: maespa
VERSION: "git"
- name: sipnet
CONTEXT: models/sipnet
DOCKERFILE: models/sipnet/Dockerfile
PLATFORM: "linux/amd64"
MODEL: sipnet
VERSION: "git"
steps:
- uses: actions/checkout@v4
# lower case name for docker
- name: docker image name
id: lower
run: echo "image_name=$(echo model-${{ matrix.MODEL }}-${{ matrix.VERSION }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
# create metadata for image
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
# list of Docker images to use as base name for tags
images: |
pecan/${{ steps.lower.outputs.image_name }}
ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ steps.lower.outputs.image_name }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# setup docker build
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Inspect Builder
run: |
echo "Name: ${{ steps.buildx.outputs.name }}"
echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
echo "Status: ${{ steps.buildx.outputs.status }}"
echo "Flags: ${{ steps.buildx.outputs.flags }}"
echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
# login to registries
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: docker image ls
run: docker image ls
# build the docker images
- name: Build and push ${{ matrix.MODEL }} ${{ matrix.VERSION }}
uses: docker/build-push-action@v6
with:
context: ${{ matrix.CONTEXT }}
file: ${{ matrix.DOCKERFILE }}
push: true
platforms: ${{ matrix.PLATFORM }}
cache-from: type=registry,ref=pecan/${{ steps.lower.outputs.image_name }}:buildcache
cache-to: type=registry,ref=pecan/${{ steps.lower.outputs.image_name }}:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
R_VERSION=${{ env.R_VERSION }}
MODEL_VERSION=${{ matrix.VERSION }}
IMAGE_VERSION=${{ steps.meta.outputs.version }}
# ----------------------------------------------------------------------
# Next are images that depend on base image
# ----------------------------------------------------------------------
baseplus:
runs-on: ubuntu-latest
needs: base
permissions:
packages: write
strategy:
fail-fast: false
matrix:
name:
- docs
- executor
- api
include:
- name: docs
CONTEXT: .
DOCKERFILE: docker/docs/Dockerfile
PLATFORM: "linux/amd64"
IMAGE: docs
- name: executor
CONTEXT: docker/executor
DOCKERFILE: docker/executor/Dockerfile
PLATFORM: "linux/amd64"
IMAGE: executor
- name: api
CONTEXT: apps/api
DOCKERFILE: apps/api/Dockerfile
PLATFORM: "linux/amd64"
IMAGE: api
steps:
- uses: actions/checkout@v4
# create metadata for image
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
# list of Docker images to use as base name for tags
images: |
pecan/${{ matrix.IMAGE }}
ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# setup docker build
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Inspect Builder
run: |
echo "Name: ${{ steps.buildx.outputs.name }}"
echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
echo "Status: ${{ steps.buildx.outputs.status }}"
echo "Flags: ${{ steps.buildx.outputs.flags }}"
echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
# login to registries
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# build the docker images
- name: Build and push ${{ matrix.name }}
uses: docker/build-push-action@v6
with:
context: ${{ matrix.CONTEXT }}
file: ${{ matrix.DOCKERFILE }}
push: true
platforms: ${{ matrix.PLATFORM }}
cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache
cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
R_VERSION=${{ env.R_VERSION }}
IMAGE_VERSION=${{ steps.meta.outputs.version }}
# ----------------------------------------------------------------------
# Next are images that do not depend on either depends or base image
# ----------------------------------------------------------------------
extras:
runs-on: ubuntu-latest
permissions:
packages: write
strategy:
fail-fast: false
matrix:
name:
- web
- dbsync
- data
- monitor
- rstudio-nginx
include:
- name: web
CONTEXT: .
DOCKERFILE: docker/web/Dockerfile
PLATFORM: "linux/amd64,linux/arm64"
IMAGE: web
- name: dbsync
CONTEXT: .
DOCKERFILE: shiny/dbsync/Dockerfile
PLATFORM: "linux/amd64"
IMAGE: shiny-dbsync
- name: data
CONTEXT: docker/data
DOCKERFILE: docker/data/Dockerfile
PLATFORM: "linux/amd64,linux/arm64"
IMAGE: data
- name: monitor
CONTEXT: docker/monitor
DOCKERFILE: docker/monitor/Dockerfile
PLATFORM: "linux/amd64,linux/arm64"
IMAGE: monitor
- name: rstudio-nginx
CONTEXT: docker/rstudio-nginx
DOCKERFILE: docker/rstudio-nginx/Dockerfile
PLATFORM: "linux/amd64,linux/arm64"
IMAGE: rstudio-nginx
steps:
- uses: actions/checkout@v4
# create metadata for image
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
# list of Docker images to use as base name for tags
images: |
pecan/${{ matrix.IMAGE }}
ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# setup docker build
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Inspect Builder
run: |
echo "Name: ${{ steps.buildx.outputs.name }}"
echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
echo "Status: ${{ steps.buildx.outputs.status }}"
echo "Flags: ${{ steps.buildx.outputs.flags }}"
echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
# login to registries
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# build the docker images
- name: Build and push ${{ matrix.name }}
uses: docker/build-push-action@v6
with:
context: ${{ matrix.CONTEXT }}
file: ${{ matrix.DOCKERFILE }}
push: true
platforms: ${{ matrix.PLATFORM }}
cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache
cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
R_VERSION=${{ env.R_VERSION }}