🤖 updated prepared dataset files #751
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| jobs: | |
| build: | |
| concurrency: build_queue | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.PA_GITHUB_TOKEN }} | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Install library | |
| run: poetry install --no-interaction | |
| - name: Get branch name | |
| id: branch-name | |
| uses: tj-actions/branch-names@v7 | |
| - name: Run tests | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_EC2_METADATA_DISABLED: true # https://github.com/aws/aws-cli/issues/5262 | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| make dvc_pull test prepare_local | |
| dvc commit -f | |
| - uses: EndBug/add-and-commit@v9 | |
| with: | |
| add: 'data/*.dvc' | |
| message: '🤖 updated prepared dataset files' | |
| default_author: github_actions | |
| new_branch: ${{ steps.branch-name.outputs.current_branch }} | |
| - name: Push to dvc | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| source .venv/bin/activate | |
| dvc push | |
| - name: image build | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| run: | | |
| # set image tag accordingly | |
| REF=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') | |
| if [ $REF = master ]; | |
| then | |
| TAG=master | |
| else | |
| TAG=dev | |
| fi | |
| # build and push | |
| make docker_build docker_push_dockerhub TAG=$TAG PLATFORM=linux/amd64 | |
| - name: run sync | |
| # only sync when there is push to master | |
| if: ${{ github.event_name == 'push' && github.event.head_commit.message != '🤖 updated raw dataset files' }} | |
| env: | |
| # Kaggle API auth | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} | |
| # data.world API auth | |
| DW_AUTH_TOKEN: ${{ secrets.DW_AUTH_TOKEN }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_EC2_METADATA_DISABLED: true # https://github.com/aws/aws-cli/issues/5262 | |
| run: | | |
| source .venv/bin/activate | |
| dvc pull data/prep | |
| make \ | |
| sync \ | |
| MESSAGE="${{ github.event.head_commit.message }}" |