-
-
Notifications
You must be signed in to change notification settings - Fork 55
98 lines (84 loc) · 2.83 KB
/
on-push.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
name: build
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
build:
concurrency: build_queue
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}
steps:
- name: checkout
uses: actions/checkout@v3
with:
token: ${{ secrets.PA_GITHUB_TOKEN }}
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
- name: Install library
run: poetry install --no-interaction
- name: Get branch name
id: branch-name
uses: tj-actions/branch-names@v7
- name: Run tests
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_EC2_METADATA_DISABLED: true # https://github.com/aws/aws-cli/issues/5262
run: |
set -e
source .venv/bin/activate
make dvc_pull test prepare_local DBT_TARGET=prod
dvc commit -f
- uses: EndBug/add-and-commit@v9
with:
add: 'data/*.dvc'
message: '🤖 updated prepared dataset files'
default_author: github_actions
new_branch: ${{ steps.branch-name.outputs.current_branch }}
- name: Push to dvc
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
source .venv/bin/activate
dvc push
- name: image build
env:
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
# set image tag accordingly
REF=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }')
if [ $REF = master ];
then
TAG=master
else
TAG=dev
fi
# build and push
make docker_build docker_push_dockerhub TAG=$TAG PLATFORM=linux/amd64
- name: run sync
# only sync when there is push to master
if: ${{ github.event_name == 'push' && github.event.head_commit.message != '🤖 updated raw dataset files' }}
env:
# Kaggle API auth
KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
# data.world API auth
DW_AUTH_TOKEN: ${{ secrets.DW_AUTH_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_EC2_METADATA_DISABLED: true # https://github.com/aws/aws-cli/issues/5262
run: |
source .venv/bin/activate
dvc pull data/prep
make \
sync \
MESSAGE="${{ github.event.head_commit.message }}"