Handle transfers file correctly in acquisition workflow (relates to #314

)

.github/workflows/acquire-transfermarkt-api.yml

@@ -18,7 +18,7 @@ env:
   DATA_DIR: data/raw/transfermarkt-api/${{ github.event_name == 'workflow_dispatch' && github.event.inputs.season || '2024' }}
 
 jobs:
-  acquire-market-values:
+  acquire:
     runs-on: ubuntu-latest
     container:
       image: dcaribou/transfermarkt-datasets:linux-amd64-master
@@ -43,6 +43,10 @@ jobs:
         with:
           name: market_values
           path: ${{ env.DATA_DIR }}/market_values.json
+      - uses: actions/upload-artifact@v3
+        with:
+          name: transfers
+          path: ${{ env.DATA_DIR }}/transfers.json
 
   dvc-push:
     runs-on: ubuntu-latest
@@ -52,7 +56,7 @@ jobs:
       run:
         shell: bash -l {0}
     needs:
-      - acquire-market-values
+      - acquire
     # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-a-workflow-based-on-the-conclusion-of-another-workflow
     if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
     steps:
@@ -66,6 +70,10 @@ jobs:
         with:
           name: market_values
           path: ${{ env.DATA_DIR }}
+      - uses: actions/download-artifact@v3
+        with:
+          name: transfers
+          path: ${{ env.DATA_DIR }}
       - name: dvc commit and push
         run: |
           dvc commit -f data/raw/transfermarkt-api && dvc push --remote s3

data/prep.dvc

@@ -1,6 +1,6 @@
 outs:
-- md5: 4f2abdfbeef0766067afba485465eb86.dir
-  size: 159682086
+- md5: 4640657d140104cc13fefc46dcbae881.dir
+  size: 160174744
   nfiles: 11
   path: prep
   hash: md5

data/raw/transfermarkt-api.dvc

@@ -1,6 +1,6 @@
 outs:
-- md5: ba9322acc167f92e79d1556ee0a9155d.dir
-  size: 573915471
+- md5: 0fdb03f9008d32b8f9b144c84e378b19.dir
+  size: 575855231
   nfiles: 15
   hash: md5
   path: transfermarkt-api

dbt/models/base/transfermarkt_api/base_market_value_development.sql

@@ -52,7 +52,12 @@ select
     player_id,
     {{ parse_market_value("json_row ->> 'mw'") }} as market_value_in_eur,
     datetime_str,
-    ("datetime")::date as "date",
+    -- we came across some dates in the future, which are clearly wrong
+    -- so are we are truncating them to the current date
+    case
+        when ("datetime")::date > current_date then current_date
+        else ("datetime")::date
+    end as "date",
     season as last_season,
     filename