diff --git a/api/app/datalake.py b/api/app/datalake.py index 801c28e..7d3b1c1 100644 --- a/api/app/datalake.py +++ b/api/app/datalake.py @@ -3,6 +3,7 @@ import uuid import glob from typing import Optional +import base64 import pandas as pd import basedosdados as bd @@ -32,6 +33,16 @@ def __init__( self._base_path = os.path.join(os.getcwd(), "/files") + def _prepare_gcp_credential(self) -> None: + base64_credential = os.environ["GCP_JSON_CREDENTIAL"] + + with open("/tmp/credentials.json", "wb") as f: + f.write(base64.b64decode(base64_credential)) + + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/tmp/credentials.json" + return + + def _split_dataframe_per_day( self, df: pd.DataFrame, @@ -79,6 +90,8 @@ def _upload_file( table_id: str, source_format: str = "parquet" ) -> None: + self._prepare_gcp_credential() + tb = bd.Table(dataset_id=dataset_id, table_id=table_id) table_staging = f"{tb.table_full_name['staging']}"