feat: Using Query Preview from Big Query

app/config/__init__.py

@@ -81,7 +81,7 @@ def inject_environment_variables(environment: str):
         f"Injecting {len(secrets)} environment variables from Infisical:")
     for secret in secrets:
         logger.info(
-            f" - {secret.secret_name}: {'*' * len(secret.secret_value)}")
+            f" - {secret.secret_name}: {len(secret.secret_value)} chars")
 
 
 environment = getenv_or_action("ENVIRONMENT", action="warn", default="dev")

app/routers/frontend.py

@@ -1,9 +1,6 @@
 # -*- coding: utf-8 -*-
-import json
-
 from typing import Annotated, List
 from fastapi import APIRouter, Depends, HTTPException
-from basedosdados import read_sql
 from tortoise.exceptions import ValidationError
 
 from app.dependencies import (
@@ -16,6 +13,7 @@
     Encounter,
     UserInfo,
 )
+from app.utils import read_bq
 from app.validators import CPFValidator
 from app.config import (
     BIGQUERY_PROJECT,
@@ -57,18 +55,14 @@ async def get_patient_header(
     except ValidationError:
         raise HTTPException(status_code=400, detail="Invalid CPF")
 
-    results_json = read_sql(
+    results = await read_bq(
         f"""
         SELECT *
         FROM `{BIGQUERY_PROJECT}`.{BIGQUERY_PATIENT_HEADER_TABLE_ID}
         WHERE cpf = '{cpf}'
         """,
         from_file="/tmp/credentials.json",
-    ).to_json(orient="records")
-    try:
-        results = json.loads(results_json)
-    except Exception:
-        results = []
+    )
 
     if len(results) == 0:
         raise HTTPException(status_code=404, detail="Patient not found")
@@ -90,15 +84,14 @@ async def get_patient_summary(
     cpf: str,
 ) -> PatientSummary:
 
-    results_json = read_sql(
+    results = await read_bq(
         f"""
         SELECT *
         FROM `{BIGQUERY_PROJECT}`.{BIGQUERY_PATIENT_SUMMARY_TABLE_ID}
         WHERE cpf = '{cpf}'
         """,
         from_file="/tmp/credentials.json",
-    ).to_json(orient="records")
-    results = json.loads(results_json)
+    )
     if len(results) == 0:
         raise HTTPException(status_code=404, detail="Patient not found")
     else:
@@ -126,13 +119,12 @@ async def get_patient_encounters(
     cpf: str,
 ) -> List[Encounter]:
 
-    results_json = read_sql(
+    results = await read_bq(
         f"""
         SELECT *
         FROM `{BIGQUERY_PROJECT}`.{BIGQUERY_PATIENT_ENCOUNTERS_TABLE_ID}
         WHERE cpf = '{cpf}' and exibicao.indicador = true
         """,
         from_file="/tmp/credentials.json",
-    ).to_json(orient="records")
-    results = json.loads(results_json)
+    )
     return results

app/utils.py

@@ -3,7 +3,11 @@
 import jwt
 import hashlib
 import json
-from typing import Literal
+import os
+
+from google.cloud import bigquery
+from google.oauth2 import service_account
+from asyncer import asyncify
 from loguru import logger
 from passlib.context import CryptContext
 
@@ -124,24 +128,19 @@ async def get_instance(Model, table, slug=None, code=None):
     return table[slug]
 
 
-def read_timestamp(timestamp: int, output_format=Literal['date','datetime']) -> str:
-    if output_format == 'date':
-        denominator = 1000
-        str_format = "%Y-%m-%d"
-    elif output_format == 'datetime':
-        denominator = 1
-        str_format = "%Y-%m-%d %H:%M:%S"
-    else:
-        raise ValueError("Invalid format")
+async def read_bq(query, from_file="/tmp/credentials.json"):
+    logger.debug(f"""Reading BigQuery with query (QUERY_PREVIEW_ENABLED={
+        os.environ['QUERY_PREVIEW_ENABLED']
+    }): {query}""")
 
-    try:
-        value = datetime(1970, 1, 1) + timedelta(seconds=timestamp/denominator)
-    except Exception as exc:
-        logger.error(f"Invalid timestamp: {timestamp} from {exc}")
-        return None
+    def execute_job():
+        credentials = service_account.Credentials.from_service_account_file(
+            from_file,
+        )
+        client = bigquery.Client(credentials=credentials)
+        row_iterator = client.query_and_wait(query)
+        return [dict(row) for row in row_iterator]
 
-    return value.strftime(str_format)
+    rows = await asyncify(execute_job)()
 
-def normalize_case(text):
-    # TODO
-    return text
+    return rows

pyproject.toml

@@ -30,6 +30,7 @@ urllib3 = "2.0.7"
 idna = "3.7"
 basedosdados = "^2.0.0b16"
 nltk = "^3.9.1"
+asyncer = "^0.0.8"
 
 
 [tool.poetry.group.dev.dependencies]