Merge pull request #194 from prefeitura-rio/development

Usando dados do Big Query

app/config/base.py

@@ -6,6 +6,9 @@
 # Logging
 LOG_LEVEL = getenv_or_action("LOG_LEVEL", default="INFO")
 
+# BigQuery Project
+BIGQUERY_PROJECT = getenv_or_action("BIGQUERY_PROJECT", action="raise")
+
 # JWT configuration
 JWT_SECRET_KEY = getenv_or_action("JWT_SECRET_KEY", default=token_bytes(32).hex())
 JWT_ALGORITHM = getenv_or_action("JWT_ALGORITHM", default="HS256")

app/datalake/models.py

@@ -164,9 +164,9 @@ class VitacareAtendimento(BaseModel):
     data__profissional__nome: Optional[str]
     data__profissional__cbo: Optional[str]
     data__profissional__cbo_descricao: Optional[str]
-    data__profissional__equipe__nome: str
-    data__profissional__equipe__cod_equipe: str
-    data__profissional__equipe__cod_ine: str
+    data__profissional__equipe__nome: Optional[str]
+    data__profissional__equipe__cod_equipe: Optional[str]
+    data__profissional__equipe__cod_ine: Optional[str]
     data__datahora_inicio_atendimento: str
     data__datahora_fim_atendimento: str
     data__datahora_marcacao_atendimento: Optional[str]

app/datalake/utils.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
+import json
 import pandas as pd
 from typing import Callable
 from loguru import logger
 
-
 REGISTERED_FORMATTERS = {}
 
 
@@ -77,9 +77,9 @@ def flatten(
                 for key, value in flattened.items():
                     updated_record[f"{field}__{key}"] = value
             else:
-                updated_record[field] = str(content)
+                updated_record[field] = json.dumps(content)
         elif isinstance(content, list) and depth >= list_max_depth:
-            updated_record[field] = str(content)
+            updated_record[field] = json.dumps(content)
         else:
             updated_record[field] = content
 

app/dependencies.py

@@ -46,3 +46,23 @@ async def get_current_active_user(current_user: Annotated[User, Depends(get_curr
     if not current_user.is_active:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Inactive user")
     return current_user
+
+
+async def get_current_pipeline_user(current_user: Annotated[User, Depends(get_current_user)]):
+    if current_user.is_superuser:
+        return current_user
+
+    if current_user.user_class.value in ["pipeline_user"]:
+        return current_user
+    else:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="User don't have permition to access Pipeline Endpoints")
+
+
+async def get_current_frontend_user(current_user: Annotated[User, Depends(get_current_user)]):
+    if current_user.is_superuser:
+        return current_user
+
+    if current_user.user_class.value in ["frontend_user"]:
+        return current_user
+    else:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="User don't have permition to access Front-end Endpoints")

app/enums.py

@@ -2,6 +2,11 @@
 from enum import Enum
 
 
+class UserClassEnum(str, Enum):
+    WEBAPP_USER = "frontend_user"
+    PIPELINE_USER = "pipeline_user"
+
+
 class ConditionCodeTypeEnum(str, Enum):
     CID = "cid"
     CIAP = "ciap"

app/main.py

@@ -9,7 +9,7 @@
 
 from app import config
 from app.db import TORTOISE_ORM
-from app.routers import auth, entities_mrg, entities_raw, entities_std, entities, frontend
+from app.routers import auth, entities_mrg, entities_raw, entities_std, frontend
 
 logger.remove()
 logger.add(sys.stdout, level=config.LOG_LEVEL)
@@ -45,7 +45,6 @@
 app.include_router(entities_raw.router)
 app.include_router(entities_std.router)
 app.include_router(entities_mrg.router)
-app.include_router(entities.router)
 app.include_router(auth.router)
 app.include_router(frontend.router)
 

app/models.py

@@ -10,6 +10,7 @@
     CategoryEnum,
     ClinicalStatusEnum,
     SystemEnum,
+    UserClassEnum
 )
 from app.validators import CPFValidator, PatientCodeValidator
 
@@ -267,12 +268,14 @@ class Meta:
 class User(Model):
     id = fields.IntField(pk=True)
     username = fields.CharField(max_length=255, unique=True)
+    role = fields.CharField(max_length=255, null=True)
     name = fields.CharField(max_length=255, null=True)
     cpf = fields.CharField(max_length=11, unique=True, null=True, validators=[CPFValidator()])
     email = fields.CharField(max_length=255, unique=True)
     password = fields.CharField(max_length=255)
     is_active = fields.BooleanField(default=True)
     is_superuser = fields.BooleanField(default=False)
+    user_class = fields.CharEnumField(enum_type=UserClassEnum, null=True, default=UserClassEnum.PIPELINE_USER)
     data_source = fields.ForeignKeyField("app.DataSource", related_name="users", null=True)
     created_at = fields.DatetimeField(auto_now_add=True)
     updated_at = fields.DatetimeField(auto_now=True)

app/routers/entities_mrg.py

@@ -7,7 +7,9 @@
 from tortoise.contrib.pydantic import pydantic_model_creator
 from tortoise.exceptions import ValidationError
 
-from app.dependencies import get_current_active_user
+from app.dependencies import (
+    get_current_pipeline_user
+)
 from app.types.pydantic_models import (
     CompletePatientModel,
     MergedPatient as PydanticMergedPatient,
@@ -47,7 +49,7 @@
 
 @router.put("/patient")
 async def create_or_update_patient(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     patients: List[PydanticMergedPatient],
 ) -> int:
 
@@ -81,7 +83,7 @@ async def create_or_update_patient(
 
 @router.put("/patientaddress")
 async def create_or_update_patientaddress(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     patientaddress_list: List[PydanticMergedPatientAddress],
 ) -> int:
     # Get list of patient codes
@@ -111,7 +113,7 @@ async def create_or_update_patientaddress(
 
 @router.put("/patienttelecom")
 async def create_or_update_patienttelecom(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     patienttelecom_list: List[PydanticMergedPatientTelecom],
 ) -> int:
     # Get list of patient codes
@@ -140,7 +142,7 @@ async def create_or_update_patienttelecom(
 
 @router.put("/patientcns")
 async def create_or_update_patientcns(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     patientcns_list: List[PydanticMergedPatientCns],
 ) -> int:
     # Get list of patient codes
@@ -169,7 +171,7 @@ async def create_or_update_patientcns(
 
 @router.get("/patient/{patient_cpf}")
 async def get_patient(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     patient_cpf: int,
 )-> list[CompletePatientModel]:
 
@@ -225,7 +227,7 @@ async def get_patient(
 
 @router.put("/professionals")
 async def create_or_update_professionals(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     professionals: List[ProfessionalModel],
 ) -> int:
 
@@ -328,7 +330,7 @@ async def create_or_update_professionals(
 
 @router.put("/teams")
 async def create_or_update_teams(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     teams: List[TeamModel],
 ) -> int:
 

app/routers/entities_raw.py

@@ -12,7 +12,7 @@
 from tortoise.exceptions import ValidationError
 
 from app.types.pydantic_models import RawDataListModel, BulkInsertOutputModel, RawDataModel
-from app.dependencies import get_current_active_user
+from app.dependencies import get_current_pipeline_user
 from app.models import User, RawPatientRecord, RawPatientCondition, DataSource, RawEncounter
 
 from app.datalake.uploader import DatalakeUploader
@@ -33,7 +33,7 @@
 
 @router.get("/{entity_name}/{filter_type}")
 async def get_raw_data(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     entity_name: Literal["patientrecords", "patientconditions", "encounter"],
     filter_type: Literal["fromEventDatetime", "fromInsertionDatetime"],
     start_datetime: dt = dt.now() - td(hours=1),
@@ -68,7 +68,7 @@ async def get_raw_data(
 @router.post("/{entity_name}", status_code=201)
 async def create_raw_data(
     entity_name: Literal["patientrecords", "patientconditions", "encounter"],
-    current_user: Annotated[User, Depends(get_current_active_user)],
+    current_user: Annotated[User, Depends(get_current_pipeline_user)],
     raw_data: RawDataListModel,
     upload_to_datalake: bool = True,
 ) -> BulkInsertOutputModel:
@@ -145,7 +145,7 @@ async def create_raw_data(
 
 @router.post("/{entity_name}/setAsInvalid", status_code=200)
 async def set_as_invalid_flag_records(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     entity_name: Literal["patientrecords", "patientconditions", "encounter"],
     raw_record_id_list: list[str],
 ):

app/routers/entities_std.py

@@ -13,7 +13,7 @@
 from tortoise.contrib.pydantic import pydantic_model_creator
 from tortoise.exceptions import ValidationError, DoesNotExist
 
-from app.dependencies import get_current_active_user
+from app.dependencies import get_current_pipeline_user
 from app.types.pydantic_models import (
     PatientMergeableRecord, StandardizedPatientRecordModel, StandardizedPatientConditionModel,
     BulkInsertOutputModel, MergeableRecord, Page
@@ -37,7 +37,7 @@
 
 @router.get("/patientrecords/updated")
 async def get_patientrecords_of_updated_patients(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     start_datetime: datetime.datetime = datetime.datetime.now() -
     datetime.timedelta(hours=1),
     end_datetime: datetime.datetime = datetime.datetime.now(),
@@ -100,7 +100,7 @@ async def get_patientrecords_of_updated_patients(
 
 @router.post("/patientrecords", status_code=201)
 async def create_standardized_patientrecords(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     records: list[StandardizedPatientRecordModel],
 ) -> BulkInsertOutputModel:
 
@@ -164,7 +164,7 @@ async def create_standardized_patientrecords(
 
 @router.get("/patientconditions")
 async def get_standardized_patientconditions(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     patient_cpf: str,
 ) -> list[MergeableRecord[StandardizedPatientConditionModel]]:
 
@@ -188,7 +188,7 @@ async def get_standardized_patientconditions(
 
 @router.post("/patientconditions", status_code=201)
 async def create_standardized_patientconditions(
-    _: Annotated[User, Depends(get_current_active_user)],
+    _: Annotated[User, Depends(get_current_pipeline_user)],
     conditions: list[StandardizedPatientConditionModel],
 ) -> BulkInsertOutputModel: