Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Datalake] Updating datalake formatter and models #179

Merged
merged 3 commits into from
Aug 8, 2024
Merged

[Datalake] Updating datalake formatter and models #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
27a8f02
Merge pull request #178 from prefeitura-rio/development
TanookiVerde Aug 5, 2024
257159c
Removing Vitacare Historico formatter
TanookiVerde Aug 8, 2024
e6ba0b4
feat: Update VitacareAtendimento model fields to set optional fields
TanookiVerde Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions app/datalake/formatters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from app.datalake.models import (
SMSRioPaciente,
VitacarePaciente,
VitacarePacienteHistorico,
VitacareAtendimento,
VitacareCondicao,
VitacareAlergia,
Expand Down Expand Up @@ -36,18 +35,14 @@ def format_vitacare_patient(raw_record: dict) -> List:

flattened = flatten(raw_record, list_max_depth=0)

# Temporary criterium to discriminate between Routine and Historic format
if 'AP' in raw_record['data'].keys():
return [VitacarePacienteHistorico(**flattened)]
else:
return [VitacarePaciente(**flattened)]
return [VitacarePaciente(**flattened)]


@register_formatter(system="vitacare", entity="encounter")
def format_vitacare_encounter(raw_record: dict) -> List:
raw_record['source_updated_at'] = str(raw_record['source_updated_at'])

flattened = flatten(raw_record,dict_max_depth=3)
flattened = flatten(raw_record, dict_max_depth=3)

rows = [VitacareAtendimento(**flattened)]

Expand Down
70 changes: 8 additions & 62 deletions app/datalake/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,75 +148,21 @@ class Config:
dataset_is_public = False


class VitacarePacienteHistorico(BaseModel):
patient_cpf: str
patient_code: str
source_updated_at: str
source_id: Optional[str]
data__AP: Optional[str]
data__SEXO: Optional[str]
data__HIST_CID: Optional[str]
data__RACA_COR: Optional[str]
data__RELIGIAO: Optional[str]
data__cpfPaciente: Optional[str]
data__ESCOLARIDADE: Optional[str]
data__dataConsulta: Optional[str]
data__NACIONALIDADE: Optional[str]
data__FREQUENTA_ESCOLA: Optional[str]
data__SITUACAO_USUARIO: Optional[str]
data__TELEFONE_CONTATO: Optional[str]
data__dataNascPaciente: Optional[str]
data__SITUACAO_FAMILIAR: Optional[str]
data__TERRITORIO_SOCIAL: Optional[str]
data__NUMERO_CNES_UNIDADE: Optional[str]
data__N_DE_CONSULTAS_2018: Optional[str]
data__N_DE_CONSULTAS_2019: Optional[str]
data__N_DE_CONSULTAS_2020: Optional[str]
data__N_DE_CONSULTAS_2021: Optional[str]
data__N_DE_CONSULTAS_2022: Optional[str]
data__N_DE_CONSULTAS_2023: Optional[str]
data__PACIENTE_TEMPORARIO: Optional[str]
data__NOME_UNIDADE_DE_SAUDE: Optional[str]
data__POSSUI_PLANO_DE_SAUDE: Optional[str]
data__SITUACAO_PROFISSIONAL: Optional[str]
data__MUNICIPIO_DE_NASCIMENTO: Optional[str]
data__N_DE_PROCEDIMENTOS_2018: Optional[str]
data__N_DE_PROCEDIMENTOS_2019: Optional[str]
data__N_DE_PROCEDIMENTOS_2020: Optional[str]
data__N_DE_PROCEDIMENTOS_2021: Optional[str]
data__N_DE_PROCEDIMENTOS_2022: Optional[str]
data__N_DE_PROCEDIMENTOS_2023: Optional[str]
data__PACIENTE_SITUACAO_RUA: Optional[str]
data__CODIGO_DA_EQUIPE_DE_SAUDE: Optional[str]
data__NOME_DA_PESSOA_CADASTRADA: Optional[str]
data__N_CNS_DA_PESSOA_CADASTRADA: Optional[str]
data__NOME_DA_MAE_PESSOA_CADASTRADA: Optional[str]
payload_cnes: str

class Config:
dataset_id = "brutos_prontuario_vitacare"
table_id = "paciente_historico_eventos"
partition_by_date = True
partition_column = "source_updated_at"
biglake_table = True
dataset_is_public = False


class VitacareAtendimento(BaseModel):
patient_cpf: str
patient_code: str
source_updated_at: str
source_id: str
data__unidade_ap: str
data__unidade_cnes: str
data__profissional__cns: str
data__profissional__cpf: str
data__profissional__nome: str
data__profissional__cbo: str
data__profissional__cbo_descricao: str
data__profissional__equipe__nome: str
data__profissional__equipe__cod_equipe: str
data__profissional__equipe__cod_ine: str
data__profissional__cns: Optional[str]
data__profissional__cpf: Optional[str]
data__profissional__nome: Optional[str]
data__profissional__cbo: Optional[str]
data__profissional__cbo_descricao: Optional[str]
data__profissional__equipe__nome: Optional[str]
data__profissional__equipe__cod_equipe: Optional[str]
data__profissional__equipe__cod_ine: Optional[str]
data__datahora_inicio_atendimento: str
data__datahora_fim_atendimento: str
data__datahora_marcacao_atendimento: Optional[str]
Expand Down
Loading