Add partitions to vitacare atendimento base files

prefeitura-rio · Sep 10, 2024 · aa425a2 · aa425a2
1 parent 80228ff
commit aa425a2
Show file tree

Hide file tree

Showing 3 changed files with 128 additions and 73 deletions.
diff --git a/models/raw/prontuario_vitacare/base/base_prontuario_vitacare__atendimento_historico.sql b/models/raw/prontuario_vitacare/base/base_prontuario_vitacare__atendimento_historico.sql
@@ -2,6 +2,11 @@
     config(
         alias="_base_atendimento_historico",
         materialized="table",
+        partition_by={
+            "field": "data_particao",
+            "data_type": "date",
+            "granularity": "day",
+        },
     )
 }}
 
@@ -225,7 +230,7 @@ with
             -- dim_procedimentos.procedimentos,
             atendimentos.updated_at,
             atendimentos.loaded_at,
-            safe_cast(atendimentos.datahora_inicio as date) as data_particao,
+            safe_cast(atendimentos.datahora_fim as date) as data_particao,
 
         from fato_atendimento as atendimentos
         left join dim_alergias using (gid)

diff --git a/models/raw/prontuario_vitacare/base/base_prontuario_vitacare__atendimento_rotineiro.sql b/models/raw/prontuario_vitacare/base/base_prontuario_vitacare__atendimento_rotineiro.sql
@@ -1,69 +1,116 @@
 {{
     config(
         alias="_base_atendimento_rotineiro",
-        materialized="table",
+        materialized="incremental",
+        incremental_strategy="insert_overwrite",
+        partition_by={
+            "field": "data_particao",
+            "data_type": "date",
+            "granularity": "day",
+        },
     )
 }}
 
 with
     bruto_atendimento_eventos_com_repeticao as (
-        select 
-            *,
-            concat(nullif(payload_cnes, ''), '.', nullif(source_id, '')) as gid
-        from {{ source("brutos_prontuario_vitacare_staging", "atendimento_eventos") }} 
+        select *, concat(nullif(payload_cnes, ''), '.', nullif(source_id, '')) as gid
+        from {{ source("brutos_prontuario_vitacare_staging", "atendimento_eventos") }}
     ),
     bruto_atendimento_eventos_ranqueados as (
-        select
-            *,
-            row_number() over (partition by gid order by datalake_loaded_at desc) as rank
+        select *,
+
         from bruto_atendimento_eventos_com_repeticao
-    )
-select
-    -- PK
-    safe_cast(gid as string) as gid,
+        qualify
+            row_number() over (partition by gid order by datalake_loaded_at desc) = 1
+    ),
 
-    -- Chaves
-    safe_cast(patient_cpf as string) as cpf,
-    safe_cast(payload_cnes as string) as cnes_unidade,
+    final as (
+        select
+            -- PK
+            safe_cast(gid as string) as gid,
+
+            -- Chaves
+            safe_cast(patient_cpf as string) as cpf,
+            safe_cast(payload_cnes as string) as cnes_unidade,
 
-    -- Profissional
-    safe_cast(data__profissional__cns as string) as cns_profissional,
-    safe_cast(data__profissional__cpf as string) as cpf_profissional,
-    safe_cast(data__profissional__nome as string) as nome_profissional,
-    safe_cast(data__profissional__cbo as string) as cbo_profissional,
-    safe_cast(data__profissional__cbo_descricao as string) as cbo_descricao_profissional,
-    safe_cast(data__profissional__equipe__cod_equipe as string) as cod_equipe_profissional,
-    safe_cast(data__profissional__equipe__cod_ine as string) as cod_ine_equipe_profissional,
-    safe_cast(data__profissional__equipe__nome as string) as nome_equipe_profissional,
+            -- Profissional
+            safe_cast(data__profissional__cns as string) as cns_profissional,
+            safe_cast(data__profissional__cpf as string) as cpf_profissional,
+            safe_cast(data__profissional__nome as string) as nome_profissional,
+            safe_cast(data__profissional__cbo as string) as cbo_profissional,
+            safe_cast(
+                data__profissional__cbo_descricao as string
+            ) as cbo_descricao_profissional,
+            safe_cast(
+                data__profissional__equipe__cod_equipe as string
+            ) as cod_equipe_profissional,
+            safe_cast(
+                data__profissional__equipe__cod_ine as string
+            ) as cod_ine_equipe_profissional,
+            safe_cast(
+                data__profissional__equipe__nome as string
+            ) as nome_equipe_profissional,
 
-    -- Dados da Consulta
-    safe_cast(data__tipo_consulta as string) as tipo,
-    safe_cast(data__eh_coleta as string) as eh_coleta,
-    safe_cast({{ process_null('data__datahora_marcacao_atendimento') }} as datetime) as datahora_marcacao,
-    safe_cast({{ process_null('data__datahora_inicio_atendimento') }} as datetime) as datahora_inicio,
-    safe_cast({{ process_null('data__datahora_fim_atendimento') }} as datetime) as datahora_fim,
+            -- Dados da Consulta
+            safe_cast(data__tipo_consulta as string) as tipo,
+            safe_cast(data__eh_coleta as string) as eh_coleta,
+            safe_cast(
+                {{ process_null("data__datahora_marcacao_atendimento") }} as datetime
+            ) as datahora_marcacao,
+            safe_cast(
+                {{ process_null("data__datahora_inicio_atendimento") }} as datetime
+            ) as datahora_inicio,
+            safe_cast(
+                {{ process_null("data__datahora_fim_atendimento") }} as datetime
+            ) as datahora_fim,
 
-    -- Campos Livres
-    safe_cast({{ process_null('data__soap_subjetivo_motivo') }} as string) as soap_subjetivo_motivo,
-    safe_cast({{ process_null('data__soap_objetivo_descricao') }} as string) as soap_objetivo_descricao,
-    safe_cast({{ process_null('data__soap_avaliacao_observacoes') }} as string) as soap_avaliacao_observacoes,
-    safe_cast({{ process_null('data__soap_plano_procedimentos_clinicos') }} as string) as soap_plano_procedimentos_clinicos,
-    safe_cast({{ process_null('data__soap_plano_observacoes') }} as string) as soap_plano_observacoes,
-    safe_cast({{ process_null('data__notas_observacoes') }} as string) as soap_notas_observacoes,
+            -- Campos Livres
+            safe_cast(
+                {{ process_null("data__soap_subjetivo_motivo") }} as string
+            ) as soap_subjetivo_motivo,
+            safe_cast(
+                {{ process_null("data__soap_objetivo_descricao") }} as string
+            ) as soap_objetivo_descricao,
+            safe_cast(
+                {{ process_null("data__soap_avaliacao_observacoes") }} as string
+            ) as soap_avaliacao_observacoes,
+            safe_cast(
+                {{ process_null("data__soap_plano_procedimentos_clinicos") }} as string
+            ) as soap_plano_procedimentos_clinicos,
+            safe_cast(
+                {{ process_null("data__soap_plano_observacoes") }} as string
+            ) as soap_plano_observacoes,
+            safe_cast(
+                {{ process_null("data__notas_observacoes") }} as string
+            ) as soap_notas_observacoes,
 
-    -- JSONs
-    safe_cast({{ dict_to_json('data__prescricoes') }} as string) as prescricoes,
-    safe_cast({{ dict_to_json('data__condicoes') }} as string) as condicoes,
-    safe_cast({{ dict_to_json('data__exames_solicitados') }} as string) as exames_solicitados,
-    safe_cast({{ dict_to_json('data__alergias_anamnese') }} as string) as alergias_anamnese,
-    safe_cast({{ dict_to_json('data__vacinas') }} as string) as vacinas,
-    safe_cast({{ dict_to_json('data__indicadores') }} as string) as indicadores,
-    safe_cast({{ dict_to_json('data__encaminhamentos') }} as string) as encaminhamentos,
+            -- JSONs
+            safe_cast({{ dict_to_json("data__prescricoes") }} as string) as prescricoes,
+            safe_cast({{ dict_to_json("data__condicoes") }} as string) as condicoes,
+            safe_cast(
+                {{ dict_to_json("data__exames_solicitados") }} as string
+            ) as exames_solicitados,
+            safe_cast(
+                {{ dict_to_json("data__alergias_anamnese") }} as string
+            ) as alergias_anamnese,
+            safe_cast({{ dict_to_json("data__vacinas") }} as string) as vacinas,
+            safe_cast({{ dict_to_json("data__indicadores") }} as string) as indicadores,
+            safe_cast(
+                {{ dict_to_json("data__encaminhamentos") }} as string
+            ) as encaminhamentos,
 
-    -- Metadados
-    safe_cast(source_updated_at as datetime) as updated_at,
-    safe_cast(datalake_loaded_at as datetime) as loaded_at,
-    safe_cast({{ process_null('data__datahora_inicio_atendimento') }} as date) as data_particao,
+            -- Metadados
+            safe_cast(source_updated_at as datetime) as updated_at,
+            safe_cast(datalake_loaded_at as datetime) as loaded_at,
+            safe_cast(
+                safe_cast(source_updated_at as datetime) as date
+            ) as data_particao,
+
+        from bruto_atendimento_eventos_ranqueados
+    )
 
-from bruto_atendimento_eventos_ranqueados 
-where rank = 1
+select *
+from final
+{% if is_incremental() %}
+        where data_particao in ({{ partitions_to_replace | join(',') }})
+{% endif %}
diff --git a/tests/generic/assert_chegando_todas_unidades_de_saude.sql b/tests/generic/assert_chegando_todas_unidades_de_saude.sql
@@ -1,25 +1,28 @@
 {% test assert_chegando_todas_unidades_de_saude(
     model, column_name, prontuario, filter
 ) %}
-        with
-            unidades_saude_presentes as (
-                select distinct {{ column_name }} from {{ model }}
-            ),
-            unidades_saude_relacao_completa as (
-                select id_cnes, area_programatica, nome_limpo, prontuario_versao,
-                from {{ ref("dim_estabelecimento") }}
-                where
-                    prontuario_tem = "sim" and prontuario_versao = '{{ prontuario }}'
-                    {% if filter is defined %} and {{ filter }} {% endif %}
-            ),
-            unidade_saude_faltantes as (
-                select relacao_completa.*
-                from unidades_saude_relacao_completa as relacao_completa
-                left join
-                    unidades_saude_presentes as presentes
-                    on presentes.{{ column_name }} = relacao_completa.id_cnes
-                where presentes.{{ column_name }} is null
-            )
-        select *
-        from unidade_saude_faltantes where FORMAT_DATE('%A', CURRENT_DATE('America/Sao_Paulo')) != 'Monday' or prontuario_versao != 'vitacare' -- filtro aplicado para evitar a execução nas unidades de atenção primaria que não funcionam aos domingos
+
+{% set modelo = model %}
+
+    with
+        unidades_saude_presentes as (
+            select distinct {{ column_name }} from {{ model }}
+        ),
+        unidades_saude_relacao_completa as (
+            select id_cnes, area_programatica, nome_limpo, prontuario_versao,
+            from {{ ref("dim_estabelecimento") }}
+            where
+                prontuario_tem = "sim" and prontuario_versao = '{{ prontuario }}'
+                {% if filter is defined %} and {{ filter }} {% endif %}
+        ),
+        unidade_saude_faltantes as (
+            select relacao_completa.*
+            from unidades_saude_relacao_completa as relacao_completa
+            left join
+                unidades_saude_presentes as presentes
+                on presentes.{{ column_name }} = relacao_completa.id_cnes
+            where presentes.{{ column_name }} is null
+        )
+    select *
+    from unidade_saude_faltantes
 {% endtest %}