From bef8e299e36a6160cab2b9662f13429ad03e7ad2 Mon Sep 17 00:00:00 2001 From: Bruno Almeida Date: Tue, 17 Sep 2024 14:35:04 -0300 Subject: [PATCH 01/12] add: pipelines and queries --- pipelines/chatbot/__init__.py | 2 + .../chatbot/dbt_chatbot_metricas/__init__.py | 0 .../chatbot/dbt_chatbot_metricas/flows.py | 41 +++ .../chatbot/dbt_chatbot_metricas/schedules.py | 53 ++++ pipelines/flows.py | 1 + .../conversas_completas_metricas.sql | 11 + queries/models/dialogflowcx/fim_conversas.sql | 146 +++++++++ .../models/dialogflowcx/fim_conversas_da.sql | 283 +++++++++++++++++ .../fim_conversas_macrofluxos.sql | 139 +++++++++ .../historico_conversas_legivel.sql | 36 +++ .../historico_conversas_macrofluxos.sql | 285 ++++++++++++++++++ 11 files changed, 997 insertions(+) create mode 100644 pipelines/chatbot/__init__.py create mode 100644 pipelines/chatbot/dbt_chatbot_metricas/__init__.py create mode 100644 pipelines/chatbot/dbt_chatbot_metricas/flows.py create mode 100644 pipelines/chatbot/dbt_chatbot_metricas/schedules.py create mode 100644 queries/models/dialogflowcx/conversas_completas_metricas.sql create mode 100644 queries/models/dialogflowcx/fim_conversas.sql create mode 100644 queries/models/dialogflowcx/fim_conversas_da.sql create mode 100644 queries/models/dialogflowcx/fim_conversas_macrofluxos.sql create mode 100644 queries/models/dialogflowcx/historico_conversas_legivel.sql create mode 100644 queries/models/dialogflowcx/historico_conversas_macrofluxos.sql diff --git a/pipelines/chatbot/__init__.py b/pipelines/chatbot/__init__.py new file mode 100644 index 0000000..6a87e03 --- /dev/null +++ b/pipelines/chatbot/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from pipelines.chatbot.dbt_chatbot_metricas.flows import * # noqa diff --git a/pipelines/chatbot/dbt_chatbot_metricas/__init__.py b/pipelines/chatbot/dbt_chatbot_metricas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py new file mode 100644 index 0000000..4e02f2b --- /dev/null +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" +DBT-related flows. +""" + +from copy import deepcopy + +from prefect.run_configs import KubernetesRun +from prefect.storage import GCS +from prefeitura_rio.pipelines_templates.run_dbt_model.flows import ( + templates__run_dbt_model__flow, +) +from prefeitura_rio.pipelines_utils.prefect import set_default_parameters +from prefeitura_rio.pipelines_utils.state_handlers import ( + handler_initialize_sentry, + handler_inject_bd_credentials, +) + +from pipelines.constants import constants +from pipelines.identidade_unica.dbt_identidade_unica.schedules import update_schedule + +run_rbt_chatbot_flow = deepcopy(templates__run_dbt_model__flow) +run_rbt_chatbot_flow.name = "Chatbot: Materializar tabelas" +run_rbt_chatbot_flow.state_handlers = [ + handler_inject_bd_credentials, + handler_initialize_sentry, +] +run_rbt_chatbot_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) +run_rbt_chatbot_flow.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) + +identidade_unica_default_parameters = { + "dataset_id": "dialogflowcx", + "upstream": False, + "dbt_alias": False, +} +run_rbt_chatbot_flow = set_default_parameters( + run_rbt_chatbot_flow, + default_parameters=identidade_unica_default_parameters, +) + +run_rbt_chatbot_flow.schedule = update_schedule diff --git a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py new file mode 100644 index 0000000..0790cba --- /dev/null +++ b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +Schedules for the database dump pipeline +""" + +from datetime import datetime, timedelta + +import pytz +from prefect.schedules import Schedule +from prefect.schedules.clocks import IntervalClock +from prefeitura_rio.pipelines_utils.io import untuple_clocks as untuple + +from pipelines.constants import constants + +##################################### +# +# Chatbot Metrics Schedules +# +##################################### + +chatbot_tables = { + "fim_conversas": { + "table_id": "fim_conversas", + "upstream": True, + }, + "historico_conversas_legivel": { + "table_id": "historico_conversas_legivel", + "upstream": False, + }, + "conversas_completas_metricas": { + "table_id": "conversas_completas_metricas", + "upstream": False, + }, +} + +chatbot_clocks = [ + IntervalClock( + interval=timedelta(hours=12), + start_date=datetime(2024, 9, 17, 19, 0, tzinfo=pytz.timezone("America/Sao_Paulo")) + + timedelta(minutes = 45 * count), + labels=[ + constants.RJ_ESCRITORIO_AGENT_LABEL.value, + ], + parameter_defaults={ + "dataset_id": "dialogflowcx", + "table_id": parameters["table_id"], + "mode": "prod", + "upstream": parameters["upstream"], + }, + ) + for count, (_, parameters) in enumerate(chatbot_tables.items()) +] +update_schedule = Schedule(clocks=untuple(chatbot_clocks)) \ No newline at end of file diff --git a/pipelines/flows.py b/pipelines/flows.py index 1e66215..36e8c1a 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -9,3 +9,4 @@ from pipelines.lgpd import * # noqa from pipelines.stress import * # noqa from pipelines.templates import * # noqa +from pipelines.chatbot import * # noqa diff --git a/queries/models/dialogflowcx/conversas_completas_metricas.sql b/queries/models/dialogflowcx/conversas_completas_metricas.sql new file mode 100644 index 0000000..a6e919a --- /dev/null +++ b/queries/models/dialogflowcx/conversas_completas_metricas.sql @@ -0,0 +1,11 @@ +SELECT + MIN(request_time) as request_time, + conversa_completa_id, + MAX(ambiente) AS ambiente, + ARRAY_AGG(DISTINCT conversation_name) AS conversation_name_list, + AVG(conversa_completa_turnos_em_menus) AS conversa_completa_turnos_em_menus, + MAX(conversa_completa_fluxos_interagidos) AS conversa_completa_fluxos_interagidos, + MAX(conversa_completa_duracao) AS conversa_completa_duracao, + MAX(conversa_completa_ultimo_fluxo_servico) AS conversa_completa_ultimo_fluxo_servico +FROM {{ ref('fim_conversas') }} +GROUP BY conversa_completa_id \ No newline at end of file diff --git a/queries/models/dialogflowcx/fim_conversas.sql b/queries/models/dialogflowcx/fim_conversas.sql new file mode 100644 index 0000000..2274c42 --- /dev/null +++ b/queries/models/dialogflowcx/fim_conversas.sql @@ -0,0 +1,146 @@ +with ultima_interacao as ( +SELECT + conversation_name, + MAX(turn_position) as last_turn +FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` +GROUP BY conversation_name +), + +# Captura apenas mensagens vindas do ASC através da mensagem que inicia a conversa +primeira_interacao AS ( + SELECT + conversation_name, + `rj-chatbot-dev.dialogflowcx.inicial_sentence_to_flow_name`(JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text'))) AS fluxo_primeira_interacao, + request_time AS hora_primeira_interacao, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) AS primeira_mensagem, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) AS ambiente + FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` + WHERE + turn_position = 1 + AND `rj-chatbot-dev.dialogflowcx.inicial_sentences`(JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text'))) +), + +fim_conversas_1746 AS ( +SELECT + hist.conversation_name, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.usuario_cpf')) AS cpf, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.phone')) as telefone, + CASE + WHEN pi.ambiente = "production" THEN "Produção" + ELSE "Homologação" + END ambiente, + INITCAP(pi.fluxo_primeira_interacao) as fluxo_nome, + FORMAT_DATETIME("%d/%m/%Y às %H:%M", DATETIME(request_time, "America/Buenos_Aires")) as horario, + DATETIME(request_time, "America/Buenos_Aires") as request_time, + ROUND(DATE_DIFF(request_time, hora_primeira_interacao, SECOND)/60,1) as duracao_minutos, + hist.turn_position as ultimo_turno, + pi.primeira_mensagem, + response, + CASE + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) = "End Session" THEN true + ELSE false + END conversa_finalizada, + CASE + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL THEN "chamado_aberto" + WHEN hist.turn_position = 1 THEN "hard_bounce" + WHEN + hist.turn_position = 2 + AND ( + ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'VOLTAR') + OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') + ) + THEN "soft_bounce" + WHEN + hist.turn_position = 2 + THEN "timeout_usuario_pos_transacao" + WHEN + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno_timeout" + THEN "timeout SGRC" + WHEN ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')),'TRANSBORDO') THEN "transbordo" + WHEN + # Casos em que o Chatbot identificou a inelegibilidade + (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) != "erro_desconhecido" + AND + (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado')) = "false" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado')) = "false") + ) + # Casos em que o SGRC identificou a inelegibilidade + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_aberto" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_aberto" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" + THEN "impedimento_regra_negocio" + WHEN + JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')) IS NULL + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "erro_desconhecido" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "erro_desconhecido" + THEN "timeout interno" + WHEN + ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'VOLTAR') + OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') + THEN "desistencia" + ELSE "timeout_usuario_pos_transacao" + END classificacao_conversa, + CASE + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL + OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'VOLTAR') + OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') + THEN "sucesso" + ELSE "falha" + END status_final_conversa, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) as passo_falha, + CASE + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1647 (RRL)' THEN 'Remoção de Resíduo' + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1614 (PAL)' THEN 'Poda de Árvore' + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1464 (VACIO)' THEN 'Verificação de Ar Condicionado Inoperante em Ônibus' + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '152 (RLU)' THEN 'Reparo de Luminária' + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '182 (RBDAP)' THEN 'Reparo de Buraco, Deformação ou Afundamento na pista' + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '3581 (FEIV)' THEN "Fiscalização de Estacionamento Irregular de Veículo" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '3802 (RSTA)' THEN "Reparo de Sinal de Trânsito Apagado" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1607 (REBI)' THEN "Remoção de Entulho e Bens Inservíveis" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '192 (DBGRR)' THEN "Desobstrução de bueiros, galerias, ramais de águas pluviais e ralos" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '2569 (RTG)' THEN "Reposição de Tampão ou Grelha" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '223 (RSTAP)' THEN "Reparo de Sinal de Trânsito em Amarelo Piscante" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1618 (CRCA)' THEN "Controle de Roedores e Caramujos Africanos" + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '3803 (RSTAAV)' THEN "Reparo de Sinal de Trânsito Abalroado ou Ausente ou Virado" + ELSE JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) + END as fluxo_falha, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) erro_abertura_ticket, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) protocolo, + SAFE_CAST(NULL AS INT64) AS turnos_em_menus, + SAFE_CAST(NULL AS INT64) AS estimativa_turnos_menu, + SAFE_CAST(NULL AS INT64) AS turnos_em_servico, + SAFE_CAST(NULL AS INT64) AS estimativa_turnos_servico, + SAFE_CAST(NULL AS INT64) AS turnos_em_endereco, + SAFE_CAST(NULL AS INT64) AS turnos_em_identificacao, + hist.conversation_name as conversa_completa_id, + SAFE_CAST(NULL AS INT64) AS conversa_completa_turnos_em_menus, + SAFE_CAST(NULL AS INT64) AS conversa_completa_fluxos_interagidos, + SAFE_CAST(NULL AS INT64) AS conversa_completa_duracao, + SAFE_CAST(NULL AS STRING) AS conversa_completa_ultimo_fluxo_servico, +FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` as hist +INNER JOIN ultima_interacao as ui + ON hist.conversation_name = ui.conversation_name AND hist.turn_position = ui.last_turn +INNER JOIN primeira_interacao as pi + ON hist.conversation_name = pi.conversation_name +ORDER BY request_time DESC) + +SELECT + *, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.codigo_servico_1746')) as codigo_servico +FROM fim_conversas_1746 + +UNION ALL + +SELECT + *, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.codigo_servico_1746')) as codigo_servico +FROM {{ ref('fim_conversas_da') }} + +UNION ALL + +SELECT + *, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.codigo_servico_1746')) as codigo_servico +FROM {{ ref('fim_conversas_macrofluxos') }} \ No newline at end of file diff --git a/queries/models/dialogflowcx/fim_conversas_da.sql b/queries/models/dialogflowcx/fim_conversas_da.sql new file mode 100644 index 0000000..7266747 --- /dev/null +++ b/queries/models/dialogflowcx/fim_conversas_da.sql @@ -0,0 +1,283 @@ +WITH filtro_divida_ativa AS ( + SELECT + conversation_name + FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` + WHERE + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) = "production" + AND turn_position = 1 + AND JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) = "dívida ativa" +), +historico AS ( + SELECT + h.conversation_name, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) as mensagem_cidadao, + JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')) as resposta_bot, + turn_position, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) as fluxo, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) as passo, + FORMAT_DATETIME("%d/%m/%Y às %H:%M", DATETIME(request_time, "America/Buenos_Aires")) as horario, + JSON_EXTRACT(response, '$.queryResult.parameters') as parametros, + request_time, + response + FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` as h + INNER JOIN filtro_divida_ativa as f ON f.conversation_name = h.conversation_name + ORDER BY conversation_name, request_time ASC +), +MarkedConversations AS ( + SELECT + *, + CASE + WHEN LAG(fluxo, 2) OVER(PARTITION BY conversation_name ORDER BY turn_position) != 'DA 0 - Menu da Dívida Ativa' + AND LAG(fluxo) OVER(PARTITION BY conversation_name ORDER BY turn_position) = 'DA 0 - Menu da Dívida Ativa' + THEN 1 + ELSE 0 + END AS isNewPart + FROM + historico +), +NumberedConversations AS ( + SELECT + *, + SUM(isNewPart) OVER(PARTITION BY conversation_name ORDER BY turn_position) AS part_number + FROM + MarkedConversations +), +ConversationsWithNewTurn AS ( + SELECT + *, + ROW_NUMBER() OVER(PARTITION BY conversation_name, part_number ORDER BY turn_position) AS new_turn + FROM + NumberedConversations +), +new_conv_id AS ( + SELECT + CONCAT(conversation_name, '-', FORMAT('%02d', part_number)) AS new_conversation_id, + mensagem_cidadao, + resposta_bot, + turn_position, + fluxo, + passo, + horario, + parametros, + request_time, + response, + new_turn, + conversation_name + FROM + ConversationsWithNewTurn +), +TypeClassification AS ( + SELECT + new_conversation_id, + CASE + WHEN COUNTIF(fluxo != 'DA 0 - Menu da Dívida Ativa') = 0 THEN 'informational' + WHEN MAX(CASE WHEN fluxo IN ('DA 1 - Consultar Débitos Dívida Ativa Solicitar Guias Exibir Informações', + 'DA 1.7. Emitir Guia de Pagamento à Vista', + 'DA 1.9. Emitir Guia de Regularização de Débitos', + 'DA 3 - Consulta de protestos', + 'DA 5 - Cadastre-se (Serviços úteis)') THEN 1 ELSE 0 END) = 1 THEN 'service' + WHEN MAX(CASE WHEN fluxo IN ('DA 2 - Como parcelar minha dívida', 'DA 4 - Informações Gerais') THEN 1 ELSE 0 END) = 1 THEN 'informational' + ELSE 'undefined' + END AS type_flow + FROM + new_conv_id + GROUP BY + new_conversation_id +) + +, DistinctFluxo AS ( + SELECT DISTINCT + new_conversation_id, + fluxo + FROM + new_conv_id + WHERE + fluxo != 'DA 0 - Menu da Dívida Ativa' +), + +FluxoMain AS ( + SELECT + d.new_conversation_id, + MIN(d.fluxo) AS main_fluxo, + STRING_AGG(d.fluxo, ', ') AS distinct_fluxo_list + FROM + DistinctFluxo d + GROUP BY + d.new_conversation_id +), + +ParametrosExtracted AS ( + SELECT + new_conversation_id, + CASE + WHEN JSON_VALUE(parametros, '$.api_resposta_sucesso') IS NULL THEN + CASE + WHEN REGEXP_CONTAINS(TO_JSON_STRING(parametros), r'\"api_resposta_sucesso\"') THEN 'replaced' + ELSE NULL + END + ELSE JSON_VALUE(parametros, '$.api_resposta_sucesso') + END AS api_resposta_sucesso, + CASE + WHEN JSON_VALUE(parametros, '$.api_descricao_erro') IS NULL THEN + CASE + WHEN REGEXP_CONTAINS(TO_JSON_STRING(parametros), r'\"api_descricao_erro\"') THEN 'replaced' + ELSE NULL + END + ELSE JSON_VALUE(parametros, '$.api_descricao_erro') + END AS api_descricao_erro + FROM + new_conv_id + WHERE + parametros IS NOT NULL +), + +DistinctApiValues AS ( + SELECT + new_conversation_id, + CASE + WHEN COUNTIF(api_resposta_sucesso = 'false') > 0 THEN 'false' + WHEN COUNTIF(api_resposta_sucesso = 'true') > 0 THEN 'true' + ELSE MAX(api_resposta_sucesso) -- Handles 'replaced' or NULL + END AS api_resposta_sucesso, + MAX(api_descricao_erro) AS api_descricao_erro + FROM + ParametrosExtracted + GROUP BY + new_conversation_id +), + +FinalTableDA AS ( + SELECT + n.*, + type_flow, + IFNULL(f.main_fluxo, 'DA 0 - Menu da Dívida Ativa') AS main_fluxo, + a.api_resposta_sucesso, + a.api_descricao_erro + FROM + new_conv_id n + JOIN + TypeClassification t ON n.new_conversation_id = t.new_conversation_id + LEFT JOIN + FluxoMain f ON n.new_conversation_id = f.new_conversation_id + LEFT JOIN + DistinctApiValues a ON n.new_conversation_id = a.new_conversation_id + WHERE t.type_flow != 'undefined' +) + +-- SELECT * FROM FinalTableDA #WHERE api_resposta_sucesso = "replaced" AND main_fluxo IN ('DA 3 - Consulta de protestos', 'DA 1 - Consultar Débitos Dívida Ativa Solicitar Guias Exibir Informações') +-- ORDER BY new_conversation_id, new_turn; + +, FinalTableOutros AS ( + # Separando fluxos de outros serviços em que o usuário começou na dívida ativa mas foi para outro serviço depois + SELECT + n.*, + t.type_flow, + f.distinct_fluxo_list + FROM + new_conv_id n + JOIN + TypeClassification t ON n.new_conversation_id = t.new_conversation_id + LEFT JOIN + FluxoMain f ON n.new_conversation_id = f.new_conversation_id + WHERE t.type_flow = 'undefined' +) + +, primeira_interacao AS ( + SELECT + new_conversation_id, + main_fluxo as fluxo_primeira_interacao, + request_time AS hora_primeira_interacao, + mensagem_cidadao AS primeira_mensagem, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) AS ambiente + FROM FinalTableDA + WHERE turn_position = 1 +) + +, ultima_interacao as ( +SELECT + new_conversation_id, + MAX(new_turn) as last_turn +FROM FinalTableDA +GROUP BY new_conversation_id +), + +fim_conversas_DA AS ( +SELECT + hist.new_conversation_id as conversation_name, + JSON_VALUE(parametros, '$.usuario_cpf') AS cpf, + JSON_VALUE(parametros, '$.phone') as telefone, + CASE + WHEN pi.ambiente = "production" THEN "Produção" + ELSE "Homologação" + END ambiente, + main_fluxo as fluxo_nome, + horario, + DATETIME(request_time, "America/Buenos_Aires") as request_time, + ROUND(DATE_DIFF(request_time, hora_primeira_interacao, SECOND)/60,1) as duracao_minutos, + hist.new_turn as ultimo_turno, + pi.primeira_mensagem, + response, + CASE + WHEN hist.passo = "End Session" OR hist.fluxo = "DA 0 - Menu da Dívida Ativa" THEN true + ELSE false + END conversa_finalizada, +##### + CASE + WHEN hist.new_turn = 1 THEN "hard_bounce" + WHEN + hist.new_turn = 2 + AND ( + ENDS_WITH(hist.resposta_bot, 'VOLTAR') + OR ENDS_WITH(hist.resposta_bot, 'SAIR') + ) + THEN "soft_bounce" + WHEN + ENDS_WITH(hist.resposta_bot, 'VOLTAR') + OR ENDS_WITH(hist.resposta_bot, 'SAIR') + THEN "desistencia" + WHEN + (hist.type_flow = "informational" AND hist.fluxo = "DA 0 - Menu da Dívida Ativa") + THEN "engajado" + WHEN + (hist.type_flow = "informational" AND hist.fluxo != "DA 0 - Menu da Dívida Ativa") #timeout informacional + OR + (hist.type_flow = "service" AND hist.api_resposta_sucesso IS NULL) + THEN "timeout_usuario_pre_transacao" + -- WHEN + -- JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado')) = "false" + -- OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado')) = "false" + -- THEN "impedimento_regra_negocio" + WHEN + (hist.type_flow = "service" AND hist.api_resposta_sucesso IN ("replaced", "true")) + THEN "transacao_realizada" + WHEN + hist.resposta_bot IS NULL + THEN "timeout interno" + WHEN hist.type_flow = "service" THEN "timeout_usuario_pos_transacao" + ELSE "investigar" + END classificacao_conversa, +##### + "sucesso" as status_final_conversa, + hist.passo as passo_falha, + hist.fluxo as fluxo_falha, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) erro_abertura_ticket, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) protocolo, + SAFE_CAST(NULL AS INT64) AS turnos_em_menus, + SAFE_CAST(NULL AS INT64) AS estimativa_turnos_menu, + SAFE_CAST(NULL AS INT64) AS turnos_em_servico, + SAFE_CAST(NULL AS INT64) AS estimativa_turnos_servico, + SAFE_CAST(NULL AS INT64) AS turnos_em_endereco, + SAFE_CAST(NULL AS INT64) AS turnos_em_identificacao, + hist.conversation_name as conversa_completa_id, + SAFE_CAST(NULL AS INT64) AS conversa_completa_turnos_em_menus, + SAFE_CAST(NULL AS INT64) AS conversa_completa_fluxos_interagidos, + SAFE_CAST(NULL AS INT64) AS conversa_completa_duracao, + SAFE_CAST(NULL AS STRING) AS conversa_completa_ultimo_fluxo_servico, +FROM FinalTableDA as hist +INNER JOIN ultima_interacao as ui + ON hist.new_conversation_id = ui.new_conversation_id AND hist.new_turn = ui.last_turn +INNER JOIN primeira_interacao as pi + ON hist.new_conversation_id = pi.new_conversation_id +ORDER BY request_time DESC) + +SELECT * FROM fim_conversas_DA WHERE fluxo_nome != 'DA 0 - Menu da Dívida Ativa' \ No newline at end of file diff --git a/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql b/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql new file mode 100644 index 0000000..aaede53 --- /dev/null +++ b/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql @@ -0,0 +1,139 @@ +WITH compilation AS ( + SELECT + * + FROM {{ ref('historico_conversas_macrofluxos') }} +), + +-- SELECT * FROM compilation +-- where +-- #NOT ENDS_WITH(new_conversation_id, "00") +-- STARTS_WITH(new_conversation_id, "projects/rj-chatbot-dev/locations/global/agents/29358e97-22d5-48e0-b6e0-fe32e70b67cd/environments/f288d64a-52f3-42f7-be7d-cac0b0f4957a/sessions/protocol-09700003664811") +-- order by conversation_name, turn_position ASC + +primeira_interacao AS ( + SELECT + new_conversation_id, + request_time AS hora_primeira_interacao, + mensagem_cidadao AS primeira_mensagem, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.intent.displayName')) as primeiro_intent, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) AS ambiente + FROM compilation + WHERE new_turn = 1 +) + +, ultima_interacao as ( +SELECT + new_conversation_id, + MAX(new_turn) as last_turn +FROM compilation +GROUP BY new_conversation_id +), + +fim_conversas_macrofluxo AS ( +SELECT + hist.new_conversation_id as conversation_name, + JSON_VALUE(parametros, '$.usuario_cpf') AS cpf, + JSON_VALUE(parametros, '$.phone') as telefone, + CASE + WHEN pi.ambiente = "production" THEN "Produção" + ELSE "Homologação" + END ambiente, + CASE + WHEN hist.nome_servico_1746 = "Serviço Não Mapeado" + THEN CONCAT("Menu ", JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.macrotema'))) + ELSE hist.nome_servico_1746 END as fluxo_nome, + horario, + DATETIME(request_time, "America/Buenos_Aires") as request_time, + ROUND(DATE_DIFF(request_time, hora_primeira_interacao, SECOND)/60,1) as duracao_minutos, + hist.new_turn as ultimo_turno, + pi.primeira_mensagem, + response, + CASE + WHEN hist.passo = "End Session" OR pi.primeiro_intent = "voltar_inicio_padrao" THEN true + ELSE false + END conversa_finalizada, +##### + CASE + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL THEN "chamado_aberto" + WHEN hist.turn_position = 1 THEN "hard_bounce" + WHEN + #hist.turn_position > 2 + hist.nome_servico_1746 = "Serviço Não Mapeado" + AND hist.fluxo = "CadÚnico" + THEN "engajado" + WHEN + JSON_VALUE(JSON_EXTRACT(hist.response, '$.queryResult.intent.displayName')) = "voltar_inicio_padrao" + THEN "desistência" + WHEN + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno_timeout" + THEN "timeout SGRC" + WHEN ENDS_WITH(resposta_bot,'TRANSBORDO') THEN "transbordo" + WHEN + # Casos em que o Chatbot identificou a inelegibilidade + (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) != "erro_desconhecido" + AND + (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado')) = "false" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado')) = "false") + ) + # Casos em que o SGRC identificou a inelegibilidade + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_aberto" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_aberto" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" + THEN "impedimento_regra_negocio" + WHEN + resposta_bot IS NULL + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "erro_desconhecido" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "erro_desconhecido" + THEN "timeout interno" + WHEN ENDS_WITH(resposta_bot, 'SAIR') THEN "engajado" + WHEN + hist.turn_position = 2 + AND hist.nome_servico_1746 = "Serviço Não Mapeado" + THEN "soft_bounce" + WHEN + hist.turn_position = 2 + AND hist.nome_servico_1746 != "Serviço Não Mapeado" + THEN "timeout_usuario_pos_transacao" + WHEN + hist.turn_position > 2 + AND hist.nome_servico_1746 = "Serviço Não Mapeado" + THEN "timeout_usuario_pre_transacao" + ELSE "timeout_usuario_pos_transacao" + END classificacao_conversa, +##### + "sucesso" as status_final_conversa, + hist.passo as passo_falha, + hist.fluxo as fluxo_falha, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) erro_abertura_ticket, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) protocolo, + CASE + WHEN hist.turns_until_service = 0 + THEN hist.new_turn + ELSE hist.turns_until_service END as turnos_em_menus, + hist.estimativa_turnos_menu, + hist.new_turn - hist.turnos_em_endereco - hist.turnos_em_identificacao - + (CASE + WHEN hist.turns_until_service = 0 + THEN hist.new_turn + ELSE hist.turns_until_service END) as turnos_em_servico, + hist.estimativa_turnos_servico, + hist.turnos_em_endereco, + hist.turnos_em_identificacao, + hist.conversation_name as conversa_completa_id, + CASE + WHEN hist.conversa_completa_turnos_em_menus = 0 + THEN hist.conversa_completa_duracao + ELSE hist.conversa_completa_turnos_em_menus END as conversa_completa_turnos_em_menus, + hist.conversa_completa_fluxos_interagidos, + hist.conversa_completa_duracao, + hist.conversa_completa_ultimo_fluxo_servico +FROM compilation as hist +INNER JOIN ultima_interacao as ui + ON hist.new_conversation_id = ui.new_conversation_id AND hist.new_turn = ui.last_turn +INNER JOIN primeira_interacao as pi + ON hist.new_conversation_id = pi.new_conversation_id +ORDER BY request_time DESC) + +SELECT * FROM fim_conversas_macrofluxo \ No newline at end of file diff --git a/queries/models/dialogflowcx/historico_conversas_legivel.sql b/queries/models/dialogflowcx/historico_conversas_legivel.sql new file mode 100644 index 0000000..0581e9c --- /dev/null +++ b/queries/models/dialogflowcx/historico_conversas_legivel.sql @@ -0,0 +1,36 @@ +WITH historico_padrao AS ( + select + conversation_name as conversa_completa_id, + turn_position as conversa_completa_turn_position, + conversation_name, + turn_position, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) as mensagem_cidadao, + JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')) as resposta_bot, + FORMAT_DATETIME("%d/%m/%Y às %H:%M", DATETIME(request_time, "America/Buenos_Aires")) as horario, + JSON_EXTRACT(response, '$.queryResult.parameters') as parametros, + request_time, + response + from `rj-chatbot-dev.dialogflowcx.historico_conversas` + ORDER BY conversation_name, request_time ASC +), + +historico_macrofluxo AS ( + SELECT + conversation_name as conversa_completa_id, + turn_position as conversa_completa_turn_position, + new_conversation_id as conversation_name, + new_turn as turn_position, + mensagem_cidadao, + resposta_bot, + horario, + parametros, + request_time, + response + FROM {{ ref('historico_conversas_macrofluxos') }} + ORDER BY conversation_name, request_time ASC +) + +(SELECT * FROM historico_padrao +UNION ALL +SELECT * FROM historico_macrofluxo) +ORDER BY conversation_name, request_time ASC \ No newline at end of file diff --git a/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql b/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql new file mode 100644 index 0000000..eba51e2 --- /dev/null +++ b/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql @@ -0,0 +1,285 @@ +WITH marked_conversations AS ( + WITH filtro_macrofluxos AS ( + SELECT DISTINCT + conversation_name + FROM rj-chatbot-dev.dialogflowcx.historico_conversas + WHERE + turn_position = 1 + AND + ( + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) IN ( + "Cuidados com a Cidade", + "Educação, Saúde e Cidadania", + "Servidor Público", + "Suporte Técnico", + "Trânsito e Transportes", + "Trânsito", + "Transportes", + "Tributos e Licenciamento" + ) + OR + ( + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) = 'Menu Principal' + AND request_time >= '2024-08-02' + ) + ) + ), + + historico AS ( + SELECT + h.conversation_name, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) AS mensagem_cidadao, + JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')) AS resposta_bot, + turn_position, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) AS fluxo, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) AS passo, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.codigo_servico_1746')) AS codigo_servico_1746, + JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.macrotema')) AS macrotema, + `rj-chatbot-dev.dialogflowcx`.get_service_data(JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.codigo_servico_1746'))) AS service_data_array, + FORMAT_DATETIME("%d/%m/%Y às %H:%M", DATETIME(request_time, "America/Buenos_Aires")) AS horario, + JSON_EXTRACT(response, '$.queryResult.parameters') AS parametros, + request_time, + response + FROM rj-chatbot-dev.dialogflowcx.historico_conversas AS h + INNER JOIN filtro_macrofluxos AS f + ON f.conversation_name = h.conversation_name + -- Remove o ORDER BY aqui e adicione apenas onde for necessário + ), + + historico_with_lag AS ( + SELECT + *, + LAG(codigo_servico_1746) OVER(PARTITION BY conversation_name ORDER BY turn_position) AS lag_codigo_servico_1746_1, + LAG(codigo_servico_1746, 2) OVER(PARTITION BY conversation_name ORDER BY turn_position) AS lag_codigo_servico_1746_2 + FROM historico + ), + + MarkedConversations AS ( + SELECT + * EXCEPT(service_data_array), + service_data_array[OFFSET(0)] AS nome_servico_1746, + service_data_array[OFFSET(1)] AS estimativa_turnos_servico, + service_data_array[OFFSET(2)] AS estimativa_turnos_menu, + CASE + WHEN lag_codigo_servico_1746_1 IS NULL AND codigo_servico_1746 IS NOT NULL THEN 0 + WHEN lag_codigo_servico_1746_2 IS NOT NULL AND codigo_servico_1746 IS NULL THEN 1 + WHEN lag_codigo_servico_1746_1 IS NOT NULL + AND lag_codigo_servico_1746_1 != codigo_servico_1746 THEN 1 + ELSE 0 + END AS isNewPart + FROM historico_with_lag + ) + + SELECT + * + FROM MarkedConversations +), + +NumberedConversations AS ( + SELECT + conversation_name, + turn_position, + fluxo, + codigo_servico_1746, + nome_servico_1746, + estimativa_turnos_servico, + estimativa_turnos_menu, + SUM(isNewPart) OVER(PARTITION BY conversation_name ORDER BY turn_position) AS part_number + FROM + marked_conversations +), +ConversationsWithNewTurn AS ( + SELECT + conversation_name, + turn_position, + part_number, + fluxo, + codigo_servico_1746, + nome_servico_1746, + estimativa_turnos_servico, + estimativa_turnos_menu, + ROW_NUMBER() OVER(PARTITION BY conversation_name, part_number ORDER BY turn_position) AS new_turn + FROM + NumberedConversations +), +new_conv_id AS ( + SELECT + CONCAT(conversation_name, '-', FORMAT('%02d', part_number)) AS new_conversation_id, + #mensagem_cidadao, + #resposta_bot, + turn_position, + #macrotema, + fluxo, + #passo, + codigo_servico_1746, + nome_servico_1746, + estimativa_turnos_servico, + estimativa_turnos_menu, + #horario, + #parametros, + #request_time, + #response, + new_turn, + conversation_name + FROM + ConversationsWithNewTurn +), + +service_names AS ( + SELECT + new_conversation_id, + CASE + WHEN COUNT(DISTINCT nome_servico_1746) = 1 THEN MAX(nome_servico_1746) + WHEN COUNT(DISTINCT nome_servico_1746) = 2 AND MAX(nome_servico_1746) = 'Serviço não mapeado' THEN MIN(nome_servico_1746) + WHEN COUNT(DISTINCT nome_servico_1746) = 2 AND MIN(nome_servico_1746) = 'Serviço não mapeado' THEN MAX(nome_servico_1746) + ELSE CONCAT('Conversa com mais de um fluxo: ', STRING_AGG(DISTINCT nome_servico_1746, ', ')) + END AS nome_servico, + MAX(SAFE_CAST(estimativa_turnos_servico AS INT64)) as estimativa_turnos_servico, + MAX(SAFE_CAST(estimativa_turnos_menu AS INT64)) as estimativa_turnos_menu + FROM + new_conv_id + GROUP BY + new_conversation_id +), + +first_service_turn AS ( + SELECT + new_conversation_id, + MIN(new_turn) AS first_service_turn_position + FROM + new_conv_id + WHERE + codigo_servico_1746 IS NOT NULL + GROUP BY + new_conversation_id +), + +compilation_0 AS ( +SELECT + n.new_conversation_id, + #INITCAP(n.macrotema) as macrotema, + INITCAP(s.nome_servico) as nome_servico_1746, + #n.mensagem_cidadao, + #n.resposta_bot, + n.turn_position, + n.fluxo, + #n.passo, + n.codigo_servico_1746, + #n.horario, + #n.parametros, + #n.request_time, + #n.response, + n.new_turn, + n.conversation_name, + COALESCE(f.first_service_turn_position - 1, 0) AS turns_until_service, + CASE WHEN n.fluxo = "Coleta Endereço" THEN true ELSE false END AS address_turn, + CASE WHEN n.fluxo = "Identificação" THEN true ELSE false END AS identification_turn, + s.estimativa_turnos_servico, + s.estimativa_turnos_menu +FROM new_conv_id as n +LEFT JOIN service_names as s + ON n.new_conversation_id = s.new_conversation_id +LEFT JOIN first_service_turn as f + ON n.new_conversation_id = f.new_conversation_id +), + +conversas_completas_metrics AS ( + SELECT + c0.conversation_name, + MAX(sq.turns_per_service) AS conversa_completa_turnos_em_menus, + COUNT(DISTINCT c0.nome_servico_1746) AS conversa_completa_fluxos_interagidos, + COUNT(CASE WHEN address_turn IS true THEN 1 ELSE NULL END) as conversa_completa_fluxos_endereco, + COUNT(CASE WHEN identification_turn IS true THEN 1 ELSE NULL END) as conversa_completa_fluxos_identificacao + FROM compilation_0 as c0 + INNER JOIN + (SELECT conversation_name, SUM(turns_per_service) as turns_per_service FROM + ( + SELECT + conversation_name, + new_conversation_id, + MAX(turns_until_service) AS turns_per_service, + FROM + compilation_0 + GROUP BY 1,2 + ) GROUP BY 1 + ) AS sq + ON c0.conversation_name = sq.conversation_name + GROUP BY + conversation_name +), + +conversas_completas_last_service AS ( + SELECT + c.conversation_name, + c.nome_servico_1746 AS ultimo_fluxo_servico, + ltp.last_turn_position + FROM + compilation_0 as c + INNER JOIN + ( + SELECT + conversation_name, + MAX(turn_position) AS last_turn_position + FROM + compilation_0 + GROUP BY + conversation_name + ) as ltp + ON c.conversation_name = ltp.conversation_name AND c.turn_position = ltp.last_turn_position +), + +conversas_completas AS ( + SELECT + m.conversation_name, + m.conversa_completa_turnos_em_menus, + m.conversa_completa_fluxos_interagidos, + m.conversa_completa_fluxos_endereco, + m.conversa_completa_fluxos_identificacao, + l.last_turn_position, + l.ultimo_fluxo_servico + FROM + conversas_completas_metrics m + LEFT JOIN + conversas_completas_last_service l + ON + m.conversation_name = l.conversation_name +), + +turnos_em_identificacao_e_endereco AS ( + SELECT + new_conversation_id, + COUNT(CASE WHEN address_turn IS true THEN 1 ELSE NULL END) as turnos_em_endereco, + COUNT(CASE WHEN identification_turn IS true THEN 1 ELSE NULL END) as turnos_em_identificacao + FROM compilation_0 + GROUP BY 1 +), + +compilation AS ( + SELECT + c.*, + INITCAP(n.macrotema) as macrotema, + n.mensagem_cidadao, + n.resposta_bot, + n.passo, + n.horario, + n.parametros, + n.request_time, + n.response, + tie.turnos_em_endereco, + tie.turnos_em_identificacao, + cc.conversa_completa_turnos_em_menus, + cc.conversa_completa_fluxos_interagidos, + cc.last_turn_position as conversa_completa_duracao, + cc.ultimo_fluxo_servico as conversa_completa_ultimo_fluxo_servico + FROM compilation_0 as c + LEFT JOIN conversas_completas as cc + ON c.conversation_name = cc.conversation_name + LEFT JOIN turnos_em_identificacao_e_endereco as tie + ON c.new_conversation_id = tie.new_conversation_id + INNER JOIN + (SELECT * FROM marked_conversations + WHERE conversation_name NOT IN ("projects/rj-chatbot-dev/locations/global/agents/29358e97-22d5-48e0-b6e0-fe32e70b67cd/environments/fcd7f325-6095-4ce1-9757-6cc028b8b554/sessions/protocol-73430003652037", "projects/rj-chatbot-dev/locations/global/agents/29358e97-22d5-48e0-b6e0-fe32e70b67cd/environments/f288d64a-52f3-42f7-be7d-cac0b0f4957a/sessions/protocol-38990003782307")) as n + ON c.conversation_name = n.conversation_name AND c.turn_position = n.turn_position +) + +SELECT * from compilation \ No newline at end of file From 47503fc549f9b2fe9d256aca6e9ff485e6a8bbf3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:35:55 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../chatbot/dbt_chatbot_metricas/schedules.py | 6 +-- pipelines/flows.py | 2 +- queries/models/dialogflowcx/fim_conversas.sql | 44 ++++++++-------- .../models/dialogflowcx/fim_conversas_da.sql | 52 +++++++++---------- .../fim_conversas_macrofluxos.sql | 44 ++++++++-------- .../historico_conversas_macrofluxos.sql | 28 +++++----- 6 files changed, 88 insertions(+), 88 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py index 0790cba..513f410 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py @@ -27,7 +27,7 @@ "table_id": "historico_conversas_legivel", "upstream": False, }, - "conversas_completas_metricas": { + "conversas_completas_metricas": { "table_id": "conversas_completas_metricas", "upstream": False, }, @@ -37,7 +37,7 @@ IntervalClock( interval=timedelta(hours=12), start_date=datetime(2024, 9, 17, 19, 0, tzinfo=pytz.timezone("America/Sao_Paulo")) - + timedelta(minutes = 45 * count), + + timedelta(minutes=45 * count), labels=[ constants.RJ_ESCRITORIO_AGENT_LABEL.value, ], @@ -50,4 +50,4 @@ ) for count, (_, parameters) in enumerate(chatbot_tables.items()) ] -update_schedule = Schedule(clocks=untuple(chatbot_clocks)) \ No newline at end of file +update_schedule = Schedule(clocks=untuple(chatbot_clocks)) diff --git a/pipelines/flows.py b/pipelines/flows.py index 36e8c1a..60982f1 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -2,6 +2,7 @@ """ Imports all flows for every project so we can register all of them. """ +from pipelines.chatbot import * # noqa from pipelines.deteccao_alagamento_cameras import * # noqa from pipelines.exemplo import * # noqa from pipelines.healthcheck import * # noqa @@ -9,4 +10,3 @@ from pipelines.lgpd import * # noqa from pipelines.stress import * # noqa from pipelines.templates import * # noqa -from pipelines.chatbot import * # noqa diff --git a/queries/models/dialogflowcx/fim_conversas.sql b/queries/models/dialogflowcx/fim_conversas.sql index 2274c42..a6701c4 100644 --- a/queries/models/dialogflowcx/fim_conversas.sql +++ b/queries/models/dialogflowcx/fim_conversas.sql @@ -1,6 +1,6 @@ with ultima_interacao as ( -SELECT - conversation_name, +SELECT + conversation_name, MAX(turn_position) as last_turn FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` GROUP BY conversation_name @@ -8,14 +8,14 @@ GROUP BY conversation_name # Captura apenas mensagens vindas do ASC através da mensagem que inicia a conversa primeira_interacao AS ( - SELECT + SELECT conversation_name, `rj-chatbot-dev.dialogflowcx.inicial_sentence_to_flow_name`(JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text'))) AS fluxo_primeira_interacao, - request_time AS hora_primeira_interacao, + request_time AS hora_primeira_interacao, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) AS primeira_mensagem, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) AS ambiente FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` - WHERE + WHERE turn_position = 1 AND `rj-chatbot-dev.dialogflowcx.inicial_sentences`(JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text'))) ), @@ -25,9 +25,9 @@ SELECT hist.conversation_name, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.usuario_cpf')) AS cpf, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.phone')) as telefone, - CASE + CASE WHEN pi.ambiente = "production" THEN "Produção" - ELSE "Homologação" + ELSE "Homologação" END ambiente, INITCAP(pi.fluxo_primeira_interacao) as fluxo_nome, FORMAT_DATETIME("%d/%m/%Y às %H:%M", DATETIME(request_time, "America/Buenos_Aires")) as horario, @@ -43,54 +43,54 @@ SELECT CASE WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL THEN "chamado_aberto" WHEN hist.turn_position = 1 THEN "hard_bounce" - WHEN - hist.turn_position = 2 + WHEN + hist.turn_position = 2 AND ( ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'VOLTAR') - OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') + OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') ) THEN "soft_bounce" - WHEN + WHEN hist.turn_position = 2 THEN "timeout_usuario_pos_transacao" - WHEN + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno_timeout" THEN "timeout SGRC" WHEN ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')),'TRANSBORDO') THEN "transbordo" - WHEN + WHEN # Casos em que o Chatbot identificou a inelegibilidade (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) != "erro_desconhecido" AND (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado')) = "false" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado')) = "false") - ) + ) # Casos em que o SGRC identificou a inelegibilidade OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_aberto" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_aberto" - OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" THEN "impedimento_regra_negocio" - WHEN + WHEN JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')) IS NULL OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "erro_desconhecido" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "erro_desconhecido" THEN "timeout interno" - WHEN + WHEN ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'VOLTAR') - OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') + OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') THEN "desistencia" ELSE "timeout_usuario_pos_transacao" END classificacao_conversa, CASE - WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'VOLTAR') OR ENDS_WITH(JSON_VALUE(JSON_EXTRACT(derived_data, '$.agentUtterances')), 'SAIR') THEN "sucesso" ELSE "falha" END status_final_conversa, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) as passo_falha, - CASE + CASE WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1647 (RRL)' THEN 'Remoção de Resíduo' WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1614 (PAL)' THEN 'Poda de Árvore' WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1464 (VACIO)' THEN 'Verificação de Ar Condicionado Inoperante em Ônibus' @@ -104,7 +104,7 @@ SELECT WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '223 (RSTAP)' THEN "Reparo de Sinal de Trânsito em Amarelo Piscante" WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '1618 (CRCA)' THEN "Controle de Roedores e Caramujos Africanos" WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) = '3803 (RSTAAV)' THEN "Reparo de Sinal de Trânsito Abalroado ou Ausente ou Virado" - ELSE JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) + ELSE JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) END as fluxo_falha, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) erro_abertura_ticket, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) protocolo, @@ -126,7 +126,7 @@ INNER JOIN primeira_interacao as pi ON hist.conversation_name = pi.conversation_name ORDER BY request_time DESC) -SELECT +SELECT *, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.codigo_servico_1746')) as codigo_servico FROM fim_conversas_1746 diff --git a/queries/models/dialogflowcx/fim_conversas_da.sql b/queries/models/dialogflowcx/fim_conversas_da.sql index 7266747..4ba175e 100644 --- a/queries/models/dialogflowcx/fim_conversas_da.sql +++ b/queries/models/dialogflowcx/fim_conversas_da.sql @@ -1,8 +1,8 @@ WITH filtro_divida_ativa AS ( - SELECT + SELECT conversation_name FROM `rj-chatbot-dev.dialogflowcx.historico_conversas` - WHERE + WHERE JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) = "production" AND turn_position = 1 AND JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.text')) = "dívida ativa" @@ -109,25 +109,25 @@ FluxoMain AS ( ParametrosExtracted AS ( SELECT new_conversation_id, - CASE - WHEN JSON_VALUE(parametros, '$.api_resposta_sucesso') IS NULL THEN - CASE + CASE + WHEN JSON_VALUE(parametros, '$.api_resposta_sucesso') IS NULL THEN + CASE WHEN REGEXP_CONTAINS(TO_JSON_STRING(parametros), r'\"api_resposta_sucesso\"') THEN 'replaced' ELSE NULL END ELSE JSON_VALUE(parametros, '$.api_resposta_sucesso') END AS api_resposta_sucesso, - CASE - WHEN JSON_VALUE(parametros, '$.api_descricao_erro') IS NULL THEN - CASE + CASE + WHEN JSON_VALUE(parametros, '$.api_descricao_erro') IS NULL THEN + CASE WHEN REGEXP_CONTAINS(TO_JSON_STRING(parametros), r'\"api_descricao_erro\"') THEN 'replaced' ELSE NULL END ELSE JSON_VALUE(parametros, '$.api_descricao_erro') END AS api_descricao_erro - FROM + FROM new_conv_id - WHERE + WHERE parametros IS NOT NULL ), @@ -164,7 +164,7 @@ FinalTableDA AS ( WHERE t.type_flow != 'undefined' ) --- SELECT * FROM FinalTableDA #WHERE api_resposta_sucesso = "replaced" AND main_fluxo IN ('DA 3 - Consulta de protestos', 'DA 1 - Consultar Débitos Dívida Ativa Solicitar Guias Exibir Informações') +-- SELECT * FROM FinalTableDA #WHERE api_resposta_sucesso = "replaced" AND main_fluxo IN ('DA 3 - Consulta de protestos', 'DA 1 - Consultar Débitos Dívida Ativa Solicitar Guias Exibir Informações') -- ORDER BY new_conversation_id, new_turn; , FinalTableOutros AS ( @@ -183,10 +183,10 @@ FinalTableDA AS ( ) , primeira_interacao AS ( - SELECT + SELECT new_conversation_id, main_fluxo as fluxo_primeira_interacao, - request_time AS hora_primeira_interacao, + request_time AS hora_primeira_interacao, mensagem_cidadao AS primeira_mensagem, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) AS ambiente FROM FinalTableDA @@ -194,8 +194,8 @@ FinalTableDA AS ( ) , ultima_interacao as ( -SELECT - new_conversation_id, +SELECT + new_conversation_id, MAX(new_turn) as last_turn FROM FinalTableDA GROUP BY new_conversation_id @@ -206,9 +206,9 @@ SELECT hist.new_conversation_id as conversation_name, JSON_VALUE(parametros, '$.usuario_cpf') AS cpf, JSON_VALUE(parametros, '$.phone') as telefone, - CASE + CASE WHEN pi.ambiente = "production" THEN "Produção" - ELSE "Homologação" + ELSE "Homologação" END ambiente, main_fluxo as fluxo_nome, horario, @@ -224,33 +224,33 @@ SELECT ##### CASE WHEN hist.new_turn = 1 THEN "hard_bounce" - WHEN - hist.new_turn = 2 + WHEN + hist.new_turn = 2 AND ( ENDS_WITH(hist.resposta_bot, 'VOLTAR') - OR ENDS_WITH(hist.resposta_bot, 'SAIR') + OR ENDS_WITH(hist.resposta_bot, 'SAIR') ) THEN "soft_bounce" - WHEN + WHEN ENDS_WITH(hist.resposta_bot, 'VOLTAR') - OR ENDS_WITH(hist.resposta_bot, 'SAIR') + OR ENDS_WITH(hist.resposta_bot, 'SAIR') THEN "desistencia" WHEN (hist.type_flow = "informational" AND hist.fluxo = "DA 0 - Menu da Dívida Ativa") THEN "engajado" - WHEN + WHEN (hist.type_flow = "informational" AND hist.fluxo != "DA 0 - Menu da Dívida Ativa") #timeout informacional OR (hist.type_flow = "service" AND hist.api_resposta_sucesso IS NULL) THEN "timeout_usuario_pre_transacao" - -- WHEN + -- WHEN -- JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado')) = "false" -- OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado')) = "false" -- THEN "impedimento_regra_negocio" - WHEN + WHEN (hist.type_flow = "service" AND hist.api_resposta_sucesso IN ("replaced", "true")) THEN "transacao_realizada" - WHEN + WHEN hist.resposta_bot IS NULL THEN "timeout interno" WHEN hist.type_flow = "service" THEN "timeout_usuario_pos_transacao" diff --git a/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql b/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql index aaede53..4b5c560 100644 --- a/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql +++ b/queries/models/dialogflowcx/fim_conversas_macrofluxos.sql @@ -4,16 +4,16 @@ WITH compilation AS ( FROM {{ ref('historico_conversas_macrofluxos') }} ), --- SELECT * FROM compilation --- where +-- SELECT * FROM compilation +-- where -- #NOT ENDS_WITH(new_conversation_id, "00") -- STARTS_WITH(new_conversation_id, "projects/rj-chatbot-dev/locations/global/agents/29358e97-22d5-48e0-b6e0-fe32e70b67cd/environments/f288d64a-52f3-42f7-be7d-cac0b0f4957a/sessions/protocol-09700003664811") -- order by conversation_name, turn_position ASC primeira_interacao AS ( - SELECT + SELECT new_conversation_id, - request_time AS hora_primeira_interacao, + request_time AS hora_primeira_interacao, mensagem_cidadao AS primeira_mensagem, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.intent.displayName')) as primeiro_intent, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.ambiente')) AS ambiente @@ -22,8 +22,8 @@ primeira_interacao AS ( ) , ultima_interacao as ( -SELECT - new_conversation_id, +SELECT + new_conversation_id, MAX(new_turn) as last_turn FROM compilation GROUP BY new_conversation_id @@ -34,9 +34,9 @@ SELECT hist.new_conversation_id as conversation_name, JSON_VALUE(parametros, '$.usuario_cpf') AS cpf, JSON_VALUE(parametros, '$.phone') as telefone, - CASE + CASE WHEN pi.ambiente = "production" THEN "Produção" - ELSE "Homologação" + ELSE "Homologação" END ambiente, CASE WHEN hist.nome_servico_1746 = "Serviço Não Mapeado" @@ -56,7 +56,7 @@ SELECT CASE WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) IS NOT NULL THEN "chamado_aberto" WHEN hist.turn_position = 1 THEN "hard_bounce" - WHEN + WHEN #hist.turn_position > 2 hist.nome_servico_1746 = "Serviço Não Mapeado" AND hist.fluxo = "CadÚnico" @@ -64,39 +64,39 @@ SELECT WHEN JSON_VALUE(JSON_EXTRACT(hist.response, '$.queryResult.intent.displayName')) = "voltar_inicio_padrao" THEN "desistência" - WHEN + WHEN JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno_timeout" THEN "timeout SGRC" WHEN ENDS_WITH(resposta_bot,'TRANSBORDO') THEN "transbordo" - WHEN + WHEN # Casos em que o Chatbot identificou a inelegibilidade (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) != "erro_desconhecido" AND (JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado')) = "false" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado')) = "false") - ) + ) # Casos em que o SGRC identificou a inelegibilidade OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_aberto" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_aberto" - OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" + OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "chamado_fechado_12_dias" THEN "impedimento_regra_negocio" - WHEN + WHEN resposta_bot IS NULL OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) = "erro_interno" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_endereco_abertura_chamado_justificativa')) = "erro_desconhecido" OR JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.rebi_elegibilidade_abertura_chamado_justificativa')) = "erro_desconhecido" THEN "timeout interno" WHEN ENDS_WITH(resposta_bot, 'SAIR') THEN "engajado" - WHEN - hist.turn_position = 2 + WHEN + hist.turn_position = 2 AND hist.nome_servico_1746 = "Serviço Não Mapeado" THEN "soft_bounce" - WHEN + WHEN hist.turn_position = 2 AND hist.nome_servico_1746 != "Serviço Não Mapeado" THEN "timeout_usuario_pos_transacao" - WHEN + WHEN hist.turn_position > 2 AND hist.nome_servico_1746 = "Serviço Não Mapeado" THEN "timeout_usuario_pre_transacao" @@ -109,13 +109,13 @@ SELECT JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_retorno')) erro_abertura_ticket, JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.parameters.solicitacao_protocolo')) protocolo, CASE - WHEN hist.turns_until_service = 0 + WHEN hist.turns_until_service = 0 THEN hist.new_turn ELSE hist.turns_until_service END as turnos_em_menus, hist.estimativa_turnos_menu, - hist.new_turn - hist.turnos_em_endereco - hist.turnos_em_identificacao - + hist.new_turn - hist.turnos_em_endereco - hist.turnos_em_identificacao - (CASE - WHEN hist.turns_until_service = 0 + WHEN hist.turns_until_service = 0 THEN hist.new_turn ELSE hist.turns_until_service END) as turnos_em_servico, hist.estimativa_turnos_servico, @@ -123,7 +123,7 @@ SELECT hist.turnos_em_identificacao, hist.conversation_name as conversa_completa_id, CASE - WHEN hist.conversa_completa_turnos_em_menus = 0 + WHEN hist.conversa_completa_turnos_em_menus = 0 THEN hist.conversa_completa_duracao ELSE hist.conversa_completa_turnos_em_menus END as conversa_completa_turnos_em_menus, hist.conversa_completa_fluxos_interagidos, diff --git a/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql b/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql index eba51e2..6ac1605 100644 --- a/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql +++ b/queries/models/dialogflowcx/historico_conversas_macrofluxos.sql @@ -3,9 +3,9 @@ WITH marked_conversations AS ( SELECT DISTINCT conversation_name FROM rj-chatbot-dev.dialogflowcx.historico_conversas - WHERE + WHERE turn_position = 1 - AND + AND ( JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentFlow.displayName')) IN ( "Cuidados com a Cidade", @@ -17,7 +17,7 @@ WITH marked_conversations AS ( "Transportes", "Tributos e Licenciamento" ) - OR + OR ( JSON_VALUE(JSON_EXTRACT(response, '$.queryResult.currentPage.displayName')) = 'Menu Principal' AND request_time >= '2024-08-02' @@ -41,7 +41,7 @@ WITH marked_conversations AS ( request_time, response FROM rj-chatbot-dev.dialogflowcx.historico_conversas AS h - INNER JOIN filtro_macrofluxos AS f + INNER JOIN filtro_macrofluxos AS f ON f.conversation_name = h.conversation_name -- Remove o ORDER BY aqui e adicione apenas onde for necessário ), @@ -63,7 +63,7 @@ WITH marked_conversations AS ( CASE WHEN lag_codigo_servico_1746_1 IS NULL AND codigo_servico_1746 IS NOT NULL THEN 0 WHEN lag_codigo_servico_1746_2 IS NOT NULL AND codigo_servico_1746 IS NULL THEN 1 - WHEN lag_codigo_servico_1746_1 IS NOT NULL + WHEN lag_codigo_servico_1746_1 IS NOT NULL AND lag_codigo_servico_1746_1 != codigo_servico_1746 THEN 1 ELSE 0 END AS isNewPart @@ -128,7 +128,7 @@ new_conv_id AS ( service_names AS ( SELECT new_conversation_id, - CASE + CASE WHEN COUNT(DISTINCT nome_servico_1746) = 1 THEN MAX(nome_servico_1746) WHEN COUNT(DISTINCT nome_servico_1746) = 2 AND MAX(nome_servico_1746) = 'Serviço não mapeado' THEN MIN(nome_servico_1746) WHEN COUNT(DISTINCT nome_servico_1746) = 2 AND MIN(nome_servico_1746) = 'Serviço não mapeado' THEN MAX(nome_servico_1746) @@ -191,8 +191,8 @@ conversas_completas_metrics AS ( COUNT(CASE WHEN address_turn IS true THEN 1 ELSE NULL END) as conversa_completa_fluxos_endereco, COUNT(CASE WHEN identification_turn IS true THEN 1 ELSE NULL END) as conversa_completa_fluxos_identificacao FROM compilation_0 as c0 - INNER JOIN - (SELECT conversation_name, SUM(turns_per_service) as turns_per_service FROM + INNER JOIN + (SELECT conversation_name, SUM(turns_per_service) as turns_per_service FROM ( SELECT conversation_name, @@ -215,14 +215,14 @@ conversas_completas_last_service AS ( ltp.last_turn_position FROM compilation_0 as c - INNER JOIN + INNER JOIN ( - SELECT - conversation_name, + SELECT + conversation_name, MAX(turn_position) AS last_turn_position - FROM + FROM compilation_0 - GROUP BY + GROUP BY conversation_name ) as ltp ON c.conversation_name = ltp.conversation_name AND c.turn_position = ltp.last_turn_position @@ -276,7 +276,7 @@ compilation AS ( ON c.conversation_name = cc.conversation_name LEFT JOIN turnos_em_identificacao_e_endereco as tie ON c.new_conversation_id = tie.new_conversation_id - INNER JOIN + INNER JOIN (SELECT * FROM marked_conversations WHERE conversation_name NOT IN ("projects/rj-chatbot-dev/locations/global/agents/29358e97-22d5-48e0-b6e0-fe32e70b67cd/environments/fcd7f325-6095-4ce1-9757-6cc028b8b554/sessions/protocol-73430003652037", "projects/rj-chatbot-dev/locations/global/agents/29358e97-22d5-48e0-b6e0-fe32e70b67cd/environments/f288d64a-52f3-42f7-be7d-cac0b0f4957a/sessions/protocol-38990003782307")) as n ON c.conversation_name = n.conversation_name AND c.turn_position = n.turn_position From f107b7c9678eb6ca69b30db76969cb12870857cb Mon Sep 17 00:00:00 2001 From: Bruno Almeida Date: Tue, 17 Sep 2024 15:14:31 -0300 Subject: [PATCH 03/12] add: fix wrong reference --- pipelines/chatbot/dbt_chatbot_metricas/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index 4e02f2b..dc71af4 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -17,7 +17,7 @@ ) from pipelines.constants import constants -from pipelines.identidade_unica.dbt_identidade_unica.schedules import update_schedule +from pipelines.chatbot.dbt_chatbot_metricas.schedules import update_schedule run_rbt_chatbot_flow = deepcopy(templates__run_dbt_model__flow) run_rbt_chatbot_flow.name = "Chatbot: Materializar tabelas" From 0c77a6a692e51355cf03a8eb48318b6189c99af5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 18:15:01 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pipelines/chatbot/dbt_chatbot_metricas/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index dc71af4..c51203d 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -16,8 +16,8 @@ handler_inject_bd_credentials, ) -from pipelines.constants import constants from pipelines.chatbot.dbt_chatbot_metricas.schedules import update_schedule +from pipelines.constants import constants run_rbt_chatbot_flow = deepcopy(templates__run_dbt_model__flow) run_rbt_chatbot_flow.name = "Chatbot: Materializar tabelas" From af1c36431b4e4937177d9e89bd5562f363a4bec0 Mon Sep 17 00:00:00 2001 From: Bruno Almeida Date: Tue, 17 Sep 2024 16:20:20 -0300 Subject: [PATCH 05/12] add: dbt yaml --- queries/dbt_project.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index f04e544..a8d729d 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -43,3 +43,6 @@ models: identidade_unica: +materialized: table +schema: identidade_unica + dialogflowcx: + +materialized: table + +schema: dialogflowcx From 37dfc68d802c9f58aee91879fa21d6f1948bd39f Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 14:46:24 -0300 Subject: [PATCH 06/12] feat: add dbt infisical_credential_path and dbt_project_materialization --- .../chatbot/dbt_chatbot_metricas/flows.py | 2 +- .../chatbot/dbt_chatbot_metricas/schedules.py | 6 +- pyproject.toml | 2 +- uv.lock | 74 ++++++++++++++++--- 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index c51203d..66ca255 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -DBT-related flows. +DBT-related flows.. """ from copy import deepcopy diff --git a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py index 513f410..0e7ec8e 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py @@ -44,8 +44,12 @@ parameter_defaults={ "dataset_id": "dialogflowcx", "table_id": parameters["table_id"], - "mode": "prod", "upstream": parameters["upstream"], + "infisical_credential_path": { + "secret_path": "/dbt-rj-chatbot-dev", + "secret_name": "SEVICE_ACCOUNT", + }, + "dbt_project_materialization": "rj-chatbot-dev", }, ) for count, (_, parameters) in enumerate(chatbot_tables.items()) diff --git a/pyproject.toml b/pyproject.toml index b618197..d2c4a56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,4 +53,4 @@ profile = "black" lint = "black . && isort . && flake8 ." [tool.uv.sources] -prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "62236b527b3528437b8d460f60c79ee39309b151" } +prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "87fc0c33b6042939d4543219ad6e56be3755201a" } diff --git a/uv.lock b/uv.lock index a5f0632..09dd895 100644 --- a/uv.lock +++ b/uv.lock @@ -11,15 +11,18 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.11' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version == '3.11.*' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "python_full_version < '3.11' and platform_python_implementation != 'PyPy' and platform_system == 'Darwin'", - "python_full_version < '3.12' and platform_python_implementation != 'PyPy' and platform_system == 'Darwin'", + "python_full_version < '3.11' and platform_python_implementation != 'PyPy' and platform_system == 'Darwin'", + "python_full_version == '3.11.*' and platform_python_implementation != 'PyPy' and platform_system == 'Darwin'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_python_implementation != 'PyPy' and platform_system == 'Darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and platform_system == 'Linux'", - "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and platform_system == 'Linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and platform_system == 'Linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and platform_system == 'Linux'", "python_full_version == '3.12.*' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and platform_system == 'Linux'", "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.11' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", - "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.11' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version == '3.11.*' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_python_implementation != 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "python_full_version < '3.11' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", @@ -32,15 +35,18 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.11' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version == '3.11.*' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "python_full_version < '3.11' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", - "python_full_version < '3.12' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", + "python_full_version < '3.11' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", + "python_full_version == '3.11.*' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and platform_system == 'Darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and platform_system == 'Linux'", - "python_full_version < '3.12' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and platform_system == 'Linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and platform_system == 'Linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and platform_system == 'Linux'", "python_full_version == '3.12.*' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and platform_system == 'Linux'", "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and platform_system == 'Linux'", "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.11' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", - "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.12' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version < '3.11' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version == '3.11.*' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin') or (python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and platform_system != 'Darwin' and platform_system != 'Linux')", ] @@ -115,7 +121,7 @@ wheels = [ [[package]] name = "basedosdados" -version = "2.0.0b15" +version = "2.0.0b21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-python-client" }, @@ -130,9 +136,9 @@ dependencies = [ { name = "tomlkit" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ff/9f/3fc634b44b15b17f99d80a3d21335402925cd3b45673f0cb09d91c390faa/basedosdados-2.0.0b15.tar.gz", hash = "sha256:4fb669a087e8d12e30978efaca375d6760324eb3da6635e7ab35ed761cfdaef3", size = 53472 } +sdist = { url = "https://files.pythonhosted.org/packages/0b/bb/1d23f6cc25c3b9164211a0c4feef6d72facbaffb378f8c868a2859eb28cb/basedosdados-2.0.0b21.tar.gz", hash = "sha256:04d136dc13c9a9cc390e8092065bbd9590a08cf418bb0bcba007d2c579d20cad", size = 31605 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/f0/70e82b2599a5164de899f2d35f25107320255bd3948e47c6d8f5ce6a42c4/basedosdados-2.0.0b15-py3-none-any.whl", hash = "sha256:2832cc2e93e6207ef4600516d90dc0858794e56d19674cc8493d77c2ae0a7e38", size = 63369 }, + { url = "https://files.pythonhosted.org/packages/8e/d4/7989d2ea36471b18dc2eedb3aa9083a6055e307d5aa325aed9eea2a12c77/basedosdados-2.0.0b21-py3-none-any.whl", hash = "sha256:433b3551c8fa72b2ca13ee1a6080eca06a5c360c7b09c6da4a2f2b18046db3a2", size = 37677 }, ] [package.optional-dependencies] @@ -2085,7 +2091,7 @@ requires-dist = [ { name = "pillow", specifier = "==10.3.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==3.7.0" }, { name = "prefect", specifier = "==1.4.1" }, - { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=62236b527b3528437b8d460f60c79ee39309b151" }, + { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=87fc0c33b6042939d4543219ad6e56be3755201a" }, { name = "requests", specifier = "==2.31.0" }, { name = "shapely", specifier = "==2.0.4" }, { name = "taskipy", marker = "extra == 'dev'", specifier = "==1.12.2" }, @@ -2154,7 +2160,7 @@ wheels = [ [[package]] name = "prefeitura-rio" version = "1.1.3a2" -source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=62236b527b3528437b8d460f60c79ee39309b151#62236b527b3528437b8d460f60c79ee39309b151" } +source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=87fc0c33b6042939d4543219ad6e56be3755201a#87fc0c33b6042939d4543219ad6e56be3755201a" } dependencies = [ { name = "hvac" }, { name = "loguru" }, @@ -2162,7 +2168,6 @@ dependencies = [ [package.optional-dependencies] pipelines = [ - { name = "basedosdados" }, { name = "basedosdados", extra = ["upload"] }, { name = "dask" }, { name = "google-cloud-storage" }, @@ -2181,6 +2186,7 @@ pipelines-templates = [ { name = "geopy" }, { name = "gspread" }, { name = "jinja2" }, + { name = "psycopg2-binary" }, { name = "pymongo" }, { name = "pymysql" }, { name = "pyodbc" }, @@ -2231,6 +2237,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/33/2d74d588408caedd065c2497bdb5ef83ce6082db01289a1e1147f6639802/psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8", size = 249898 }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/07/e720e53bfab016ebcc34241695ccc06a9e3d91ba19b40ca81317afbdc440/psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c", size = 384973 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7c/6aaf8c3cb05d86d2c3f407b95bac0c71a43f2718e38c1091972aacb5e1b2/psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202", size = 2822503 }, + { url = "https://files.pythonhosted.org/packages/72/3d/acab427845198794aafd963dd073ee35810e2c52606e8a28c12db39821fb/psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7", size = 2552645 }, + { url = "https://files.pythonhosted.org/packages/ed/be/6c787962d706e55a528ef1693dd7251de657ae60e4d9d767ed61e8e2975c/psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b", size = 2850980 }, + { url = "https://files.pythonhosted.org/packages/83/50/a054076c6358753661cd1da59f4dabc03e83d51690371f3fd1edb9e2cf72/psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9", size = 3080543 }, + { url = "https://files.pythonhosted.org/packages/9c/02/826dc5cdfc9515423ec912ba00cc2e4eb09f69e0339b177c9c742f2a09a2/psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84", size = 3264316 }, + { url = "https://files.pythonhosted.org/packages/bc/0d/486e3fa27f39a00168abfcf14a3d8444f437f4b755cc34316da1124f293d/psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e", size = 3019508 }, + { url = "https://files.pythonhosted.org/packages/41/af/bce37630c525d2b9cf93f930110fc98616d6aca308d59b833b83b3a38176/psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98", size = 2355821 }, + { url = "https://files.pythonhosted.org/packages/3b/76/e46dae1b2273814ef80231f86d59cadf94ec36fd757045ed713c5b75cde7/psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245", size = 2534855 }, + { url = "https://files.pythonhosted.org/packages/0e/6d/e97245eabff29d7c2de5fc1fc17cf7ef427beee93d20a5ae114c6e6718bd/psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e", size = 2486614 }, + { url = "https://files.pythonhosted.org/packages/70/a7/2cd2c9d5e23b556c11e3b7da41895808d9b056f8f34f50de4375a35b4951/psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f", size = 2454928 }, + { url = "https://files.pythonhosted.org/packages/63/41/815d19767e2adb1a585213b801c954f46102f305c352c4a4f96287342d44/psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682", size = 1025249 }, + { url = "https://files.pythonhosted.org/packages/5e/4c/9233e0e206634a5387f3ab40f334a5325fb8bef2ca4e12ee7dbdeaf96afc/psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0", size = 1163645 }, + { url = "https://files.pythonhosted.org/packages/a5/ac/702d300f3df169b9d0cbef0340d9f34a78bc18dc2dbafbcb39ff0f165cf8/psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26", size = 2822581 }, + { url = "https://files.pythonhosted.org/packages/7a/1f/a6cf0cdf944253f7c45d90fbc876cc8bed5cc9942349306245715c0d88d6/psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f", size = 2552633 }, + { url = "https://files.pythonhosted.org/packages/81/0b/3adf561107c865928455891156d1dde5325253f7f4316fe56cd2c3f73570/psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2", size = 2851075 }, + { url = "https://files.pythonhosted.org/packages/f7/98/c2fedcbf0a9607519a010dcf88571138b2251062dbde3610cdba5ba1eee1/psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0", size = 3080509 }, + { url = "https://files.pythonhosted.org/packages/c2/05/81e8bc7fca95574c9323e487d9ce1b58a4cfcc17f89b8fe843af46361211/psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53", size = 3264303 }, + { url = "https://files.pythonhosted.org/packages/ce/85/62825cabc6aad53104b7b6d12eb2ad74737d268630032d07b74d4444cb72/psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be", size = 3019515 }, + { url = "https://files.pythonhosted.org/packages/e9/b0/9ca2b8e01a0912c9a14234fd5df7a241a1e44778c5797bf4b8eaa8dc3d3a/psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27", size = 2355892 }, + { url = "https://files.pythonhosted.org/packages/73/17/ba28bb0022db5e2015a82d2df1c4b0d419c37fa07a588b3aff3adc4939f6/psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359", size = 2534903 }, + { url = "https://files.pythonhosted.org/packages/3b/92/b463556409cdc12791cd8b1dae0072bf8efe817ef68b7ea3d9cf7d0e5656/psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2", size = 2486597 }, + { url = "https://files.pythonhosted.org/packages/92/57/96576e07132d7f7a1ac1df939575e6fdd8951aea337ee152b586bb51a971/psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc", size = 2454908 }, + { url = "https://files.pythonhosted.org/packages/7c/ae/cedd56e1f4a2b0e37213283caf3733a875c4c76f3372241e19c0d2a87355/psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d", size = 1024240 }, + { url = "https://files.pythonhosted.org/packages/25/1f/7ae31759142999a8d06b3e250c1346c4abcdcada8fa884376775dc1de686/psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417", size = 1163655 }, + { url = "https://files.pythonhosted.org/packages/a7/d0/5f2db14e7b53552276ab613399a83f83f85b173a862d3f20580bc7231139/psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf", size = 2823784 }, + { url = "https://files.pythonhosted.org/packages/18/ca/da384fd47233e300e3e485c90e7aab5d7def896d1281239f75901faf87d4/psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d", size = 2553308 }, + { url = "https://files.pythonhosted.org/packages/50/66/fa53d2d3d92f6e1ef469d92afc6a4fe3f6e8a9a04b687aa28fb1f1d954ee/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212", size = 2851283 }, + { url = "https://files.pythonhosted.org/packages/04/37/2429360ac5547378202db14eec0dde76edbe1f6627df5a43c7e164922859/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493", size = 3081839 }, + { url = "https://files.pythonhosted.org/packages/62/2a/c0530b59d7e0d09824bc2102ecdcec0456b8ca4d47c0caa82e86fce3ed4c/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996", size = 3264488 }, + { url = "https://files.pythonhosted.org/packages/19/57/9f172b900795ea37246c78b5f52e00f4779984370855b3e161600156906d/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119", size = 3020700 }, + { url = "https://files.pythonhosted.org/packages/94/68/1176fc14ea76861b7b8360be5176e87fb20d5091b137c76570eb4e237324/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba", size = 2355968 }, + { url = "https://files.pythonhosted.org/packages/70/bb/aec2646a705a09079d008ce88073401cd61fc9b04f92af3eb282caa3a2ec/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07", size = 2536101 }, + { url = "https://files.pythonhosted.org/packages/14/33/12818c157e333cb9d9e6753d1b2463b6f60dbc1fade115f8e4dc5c52cac4/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb", size = 2487064 }, + { url = "https://files.pythonhosted.org/packages/56/a2/7851c68fe8768f3c9c246198b6356ee3e4a8a7f6820cc798443faada3400/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe", size = 2456257 }, + { url = "https://files.pythonhosted.org/packages/6f/ee/3ba07c6dc7c3294e717e94720da1597aedc82a10b1b180203ce183d4631a/psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93", size = 1024709 }, + { url = "https://files.pythonhosted.org/packages/7b/08/9c66c269b0d417a0af9fb969535f0371b8c538633535a7a6a5ca3f9231e2/psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab", size = 1163864 }, +] + [[package]] name = "pyarrow" version = "17.0.0" From c2d35d33508c2d4da33969b5beddff4906e414b9 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 14:47:33 -0300 Subject: [PATCH 07/12] feat: add dbt infisical_credential_path and dbt_project_materialization --- pipelines/chatbot/dbt_chatbot_metricas/schedules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py index 0e7ec8e..c3e0ad0 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py @@ -46,8 +46,8 @@ "table_id": parameters["table_id"], "upstream": parameters["upstream"], "infisical_credential_path": { - "secret_path": "/dbt-rj-chatbot-dev", - "secret_name": "SEVICE_ACCOUNT", + "secret_path": "/rj-chatbot-dev", + "secret_name": "DBT_SEVICE_ACCOUNT", }, "dbt_project_materialization": "rj-chatbot-dev", }, From e60dcd5bdffc8ae7568cd637796aeb20981af903 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 15:06:54 -0300 Subject: [PATCH 08/12] feat: add ruamel to pipelines and templates dependencies --- .../chatbot/dbt_chatbot_metricas/flows.py | 2 +- pipelines/templates/run_dbt_model/flows.py | 2 +- pyproject.toml | 2 +- uv.lock | 49 ++++++++++++++++++- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index 66ca255..b72d208 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -DBT-related flows.. +DBT-related flows... """ from copy import deepcopy diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index 92f9949..716fef5 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT. +MATERIALIZA MODELOS DO DBT.. """ from copy import deepcopy diff --git a/pyproject.toml b/pyproject.toml index d2c4a56..0857abe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,4 +53,4 @@ profile = "black" lint = "black . && isort . && flake8 ." [tool.uv.sources] -prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "87fc0c33b6042939d4543219ad6e56be3755201a" } +prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "7cdfdddf921215b913ba132fb3244cce9b0c6c56" } diff --git a/uv.lock b/uv.lock index 09dd895..6e4b4aa 100644 --- a/uv.lock +++ b/uv.lock @@ -2091,7 +2091,7 @@ requires-dist = [ { name = "pillow", specifier = "==10.3.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==3.7.0" }, { name = "prefect", specifier = "==1.4.1" }, - { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=87fc0c33b6042939d4543219ad6e56be3755201a" }, + { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=7cdfdddf921215b913ba132fb3244cce9b0c6c56" }, { name = "requests", specifier = "==2.31.0" }, { name = "shapely", specifier = "==2.0.4" }, { name = "taskipy", marker = "extra == 'dev'", specifier = "==1.12.2" }, @@ -2160,7 +2160,7 @@ wheels = [ [[package]] name = "prefeitura-rio" version = "1.1.3a2" -source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=87fc0c33b6042939d4543219ad6e56be3755201a#87fc0c33b6042939d4543219ad6e56be3755201a" } +source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=7cdfdddf921215b913ba132fb3244cce9b0c6c56#7cdfdddf921215b913ba132fb3244cce9b0c6c56" } dependencies = [ { name = "hvac" }, { name = "loguru" }, @@ -2175,6 +2175,7 @@ pipelines = [ { name = "jinja2" }, { name = "pandas" }, { name = "prefect" }, + { name = "ruamel-yaml" }, { name = "sentry-sdk" }, ] pipelines-templates = [ @@ -2858,6 +2859,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/97/fa78e3d2f65c02c8e1268b9aba606569fe97f6c8f7c2d74394553347c145/rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", size = 34315 }, ] +[[package]] +name = "ruamel-yaml" +version = "0.18.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ruamel-yaml-clib", marker = "python_full_version < '3.13' and platform_python_implementation == 'CPython'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/81/4dfc17eb6ebb1aac314a3eb863c1325b907863a1b8b1382cdffcb6ac0ed9/ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b", size = 143362 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/67/8ece580cc363331d9a53055130f86b096bf16e38156e33b1d3014fffda6b/ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636", size = 117761 }, +] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/ab/bab9eb1566cd16f060b54055dd39cf6a34bfa0240c53a7218c43e974295b/ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512", size = 213824 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/01/37ac131614f71b98e9b148b2d7790662dcee92217d2fb4bac1aa377def33/ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d", size = 148236 }, + { url = "https://files.pythonhosted.org/packages/61/ee/4874c9fc96010fce85abefdcbe770650c5324288e988d7a48b527a423815/ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462", size = 133996 }, + { url = "https://files.pythonhosted.org/packages/d3/62/c60b034d9a008bbd566eeecf53a5a4c73d191c8de261290db6761802b72d/ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412", size = 526680 }, + { url = "https://files.pythonhosted.org/packages/90/8c/6cdb44f548b29eb6328b9e7e175696336bc856de2ff82e5776f860f03822/ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f", size = 605853 }, + { url = "https://files.pythonhosted.org/packages/88/30/fc45b45d5eaf2ff36cffd215a2f85e9b90ac04e70b97fd4097017abfb567/ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334", size = 655206 }, + { url = "https://files.pythonhosted.org/packages/af/dc/133547f90f744a0c827bac5411d84d4e81da640deb3af1459e38c5f3b6a0/ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d", size = 689649 }, + { url = "https://files.pythonhosted.org/packages/23/1d/589139191b187a3c750ae8d983c42fd799246d5f0dd84451a0575c9bdbe9/ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d", size = 100044 }, + { url = "https://files.pythonhosted.org/packages/4f/5b/744df20285a75ac4c606452ce9a0fcc42087d122f42294518ded1017697c/ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31", size = 117825 }, + { url = "https://files.pythonhosted.org/packages/b1/15/971b385c098e8d0d170893f5ba558452bb7b776a0c90658b8f4dd0e3382b/ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069", size = 148870 }, + { url = "https://files.pythonhosted.org/packages/01/b0/4ddef56e9f703d7909febc3a421d709a3482cda25826816ec595b73e3847/ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248", size = 134475 }, + { url = "https://files.pythonhosted.org/packages/a4/f7/22d6b620ed895a05d40802d8281eff924dc6190f682d933d4efff60db3b5/ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b", size = 544020 }, + { url = "https://files.pythonhosted.org/packages/7c/e4/0d19d65e340f93df1c47f323d95fa4b256bb28320290f5fddef90837853a/ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe", size = 642643 }, + { url = "https://files.pythonhosted.org/packages/c9/ff/f781eb5e2ae011e586d5426e2086a011cf1e0f59704a6cad1387975c5a62/ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899", size = 695832 }, + { url = "https://files.pythonhosted.org/packages/e3/41/f62e67ac651358b8f0d60cfb12ab2daf99b1b69eeaa188d0cec809d943a6/ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9", size = 730923 }, + { url = "https://files.pythonhosted.org/packages/9f/f0/19ab8acbf983cd1b37f47d27ceb8b10a738d60d36316a54bad57e0d73fbb/ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7", size = 99999 }, + { url = "https://files.pythonhosted.org/packages/ec/54/d8a795997921d87224c65d44499ca595a833093fb215b133f920c1062956/ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb", size = 118008 }, + { url = "https://files.pythonhosted.org/packages/7a/a2/eb5e9d088cb9d15c24d956944c09dca0a89108ad6e2e913c099ef36e3f0d/ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1", size = 144636 }, + { url = "https://files.pythonhosted.org/packages/66/98/8de4f22bbfd9135deb3422e96d450c4bc0a57d38c25976119307d2efe0aa/ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2", size = 135684 }, + { url = "https://files.pythonhosted.org/packages/30/d3/5fe978cd01a61c12efd24d65fa68c6f28f28c8073a06cf11db3a854390ca/ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92", size = 734571 }, + { url = "https://files.pythonhosted.org/packages/55/b3/e2531a050758b717c969cbf76c103b75d8a01e11af931b94ba656117fbe9/ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62", size = 643946 }, + { url = "https://files.pythonhosted.org/packages/0d/aa/06db7ca0995b513538402e11280282c615b5ae5f09eb820460d35fb69715/ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9", size = 692169 }, + { url = "https://files.pythonhosted.org/packages/27/38/4cf4d482b84ecdf51efae6635cc5483a83cf5ca9d9c13e205a750e251696/ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d", size = 740325 }, + { url = "https://files.pythonhosted.org/packages/6f/67/c62c6eea53a4feb042727a3d6c18f50dc99683c2b199c06bd2a9e3db8e22/ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa", size = 98639 }, + { url = "https://files.pythonhosted.org/packages/10/d2/52a3d810d0b5b3720725c0504a27b3fced7b6f310fe928f7019d79387bc1/ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b", size = 115305 }, +] + [[package]] name = "sentry-sdk" version = "1.45.1" From bb31c684ac75388258937e72738c38c6e02f2b49 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 15:14:25 -0300 Subject: [PATCH 09/12] feat: add ruamel to pipelines and templates dependencies --- pipelines/chatbot/dbt_chatbot_metricas/flows.py | 2 +- .../deteccao_alagamento_cameras/flooding_detection/tasks.py | 3 +-- pipelines/templates/run_dbt_model/flows.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index b72d208..0e7cf83 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -DBT-related flows... +DBT-related flows.... """ from copy import deepcopy diff --git a/pipelines/deteccao_alagamento_cameras/flooding_detection/tasks.py b/pipelines/deteccao_alagamento_cameras/flooding_detection/tasks.py index 72a012d..54eac06 100644 --- a/pipelines/deteccao_alagamento_cameras/flooding_detection/tasks.py +++ b/pipelines/deteccao_alagamento_cameras/flooding_detection/tasks.py @@ -22,9 +22,8 @@ from prefect import task from prefeitura_rio.pipelines_utils.gcs import upload_file_to_bucket from prefeitura_rio.pipelines_utils.infisical import get_secret -from prefeitura_rio.pipelines_utils.io import to_partitions from prefeitura_rio.pipelines_utils.logging import log -from prefeitura_rio.pipelines_utils.pandas import parse_date_columns +from prefeitura_rio.pipelines_utils.pandas import parse_date_columns, to_partitions from prefeitura_rio.pipelines_utils.redis_pal import get_redis_client from prefeitura_rio.pipelines_utils.time import TimeoutError from redis_pal import RedisPal diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index 716fef5..a6ab20f 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT.. +MATERIALIZA MODELOS DO DBT... """ from copy import deepcopy From 15b14ebe158a746fd02e530fa0d9d5193b2a0967 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 15:27:05 -0300 Subject: [PATCH 10/12] feat: add ruamel to pipelines and templates dependencies --- pipelines/chatbot/dbt_chatbot_metricas/flows.py | 7 +++++-- pipelines/templates/run_dbt_model/flows.py | 2 +- pyproject.toml | 2 +- uv.lock | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index 0e7cf83..257c54b 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -DBT-related flows.... +DBT-related flows..... """ from copy import deepcopy @@ -26,7 +26,10 @@ handler_initialize_sentry, ] run_rbt_chatbot_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) -run_rbt_chatbot_flow.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) +run_rbt_chatbot_flow.run_config = KubernetesRun( + image=constants.DOCKER_IMAGE.value, + labels=[constants.RJ_ESCRITORIO_AGENT_LABEL.value], +) identidade_unica_default_parameters = { "dataset_id": "dialogflowcx", diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index a6ab20f..026bc1d 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT... +MATERIALIZA MODELOS DO DBT..., """ from copy import deepcopy diff --git a/pyproject.toml b/pyproject.toml index 0857abe..d76a230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,4 +53,4 @@ profile = "black" lint = "black . && isort . && flake8 ." [tool.uv.sources] -prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "7cdfdddf921215b913ba132fb3244cce9b0c6c56" } +prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "55e9d3d360a22ca883b7bd8a69e43eba46e480dd" } diff --git a/uv.lock b/uv.lock index 6e4b4aa..4ae69af 100644 --- a/uv.lock +++ b/uv.lock @@ -2091,7 +2091,7 @@ requires-dist = [ { name = "pillow", specifier = "==10.3.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==3.7.0" }, { name = "prefect", specifier = "==1.4.1" }, - { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=7cdfdddf921215b913ba132fb3244cce9b0c6c56" }, + { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=55e9d3d360a22ca883b7bd8a69e43eba46e480dd" }, { name = "requests", specifier = "==2.31.0" }, { name = "shapely", specifier = "==2.0.4" }, { name = "taskipy", marker = "extra == 'dev'", specifier = "==1.12.2" }, @@ -2160,7 +2160,7 @@ wheels = [ [[package]] name = "prefeitura-rio" version = "1.1.3a2" -source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=7cdfdddf921215b913ba132fb3244cce9b0c6c56#7cdfdddf921215b913ba132fb3244cce9b0c6c56" } +source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=55e9d3d360a22ca883b7bd8a69e43eba46e480dd#55e9d3d360a22ca883b7bd8a69e43eba46e480dd" } dependencies = [ { name = "hvac" }, { name = "loguru" }, From 8190c7922929d85d0c15c497e649a05dfb4f9cff Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 15:38:42 -0300 Subject: [PATCH 11/12] feat: add ruamel to pipelines and templates dependencies --- pipelines/chatbot/dbt_chatbot_metricas/flows.py | 2 +- pipelines/templates/run_dbt_model/flows.py | 2 +- pyproject.toml | 2 +- uv.lock | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index 257c54b..e81b6fa 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -DBT-related flows..... +DBT-related flows...... """ from copy import deepcopy diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index 026bc1d..ef49afe 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT..., +MATERIALIZA MODELOS DO DBT.... """ from copy import deepcopy diff --git a/pyproject.toml b/pyproject.toml index d76a230..b15c220 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,4 +53,4 @@ profile = "black" lint = "black . && isort . && flake8 ." [tool.uv.sources] -prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "55e9d3d360a22ca883b7bd8a69e43eba46e480dd" } +prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", rev = "1e20ad960b680d8982046715e18e1a66bd4386ea" } diff --git a/uv.lock b/uv.lock index 4ae69af..1625ab7 100644 --- a/uv.lock +++ b/uv.lock @@ -2091,7 +2091,7 @@ requires-dist = [ { name = "pillow", specifier = "==10.3.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==3.7.0" }, { name = "prefect", specifier = "==1.4.1" }, - { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=55e9d3d360a22ca883b7bd8a69e43eba46e480dd" }, + { name = "prefeitura-rio", extras = ["pipelines", "pipelines-templates"], git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=1e20ad960b680d8982046715e18e1a66bd4386ea" }, { name = "requests", specifier = "==2.31.0" }, { name = "shapely", specifier = "==2.0.4" }, { name = "taskipy", marker = "extra == 'dev'", specifier = "==1.12.2" }, @@ -2160,7 +2160,7 @@ wheels = [ [[package]] name = "prefeitura-rio" version = "1.1.3a2" -source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=55e9d3d360a22ca883b7bd8a69e43eba46e480dd#55e9d3d360a22ca883b7bd8a69e43eba46e480dd" } +source = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git?rev=1e20ad960b680d8982046715e18e1a66bd4386ea#1e20ad960b680d8982046715e18e1a66bd4386ea" } dependencies = [ { name = "hvac" }, { name = "loguru" }, From 61195a097375c846b653a312cc8e91301ceafb4d Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 19 Sep 2024 15:41:36 -0300 Subject: [PATCH 12/12] feat: add ruamel to pipelines and templates dependencies --- pipelines/chatbot/dbt_chatbot_metricas/flows.py | 2 +- pipelines/chatbot/dbt_chatbot_metricas/schedules.py | 2 +- pipelines/templates/run_dbt_model/flows.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/chatbot/dbt_chatbot_metricas/flows.py b/pipelines/chatbot/dbt_chatbot_metricas/flows.py index e81b6fa..a106a54 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/flows.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -DBT-related flows...... +DBT-related flows....... """ from copy import deepcopy diff --git a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py index c3e0ad0..84e33ab 100644 --- a/pipelines/chatbot/dbt_chatbot_metricas/schedules.py +++ b/pipelines/chatbot/dbt_chatbot_metricas/schedules.py @@ -45,7 +45,7 @@ "dataset_id": "dialogflowcx", "table_id": parameters["table_id"], "upstream": parameters["upstream"], - "infisical_credential_path": { + "infisical_credential_dict": { "secret_path": "/rj-chatbot-dev", "secret_name": "DBT_SEVICE_ACCOUNT", }, diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index ef49afe..5fabddb 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT.... +MATERIALIZA MODELOS DO DBT..... """ from copy import deepcopy