Skip to content

Commit

Permalink
Merge branch 'master' into pre-commit-ci-update-config
Browse files Browse the repository at this point in the history
  • Loading branch information
@mergify
mergify[bot] committed Sep 19, 2023
2 parents 82f24fb + d35dee5 commit 1b348d6
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 2 deletions.
41 changes: 40 additions & 1 deletion pipelines/rj_smfp/dump_db_ergon/schedules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,125 +22,148 @@
ergon_queries = {
"cargo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_CARGOS_",
},
"categoria": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_CATEGORIAS_",
},
"empresa": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_EMPRESAS",
},
"matricula": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_ERG_MATRICULAS",
},
"fita_banco": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"partition_columns": "MES_ANO",
"dump_mode": "append",
"lower_bound_date": "current_month",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FITA_BANCO",
},
"folha_empresa": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"partition_columns": "MES_ANO",
"dump_mode": "append",
"lower_bound_date": "current_month",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FOLHAS_EMP",
},
"forma_provimento": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FORMAS_PROV_",
},
"funcionario": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FUNCIONARIOS",
},
"horario_trabalho": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_HORARIO_TRAB_",
},
"setor": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_HSETOR_",
},
"jornada": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_JORNADAS_",
},
"orgaos_externos": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_ORGAOS_EXTERNOS",
},
"orgaos_regime_juridico": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_ORGAOS_REGIMES_JUR_",
},
"provimento": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_PROVIMENTOS_EV",
},
"regime_juridico": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_REGIMES_JUR_",
},
"tipo_folha": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_TIPO_FOLHA",
},
"tipo_orgao": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_TIPO_ORGAO",
},
"tipo_vinculo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_TIPO_VINC_",
},
"vinculo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_VINCULOS",
},
"licenca_afastamento": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"partition_columns": "DTINI",
"dump_mode": "append",
"lower_bound_date": "current_month",
"execute_query": """
SELECT NUMFUNC,NUMVINC,DTINI,DTFIM,TIPOFREQ,CODFREQ,MOTIVO,DTPREVFIM,FLEX_CAMPO_01,
FLEX_CAMPO_02,EMP_CODIGO,FLEX_CAMPO_07
Expand All @@ -149,19 +172,23 @@
},
"frequencia": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"partition_columns": "DTINI",
"dump_mode": "append",
"lower_bound_date": "current_month",
"execute_query": """
SELECT NUMFUNC,NUMVINC,DTINI,DTFIM,TIPOFREQ,CODFREQ,OBS,EMP_CODIGO
FROM ERGON.FREQUENCIAS
""",
},
"vantagens": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"partition_columns": "DTINI",
"dump_mode": "append",
"lower_bound_date": "current_month",
"execute_query": """
SELECT NUMFUNC,NUMVINC,VANTAGEM,DTINI,DTFIM,VALOR,INFO,TIPO_INCORPORACAO,PERC_INC_FUNCAO,
INC_TABELAVENC,INC_REFERENCIA,OBS,VALOR2,INFO2,VALOR3,INFO3,VALOR4,INFO4,VALOR5,INFO5,
Expand All @@ -171,6 +198,7 @@
},
"total_contagem": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -181,6 +209,7 @@
},
"pre_contagem": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -190,6 +219,7 @@
},
"averbacoes": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -200,6 +230,7 @@
},
"averbacoes_contagem": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -209,6 +240,7 @@
},
"frequencia_antigo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -218,6 +250,7 @@
},
"afastamento_antigo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -227,6 +260,7 @@
},
"afastamento_antigo_nomes": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -237,6 +271,7 @@
},
"tipo_tempo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"execute_query": """
Expand All @@ -247,8 +282,10 @@
},
"ficha_financeira": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "append",
"lower_bound_date": "current_month",
"partition_columns": "MES_ANO_FOLHA",
"execute_query": """
SELECT MES_ANO_FOLHA,NUM_FOLHA,LANCAMENTO,NUMFUNC,NUMVINC,NUMPENS,MES_ANO_DIREITO,
Expand All @@ -258,8 +295,10 @@
},
"ficha_financeira_contabil": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "append",
"lower_bound_date": "current_month",
"partition_columns": "MES_ANO_FOLHA",
"execute_query": """
SELECT MES_ANO_FOLHA,NUM_FOLHA,NUMFUNC,NUMVINC,NUMPENS,SETOR,SECRETARIA,TIPO_FUNC,
Expand All @@ -272,7 +311,7 @@

ergon_clocks = generate_dump_db_schedules(
interval=timedelta(days=1),
start_date=datetime(2022, 11, 9, 10, 30, tzinfo=pytz.timezone("America/Sao_Paulo")),
start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")),
labels=[
constants.RJ_SMFP_AGENT_LABEL.value,
],
Expand Down
15 changes: 14 additions & 1 deletion pipelines/rj_smfp/dump_db_ergon_comlurb/schedules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
ergon_queries = {
"cargo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -105,6 +106,7 @@
},
"dependente": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -216,6 +218,7 @@
},
"funcionario_evento": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -333,6 +336,7 @@
},
"ficha_financeira": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand All @@ -359,8 +363,10 @@
},
"fita_banco": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "append",
"lower_bound_date": "current_month",
"partition_columns": "MES_ANO",
"dbt_alias": True,
"execute_query": """
Expand Down Expand Up @@ -449,6 +455,7 @@
},
"frequencia": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -524,6 +531,7 @@
},
"funcionario": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -729,6 +737,7 @@
},
"setor_h": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand All @@ -739,6 +748,7 @@
},
"licenca_afastamento": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -829,6 +839,7 @@
},
"setor": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -882,6 +893,7 @@
},
"vantagens": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -958,6 +970,7 @@
},
"vinculo": {
"materialize_after_dump": True,
"biglake_table": True,
"materialization_mode": "prod",
"dump_mode": "overwrite",
"dbt_alias": True,
Expand Down Expand Up @@ -1121,7 +1134,7 @@
ergon_clocks = generate_dump_db_schedules(
interval=timedelta(days=1),
start_date=datetime(
2022, 10, 25, 18, 30, tzinfo=pytz.timezone("America/Sao_Paulo")
2022, 10, 25, 23, 30, tzinfo=pytz.timezone("America/Sao_Paulo")
),
labels=[
constants.RJ_SMFP_AGENT_LABEL.value,
Expand Down

0 comments on commit 1b348d6

Please sign in to comment.