diff --git a/.flake8 b/.flake8 index 51b50a0..2f6132b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ [flake8] +ignore = E203,F401,F403, W503 max-line-length = 100 \ No newline at end of file diff --git a/pipelines/__init__.py b/pipelines/__init__.py index e69de29..a44e546 100644 --- a/pipelines/__init__.py +++ b/pipelines/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +""" +Imports all flows for every project so we can register all of them. +""" +from pipelines.meteorologia import * +from pipelines.templates import * diff --git a/pipelines/meteorologia/meteorologia_inmet/flows.py b/pipelines/meteorologia/meteorologia_inmet/flows.py index 982639a..faa959d 100644 --- a/pipelines/meteorologia/meteorologia_inmet/flows.py +++ b/pipelines/meteorologia/meteorologia_inmet/flows.py @@ -9,6 +9,7 @@ from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants from pipelines.meteorologia.meteorologia_inmet.schedules import hour_schedule @@ -29,9 +30,7 @@ with Flow( name="COR: Meteorologia - Meteorologia INMET", - code_owners=[ - "paty", - ], + state_handlers=[handler_inject_bd_credentials], ) as cor_meteorologia_meteorologia_inmet: DATASET_ID = "clima_estacao_meteorologica" TABLE_ID = "meteorologia_inmet" diff --git a/pipelines/meteorologia/meteorologia_inmet/tasks.py b/pipelines/meteorologia/meteorologia_inmet/tasks.py index 14a7fcf..7afa602 100644 --- a/pipelines/meteorologia/meteorologia_inmet/tasks.py +++ b/pipelines/meteorologia/meteorologia_inmet/tasks.py @@ -11,10 +11,11 @@ import pendulum import requests from prefect import task +from prefeitura_rio.pipelines_utils.infisical import get_secret from pipelines.constants import constants from pipelines.meteorologia.precipitacao_alertario.utils import parse_date_columns -from pipelines.utils.utils import get_vault_secret, log, to_partitions +from pipelines.utils.utils import log, to_partitions # from pipelines.rj_cor.meteorologia.meteorologia_inmet.meteorologia_utils import converte_timezone @@ -79,8 +80,7 @@ def download(data_inicio: str, data_fim: str) -> pd.DataFrame: # no UTC, visto que ele só traria dados do novo dia e substituiria # no arquivo da partição do dia atual no nosso timezone - dicionario = get_vault_secret("inmet_api") - token = dicionario["data"]["token"] + token = get_secret("INMET_API")["INMET_API"] raw = [] for id_estacao in estacoes_unicas: diff --git a/pipelines/meteorologia/meteorologia_redemet/flows.py b/pipelines/meteorologia/meteorologia_redemet/flows.py index 04e8bc0..022d79f 100644 --- a/pipelines/meteorologia/meteorologia_redemet/flows.py +++ b/pipelines/meteorologia/meteorologia_redemet/flows.py @@ -9,6 +9,7 @@ from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants from pipelines.meteorologia.meteorologia_redemet.schedules import ( @@ -35,9 +36,7 @@ with Flow( name="COR: Meteorologia - Meteorologia REDEMET", - code_owners=[ - "paty", - ], + state_handlers=[handler_inject_bd_credentials], ) as cor_meteorologia_meteorologia_redemet: DUMP_MODE = Parameter("dump_mode", default="append", required=True) DATASET_ID = Parameter("dataset_id", default="clima_estacao_meteorologica", required=True) @@ -49,7 +48,7 @@ # Materialization parameters MATERIALIZE_AFTER_DUMP = Parameter("materialize_after_dump", default=True, required=False) - MATERIALIZE_TO_DATARIO = Parameter("materialize_to_datario", default=True, required=False) + MATERIALIZE_TO_DATARIO = Parameter("materialize_to_datario", default=False, required=False) MATERIALIZATION_MODE = Parameter("mode", default="prod", required=False) # Dump to GCS after? Should only dump to GCS if materializing to datario diff --git a/pipelines/meteorologia/meteorologia_redemet/schedules.py b/pipelines/meteorologia/meteorologia_redemet/schedules.py index 4ed1f73..2f401d6 100644 --- a/pipelines/meteorologia/meteorologia_redemet/schedules.py +++ b/pipelines/meteorologia/meteorologia_redemet/schedules.py @@ -43,7 +43,7 @@ parameter_defaults={ "materialize_after_dump": True, "mode": "prod", - "materialize_to_datario": True, + "materialize_to_datario": False, "dump_to_gcs": False, # "dump_mode": "overwrite", # "dataset_id": "clima_estacao_meteorologica", diff --git a/pipelines/meteorologia/meteorologia_redemet/tasks.py b/pipelines/meteorologia/meteorologia_redemet/tasks.py index 2ca918e..a69aa04 100644 --- a/pipelines/meteorologia/meteorologia_redemet/tasks.py +++ b/pipelines/meteorologia/meteorologia_redemet/tasks.py @@ -13,15 +13,11 @@ from prefect import task from prefect.engine.signals import ENDRUN from prefect.engine.state import Failed +from prefeitura_rio.pipelines_utils.infisical import get_secret from unidecode import unidecode from pipelines.constants import constants -from pipelines.utils.utils import ( - get_vault_secret, - log, - parse_date_columns, - to_partitions, -) +from pipelines.utils.utils import log, parse_date_columns, to_partitions @task(nout=3) @@ -62,8 +58,7 @@ def download_data(first_date: str, last_date: str) -> pd.DataFrame: "SBSC", ] - redemet_token = get_vault_secret("redemet-token") - redemet_token = redemet_token["data"]["token"] + redemet_token = get_secret("REDEMET-TOKEN")["REDEMET-TOKEN"] # Converte datas em int para cálculo de faixas. first_date_int = int(first_date.replace("-", "")) @@ -80,6 +75,7 @@ def download_data(first_date: str, last_date: str) -> pd.DataFrame: res = requests.get(url) if res.status_code != 200: log(f"Problema no id: {id_estacao}, {res.status_code}") + log(f"Data: {data}, Hora: {hora}") continue res_data = json.loads(res.text) if res_data["status"] is not True: @@ -90,12 +86,17 @@ def download_data(first_date: str, last_date: str) -> pd.DataFrame: continue raw.append(res_data) + log(f"Raw data size: {len(raw)}") + # Extrai objetos de dataframe raw = [res_data["data"] for res_data in raw] # converte para dataframe dataframe = pd.DataFrame(raw) + # Log dataframe size + log(f"Daframe shape: {dataframe.shape}") + return dataframe @@ -205,8 +206,7 @@ def download_stations_data() -> pd.DataFrame: Download station information """ - redemet_token = get_vault_secret("redemet-token") - redemet_token = redemet_token["data"]["token"] + redemet_token = get_secret("REDEMET-TOKEN")["REDEMET-TOKEN"] base_url = f"https://api-redemet.decea.mil.br/aerodromos/?api_key={redemet_token}" # noqa url = f"{base_url}&pais=Brasil" res = requests.get(url) diff --git a/pipelines/meteorologia/precipitacao_alertario/flows.py b/pipelines/meteorologia/precipitacao_alertario/flows.py index 26d5d4c..e961b01 100644 --- a/pipelines/meteorologia/precipitacao_alertario/flows.py +++ b/pipelines/meteorologia/precipitacao_alertario/flows.py @@ -10,6 +10,7 @@ from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants from pipelines.meteorologia.precipitacao_alertario.constants import ( @@ -42,10 +43,7 @@ with Flow( name="COR: Meteorologia - Precipitacao e Meteorologia ALERTARIO", - code_owners=[ - "paty", - ], - # skip_if_running=True, + state_handlers=[handler_inject_bd_credentials], ) as cor_meteorologia_precipitacao_alertario: DATASET_ID_PLUVIOMETRIC = alertario_constants.DATASET_ID_PLUVIOMETRIC.value TABLE_ID_PLUVIOMETRIC = alertario_constants.TABLE_ID_PLUVIOMETRIC.value diff --git a/pipelines/meteorologia/precipitacao_alertario/schedules.py b/pipelines/meteorologia/precipitacao_alertario/schedules.py index 104e534..ad90fbd 100644 --- a/pipelines/meteorologia/precipitacao_alertario/schedules.py +++ b/pipelines/meteorologia/precipitacao_alertario/schedules.py @@ -24,7 +24,7 @@ "materialize_after_dump_old_api": True, "materialize_to_datario_old_api": True, "materialize_after_dump": True, - "materialize_to_datario": True, + "materialize_to_datario": False, "mode": "prod", "dump_to_gcs": False, }, diff --git a/pipelines/meteorologia/precipitacao_alertario/tasks.py b/pipelines/meteorologia/precipitacao_alertario/tasks.py index c3e00b7..d544738 100644 --- a/pipelines/meteorologia/precipitacao_alertario/tasks.py +++ b/pipelines/meteorologia/precipitacao_alertario/tasks.py @@ -13,6 +13,7 @@ import requests from bs4 import BeautifulSoup from prefect import task +from prefeitura_rio.pipelines_utils.infisical import get_secret from pipelines.constants import constants from pipelines.meteorologia.precipitacao_alertario.utils import ( @@ -23,7 +24,6 @@ build_redis_key, compare_dates_between_tables_redis, get_redis_output, - get_vault_secret, log, parse_date_columns, save_str_on_redis, @@ -42,8 +42,7 @@ def download_data() -> pd.DataFrame: Request data from API and return each data in a different dataframe. """ - dicionario = get_vault_secret("alertario_api") - url = dicionario["data"]["url"] + url = get_secret("ALERTARIO_API")["ALERTARIO_API"] try: response = requests.get(url) diff --git a/pipelines/meteorologia/precipitacao_cemaden/flows.py b/pipelines/meteorologia/precipitacao_cemaden/flows.py index cbf4e2e..7cd5c52 100644 --- a/pipelines/meteorologia/precipitacao_cemaden/flows.py +++ b/pipelines/meteorologia/precipitacao_cemaden/flows.py @@ -9,6 +9,7 @@ from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants from pipelines.meteorologia.precipitacao_cemaden.constants import ( @@ -38,9 +39,7 @@ with Flow( name="COR: Meteorologia - Precipitacao CEMADEN", - code_owners=[ - "paty", - ], + state_handlers=[handler_inject_bd_credentials], # skip_if_running=True, ) as cor_meteorologia_precipitacao_cemaden: DUMP_MODE = Parameter("dump_mode", default="append", required=True) @@ -48,7 +47,7 @@ TABLE_ID = Parameter("table_id", default="taxa_precipitacao_cemaden", required=True) # Materialization parameters MATERIALIZE_AFTER_DUMP = Parameter("materialize_after_dump", default=True, required=False) - MATERIALIZE_TO_DATARIO = Parameter("materialize_to_datario", default=True, required=False) + MATERIALIZE_TO_DATARIO = Parameter("materialize_to_datario", default=False, required=False) MATERIALIZATION_MODE = Parameter("mode", default="prod", required=False) TRIGGER_RAIN_DASHBOARD_UPDATE = Parameter( "trigger_rain_dashboard_update", default=False, required=False diff --git a/pipelines/meteorologia/precipitacao_cemaden/schedules.py b/pipelines/meteorologia/precipitacao_cemaden/schedules.py index eb24de1..0de79b1 100644 --- a/pipelines/meteorologia/precipitacao_cemaden/schedules.py +++ b/pipelines/meteorologia/precipitacao_cemaden/schedules.py @@ -23,7 +23,7 @@ # "trigger_rain_dashboard_update": True, "materialize_after_dump": True, "mode": "prod", - "materialize_to_datario": True, + "materialize_to_datario": False, "dump_to_gcs": False, "dump_mode": "append", "dataset_id": "clima_pluviometro", diff --git a/pipelines/meteorologia/precipitacao_inea/__init__.py b/pipelines/meteorologia/precipitacao_inea/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/meteorologia/precipitacao_inea/flows.py b/pipelines/meteorologia/precipitacao_inea/flows.py index aff6427..5a3ab22 100644 --- a/pipelines/meteorologia/precipitacao_inea/flows.py +++ b/pipelines/meteorologia/precipitacao_inea/flows.py @@ -9,6 +9,7 @@ from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants from pipelines.meteorologia.precipitacao_inea.schedules import minute_schedule @@ -34,9 +35,7 @@ with Flow( name="COR: Meteorologia - Precipitacao e Fluviometria INEA", - code_owners=[ - "paty", - ], + state_handlers=[handler_inject_bd_credentials], # skip_if_running=True, ) as cor_meteorologia_precipitacao_inea: DUMP_MODE = Parameter("dump_mode", default="append", required=True) @@ -55,7 +54,7 @@ # Materialization parameters MATERIALIZE_AFTER_DUMP = Parameter("materialize_after_dump", default=True, required=False) - MATERIALIZE_TO_DATARIO = Parameter("materialize_to_datario", default=True, required=False) + MATERIALIZE_TO_DATARIO = Parameter("materialize_to_datario", default=False, required=False) MATERIALIZATION_MODE = Parameter("mode", default="prod", required=False) # Dump to GCS after? Should only dump to GCS if materializing to datario diff --git a/pipelines/meteorologia/precipitacao_inea/schedules.py b/pipelines/meteorologia/precipitacao_inea/schedules.py index 586d9d9..139a57f 100644 --- a/pipelines/meteorologia/precipitacao_inea/schedules.py +++ b/pipelines/meteorologia/precipitacao_inea/schedules.py @@ -23,7 +23,7 @@ # "trigger_rain_dashboard_update": True, "materialize_after_dump": True, "mode": "prod", - "materialize_to_datario": True, + "materialize_to_datario": False, "dump_to_gcs": False, "dump_mode": "append", "dataset_id_pluviometric": "clima_pluviometro", diff --git a/pipelines/meteorologia/precipitacao_websirene/flows.py b/pipelines/meteorologia/precipitacao_websirene/flows.py index f329652..ddc82ea 100644 --- a/pipelines/meteorologia/precipitacao_websirene/flows.py +++ b/pipelines/meteorologia/precipitacao_websirene/flows.py @@ -7,6 +7,7 @@ from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants from pipelines.meteorologia.precipitacao_websirene.schedules import MINUTE_SCHEDULE @@ -24,9 +25,7 @@ with Flow( "COR: Meteorologia - Precipitacao WEBSIRENE", - code_owners=[ - "paty", - ], + state_handlers=[handler_inject_bd_credentials], ) as cor_meteorologia_precipitacao_websirene: DATASET_ID = "clima_pluviometro" TABLE_ID = "taxa_precipitacao_websirene" @@ -85,6 +84,7 @@ # para rodar na cloud cor_meteorologia_precipitacao_websirene.storage = GCS(constants.GCS_FLOWS_BUCKET.value) cor_meteorologia_precipitacao_websirene.run_config = KubernetesRun( - image=constants.DOCKER_IMAGE.value + image=constants.DOCKER_IMAGE.value, + labels=[constants.RJ_COR_AGENT_LABEL.value], ) cor_meteorologia_precipitacao_websirene.schedule = MINUTE_SCHEDULE diff --git a/pipelines/meteorologia/radar/mendanha/flows.py b/pipelines/meteorologia/radar/mendanha/flows.py index b98cfe1..a3cddf4 100644 --- a/pipelines/meteorologia/radar/mendanha/flows.py +++ b/pipelines/meteorologia/radar/mendanha/flows.py @@ -7,7 +7,7 @@ from prefect import Parameter from prefect.run_configs import KubernetesRun from prefect.storage import GCS -from prefeitura_rio.pipelines_utils.custom import Flow # pylint: disable=E0611, E0401 +from prefeitura_rio.pipelines_utils.custom import Flow from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials from pipelines.constants import constants diff --git a/pipelines/meteorologia/satelite/flows.py b/pipelines/meteorologia/satelite/flows.py index b748879..3de3b74 100644 --- a/pipelines/meteorologia/satelite/flows.py +++ b/pipelines/meteorologia/satelite/flows.py @@ -9,6 +9,7 @@ from prefect import Parameter, case from prefect.run_configs import KubernetesRun from prefect.storage import GCS +from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from prefeitura_rio.pipelines_utils.custom import Flow # pylint: disable=E0611, E0401 from prefeitura_rio.pipelines_utils.state_handlers import ( handler_initialize_sentry, diff --git a/pipelines/rj_escritorio/__init__.py b/pipelines/rj_escritorio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/rj_escritorio/rain_dashboard/__init__.py b/pipelines/rj_escritorio/rain_dashboard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/tasks.py b/pipelines/tasks.py index a087730..2576c8b 100644 --- a/pipelines/tasks.py +++ b/pipelines/tasks.py @@ -14,7 +14,6 @@ from prefect import task from prefect.triggers import all_successful from prefeitura_rio.pipelines_utils.infisical import get_secret -from prefeitura_rio.pipelines_utils.logging import log from prefeitura_rio.pipelines_utils.pandas import ( # pylint: disable=E0611, E0401 parse_date_columns, to_partitions, @@ -23,13 +22,33 @@ get_redis_client, ) -from pipelines.utils_rj_cor import ( # get_redis_client_from_infisical, - treat_redis_output, -) +from pipelines.utils.utils import build_redis_key, log +from pipelines.utils_rj_cor import treat_redis_output # from redis_pal import RedisPal +@task(trigger=all_successful) +def save_on_redis( + dataset_id: str, + table_id: str, + mode: str = "prod", + files: list = [], + keep_last: int = 50, + wait=None, +) -> None: + """ + Set the last updated time on Redis. + """ + redis_client = get_redis_client() + key = build_redis_key(dataset_id, table_id, "files", mode) + files = list(set(files)) + print(">>>> save on redis files ", files) + files.sort() + files = files[-keep_last:] + redis_client.set(key, files) + + @task(checkpoint=False) def task_get_redis_client( infisical_host_env: str = "REDIS_HOST", @@ -79,24 +98,24 @@ def task_build_redis_hash(dataset_id: str, table_id: str, name: str = None, mode return redis_hash -# @task -# def get_on_redis( -# dataset_id: str, -# table_id: str, -# mode: str = "prod", -# wait=None, -# ) -> list: -# """ -# Get filenames saved on Redis. -# converti em get_redis_client_from_infisical -# """ -# redis_client = get_redis_client() -# key = build_redis_key(dataset_id, table_id, "files", mode) -# files_on_redis = redis_client.get(key) -# files_on_redis = [] if files_on_redis is None else files_on_redis -# files_on_redis = list(set(files_on_redis)) -# files_on_redis.sort() -# return files_on_redis +@task +def get_on_redis( + dataset_id: str, + table_id: str, + mode: str = "prod", + wait=None, +) -> list: + """ + Get filenames saved on Redis. + converti em task_get_redis_output + """ + redis_client = get_redis_client() + key = build_redis_key(dataset_id, table_id, "files", mode) + files_on_redis = redis_client.get(key) + files_on_redis = [] if files_on_redis is None else files_on_redis + files_on_redis = list(set(files_on_redis)) + files_on_redis.sort() + return files_on_redis @task diff --git a/pipelines/utils/utils.py b/pipelines/utils/utils.py index 87b0c62..0e79f6d 100644 --- a/pipelines/utils/utils.py +++ b/pipelines/utils/utils.py @@ -24,9 +24,9 @@ import pandas as pd import pendulum import prefect - -# from redis_pal import RedisPal import requests + +# import telegram from google.cloud import storage from google.cloud.storage.blob import Blob from google.oauth2 import service_account @@ -34,8 +34,9 @@ from prefect.engine.state import Skipped, State from prefect.run_configs import KubernetesRun from prefect.utilities.graphql import with_args +from prefeitura_rio.pipelines_utils.infisical import get_secret -# import telegram +# from redis_pal import RedisPal from prefeitura_rio.pipelines_utils.redis_pal import get_redis_client from pipelines.constants import constants @@ -935,7 +936,7 @@ def get_redis_output(redis_key): Get Redis output Example: {b'date': b'2023-02-27 07:29:04'} """ - redis_client = get_redis_client() + redis_client = get_redis_client_from_infisical() output = redis_client.hgetall(redis_key) if len(output) > 0: output = treat_redis_output(output) @@ -994,7 +995,7 @@ def save_updated_rows_on_redis( # pylint: disable=R0914 updated unique_id as a DataFrame and save new dates on redis """ - redis_client = get_redis_client() + redis_client = get_redis_client_from_infisical() key = dataset_id + "." + table_id if mode == "dev": @@ -1070,3 +1071,38 @@ def save_updated_rows_on_redis( # pylint: disable=R0914 [redis_client.hset(key, k, v) for k, v in new_updates.items()] return dataframe.reset_index() + + +def get_redis_client_from_infisical( + infisical_host_env: str = "REDIS_HOST", + infisical_port_env: str = "REDIS_PORT", + infisical_db_env: str = "REDIS_DB", + infisical_password_env: str = "REDIS_PASSWORD", + infisical_secrets_path: str = "/new_redis_cor", +): + """ + Gets a Redis client. + + Args: + infisical_host_env: The environment variable for the Redis host. + infisical_port_env: The environment variable for the Redis port. + infisical_db_env: The environment variable for the Redis database. + infisical_password_env: The environment variable for the Redis password. + + Returns: + The Redis client. + """ + redis_host = get_secret(infisical_host_env, path=infisical_secrets_path)[infisical_host_env] + redis_port = int( + get_secret(infisical_port_env, path=infisical_secrets_path)[infisical_port_env] + ) + redis_db = int(get_secret(infisical_db_env, path=infisical_secrets_path)[infisical_db_env]) + redis_password = get_secret(infisical_password_env, path=infisical_secrets_path)[ + infisical_password_env + ] + return get_redis_client( + host=redis_host, + port=redis_port, + db=redis_db, + password=redis_password, + ) diff --git a/pipelines/utils_rj_cor.py b/pipelines/utils_rj_cor.py index 88cec15..8a97277 100644 --- a/pipelines/utils_rj_cor.py +++ b/pipelines/utils_rj_cor.py @@ -326,18 +326,18 @@ def compare_actual_df_with_redis_df( return dfr_diff, updated_dfr_redis -def bq_project(kind: str = "bigquery_prod"): - """Get the set BigQuery project_id - - Args: - kind (str, optional): Which client to get the project name from. - Options are 'bigquery_staging', 'bigquery_prod' and 'storage_staging' - Defaults to 'bigquery_prod'. - - Returns: - str: the requested project_id - """ - return bd.upload.base.Base().client[kind].project +# def bq_project(kind: str = "bigquery_prod"): +# """Get the set BigQuery project_id +# +# Args: +# kind (str, optional): Which client to get the project name from. +# Options are 'bigquery_staging', 'bigquery_prod' and 'storage_staging' +# Defaults to 'bigquery_prod'. +# +# Returns: +# str: the requested project_id +# """ +# return bd.upload.base.Base().client[kind].project def wait_task_run(api, task_id) -> Dict: diff --git a/poetry.lock b/poetry.lock index 77a4140..8cbe617 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2780,6 +2780,160 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] +[[package]] +name = "lxml" +version = "5.3.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"}, + {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"}, + {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"}, + {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"}, + {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"}, + {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"}, + {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"}, + {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"}, + {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"}, + {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"}, + {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"}, + {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"}, + {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"}, + {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"}, + {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"}, + {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"}, + {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"}, + {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"}, + {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"}, + {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"}, + {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"}, + {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.11)"] + [[package]] name = "markupsafe" version = "2.1.5" diff --git a/pyproject.toml b/pyproject.toml index d25ccde..9bf4c48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio.git", ] } openpyxl = "^3.1.5" arm-pyart = "^1.19.1" +lxml = "^5.3.0" gdal = {url = "https://prefeitura-rio.github.io/storage/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl"} numpy = "<2" @@ -70,4 +71,4 @@ develop = false [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/run_local.py b/run_local.py deleted file mode 100644 index 624a585..0000000 --- a/run_local.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa: 402 -import base64 -import os -import sys -from os import getenv -from pathlib import Path - -from dotenv import load_dotenv - -from pipelines.meteorologia.radar.mendanha.flows import ( - cor_meteorologia_refletividade_radar_flow as flow, # TODO: import your flow here -) - -# Adiciona o diretório `/algum/diretorio/` ao sys.path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../prefeitura-rio"))) -# sys.path.insert(0, '/home/patricia/Documentos/escritorio_dados/prefeitura-rio') -print("sys.path:", sys.path) - -from prefeitura_rio.pipelines_utils.custom import Flow # noqa - - -def run_local(flow: Flow): - load_dotenv() - envs = [ - "INFISICAL_TOKEN", - "INFISICAL_ADDRESS", - "PREFECT__BACKEND", - "PREFECT__CLOUD__API", - "PREFECT_AUTH_TOML_B64", - ] - for env in envs: - if not getenv(env): - raise SystemError(f"Environment variable {env} not set.") - - prefect_dir = Path.home() / ".prefect" - prefect_dir.mkdir(parents=True, exist_ok=True) - - with open(prefect_dir / "auth.toml", "w") as f: - f.write(base64.b64decode(getenv("PREFECT_AUTH_TOML_B64")).decode()) - - flow.storage = None - flow.run_config = None - flow.schedule = None - flow.state_handlers = [] - - # flow.run(parameters={ - # "mode": "prod", - # "radar_name": "men" - # }) # TODO: add your parameters here - flow.run() - - -run_local(flow)