Cria ambiente hmg

queries/.gitignore

@@ -2,4 +2,6 @@
 target/
 dbt_packages/
 logs/
-*dev/
+*dev/
+target-base/
+package-lock.yml

queries/dev/profiles-example.yml

@@ -1,4 +1,4 @@
-default:
+queries:
   target: dev
   outputs:
     dev:
@@ -7,13 +7,70 @@ default:
       project: rj-smtr-dev
       dataset: dbt
       location: US
-      threads: 2
+      threads: 1
       keyfile: # caminho/para/sua/credencial.json
+      priority: interactive
+      job_retries: 1
+
+      # for dbt Python models to be run on Dataproc Serverless
+      gcs_bucket: rj-smtr
+      dataproc_region: us-central1
+      submission_method: serverless
+      dataproc_batch:
+        environment_config:
+          execution_config:
+            service_account: # sua-conta-de-servico
+
+        runtime_config:
+          properties:
+            spark.executor.instances: "2"
+            spark.driver.memory: 4g
+            spark.driver.memoryOverhead: 1g
+    hmg:
+      type: bigquery
+      method: service-account
+      project: rj-smtr-dev
+      dataset: dbt
+      location: US
+      threads: 1
+      keyfile: # caminho/para/sua/credencial.json
+      priority: interactive
+      job_retries: 1
+
+      # for dbt Python models to be run on Dataproc Serverless
+      gcs_bucket: rj-smtr
+      dataproc_region: us-central1
+      submission_method: serverless
+      dataproc_batch:
+        environment_config:
+          execution_config:
+            service_account: # sua-conta-de-servico
+
+        runtime_config:
+          properties:
+            spark.executor.instances: "2"
+            spark.driver.memory: 4g
+            spark.driver.memoryOverhead: 1g
     prod:
       type: bigquery
       method: service-account
       project: rj-smtr
       dataset: dbt
       location: US
-      threads: 2
-      keyfile: # caminho/para/sua/credencial.json
+      threads: 1
+      keyfile: # caminho/para/sua/credencial.json
+
+      # for dbt Python models to be run on Dataproc Serverless
+      gcs_bucket: rj-smtr
+      dataproc_region: us-central1
+      submission_method: serverless
+      dataproc_batch:
+        environment_config:
+          execution_config:
+            service_account: # sua-conta-de-servico
+
+        runtime_config:
+          properties:
+            spark.executor.instances: "2"
+            spark.driver.memory: 4g
+            spark.driver.memoryOverhead: 1g

queries/dev/utils.py

@@ -63,6 +63,7 @@ def run_dbt_model(
         run_command += f" {flags}"
 
     print(f"\n>>> RUNNING: {run_command}\n")
+    os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
     os.system(run_command)
 
 

queries/macros/generate_database_name.sql

@@ -0,0 +1,24 @@
+-- fmt: off
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+
+    {%- set default_database = target.database -%}
+    {% set dev_database = "rj-smtr-dev" %}
+    {%- if custom_database_name is none -%}
+
+        {% if target.name in ("dev", "hmg") %}
+
+            {{ dev_database }}
+
+        {% else %}
+
+            {{ default_database }}
+
+        {% endif %}
+
+    {%- else -%}
+
+        {{ custom_database_name | trim }}
+
+    {%- endif -%}
+
+{%- endmacro %}

queries/macros/generate_schema_name.sql

@@ -1,14 +1,25 @@
+-- fmt: off
 {% macro generate_schema_name(custom_schema_name, node) -%}
 
     {%- set default_schema = target.schema -%}
     {%- if custom_schema_name is none -%}
 
-        {{ default_schema }}
+        {% set schema_name = default_schema %}
 
     {%- else -%}
 
-        {{ custom_schema_name | trim }}
+        {% set schema_name = custom_schema_name | trim %}
 
     {%- endif -%}
 
+    {% if target.name == "dev" %}
+        {% set schema_name = env_var("DBT_USER") + "__" + schema_name %}
+    {% endif %}
+
+    {% if target.name == "hmg" and schema_name.endswith("_staging") %}
+        {% set schema_name = schema_name + "_dbt" %}
+    {% endif %}
+
+    {{ schema_name }}
+
 {%- endmacro %}

queries/packages.yml

@@ -0,0 +1,5 @@
+packages:
+  - package: dbt-labs/audit_helper
+    version: 0.12.0
+  - package: data-mie/dbt_profiler
+    version: 0.8.2

queries/profiles.yml

@@ -11,6 +11,47 @@ queries:
       project: rj-smtr-dev
       threads: 1
       type: bigquery
+
+      # for dbt Python models to be run on Dataproc Serverless
+      gcs_bucket: rj-smtr
+      dataproc_region: us-central1
+      submission_method: serverless
+      dataproc_batch:
+        environment_config:
+          execution_config:
+            service_account: [email protected]
+
+        runtime_config:
+          properties:
+            spark.executor.instances: "2"
+            spark.driver.memory: 4g
+            spark.driver.memoryOverhead: 1g
+    hmg:
+      dataset: dbt
+      job_execution_timeout_seconds: 600
+      job_retries: 1
+      keyfile: /tmp/credentials.json
+      location: us
+      method: service-account
+      priority: interactive
+      project: rj-smtr-dev
+      threads: 1
+      type: bigquery
+
+      # for dbt Python models to be run on Dataproc Serverless
+      gcs_bucket: rj-smtr
+      dataproc_region: us-central1
+      submission_method: serverless
+      dataproc_batch:
+        environment_config:
+          execution_config:
+            service_account: [email protected]
+
+        runtime_config:
+          properties:
+            spark.executor.instances: "2"
+            spark.driver.memory: 4g
+            spark.driver.memoryOverhead: 1g
     prod:
       dataset: dbt
       job_execution_timeout_seconds: 600
@@ -22,4 +63,19 @@ queries:
       project: rj-smtr
       threads: 1
       type: bigquery
-  target: prod
+
+      # for dbt Python models to be run on Dataproc Serverless
+      gcs_bucket: rj-smtr
+      dataproc_region: us-central1
+      submission_method: serverless
+      dataproc_batch:
+        environment_config:
+          execution_config:
+            service_account: [email protected]
+
+        runtime_config:
+          properties:
+            spark.executor.instances: "2"
+            spark.driver.memory: 4g
+            spark.driver.memoryOverhead: 1g
+  target: prod

queries/recce.yml

@@ -0,0 +1,13 @@
+# Preset Checks
+# Please see https://datarecce.io/docs/features/preset-checks/
+checks:
+- name: Row count diff
+  description: Check the row count diff for all table models.
+  type: row_count_diff
+  params:
+    select: state:modified,config.materialized:table
+- name: Schema diff
+  description: Check the schema diff for all nodes.
+  type: schema_diff
+  params:
+    select: state:modified