From caea701d688ec2b287b9cdb79d4ff777af7c2a13 Mon Sep 17 00:00:00 2001 From: Antony Southworth Date: Fri, 24 May 2024 11:31:18 +1200 Subject: [PATCH 1/4] init --- .../materializations/models/table/create_table_as.sql | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql index e20c4020..ea0be233 100644 --- a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql @@ -1,9 +1,9 @@ -{% macro create_table_as(temporary, relation, compiled_code, language='sql', skip_partitioning=false) -%} - {{ adapter.dispatch('create_table_as', 'athena')(temporary, relation, compiled_code, language, skip_partitioning) }} +{% macro create_table_as(temporary, relation, compiled_code, language='sql', skip_partitioning=false, with_no_data=false) -%} + {{ adapter.dispatch('create_table_as', 'athena')(temporary, relation, compiled_code, language, skip_partitioning, with_no_data) }} {%- endmacro %} -{% macro athena__create_table_as(temporary, relation, compiled_code, language='sql', skip_partitioning=false) -%} +{% macro athena__create_table_as(temporary, relation, compiled_code, language='sql', skip_partitioning=false, with_no_data=false) -%} {%- set materialized = config.get('materialized', default='table') -%} {%- set external_location = config.get('external_location', default=none) -%} {%- do log("Skip partitioning: " ~ skip_partitioning) -%} @@ -143,6 +143,11 @@ ) as {{ compiled_code }} + + {% if with_no_data %} + WITH NO DATA + {% endif %} + {%- endif -%} {%- endmacro -%} From ab08613d520bca17d301dba115c0153bb0f2963f Mon Sep 17 00:00:00 2001 From: Antony Southworth Date: Fri, 24 May 2024 11:45:03 +1200 Subject: [PATCH 2/4] create table with no data before batch inserts --- .../models/table/create_table_as.sql | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql index ea0be233..97a432fc 100644 --- a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql @@ -175,27 +175,24 @@ {%- set dest_columns = adapter.get_columns_in_relation(tmp_relation) -%} {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} + {%- set create_target_relation_sql -%} + select {{ dest_cols_csv }} + from {{ tmp_relation }} + where {{ batch }} + {%- endset -%} + {%- do run_query(create_table_as(temporary, relation, create_target_relation_sql, language, with_no_data=true)) -%} + {%- for batch in partitions_batches -%} {%- do log('BATCH PROCESSING: ' ~ loop.index ~ ' OF ' ~ partitions_batches | length) -%} - {%- if loop.index == 1 -%} - {%- set create_target_relation_sql -%} - select {{ dest_cols_csv }} - from {{ tmp_relation }} - where {{ batch }} - {%- endset -%} - {%- do run_query(create_table_as(temporary, relation, create_target_relation_sql, language)) -%} - {%- else -%} - {%- set insert_batch_partitions_sql -%} - insert into {{ relation }} ({{ dest_cols_csv }}) - select {{ dest_cols_csv }} - from {{ tmp_relation }} - where {{ batch }} - {%- endset -%} - - {%- do run_query(insert_batch_partitions_sql) -%} - {%- endif -%} + {%- set insert_batch_partitions_sql -%} + insert into {{ relation }} ({{ dest_cols_csv }}) + select {{ dest_cols_csv }} + from {{ tmp_relation }} + where {{ batch }} + {%- endset -%} + {%- do run_query(insert_batch_partitions_sql) -%} {%- endfor -%} From be587d3da7a92484d4dd281bb764af34ba38d0d2 Mon Sep 17 00:00:00 2001 From: Antony Southworth Date: Wed, 12 Jun 2024 15:27:02 +1200 Subject: [PATCH 3/4] try something else --- .../macros/materializations/models/table/create_table_as.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql index 97a432fc..bf0bd0fb 100644 --- a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql @@ -142,7 +142,10 @@ {% endif %} ) as - {{ compiled_code }} + SELECT * + FROM ( + {{ compiled_code }} + ) {% if with_no_data %} WITH NO DATA From 728c5689ddbb04e58ffdad5931f88290d88325f9 Mon Sep 17 00:00:00 2001 From: Antony Southworth Date: Wed, 12 Jun 2024 15:59:23 +1200 Subject: [PATCH 4/4] try semicolon? --- .../macros/materializations/models/table/create_table_as.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql index bf0bd0fb..2586a416 100644 --- a/dbt/include/athena/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/athena/macros/materializations/models/table/create_table_as.sql @@ -148,7 +148,7 @@ ) {% if with_no_data %} - WITH NO DATA + WITH NO DATA; {% endif %} {%- endif -%}