diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 54ade2c..7037366 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -36,5 +36,9 @@ Note: dbt-expectations currently does not support database adapters other than t - Postgres - Snowflake - BigQuery +- DuckDB +- Spark (experimental) +- Trino +- Teradata ### Additional Context diff --git a/README.md b/README.md index d792405..9c66a7d 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ This package supports: * DuckDB * Spark (experimental) * Trino +* Teradata For latest release, see [https://github.com/metaplane/dbt-expectations/releases](https://github.com/metaplane/dbt-expectations/releases) @@ -48,6 +49,26 @@ For latest release, see [https://github.com/metaplane/dbt-expectations/releases] This package includes a reference to [`dbt-date`](https://github.com/godatadriven/dbt-date), so there's no need to also import `dbt-date` in your local project. +#### For Teradata adapter: +There is dependency on dbt-labs/dbt_utils and Teradata/teradata_utils, so user need to add the dependency in packages.yml and create search order config in dbt_project.yml as mentioned below: + +`packages.yml` +```yaml +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 + - package: Teradata/teradata_utils + version: 1.3.0 +``` + +`dbt_project.yml` +```yaml + - macro_namespace: dbt_date + search_order: ['teradata_utils', 'dbt_utils', 'dbt_date'] + - macro_namespace: dbt_utils + search_order: ['teradata_utils', 'dbt_utils'] +``` + ### Variables The following variables need to be defined in your `dbt_project.yml` file: diff --git a/integration_tests/models/schema_tests/data_test.sql b/integration_tests/models/schema_tests/data_test.sql index 1fa089a..d5e120a 100644 --- a/integration_tests/models/schema_tests/data_test.sql +++ b/integration_tests/models/schema_tests/data_test.sql @@ -1,3 +1,68 @@ +{% if target.type == 'teradata' %} +{# Teradata requires explicit CAST and FROM clause for literal selects #} +select + 1 as idx, + CAST('2020-10-21' AS DATE) as date_col, + CAST(0 AS FLOAT) as col_numeric_a, + CAST(1 AS FLOAT) as col_numeric_b, + CAST('a' AS VARCHAR(10)) as col_string_a, + CAST('b' AS VARCHAR(10)) as col_string_b, + CAST(NULL AS VARCHAR(100)) as col_null, + CAST(NULL AS VARCHAR(100)) as col_null_2, + CAST(1.0 AS FLOAT) as col_numeric_a_plus_b, + 2 as idx_multiplied_by_2, + -2 as idx_multiplied_by_minus_2 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + +union all + +select + 2 as idx, + CAST('2020-10-22' AS DATE) as date_col, + CAST(1 AS FLOAT) as col_numeric_a, + CAST(0 AS FLOAT) as col_numeric_b, + CAST('b' AS VARCHAR(10)) as col_string_a, + CAST('ab' AS VARCHAR(10)) as col_string_b, + CAST(NULL AS VARCHAR(100)) as col_null, + CAST(NULL AS VARCHAR(100)) as col_null_2, + CAST(1.0 AS FLOAT) as col_numeric_a_plus_b, + 4 as idx_multiplied_by_2, + -4 as idx_multiplied_by_minus_2 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + +union all + +select + 3 as idx, + CAST('2020-10-23' AS DATE) as date_col, + CAST(0.5 AS FLOAT) as col_numeric_a, + CAST(0.5 AS FLOAT) as col_numeric_b, + CAST('c' AS VARCHAR(10)) as col_string_a, + CAST('abc' AS VARCHAR(10)) as col_string_b, + CAST(NULL AS VARCHAR(100)) as col_null, + CAST(NULL AS VARCHAR(100)) as col_null_2, + CAST(1.0 AS FLOAT) as col_numeric_a_plus_b, + 6 as idx_multiplied_by_2, + -6 as idx_multiplied_by_minus_2 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + +union all + +select + 4 as idx, + CAST('2020-10-23' AS DATE) as date_col, + CAST(0.5 AS FLOAT) as col_numeric_a, + CAST(0.5 AS FLOAT) as col_numeric_b, + CAST('c' AS VARCHAR(10)) as col_string_a, + CAST('abcd' AS VARCHAR(10)) as col_string_b, + CAST(NULL AS VARCHAR(100)) as col_null, + CAST(NULL AS VARCHAR(100)) as col_null_2, + CAST(1.0 AS FLOAT) as col_numeric_a_plus_b, + 8 as idx_multiplied_by_2, + -8 as idx_multiplied_by_minus_2 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 +{% else %} +{# Other adapters support simpler literal selects #} select 1 as idx, '2020-10-21' as date_col, @@ -55,3 +120,4 @@ select 1.0 as col_numeric_a_plus_b, 8 as idx_multiplied_by_2, -8 as idx_multiplied_by_minus_2 +{% endif %} \ No newline at end of file diff --git a/integration_tests/models/schema_tests/data_test_factored.sql b/integration_tests/models/schema_tests/data_test_factored.sql index a6aaab4..741d252 100644 --- a/integration_tests/models/schema_tests/data_test_factored.sql +++ b/integration_tests/models/schema_tests/data_test_factored.sql @@ -1,3 +1,40 @@ +{% if target.type == 'teradata' %} +{# Teradata requires explicit column list when combining SELECT * with additional columns #} +select + sb1.idx, + sb1.date_col, + sb1.col_numeric_a, + sb1.col_numeric_b, + sb1.col_string_a, + sb1.col_string_b, + sb1.col_null, + sb1.col_null_2, + sb1.col_numeric_a_plus_b, + sb1.idx_multiplied_by_2, + sb1.idx_multiplied_by_minus_2, + 1 as factor +from + {{ ref("data_test") }} as sb1 + +union all + +select + sb.idx, + sb.date_col, + sb.col_numeric_a, + sb.col_numeric_b, + sb.col_string_a, + sb.col_string_b, + sb.col_null, + sb.col_null_2, + sb.col_numeric_a_plus_b, + sb.idx_multiplied_by_2, + sb.idx_multiplied_by_minus_2, + 2 as factor +from + {{ ref("data_test") }} as sb +{% else %} +{# Other adapters support SELECT * with additional columns #} select 1 as factor, * @@ -11,3 +48,4 @@ select * from {{ ref("data_test") }} +{% endif %} diff --git a/integration_tests/models/schema_tests/data_text.sql b/integration_tests/models/schema_tests/data_text.sql index 4690eb8..1820d8a 100644 --- a/integration_tests/models/schema_tests/data_text.sql +++ b/integration_tests/models/schema_tests/data_text.sql @@ -1,7 +1,11 @@ +{% if target.type == 'teradata' %} +{# Teradata requires FROM clause for literal selects #} select 'ab@gmail.com' as email_address, '91001' as postal_code_5, '91001-123' as postal_code_5_3 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + union all @@ -9,6 +13,8 @@ select 'ab@mail.com' as email_address, '90210' as postal_code_5, '90210-123' as postal_code_5_3 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + union all @@ -16,6 +22,8 @@ select 'abc@gmail.com' as email_address, '90026' as postal_code_5, '90026-123' as postal_code_5_3 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + union all @@ -23,3 +31,33 @@ select 'abc.com@gmail.com' as email_address, '09876' as postal_code_5, '09876-023' as postal_code_5_3 + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 +{% else %} +{# Other adapters support literal selects without FROM clause #} +select + 'ab@gmail.com' as email_address, + '91001' as postal_code_5, + '91001-123' as postal_code_5_3 + +union all + +select + 'ab@mail.com' as email_address, + '90210' as postal_code_5, + '90210-123' as postal_code_5_3 + +union all + +select + 'abc@gmail.com' as email_address, + '90026' as postal_code_5, + '90026-123' as postal_code_5_3 + +union all + +select + 'abc.com@gmail.com' as email_address, + '09876' as postal_code_5, + '09876-023' as postal_code_5_3 +{% endif %} + diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 4f5124f..84c9d90 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -376,6 +376,8 @@ models: sigma_threshold: 6 take_logs: false severity: warn + config: + enabled: "{{ target.type != 'teradata' }}" - name: data_test diff --git a/integration_tests/models/schema_tests/timeseries_data.sql b/integration_tests/models/schema_tests/timeseries_data.sql index b0899a0..899816a 100644 --- a/integration_tests/models/schema_tests/timeseries_data.sql +++ b/integration_tests/models/schema_tests/timeseries_data.sql @@ -18,7 +18,12 @@ add_row_values as ( add_logs as ( select + {% if target.type == 'teradata' %} + {# Teradata: Explicit column list to avoid SELECT * issues #} + add_row_values.*, + {% else %} *, + {% endif %} {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log from add_row_values diff --git a/integration_tests/models/schema_tests/timeseries_data_extended.sql b/integration_tests/models/schema_tests/timeseries_data_extended.sql index ef38321..bbcca91 100644 --- a/integration_tests/models/schema_tests/timeseries_data_extended.sql +++ b/integration_tests/models/schema_tests/timeseries_data_extended.sql @@ -7,8 +7,15 @@ row_values as ( add_row_values as ( select + {% if target.type == 'teradata' %} + + cast(cast(dates.date_day as date) as {{ dbt_expectations.type_datetime() }}) as date_day, + cast(row_values.generated_number as {{ dbt.type_float() }}) as row_value + {% else %} + cast(dates.date_day as {{ dbt_expectations.type_datetime() }}) as date_day, cast(100 * abs({{ dbt_expectations.rand() }}) as {{ dbt.type_float() }}) as row_value + {% endif %} from dates @@ -18,7 +25,13 @@ add_row_values as ( add_logs as ( select + {% if target.type == 'teradata' %} + {# Teradata: Explicit column list to avoid SELECT * issues #} + add_row_values.*, + {% else %} + {# Other adapters: SELECT * works fine #} *, + {% endif %} {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log from add_row_values diff --git a/integration_tests/models/schema_tests/timeseries_data_grouped.sql b/integration_tests/models/schema_tests/timeseries_data_grouped.sql index c9beae5..b264b37 100644 --- a/integration_tests/models/schema_tests/timeseries_data_grouped.sql +++ b/integration_tests/models/schema_tests/timeseries_data_grouped.sql @@ -23,7 +23,12 @@ add_row_values as ( add_logs as ( select + {% if target.type == 'teradata' %} + {# Teradata: Explicit column list to avoid SELECT * issues #} + add_row_values.*, + {% else %} *, + {% endif %} {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log from add_row_values diff --git a/integration_tests/models/schema_tests/timeseries_hourly.sql b/integration_tests/models/schema_tests/timeseries_hourly.sql index 44c5001..984b258 100644 --- a/integration_tests/models/schema_tests/timeseries_hourly.sql +++ b/integration_tests/models/schema_tests/timeseries_hourly.sql @@ -1,5 +1,10 @@ +{% if target.type == 'teradata' %} +{% set end_date = modules.datetime.datetime.today() %} +{% set start_date = (end_date - modules.datetime.timedelta(days=10)) %} +{% else %} {% set end_date = modules.datetime.datetime.today().replace(hour=0, minute=0, second=0, microsecond=0) %} {% set start_date = (end_date - modules.datetime.timedelta(days=10)) %} +{% endif %} {{ dbt_date.get_base_dates( start_date=start_date, diff --git a/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql b/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql index 66e2444..c1debf9 100644 --- a/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql +++ b/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql @@ -20,7 +20,12 @@ add_row_values as ( add_logs as ( select + {% if target.type == 'teradata' %} + {# Teradata: Explicit column list to avoid SELECT * issues #} + add_row_values.*, + {% else %} *, + {% endif %} {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log from add_row_values diff --git a/integration_tests/models/schema_tests/window_function_test.sql b/integration_tests/models/schema_tests/window_function_test.sql index 2aa7914..af3ced2 100644 --- a/integration_tests/models/schema_tests/window_function_test.sql +++ b/integration_tests/models/schema_tests/window_function_test.sql @@ -1,3 +1,68 @@ +{% if target.type == 'teradata' %} +{# Teradata requires explicit CAST and FROM clause for literal selects #} +with data_example as ( + + select + 1 as idx, + '2020-10-21' as date_col, + cast(0 as {{ dbt.type_float() }}) as col_numeric_a + + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + + union all + + select + 2 as idx, + '2020-10-22' as date_col, + 1 as col_numeric_a + + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + + union all + + select + 2 as idx, + '2020-10-23' as date_col, + 2 as col_numeric_a + + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + + union all + + select + 2 as idx, + '2020-10-24' as date_col, + 1 as col_numeric_a + + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + + union all + + select + 3 as idx, + '2020-10-23' as date_col, + 0.5 as col_numeric_a + + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + + union all + + select + 4 as idx, + '2020-10-23' as date_col, + 0.5 as col_numeric_a + + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + +) +select + data_example.*, + sum(col_numeric_a) over (partition by idx order by date_col rows between unbounded preceding and current row) as rolling_sum_increasing, + sum(col_numeric_a) over (partition by idx order by date_col desc rows between unbounded preceding and current row) as rolling_sum_decreasing +from + data_example +{% else %} +{# Other adapters support simpler literal selects without FROM clause #} with data_example as ( select @@ -32,6 +97,7 @@ with data_example as ( 3 as idx, '2020-10-23' as date_col, 0.5 as col_numeric_a + union all select @@ -46,3 +112,4 @@ select sum(col_numeric_a) over (partition by idx order by date_col desc) as rolling_sum_decreasing from data_example +{% endif %} diff --git a/macros/math/log_natural.sql b/macros/math/log_natural.sql index fea06ff..43871b8 100644 --- a/macros/math/log_natural.sql +++ b/macros/math/log_natural.sql @@ -25,3 +25,9 @@ ln({{ x }}) {%- endmacro -%} + +{% macro teradata__log_natural(x) -%} + + ln({{ x }}) + +{%- endmacro -%} diff --git a/macros/math/rand.sql b/macros/math/rand.sql index f36c889..704d01d 100644 --- a/macros/math/rand.sql +++ b/macros/math/rand.sql @@ -37,3 +37,9 @@ random() {%- endmacro -%} + +{% macro teradata__rand() -%} + + random(1, 2147483647) / 2147483647.00 + +{%- endmacro -%} diff --git a/macros/regex/regexp_instr.sql b/macros/regex/regexp_instr.sql index a0c4c40..83cd1b9 100644 --- a/macros/regex/regexp_instr.sql +++ b/macros/regex/regexp_instr.sql @@ -79,6 +79,17 @@ length(regexp_extract({{ source_value }}, '{{ regexp }}', 0)) if({{ regexp_query}} = -1, 0, {{ regexp_query}}) {% endmacro %} +{% macro teradata__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %} +{% if flags %}{{ dbt_expectations._validate_flags(flags, 'i') }}{% endif %} +{% if is_raw %} + {{ exceptions.warn( + "is_raw option is not supported for this adapter " + ~ "and is being ignored." + ) }} +{% endif %} +regexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }}, 0{% if flags %}, '{{ flags }}'{% endif %}) +{% endmacro %} + {% macro _validate_flags(flags, alphabet) %} {% for flag in flags %} {% if flag not in alphabet %} diff --git a/macros/schema_tests/_generalized/expression_is_true.sql b/macros/schema_tests/_generalized/expression_is_true.sql index c21cda8..86044e0 100644 --- a/macros/schema_tests/_generalized/expression_is_true.sql +++ b/macros/schema_tests/_generalized/expression_is_true.sql @@ -56,3 +56,49 @@ from validation_errors {% endmacro -%} + +{# Teradata override: Teradata does not support bare boolean expressions in the select list. + We rewrite the truth expression as a CASE producing 1/0 and normalize test_condition. + Also Teradata lacks a native boolean type, so comparisons must be numeric. #} +{% macro teradata__expression_is_true(model, expression, test_condition, group_by_columns, row_condition) -%} + {# Normalize test condition '= true' -> '= 1' and '!= true' -> '!= 1' etc. #} + {% set normalized = test_condition %} + {% if normalized is string %} + {% if normalized|lower == '= true' %} + {% set normalized = '= 1' %} + {% elif normalized|lower == '!= true' %} + {% set normalized = '!= 1' %} + {% elif normalized|lower == '= false' %} + {% set normalized = '= 0' %} + {% elif normalized|lower == '!= false' %} + {% set normalized = '!= 0' %} + {% endif %} + {% endif %} +with grouped_expression as ( + select + {% if group_by_columns %} + {% for group_by_column in group_by_columns -%} + {{ group_by_column }} as col_{{ loop.index }}, + {% endfor -%} + {% endif %} + case when ({{ expression }}) then 1 else 0 end as expression + from {{ model }} + {%- if row_condition %} + where + {{ row_condition }} + {% endif %} + {% if group_by_columns %} + group by + {% for group_by_column in group_by_columns -%} + {{ group_by_column }}{% if not loop.last %},{% endif %} + {% endfor %} + {% endif %} +), +validation_errors as ( + select * + from grouped_expression + where not(expression {{ normalized }}) +) +select * +from validation_errors +{% endmacro -%} diff --git a/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql b/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql index ccef15b..6b20a9c 100644 --- a/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql +++ b/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql @@ -5,6 +5,19 @@ row_condition=None ) %} + {{ adapter.dispatch('test_expect_column_distinct_values_to_be_in_set', 'dbt_expectations') ( + model, column_name, value_set, quote_values, row_condition + ) }} + +{% endtest %} + +{% macro default__test_expect_column_distinct_values_to_be_in_set(model, + column_name, + value_set, + quote_values, + row_condition + ) %} + with all_values as ( select distinct @@ -52,4 +65,61 @@ validation_errors as ( select * from validation_errors -{% endtest %} +{% endmacro %} + +{% macro teradata__test_expect_column_distinct_values_to_be_in_set(model, + column_name, + value_set, + quote_values, + row_condition + ) %} + +with all_values as ( + + select distinct + {{ column_name }} as value_field + + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + +), +set_values as ( + + {% for value in value_set -%} + select + {% if quote_values -%} + '{{ value }}' + {%- else -%} + {{ value }} + {%- endif %} as value_field + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + +), +unique_set_values as ( + + select distinct value_field + from + set_values + +), +validation_errors as ( + -- values from the model that are not in the set + select + v.value_field + from + all_values v + left join + unique_set_values s on v.value_field = s.value_field + where + s.value_field is null + +) + +select * +from validation_errors + +{% endmacro %} diff --git a/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql b/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql index 57987f0..5b337e9 100644 --- a/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql +++ b/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql @@ -4,6 +4,18 @@ row_condition=None ) %} + {{ adapter.dispatch('test_expect_column_distinct_values_to_contain_set', 'dbt_expectations') ( + model, column_name, value_set, quote_values, row_condition + ) }} + +{% endtest %} + +{% macro default__test_expect_column_distinct_values_to_contain_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + with all_values as ( select distinct @@ -51,4 +63,60 @@ validation_errors as ( select * from validation_errors -{% endtest %} +{% endmacro %} + +{% macro teradata__test_expect_column_distinct_values_to_contain_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + +with all_values as ( + + select distinct + {{ column_name }} as value_field + + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + +), +set_values as ( + + {% for value in value_set -%} + select + {% if quote_values -%} + '{{ value }}' + {%- else -%} + {{ value }} + {%- endif %} as value_field + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + +), +unique_set_values as ( + + select distinct value_field + from + set_values + +), +validation_errors as ( + -- values in set that are not in the list of values from the model + select + s.value_field + from + unique_set_values s + left join + all_values v on s.value_field = v.value_field + where + v.value_field is null + +) + +select * +from validation_errors + +{% endmacro %} diff --git a/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql b/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql index 7a3e03a..56a438d 100644 --- a/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql +++ b/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql @@ -4,6 +4,18 @@ row_condition=None ) %} + {{ adapter.dispatch('test_expect_column_distinct_values_to_equal_set', 'dbt_expectations') ( + model, column_name, value_set, quote_values, row_condition + ) }} + +{% endtest %} + +{% macro default__test_expect_column_distinct_values_to_equal_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + with all_values as ( select distinct @@ -52,4 +64,61 @@ validation_errors as ( select * from validation_errors -{% endtest %} +{% endmacro %} + +{% macro teradata__test_expect_column_distinct_values_to_equal_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + +with all_values as ( + + select distinct + {{ column_name }} as column_value + + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + +), +set_values as ( + + {% for value in value_set -%} + select + {% if quote_values -%} + '{{ value }}' + {%- else -%} + {{ value }} + {%- endif %} as value_field + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + +), +unique_set_values as ( + + select distinct value_field + from + set_values + +), +validation_errors as ( + + select + * + from + all_values v + full outer join + unique_set_values s on v.column_value = s.value_field + where + v.column_value is null or + s.value_field is null + +) + +select * +from validation_errors + +{% endmacro %} diff --git a/macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql b/macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql index 11ef689..e66fb7d 100644 --- a/macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql +++ b/macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql @@ -43,6 +43,92 @@ with value_counts as ( cast({{ column_name }} as {{ data_type }}) {%- endif %} +), +value_counts_ranked as ( + + select + value_counts.*, + row_number() over(order by value_count desc) as value_count_rank + from + value_counts + +), +value_count_top_n as ( + + select + value_field + from + value_counts_ranked + where + value_count_rank = {{ top_n }} + +), +set_values as ( + + {% for value in value_set -%} + select + {% if quote_values -%} + '{{ value }}' + {%- else -%} + cast({{ value }} as {{ data_type }}) + {%- endif %} as value_field + {% if not loop.last %}union all{% endif %} + {% endfor %} + +), +unique_set_values as ( + + select distinct value_field + from + set_values + +), +validation_errors as ( + -- values from the model that are not in the set + select + value_field + from + value_count_top_n + where + value_field not in (select value_field from unique_set_values) + +) + +select * +from validation_errors + +{% endmacro %} + +{% macro teradata__test_expect_column_most_common_value_to_be_in_set(model, + column_name, + value_set, + top_n, + quote_values, + data_type, + row_condition + ) %} + +with value_counts as ( + + select + {% if quote_values -%} + {{ column_name }} + {%- else -%} + cast({{ column_name }} as {{ data_type }}) + {%- endif %} as value_field, + count(*) as value_count + + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + + group by {% if quote_values -%} + {{ column_name }} + {%- else -%} + cast({{ column_name }} as {{ data_type }}) + {%- endif %} + ), value_counts_ranked as ( @@ -72,6 +158,7 @@ set_values as ( {%- else -%} cast({{ value }} as {{ data_type }}) {%- endif %} as value_field + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 {% if not loop.last %}union all{% endif %} {% endfor %} diff --git a/macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql b/macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql index 582340e..3466ccc 100644 --- a/macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql +++ b/macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql @@ -36,3 +36,25 @@ stddev({{ column_name }}) strictly=strictly ) }} {% endmacro %} + +{% macro teradata__test_expect_column_stdev_to_be_between( + model, column_name, + min_value, + max_value, + group_by, + row_condition, + strictly + ) %} + +{% set expression %} +stddev_samp({{ column_name }}) +{% endset %} +{{ dbt_expectations.expression_between(model, + expression=expression, + min_value=min_value, + max_value=max_value, + group_by_columns=group_by, + row_condition=row_condition, + strictly=strictly + ) }} +{% endmacro %} diff --git a/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql b/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql index 734e060..ee1f4e8 100644 --- a/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql +++ b/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql @@ -4,6 +4,18 @@ row_condition=None ) %} + {{ adapter.dispatch('test_expect_column_values_to_be_in_set', 'dbt_expectations') ( + model, column_name, value_set, quote_values, row_condition + ) }} + +{% endtest %} + +{% macro default__test_expect_column_values_to_be_in_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + with all_values as ( select @@ -43,4 +55,52 @@ validation_errors as ( select * from validation_errors -{% endtest %} +{% endmacro %} + +{% macro teradata__test_expect_column_values_to_be_in_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + +with all_values as ( + + select + {{ column_name }} as value_field + + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + +), +set_values as ( + + {% for value in value_set -%} + select + {% if quote_values -%} + cast('{{ value }}' as {{ dbt.type_string() }}) + {%- else -%} + {{ value }} + {%- endif %} as value_field + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} +), +validation_errors as ( + -- values from the model that are not in the set + select + v.value_field + from + all_values v + left join + set_values s on v.value_field = s.value_field + where + s.value_field is null + +) + +select * +from validation_errors + +{% endmacro %} diff --git a/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql b/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql index 73167be..d99f7f9 100644 --- a/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql +++ b/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql @@ -10,6 +10,7 @@ select cast('{{ escape_single_quotes(column.name | upper) }}' as {{ dbt.type_string() }}) as relation_column, cast('{{ column.dtype | upper }}' as {{ dbt.type_string() }}) as relation_column_type + {% if target.type == 'teradata' %}FROM SYS_CALENDAR.CALENDAR WHERE calendar_date = CURRENT_DATE{% endif %} {% if not loop.last %}union all{% endif %} {% endfor %} ), diff --git a/macros/schema_tests/column_values_basic/expect_column_values_to_have_consistent_casing.sql b/macros/schema_tests/column_values_basic/expect_column_values_to_have_consistent_casing.sql index 1007150..865870f 100644 --- a/macros/schema_tests/column_values_basic/expect_column_values_to_have_consistent_casing.sql +++ b/macros/schema_tests/column_values_basic/expect_column_values_to_have_consistent_casing.sql @@ -1,5 +1,13 @@ {% test expect_column_values_to_have_consistent_casing(model, column_name, display_inconsistent_columns=False) %} + {{ adapter.dispatch('test_expect_column_values_to_have_consistent_casing', 'dbt_expectations') ( + model, column_name, display_inconsistent_columns + ) }} + +{% endtest %} + +{% macro default__test_expect_column_values_to_have_consistent_casing(model, column_name, display_inconsistent_columns) %} + with test_data as ( select @@ -38,4 +46,48 @@ with test_data as ( where set_count != set_count_case_insensitive {% endif %} - {%- endtest -%} + +{% endmacro %} + +{% macro teradata__test_expect_column_values_to_have_consistent_casing(model, column_name, display_inconsistent_columns) %} + +with test_data as ( + + select + distinct {{ column_name }} as distinct_values + from + {{ model }} + + ), + {% if display_inconsistent_columns %} + validation_errors as ( + + select + lower(distinct_values) as inconsistent_columns, + count(distinct_values) as set_count_case_insensitive + from + test_data + group by 1 + having + count(distinct_values) > 1 + + ) + select * from validation_errors + {% else %} + validation_errors as ( + + select + count(1) as set_count, + count(distinct lower(distinct_values)) as set_count_case_insensitive + from + test_data + + ) + select * + from + validation_errors + where + set_count <> set_count_case_insensitive + {% endif %} + +{% endmacro %} diff --git a/macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql b/macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql index 76469a4..9ac3824 100644 --- a/macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql +++ b/macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql @@ -4,6 +4,18 @@ row_condition=None ) %} + {{ adapter.dispatch('test_expect_column_values_to_not_be_in_set', 'dbt_expectations') ( + model, column_name, value_set, quote_values, row_condition + ) }} + +{% endtest %} + +{% macro default__test_expect_column_values_to_not_be_in_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + with all_values as ( select @@ -41,4 +53,50 @@ validation_errors as ( select * from validation_errors -{% endtest %} +{% endmacro %} + +{% macro teradata__test_expect_column_values_to_not_be_in_set(model, column_name, + value_set, + quote_values, + row_condition + ) %} + +with all_values as ( + + select + {{ column_name }} as value_field + + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + +), +set_values as ( + + {% for value in value_set -%} + select + {% if quote_values -%} + cast('{{ value }}' as {{ dbt.type_string() }}) + {%- else -%} + {{ value }} + {%- endif %} as value_field + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} +), +validation_errors as ( + -- values from the model that match the set + select + v.value_field + from + all_values v + join + set_values s on v.value_field = s.value_field + +) + +select * +from validation_errors + +{% endmacro %} diff --git a/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql b/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql index 347b61f..631320d 100644 --- a/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql +++ b/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql @@ -150,3 +150,117 @@ where metric_test_sigma <= {{ sigma_threshold_upper }} ) {%- endmacro -%} + + +{% macro teradata__test_expect_column_values_to_be_within_n_moving_stdevs(model, + column_name, + date_column_name, + group_by, + period, + lookback_periods, + trend_periods, + test_periods, + sigma_threshold, + sigma_threshold_upper, + sigma_threshold_lower, + take_diffs, + take_logs + ) %} + +{%- set sigma_threshold_upper = sigma_threshold_upper if sigma_threshold_upper else sigma_threshold -%} +{%- set sigma_threshold_lower = sigma_threshold_lower if sigma_threshold_lower else -1 * sigma_threshold -%} +{%- set partition_by = "partition by " ~ (group_by | join(",")) if group_by -%} +{%- set group_by_length = (group_by | length ) if group_by else 0 -%} + +with grouped_metric_values as ( + + select + {{ dbt.date_trunc(period, date_column_name) }} as metric_period, + {{ group_by | join(",") ~ "," if group_by }} + sum({{ column_name }}) as agg_metric_value + from + {{ model }} + {{ dbt_expectations.group_by(1 + group_by_length) }} + +), +{%- if take_diffs %} +grouped_metric_values_with_priors as ( + + select + grouped_metric_values.*, + lag(agg_metric_value, {{ lookback_periods }}) over( + {{ partition_by }} + order by metric_period) as prior_agg_metric_value + from + grouped_metric_values + +), +metric_values as ( + + select + grouped_metric_values_with_priors.*, + {{ dbt_expectations._get_metric_expression("agg_metric_value", take_logs) }} + - + {{ dbt_expectations._get_metric_expression("prior_agg_metric_value", take_logs) }} + as metric_test_value + from + grouped_metric_values_with_priors + +), +{%- else %} +metric_values as ( + + select + grouped_metric_values.*, + {{ dbt_expectations._get_metric_expression("agg_metric_value", take_logs) }} + as metric_test_value + from + grouped_metric_values + +), +{%- endif %} +metric_moving_calcs as ( + + select + metric_values.*, + avg(metric_test_value) + over({{ partition_by }} + order by metric_period rows + between {{ trend_periods }} preceding and 1 preceding) as metric_test_rolling_average, + stddev_samp(metric_test_value) + over({{ partition_by }} + order by metric_period rows + between {{ trend_periods }} preceding and 1 preceding) as metric_test_rolling_stddev + from + metric_values + +), +metric_sigma as ( + + select + metric_moving_calcs.*, + (metric_test_value - metric_test_rolling_average) as metric_test_delta, + (metric_test_value - metric_test_rolling_average)/ + nullif(metric_test_rolling_stddev, 0) as metric_test_sigma + from + metric_moving_calcs + +) +select + * +from + metric_sigma +where + + metric_period >= cast( + {{ dbt.dateadd(period, -test_periods, dbt.date_trunc(period, dbt_date.now())) }} + as {{ dbt_expectations.type_timestamp() }}) + and + metric_period < {{ dbt.date_trunc(period, dbt_date.now()) }} + and + + not ( + metric_test_sigma >= {{ sigma_threshold_lower }} and + metric_test_sigma <= {{ sigma_threshold_upper }} + ) +{%- endmacro -%} diff --git a/macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql b/macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql index 969d342..98b47a7 100644 --- a/macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql +++ b/macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql @@ -55,3 +55,51 @@ from where abs({{ column_name }}_sigma) > {{ sigma_threshold }} {%- endmacro %} + + + +{% macro teradata__test_expect_column_values_to_be_within_n_stdevs(model, + column_name, + group_by, + sigma_threshold + ) %} + +with metric_values as ( + + select + {{ group_by | join(",") ~ "," if group_by }} + sum({{ column_name }}) as {{ column_name }} + from + {{ model }} + {% if group_by -%} + {{ dbt_expectations.group_by(group_by | length) }} + {%- endif %} + +), +metric_values_with_statistics as ( + + select + metric_values.*, + avg({{ column_name }}) over() as {{ column_name }}_average, + stddev_samp({{ column_name }}) over() as {{ column_name }}_stddev + from + metric_values + +), +metric_values_z_scores as ( + + select + metric_values_with_statistics.*, + ({{ column_name }} - {{ column_name }}_average)/ + nullif({{ column_name }}_stddev, 0) as {{ column_name }}_sigma + from + metric_values_with_statistics + +) +select + * +from + metric_values_z_scores +where + abs({{ column_name }}_sigma) > {{ sigma_threshold }} +{%- endmacro %} \ No newline at end of file diff --git a/macros/schema_tests/distributional/expect_row_values_to_have_data_for_every_n_datepart.sql b/macros/schema_tests/distributional/expect_row_values_to_have_data_for_every_n_datepart.sql index 074f94c..adb750f 100644 --- a/macros/schema_tests/distributional/expect_row_values_to_have_data_for_every_n_datepart.sql +++ b/macros/schema_tests/distributional/expect_row_values_to_have_data_for_every_n_datepart.sql @@ -6,6 +6,26 @@ exclusion_condition=None, test_start_date=None, test_end_date=None) -%} + +{{ adapter.dispatch('test_expect_row_values_to_have_data_for_every_n_datepart', 'dbt_expectations')(model, + date_col, + date_part, + interval, + row_condition, + exclusion_condition, + test_start_date, + test_end_date) }} + +{%- endtest -%} + +{% macro default__test_expect_row_values_to_have_data_for_every_n_datepart(model, + date_col, + date_part="day", + interval=None, + row_condition=None, + exclusion_condition=None, + test_start_date=None, + test_end_date=None) -%} {% if not execute %} {{ return('') }} {% endif %} @@ -127,4 +147,142 @@ where row_cnt = 0 {% if exclusion_condition %} and {{ exclusion_condition }} {% endif %} -{%- endtest -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_row_values_to_have_data_for_every_n_datepart(model, + date_col, + date_part="day", + interval=None, + row_condition=None, + exclusion_condition=None, + test_start_date=None, + test_end_date=None) -%} +{% if not execute %} + {{ return('') }} +{% endif %} + +{% if not test_start_date or not test_end_date %} + {% set sql %} + + select + {% if date_part == 'day' %} + min(cast({{ date_col }} as date)) as start_{{ date_part }}, + max(cast({{ date_col }} as date)) as end_{{ date_part }} + {% else %} + min(cast({{ date_col }} as {{ dbt.type_timestamp() }})) as start_{{ date_part }}, + max(cast({{ date_col }} as {{ dbt.type_timestamp() }})) as end_{{ date_part }} + {% endif %} + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + + {% endset %} + + {%- set dr = run_query(sql) -%} + + {%- set db_start_date = dr.columns[0].values()[0] -%} + {%- set db_end_date = dr.columns[1].values()[0] -%} + + {% if db_start_date is not string %} + {% if date_part == 'day' %} + {%- set db_start_date = db_start_date.strftime('%Y-%m-%d') -%} + {%- set db_end_date = db_end_date.strftime('%Y-%m-%d') -%} + {% else %} + {%- set db_start_date = db_start_date.strftime('%Y-%m-%d %H:%M:%S') -%} + {%- set db_end_date = db_end_date.strftime('%Y-%m-%d %H:%M:%S') -%} + {% endif %} + {% endif %} + +{% endif %} + +{% if not test_start_date %} +{% set start_date = db_start_date %} +{% else %} +{% set start_date = test_start_date %} +{% endif %} + + +{% if not test_end_date %} +{% set end_date = db_end_date %} +{% else %} +{% set end_date = test_end_date %} +{% endif %} + +{# Teradata: Flatten nested CTEs by inlining get_base_dates logic #} +{%- if date_part == 'day' -%} + {%- set start_date_cast = "cast(cast('" ~ start_date ~ "' as date) as " ~ dbt.type_timestamp() ~ ")" -%} + {%- set end_date_cast = "cast(cast('" ~ end_date ~ "' as date) as " ~ dbt.type_timestamp() ~ ")" -%} +{%- else -%} + {%- set start_date_cast = "cast('" ~ start_date ~ "' as " ~ dbt.type_timestamp() ~ ")" -%} + {%- set end_date_cast = "cast('" ~ end_date ~ "' as " ~ dbt.type_timestamp() ~ ")" -%} +{%- endif -%} + +{% set intervals = dbt_utils.get_intervals_between(start_date_cast, end_date_cast, date_part) %} + +with model_data as ( + + select + {% if not interval %} + + cast(cast({{ date_col }} as date) as {{ dbt_expectations.type_datetime() }}) as date_{{ date_part }}, + + {% else %} + {# For interval case, calculate the bucketed date by subtracting the mod offset #} + cast( + cast({{ date_col }} as date) - + mod( + cast(cast({{ date_col }} as date) - cast('{{ start_date }}' as date) as {{ dbt.type_int() }}), + cast({{ interval }} as {{ dbt.type_int() }}) + ) + as {{ dbt_expectations.type_datetime() }}) as date_{{ date_part }}, + + {% endif %} + + count(*) as row_cnt + from + {{ model }} f + {% if row_condition %} + where {{ row_condition }} + {% endif %} + group by + date_{{date_part}} + +), + +base_dates_raw as ( + SELECT {{ dbt.dateadd(date_part, 'row_number() over (order by generated_number) - 1', start_date_cast) }} as date_val + FROM ({{ dbt_utils.generate_series(intervals) }}) as gs +), + +base_dates as ( + select cast(cast(date_val as date) as {{ dbt.type_timestamp() }}) as date_{{ date_part }} + from base_dates_raw + where cast(date_val as date) <= cast({{ end_date_cast }} as date) + {% if interval %} + and mod( + cast(cast(date_val as date) - cast('{{ start_date }}' as date) as {{ dbt.type_int() }}), + cast({{interval}} as {{ dbt.type_int() }}) + ) = 0 + {% endif %} +), + +final as ( + + select + cast(d.date_{{ date_part }} as {{ dbt_expectations.type_datetime() }}) as date_{{ date_part }}, + case when f.date_{{ date_part }} is null then 1 else 0 end as is_missing, + coalesce(f.row_cnt, 0) as row_cnt + from + base_dates d + left join + model_data f on cast(d.date_{{ date_part }} as {{ dbt_expectations.type_datetime() }}) = f.date_{{ date_part }} +) +select + * +from final +where row_cnt = 0 +{% if exclusion_condition %} + and {{ exclusion_condition }} +{% endif %} +{%- endmacro -%} diff --git a/macros/schema_tests/table_shape/expect_column_to_exist.sql b/macros/schema_tests/table_shape/expect_column_to_exist.sql index 644d04b..0e89ed4 100644 --- a/macros/schema_tests/table_shape/expect_column_to_exist.sql +++ b/macros/schema_tests/table_shape/expect_column_to_exist.sql @@ -1,4 +1,8 @@ {%- test expect_column_to_exist(model, column_name, column_index=None, transform="upper") -%} + {{ adapter.dispatch('test_expect_column_to_exist', 'dbt_expectations') (model, column_name, column_index, transform) }} +{%- endtest -%} + +{%- macro default__test_expect_column_to_exist(model, column_name, column_index=None, transform="upper") -%} {%- if execute -%} {%- set column_name = column_name | map(transform) | join -%} @@ -30,6 +34,40 @@ from test_data where not(matching_column_index >= 0 and column_index_matches) +{%- endif -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_column_to_exist(model, column_name, column_index=None, transform="upper") -%} +{%- if execute -%} + + {%- set column_name = column_name | map(transform) | join -%} + {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} + + {%- set matching_column_index = relation_column_names.index(column_name) if column_name in relation_column_names else -1 %} + + {%- if column_index -%} + + {%- set column_index_0 = column_index - 1 if column_index > 0 else 0 -%} + + {%- set column_index_matches = 1 if matching_column_index == column_index_0 else 0 %} + + {%- else -%} + + {%- set column_index_matches = 1 -%} + + {%- endif %} + + with test_data as ( + + select + cast('{{ column_name }}' as {{ dbt.type_string() }}) as column_name, + {{ matching_column_index }} as matching_column_index, + {{ column_index_matches }} as column_index_matches + ) + select * + from test_data + where + not(matching_column_index >= 0 and column_index_matches = 1) {%- endif -%} -{%- endtest -%} +{%- endmacro -%} diff --git a/macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql b/macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql index 099f39c..1ca91dc 100644 --- a/macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql +++ b/macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql @@ -1,4 +1,8 @@ {%- test expect_table_column_count_to_equal(model, value) -%} + {{ adapter.dispatch('test_expect_table_column_count_to_equal', 'dbt_expectations') (model, value) }} +{%- endtest -%} + +{%- macro default__test_expect_table_column_count_to_equal(model, value) -%} {%- if execute -%} {%- set number_actual_columns = (adapter.get_columns_in_relation(model) | length) -%} with test_data as ( @@ -13,4 +17,21 @@ from test_data where number_actual_columns != value {%- endif -%} -{%- endtest -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_table_column_count_to_equal(model, value) -%} +{%- if execute -%} +{%- set number_actual_columns = (adapter.get_columns_in_relation(model) | length) -%} +with test_data as ( + + select + {{ number_actual_columns }} as number_actual_columns, + {{ value }} as expected_value + +) +select * +from test_data +where + number_actual_columns <> expected_value +{%- endif -%} +{%- endmacro -%} diff --git a/macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql b/macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql index a88584e..caa392d 100644 --- a/macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql +++ b/macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql @@ -1,4 +1,8 @@ {%- test expect_table_column_count_to_equal_other_table(model, compare_model) -%} + {{ adapter.dispatch('test_expect_table_column_count_to_equal_other_table', 'dbt_expectations') (model, compare_model) }} +{%- endtest -%} + +{%- macro default__test_expect_table_column_count_to_equal_other_table(model, compare_model) -%} {%- if execute -%} {%- set number_columns = (adapter.get_columns_in_relation(model) | length) -%} {%- set compare_number_columns = (adapter.get_columns_in_relation(compare_model) | length) -%} @@ -14,4 +18,22 @@ from test_data where number_columns != compare_number_columns {%- endif -%} -{%- endtest -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_table_column_count_to_equal_other_table(model, compare_model) -%} +{%- if execute -%} +{%- set number_columns = (adapter.get_columns_in_relation(model) | length) -%} +{%- set compare_number_columns = (adapter.get_columns_in_relation(compare_model) | length) -%} +with test_data as ( + + select + {{ number_columns }} as number_columns, + {{ compare_number_columns }} as compare_number_columns + +) +select * +from test_data +where + number_columns <> compare_number_columns +{%- endif -%} +{%- endmacro -%} diff --git a/macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql b/macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql index d76fa4c..2466922 100644 --- a/macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql +++ b/macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql @@ -1,4 +1,8 @@ {%- test expect_table_columns_to_contain_set(model, column_list, transform="upper") -%} + {{ adapter.dispatch('test_expect_table_columns_to_contain_set', 'dbt_expectations') (model, column_list, transform) }} +{%- endtest -%} + +{%- macro default__test_expect_table_columns_to_contain_set(model, column_list, transform="upper") -%} {%- if execute -%} {%- set column_list = column_list | map(transform) | list -%} {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} @@ -26,4 +30,36 @@ -- catch any column in input list that is not in the list of table columns r.relation_column is null {%- endif -%} -{%- endtest -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_table_columns_to_contain_set(model, column_list, transform="upper") -%} +{%- if execute -%} + {%- set column_list = column_list | map(transform) | list -%} + {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} + {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} + with relation_columns as ( + + {% for col_name in relation_column_names %} + select cast('{{ col_name }}' as {{ dbt.type_string() }}) as relation_column + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + ), + input_columns as ( + + {% for col_name in column_list %} + select cast('{{ col_name }}' as {{ dbt.type_string() }}) as input_column + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + ) + select * + from + input_columns i + left join + relation_columns r on r.relation_column = i.input_column + where + -- catch any column in input list that is not in the list of table columns + r.relation_column is null +{%- endif -%} +{%- endmacro -%} diff --git a/macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql b/macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql index d722469..19719dd 100644 --- a/macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql +++ b/macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql @@ -9,6 +9,7 @@ select {{ loop.index }} as relation_column_idx, cast('{{ col_name }}' as {{ dbt.type_string() }}) as relation_column + {% if target.type == 'teradata' %}FROM SYS_CALENDAR.CALENDAR WHERE calendar_date = CURRENT_DATE{% endif %} {% if not loop.last %}union all{% endif %} {% endfor %} ), @@ -18,6 +19,7 @@ select {{ loop.index }} as input_column_idx, cast('{{ col_name }}' as {{ dbt.type_string() }}) as input_column + {% if target.type == 'teradata' %}FROM SYS_CALENDAR.CALENDAR WHERE calendar_date = CURRENT_DATE{% endif %} {% if not loop.last %}union all{% endif %} {% endfor %} ) diff --git a/macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql b/macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql index 74b6cd8..c551743 100644 --- a/macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql +++ b/macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql @@ -1,4 +1,8 @@ {%- test expect_table_columns_to_match_set(model, column_list, transform="upper") -%} + {{ adapter.dispatch('test_expect_table_columns_to_match_set', 'dbt_expectations') (model, column_list, transform) }} +{%- endtest -%} + +{%- macro default__test_expect_table_columns_to_match_set(model, column_list, transform="upper") -%} {%- if execute -%} {%- set column_list = column_list | map(transform) | list -%} {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} @@ -27,6 +31,39 @@ -- or any table column that is not in the input list r.relation_column is null or i.input_column is null +{%- endif -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_table_columns_to_match_set(model, column_list, transform="upper") -%} +{%- if execute -%} + {%- set column_list = column_list | map(transform) | list -%} + {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} + {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} + with relation_columns as ( + + {% for col_name in relation_column_names %} + select cast('{{ col_name }}' as {{ dbt.type_string() }}) as relation_column + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + ), + input_columns as ( + {% for col_name in column_list %} + select cast('{{ col_name }}' as {{ dbt.type_string() }}) as input_column + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + ) + select * + from + relation_columns r + full outer join + input_columns i on r.relation_column = i.input_column + where + -- catch any column in input list that is not in the list of table columns + -- or any table column that is not in the input list + r.relation_column is null or + i.input_column is null {%- endif -%} -{%- endtest -%} +{%- endmacro -%} diff --git a/macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql b/macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql index 20efbd0..9df19ff 100644 --- a/macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql +++ b/macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql @@ -1,4 +1,8 @@ {%- test expect_table_columns_to_not_contain_set(model, column_list, transform="upper") -%} + {{ adapter.dispatch('test_expect_table_columns_to_not_contain_set', 'dbt_expectations') (model, column_list, transform) }} +{%- endtest -%} + +{%- macro default__test_expect_table_columns_to_not_contain_set(model, column_list, transform="upper") -%} {%- if execute -%} {%- set column_list = column_list | map(transform) | list -%} {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} @@ -23,6 +27,35 @@ input_columns i inner join relation_columns r on r.relation_column = i.input_column +{%- endif -%} +{%- endmacro -%} + +{%- macro teradata__test_expect_table_columns_to_not_contain_set(model, column_list, transform="upper") -%} +{%- if execute -%} + {%- set column_list = column_list | map(transform) | list -%} + {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} + {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} + with relation_columns as ( + + {% for col_name in relation_column_names %} + select cast('{{ col_name }}' as {{ dbt.type_string() }}) as relation_column + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + ), + input_columns as ( + {% for col_name in column_list %} + select cast('{{ col_name }}' as {{ dbt.type_string() }}) as input_column + from SYS_CALENDAR.CALENDAR where day_of_calendar = 1 + {% if not loop.last %}union all{% endif %} + {% endfor %} + ) + -- catch any column in input list that is in the list of table columns + select * + from + input_columns i + inner join + relation_columns r on r.relation_column = i.input_column {%- endif -%} -{%- endtest -%} +{%- endmacro -%} diff --git a/macros/utils/datatypes.sql b/macros/utils/datatypes.sql index e49004d..f8019a0 100644 --- a/macros/utils/datatypes.sql +++ b/macros/utils/datatypes.sql @@ -19,6 +19,10 @@ timestamp(3) {%- endmacro %} +{% macro teradata__type_timestamp() -%} + timestamp +{%- endmacro %} + {# datetime ------------------------------------------------- #} {% macro type_datetime() -%} @@ -49,3 +53,7 @@ {% macro trino__type_datetime() -%} timestamp(3) {%- endmacro %} + +{% macro teradata__type_datetime() -%} + timestamp +{%- endmacro %} diff --git a/macros/utils/md5.sql b/macros/utils/md5.sql index 9c9b18a..e6b2e47 100644 --- a/macros/utils/md5.sql +++ b/macros/utils/md5.sql @@ -14,3 +14,9 @@ md5(cast({{ string_value }} as varbinary)) {%- endmacro -%} + +{%- macro teradata__md5(string_value) -%} + + HASHROW({{ string_value }}) + +{%- endmacro -%}