diff --git a/models/build/edfi_3/assessments/bld_ef3__student_assessments_long_results.sql b/models/build/edfi_3/assessments/bld_ef3__student_assessments_long_results.sql index 39e4b74f..0a19af26 100644 --- a/models/build/edfi_3/assessments/bld_ef3__student_assessments_long_results.sql +++ b/models/build/edfi_3/assessments/bld_ef3__student_assessments_long_results.sql @@ -4,6 +4,12 @@ with score_results as ( xwalk_scores as ( select * from {{ ref('xwalk_assessment_scores') }} ), +xwalk_score_values as ( + select * from {{ ref('xwalk_assessment_score_values') }} +), +xwalk_score_value_thresholds as ( + select * from {{ ref('xwalk_assessment_score_value_thresholds') }} +), performance_levels as ( select tenant_code, @@ -34,16 +40,35 @@ dedupe_results as ( ), merged_xwalk as ( select - tenant_code, - api_year, - k_student_assessment, - score_name as original_score_name, - coalesce(normalized_score_name, 'other') as normalized_score_name, - score_result + dedupe_results.tenant_code, + dedupe_results.api_year, + dedupe_results.k_student_assessment, + dedupe_results.score_name as original_score_name, + coalesce(xwalk_scores.normalized_score_name, 'other') as normalized_score_name, + dedupe_results.score_result, + coalesce(xwalk_score_value_thresholds.normalized_score_result::varchar, + xwalk_score_values.normalized_score_result::varchar, + score_result::varchar + ) as normalized_score_result from dedupe_results left join xwalk_scores on dedupe_results.assessment_identifier = xwalk_scores.assessment_identifier and dedupe_results.namespace = xwalk_scores.namespace and dedupe_results.score_name = xwalk_scores.original_score_name + left join xwalk_score_values + on dedupe_results.assessment_identifier = xwalk_score_values.assessment_identifier + and dedupe_results.namespace = xwalk_score_values.namespace + and xwalk_scores.normalized_score_name = xwalk_score_values.normalized_score_name + and dedupe_results.score_result = xwalk_score_values.original_score_result + left join xwalk_score_value_thresholds + on dedupe_results.assessment_identifier = xwalk_score_value_thresholds.assessment_identifier + and dedupe_results.namespace = xwalk_score_value_thresholds.namespace + and xwalk_scores.normalized_score_name = xwalk_score_value_thresholds.normalized_score_name + -- todo check these comparators -- what if there's a value between the upper and next lower? eg value is 20.4 and the cutoffs are 20 and 21 + -- todo review my use of try_to_numeric here -- the idea is to allow numeric values to merge, otherwise don't merge without error + and try_to_numeric(dedupe_results.score_result) >= xwalk_score_value_thresholds.lower_bound + and try_to_numeric(dedupe_results.score_result) <= xwalk_score_value_thresholds.upper_bound + -- todo in future, may need to include subject & grade level in this join (with options to join across subjects) + ) select * from merged_xwalk \ No newline at end of file diff --git a/models/build/edfi_3/assessments/bld_ef3__student_objective_assessments_long_results.sql b/models/build/edfi_3/assessments/bld_ef3__student_objective_assessments_long_results.sql index 9061582f..c155a91c 100644 --- a/models/build/edfi_3/assessments/bld_ef3__student_objective_assessments_long_results.sql +++ b/models/build/edfi_3/assessments/bld_ef3__student_objective_assessments_long_results.sql @@ -4,6 +4,12 @@ with score_results as ( xwalk_scores as ( select * from {{ ref('xwalk_objective_assessment_scores') }} ), +xwalk_score_values as ( + select * from {{ ref('xwalk_assessment_score_values') }} +), +xwalk_score_value_thresholds as ( + select * from {{ ref('xwalk_assessment_score_value_thresholds') }} +), performance_levels as ( select tenant_code, @@ -39,13 +45,31 @@ merged_xwalk as ( api_year, k_student_objective_assessment, score_name as original_score_name, - coalesce(normalized_score_name, 'other') as normalized_score_name, - score_result + coalesce(xwalk_scores.normalized_score_name, 'other') as normalized_score_name, + score_result, + coalesce(xwalk_score_value_thresholds.normalized_score_result::varchar, + xwalk_score_values.normalized_score_result::varchar, + score_result::varchar + ) as normalized_score_result from dedupe_results left join xwalk_scores on dedupe_results.assessment_identifier = xwalk_scores.assessment_identifier and dedupe_results.namespace = xwalk_scores.namespace and dedupe_results.objective_assessment_identification_code = xwalk_scores.objective_assessment_identification_code and dedupe_results.score_name = xwalk_scores.original_score_name + left join xwalk_score_values + on dedupe_results.assessment_identifier = xwalk_score_values.assessment_identifier + and dedupe_results.namespace = xwalk_score_values.namespace + and xwalk_scores.normalized_score_name = xwalk_score_values.normalized_score_name + and dedupe_results.score_result = xwalk_score_values.original_score_result + left join xwalk_score_value_thresholds + on dedupe_results.assessment_identifier = xwalk_score_value_thresholds.assessment_identifier + and dedupe_results.namespace = xwalk_score_value_thresholds.namespace + and xwalk_scores.normalized_score_name = xwalk_score_value_thresholds.normalized_score_name + -- todo check these comparators -- what if there's a value between the upper and next lower? eg value is 20.4 and the cutoffs are 20 and 21 + -- todo review my use of try_to_numeric here -- the idea is to allow numeric values to merge, otherwise don't merge without error + and try_to_numeric(dedupe_results.score_result) >= xwalk_score_value_thresholds.lower_bound + and try_to_numeric(dedupe_results.score_result) <= xwalk_score_value_thresholds.upper_bound + -- todo in future, may need to include subject & grade level in this join (with options to join across subjects) ) select * from merged_xwalk \ No newline at end of file diff --git a/models/core_warehouse/fct_student_assessment.sql b/models/core_warehouse/fct_student_assessment.sql index a767c843..eb8ebb53 100644 --- a/models/core_warehouse/fct_student_assessment.sql +++ b/models/core_warehouse/fct_student_assessment.sql @@ -1,3 +1,5 @@ +-- depends_on: {{ ref('xwalk_assessment_score_values') }} +-- depends_on: {{ ref('xwalk_assessment_score_value_thresholds') }} {{ config( post_hook=[ @@ -30,9 +32,9 @@ student_assessments_wide as ( student_assessments.tenant_code, student_assessments.student_assessment_identifier, student_assessments.serial_number, - school_year, - administration_date, - administration_end_date, + student_assessments.school_year, + student_assessments.administration_date, + student_assessments.administration_end_date, event_description, administration_environment, administration_language, @@ -50,6 +52,18 @@ student_assessments_wide as ( else_value='NULL', agg='max', quote_identifiers=False + ) }}, + {#- find distinct score names that are in one of the normalize_result xwalks (distinct scores to add normalized_ column for) -#} + {% set normalized_names_values = dbt_utils.get_column_values(ref('xwalk_assessment_score_values'), 'normalized_score_name') or [] %} + {% set normalized_names_thresholds = dbt_utils.get_column_values(ref('xwalk_assessment_score_value_thresholds'), 'normalized_score_name') or [] %} + {{ dbt_utils.pivot( + 'normalized_score_name', + (normalized_names_values + normalized_names_thresholds) | unique, + then_value='normalized_score_result', + else_value='NULL', + prefix='normalized_', + agg='max', + quote_identifiers=False ) }} {%- endif %} from student_assessments diff --git a/models/core_warehouse/fct_student_objective_assessment.sql b/models/core_warehouse/fct_student_objective_assessment.sql index 93c4611f..67c2639a 100644 --- a/models/core_warehouse/fct_student_objective_assessment.sql +++ b/models/core_warehouse/fct_student_objective_assessment.sql @@ -1,3 +1,5 @@ +-- depends_on: {{ ref('xwalk_objective_assessment_score_values') }} +-- depends_on: {{ ref('xwalk_objective_assessment_score_value_thresholds') }} {{ config( post_hook=[ @@ -51,6 +53,18 @@ student_obj_assessments_wide as ( else_value='NULL', agg='max', quote_identifiers=False + ) }}, + {#- find distinct score names that are in one of the normalize_result xwalks (distinct scores to add normalized_ column for) -#} + {% set normalized_names_values = dbt_utils.get_column_values(ref('xwalk_objective_assessment_score_values'), 'normalized_score_name') or [] %} + {% set normalized_names_thresholds = dbt_utils.get_column_values(ref('xwalk_objective_assessment_score_value_thresholds'), 'normalized_score_name') or [] %} + {{ dbt_utils.pivot( + 'normalized_score_name', + (normalized_names_values + normalized_names_thresholds) | unique, + then_value='normalized_score_result', + else_value='NULL', + prefix='normalized_', + agg='max', + quote_identifiers=False ) }} {%- endif %} from student_obj_assessments