diff --git a/.gitignore b/.gitignore index 8b25ed2..baf5974 100644 --- a/.gitignore +++ b/.gitignore @@ -217,4 +217,9 @@ dbt_internal_packages/ package-lock.yml logs/ -/airstats +# /airstats +profiles.yml +# dbt_project.yml +.gitignore +.user.yml +.gitkeep diff --git a/airstats/README.md b/airstats/README.md new file mode 100644 index 0000000..7874ac8 --- /dev/null +++ b/airstats/README.md @@ -0,0 +1,15 @@ +Welcome to your new dbt project! + +### Using the starter project + +Try running the following commands: +- dbt run +- dbt test + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/airstats/analyses/la_heliport_closed.sql b/airstats/analyses/la_heliport_closed.sql new file mode 100644 index 0000000..c88aadd --- /dev/null +++ b/airstats/analyses/la_heliport_closed.sql @@ -0,0 +1,2 @@ +-- Validate snapshot captured the closure of Los Angeles County Sheriff's Department Heliport +SELECT * FROM {{ ref('scd_silver_airports') }} WHERE airport_ident = '01CN' \ No newline at end of file diff --git a/airstats/dbt_project.yml b/airstats/dbt_project.yml new file mode 100644 index 0000000..7b899bc --- /dev/null +++ b/airstats/dbt_project.yml @@ -0,0 +1,45 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airstats' +version: '1.0.0' + +# This setting configures which "profile" dbt uses for this project. +profile: 'airstats' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +data_tests: + +store_failures: true + +schema: _test_failures + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +# models: +# airstats: +# # Config indicated by + and applies to all files under models/example/ +# example: +# +materialized: view +models: + airstats: + silver: + +materialized: table +flags: + require_generic_test_arguments_property: true diff --git a/airstats/models/bronze/src_airport_comments.sql b/airstats/models/bronze/src_airport_comments.sql new file mode 100644 index 0000000..2cbe54b --- /dev/null +++ b/airstats/models/bronze/src_airport_comments.sql @@ -0,0 +1,17 @@ +{{ + config(materialized='ephemeral') +}} + +WITH airport_comments AS ( + SELECT * FROM {{ source('airstats', 'comments') }} +) + +SELECT + id AS comment_id, + airport_ident, + date AS comment_timestamp, + member_nickname, + subject AS comment_subject, + body AS comment_body + +FROM airport_comments \ No newline at end of file diff --git a/airstats/models/bronze/src_airports.sql b/airstats/models/bronze/src_airports.sql new file mode 100644 index 0000000..19bba53 --- /dev/null +++ b/airstats/models/bronze/src_airports.sql @@ -0,0 +1,18 @@ +{{ + config(materialized='ephemeral') +}} + +WITH airports AS ( + SELECT * FROM {{ source('airstats', 'airports') }} +) + +SELECT + ident AS airport_ident, + type AS airport_type, + name AS airport_name, + latitude_deg AS airport_lat, + longitude_deg AS airport_long, + continent, + iso_country, + iso_region +FROM airports \ No newline at end of file diff --git a/airstats/models/bronze/src_runways.sql b/airstats/models/bronze/src_runways.sql new file mode 100644 index 0000000..1f2e6a7 --- /dev/null +++ b/airstats/models/bronze/src_runways.sql @@ -0,0 +1,18 @@ +{{ + config(materialized='ephemeral') +}} + +WITH runways AS ( + SELECT * FROM {{ source('airstats', 'runways') }} +) + +SELECT + id AS runway_id, + airport_ident, + length_ft AS runway_length_ft, + width_ft AS runway_width_ft, + surface AS runway_surface, + lighted AS runway_lighted, + closed AS runway_closed + +FROM runways \ No newline at end of file diff --git a/airstats/models/silver/_docs.md b/airstats/models/silver/_docs.md new file mode 100644 index 0000000..bd6d14e --- /dev/null +++ b/airstats/models/silver/_docs.md @@ -0,0 +1,21 @@ +{% docs silver_airports_description %} +The silver_airports table contains cleaned and standardized airport data from OurAirports. +It includes geographic coordinates, airport classifications, and location information for over 70,000 airports worldwide. + +**Data Source:** OurAirports (ourairports.com/data/) + +**Key transformations:** +- Column names standardized with airport_ prefix +- No data filtering applied - all airports included +{% enddocs %} + +{% docs airport_type_description %} +Classification of the airport by size and type: +- **small_airport**: Regional or community airports +- **medium_airport**: Regional commercial airports +- **large_airport**: Major commercial airports +- **heliport**: Helicopter landing sites +- **seaplane_base**: Water landing sites +- **balloonport**: Balloon launch sites +- **closed**: Non-operational airports +{% enddocs %} diff --git a/airstats/models/silver/overview.md b/airstats/models/silver/overview.md new file mode 100644 index 0000000..8810cf2 --- /dev/null +++ b/airstats/models/silver/overview.md @@ -0,0 +1,33 @@ +# Silver Layer Overview + +## Tables + +The silver layer contains three interconnected tables that provide a clean, analysis-ready view of airport data. + +### silver_airports +The core dimension table containing airport information including identifiers, names, types, and geographic coordinates. This is the parent table that other tables reference. + +### silver_runways +Contains runway specifications for airports. Each runway belongs to exactly one airport, linked via `airport_ident`. Includes dimensions (length, width), surface type, and operational status. + +### silver_airport_comments +An incremental table storing user-submitted comments about airports. Comments are linked to airports via `airport_ident`. Records are filtered to exclude empty comment bodies, and a `loaded_at` timestamp tracks when each record was loaded. + +## Relationships + +All three tables are connected through the `airport_ident` field: + +``` +silver_airports (parent) + | + +-- silver_runways (one airport has many runways) + | + +-- silver_airport_comments (one airport has many comments) +``` + +## Data Quality + +- Unknown runway surfaces are standardized to `__UNKNOWN__` +- Unknown commenter nicknames are standardized to `__UNKNOWN__` +- Empty or null comment bodies are filtered out +- All primary keys are unique and not null diff --git a/airstats/models/silver/schema.yml b/airstats/models/silver/schema.yml new file mode 100644 index 0000000..ff0863f --- /dev/null +++ b/airstats/models/silver/schema.yml @@ -0,0 +1,129 @@ +version: 2 + +models: + - name: silver_airports + description: '{{ doc("silver_airports_description") }}' + columns: + - name: airport_ident + description: "Unique ICAO identifier for the airport" + data_tests: + - unique + - not_null + + - name: airport_type + description: '{{ doc("airport_type_description") }}' + data_tests: + - not_null + - accepted_values: + arguments: + values: + - small_airport + - medium_airport + - large_airport + - heliport + - seaplane_base + - balloonport + - closed + + - name: airport_name + description: "Official name of the airport" + data_tests: + - not_null + + - name: airport_lat + description: "Latitude coordinate in decimal degrees" + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + arguments: + min_value: -90 + max_value: 90 + - name: airport_long + description: "Longitude coordinate in decimal degrees" + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + arguments: + min_value: -180 + max_value: 180 + - name: continent + description: "Continent code where the airport is located" + + - name: iso_country + description: "ISO country code" + + - name: iso_region + description: "ISO region code" + + - name: silver_runways + description: "Runway specifications linked to airports, including dimensions and surface type" + columns: + - name: runway_id + description: "Unique identifier for the runway" + data_tests: + - unique + - not_null + + - name: airport_ident + description: "Reference to the parent airport" + data_tests: + - not_null + - relationships: + arguments: + to: ref('silver_airports') + field: airport_ident + config: + severity: warn + - name: runway_length_ft + description: "Length of the runway in feet" + + - name: runway_width_ft + description: "Width of the runway in feet" + + - name: runway_surface + description: "Surface material of the runway (e.g., asphalt, concrete, grass). Unknown values are marked as __UNKNOWN__" + data_tests: + - not_null + + - name: runway_lighted + description: "Whether the runway has lighting (1=yes, 0=no)" + + - name: runway_closed + description: "Whether the runway is closed (1=yes, 0=no)" + + - name: silver_airport_comments + description: "User-submitted comments about airports, loaded incrementally" + columns: + - name: comment_id + description: "Unique identifier for the comment" + data_tests: + - unique + - not_null + + - name: airport_ident + description: "Reference to the airport being commented on" + data_tests: + - not_null + - relationships: + arguments: + to: ref('silver_airports') + field: airport_ident + config: + severity: warn + - name: comment_timestamp + description: "When the comment was posted" + data_tests: + - dbt_expectations.expect_column_values_to_be_of_type: + arguments: + column_type: timestamp_ntz + - name: member_nickname + description: "Username of the commenter. Unknown users are marked as __UNKNOWN__" + + - name: comment_subject + description: "Subject line of the comment" + + - name: comment_body + description: "Full text of the comment" + data_tests: + - not_null + + - name: loaded_at + description: "Timestamp when the record was loaded into the silver layer" \ No newline at end of file diff --git a/airstats/models/silver/silver_airport_comments.sql b/airstats/models/silver/silver_airport_comments.sql new file mode 100644 index 0000000..2567487 --- /dev/null +++ b/airstats/models/silver/silver_airport_comments.sql @@ -0,0 +1,27 @@ +{{ + config( + materialized = 'incremental', + on_schema_change='fail' + ) +}} + +WITH src_airport_comments AS ( + SELECT * FROM {{ ref('src_airport_comments') }} +) + +SELECT + comment_id, + airport_ident, + comment_timestamp, + CASE + WHEN member_nickname IS NULL THEN '__UNKNOWN__' ELSE member_nickname + END AS member_nickname, + comment_subject, + comment_body, + current_timestamp() AS loaded_at +FROM src_airport_comments +WHERE (comment_body IS NOT NULL AND comment_body != '') + +{% if is_incremental() %} + AND comment_id > (select max(t.comment_id) from {{ this }} t) +{% endif %} \ No newline at end of file diff --git a/airstats/models/silver/silver_airports.sql b/airstats/models/silver/silver_airports.sql new file mode 100644 index 0000000..1e71e52 --- /dev/null +++ b/airstats/models/silver/silver_airports.sql @@ -0,0 +1,4 @@ +WITH src_airports AS ( + SELECT * FROM {{ ref('src_airports') }} +) +SELECT * FROM src_airports \ No newline at end of file diff --git a/airstats/models/silver/silver_runways.sql b/airstats/models/silver/silver_runways.sql new file mode 100644 index 0000000..1323bfb --- /dev/null +++ b/airstats/models/silver/silver_runways.sql @@ -0,0 +1,15 @@ +WITH src_runways AS ( + SELECT * FROM {{ ref('src_runways') }} +) + +SELECT + runway_id, + airport_ident, + runway_length_ft, + runway_width_ft, + CASE + WHEN runway_surface IS NULL OR runway_surface = '' THEN '__UNKNOWN__' ELSE runway_surface + END AS runway_surface, + runway_lighted, + runway_closed +FROM src_runways \ No newline at end of file diff --git a/airstats/models/sources.yml b/airstats/models/sources.yml new file mode 100644 index 0000000..63ad0ca --- /dev/null +++ b/airstats/models/sources.yml @@ -0,0 +1,10 @@ +sources: + - name: airstats + schema: raw + tables: + - name: airports + identifier: airports + - name: runways + identifier: runways + - name: comments + identifier: airport_comments \ No newline at end of file diff --git a/airstats/snapshots/snapshots.yml b/airstats/snapshots/snapshots.yml new file mode 100644 index 0000000..2cf53e0 --- /dev/null +++ b/airstats/snapshots/snapshots.yml @@ -0,0 +1,15 @@ +snapshots: + - name: scd_silver_airports + relation: ref('silver_airports') + config: + unique_key: airport_ident + strategy: check + check_cols: all + hard_deletes: invalidate + - name: scd_silver_runways + relation: ref('silver_runways') + config: + unique_key: runway_id + strategy: check + check_cols: all + hard_deletes: invalidate \ No newline at end of file diff --git a/airstats/tests/test_airports_valid_coordinates.sql b/airstats/tests/test_airports_valid_coordinates.sql new file mode 100644 index 0000000..f9e8ca1 --- /dev/null +++ b/airstats/tests/test_airports_valid_coordinates.sql @@ -0,0 +1,14 @@ +-- Test that all airport coordinates are within valid geographic ranges +-- Latitude must be between -90 and 90 +-- Longitude must be between -180 and 180 +-- This test fails if any rows are returned + +SELECT + airport_ident, + airport_name, + airport_lat, + airport_long +FROM {{ ref('silver_airports') }} +WHERE + (airport_lat IS NOT NULL AND (airport_lat < -90 OR airport_lat > 90)) + OR (airport_long IS NOT NULL AND (airport_long < -180 OR airport_long > 180)) \ No newline at end of file diff --git a/airstats/tests/test_runways_positive_dimension.sql b/airstats/tests/test_runways_positive_dimension.sql new file mode 100644 index 0000000..4786307 --- /dev/null +++ b/airstats/tests/test_runways_positive_dimension.sql @@ -0,0 +1,17 @@ +{{ + config(severity="warn") +}} + +-- Test that runway dimensions are positive when specified +-- Length and width should be greater than 0 if not null +-- This test fails if any rows are returned + +SELECT + runway_id, + airport_ident, + runway_length_ft, + runway_width_ft +FROM {{ ref('silver_runways') }} +WHERE + (runway_length_ft IS NOT NULL AND runway_length_ft <= 0) + OR (runway_width_ft IS NOT NULL AND runway_width_ft <= 0) \ No newline at end of file