Airstats capstone

.gitignore

@@ -217,4 +217,9 @@ dbt_internal_packages/
 package-lock.yml
 logs/
 
-/airstats
+# /airstats
+profiles.yml
+# dbt_project.yml
+.gitignore
+.user.yml
+.gitkeep

airstats/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

airstats/analyses/la_heliport_closed.sql

@@ -0,0 +1,2 @@
+-- Validate snapshot captured the closure of Los Angeles County Sheriff's Department Heliport
+SELECT * FROM {{ ref('scd_silver_airports') }} WHERE airport_ident = '01CN'

airstats/dbt_project.yml

@@ -0,0 +1,45 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'airstats'
+version: '1.0.0'
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'airstats'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+data_tests:
+  +store_failures: true
+  +schema: _test_failures
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+# models:
+#   airstats:
+#     # Config indicated by + and applies to all files under models/example/
+#     example:
+#       +materialized: view
+models:
+  airstats:
+    silver:
+      +materialized: table
+flags:
+  require_generic_test_arguments_property: true

airstats/models/bronze/src_airport_comments.sql

@@ -0,0 +1,17 @@
+{{ 
+    config(materialized='ephemeral') 
+}}
+
+WITH airport_comments AS (
+    SELECT * FROM {{ source('airstats', 'comments') }}
+)
+
+SELECT
+    id AS comment_id,
+    airport_ident,
+    date AS comment_timestamp,
+    member_nickname,
+    subject AS comment_subject,
+    body AS comment_body
+
+FROM airport_comments

airstats/models/bronze/src_airports.sql

@@ -0,0 +1,18 @@
+{{ 
+    config(materialized='ephemeral') 
+}}
+
+WITH airports AS (
+    SELECT * FROM {{ source('airstats', 'airports') }}
+)
+
+SELECT
+    ident AS airport_ident,
+    type AS airport_type,
+    name AS airport_name,
+    latitude_deg AS airport_lat,
+    longitude_deg AS airport_long,
+    continent,
+    iso_country,
+    iso_region
+FROM airports

airstats/models/bronze/src_runways.sql

@@ -0,0 +1,18 @@
+{{ 
+    config(materialized='ephemeral') 
+}}
+
+WITH runways AS (
+    SELECT * FROM {{ source('airstats', 'runways') }}
+)
+
+SELECT
+    id AS runway_id,
+    airport_ident,
+    length_ft AS runway_length_ft,
+    width_ft AS runway_width_ft,
+    surface AS runway_surface,
+    lighted AS runway_lighted,
+    closed AS runway_closed
+
+FROM runways

airstats/models/silver/_docs.md

@@ -0,0 +1,21 @@
+{% docs silver_airports_description %}
+The silver_airports table contains cleaned and standardized airport data from OurAirports.
+It includes geographic coordinates, airport classifications, and location information for over 70,000 airports worldwide.
+
+**Data Source:** OurAirports (ourairports.com/data/)
+
+**Key transformations:**
+- Column names standardized with airport_ prefix
+- No data filtering applied - all airports included
+{% enddocs %}
+
+{% docs airport_type_description %}
+Classification of the airport by size and type:
+- **small_airport**: Regional or community airports
+- **medium_airport**: Regional commercial airports
+- **large_airport**: Major commercial airports
+- **heliport**: Helicopter landing sites
+- **seaplane_base**: Water landing sites
+- **balloonport**: Balloon launch sites
+- **closed**: Non-operational airports
+{% enddocs %}

airstats/models/silver/overview.md

@@ -0,0 +1,33 @@
+# Silver Layer Overview
+
+## Tables
+
+The silver layer contains three interconnected tables that provide a clean, analysis-ready view of airport data.
+
+### silver_airports
+The core dimension table containing airport information including identifiers, names, types, and geographic coordinates. This is the parent table that other tables reference.
+
+### silver_runways
+Contains runway specifications for airports. Each runway belongs to exactly one airport, linked via `airport_ident`. Includes dimensions (length, width), surface type, and operational status.
+
+### silver_airport_comments
+An incremental table storing user-submitted comments about airports. Comments are linked to airports via `airport_ident`. Records are filtered to exclude empty comment bodies, and a `loaded_at` timestamp tracks when each record was loaded.
+
+## Relationships
+
+All three tables are connected through the `airport_ident` field:
+
+```
+silver_airports (parent)
+    |
+    +-- silver_runways (one airport has many runways)
+    |
+    +-- silver_airport_comments (one airport has many comments)
+```
+
+## Data Quality
+
+- Unknown runway surfaces are standardized to `__UNKNOWN__`
+- Unknown commenter nicknames are standardized to `__UNKNOWN__`
+- Empty or null comment bodies are filtered out
+- All primary keys are unique and not null

airstats/models/silver/schema.yml

@@ -0,0 +1,129 @@
+version: 2
+
+models:
+  - name: silver_airports
+    description: '{{ doc("silver_airports_description") }}'
+    columns:
+      - name: airport_ident
+        description: "Unique ICAO identifier for the airport"
+        data_tests:
+          - unique
+          - not_null
+
+      - name: airport_type
+        description: '{{ doc("airport_type_description") }}'
+        data_tests:
+          - not_null
+          - accepted_values:
+              arguments:
+                values:
+                  - small_airport
+                  - medium_airport
+                  - large_airport
+                  - heliport
+                  - seaplane_base
+                  - balloonport
+                  - closed
+
+      - name: airport_name
+        description: "Official name of the airport"
+        data_tests:
+          - not_null
+
+      - name: airport_lat
+        description: "Latitude coordinate in decimal degrees"
+        data_tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              arguments:
+                min_value: -90
+                max_value: 90
+      - name: airport_long
+        description: "Longitude coordinate in decimal degrees"
+        data_tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              arguments:
+                min_value: -180
+                max_value: 180
+      - name: continent
+        description: "Continent code where the airport is located"
+
+      - name: iso_country
+        description: "ISO country code"
+
+      - name: iso_region
+        description: "ISO region code"
+
+  - name: silver_runways
+    description: "Runway specifications linked to airports, including dimensions and surface type"
+    columns:
+      - name: runway_id
+        description: "Unique identifier for the runway"
+        data_tests:
+          - unique
+          - not_null
+
+      - name: airport_ident
+        description: "Reference to the parent airport"
+        data_tests:
+          - not_null
+          - relationships:
+              arguments:
+                to: ref('silver_airports')
+                field: airport_ident
+              config:
+                severity: warn
+      - name: runway_length_ft
+        description: "Length of the runway in feet"
+
+      - name: runway_width_ft
+        description: "Width of the runway in feet"
+
+      - name: runway_surface
+        description: "Surface material of the runway (e.g., asphalt, concrete, grass). Unknown values are marked as __UNKNOWN__"
+        data_tests:
+          - not_null
+
+      - name: runway_lighted
+        description: "Whether the runway has lighting (1=yes, 0=no)"
+
+      - name: runway_closed
+        description: "Whether the runway is closed (1=yes, 0=no)"
+
+  - name: silver_airport_comments
+    description: "User-submitted comments about airports, loaded incrementally"
+    columns:
+      - name: comment_id
+        description: "Unique identifier for the comment"
+        data_tests:
+          - unique
+          - not_null
+
+      - name: airport_ident
+        description: "Reference to the airport being commented on"
+        data_tests:
+          - not_null
+          - relationships:
+              arguments:
+                to: ref('silver_airports')
+                field: airport_ident
+              config:
+                severity: warn
+      - name: comment_timestamp
+        description: "When the comment was posted"
+        data_tests:
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              arguments:
+                column_type: timestamp_ntz
+      - name: member_nickname
+        description: "Username of the commenter. Unknown users are marked as __UNKNOWN__"
+
+      - name: comment_subject
+        description: "Subject line of the comment"
+
+      - name: comment_body
+        description: "Full text of the comment"
+        data_tests:
+          - not_null
+
+      - name: loaded_at
+        description: "Timestamp when the record was loaded into the silver layer"

airstats/models/silver/silver_airport_comments.sql

@@ -0,0 +1,27 @@
+{{
+  config(
+    materialized = 'incremental',
+    on_schema_change='fail'
+    )
+}}
+
+WITH src_airport_comments AS (
+    SELECT * FROM {{ ref('src_airport_comments') }}
+)
+
+SELECT
+    comment_id, 
+    airport_ident, 
+    comment_timestamp, 
+    CASE
+        WHEN member_nickname IS NULL THEN '__UNKNOWN__' ELSE member_nickname
+    END AS member_nickname,
+    comment_subject,
+    comment_body,
+    current_timestamp() AS loaded_at
+FROM src_airport_comments
+WHERE (comment_body IS NOT NULL AND comment_body != '')
+
+{% if is_incremental() %}
+    AND comment_id > (select max(t.comment_id) from {{ this }} t)
+{% endif %}

airstats/models/silver/silver_airports.sql

@@ -0,0 +1,4 @@
+WITH src_airports AS (
+    SELECT * FROM {{ ref('src_airports') }}
+)
+SELECT * FROM src_airports

airstats/models/silver/silver_runways.sql

@@ -0,0 +1,15 @@
+WITH src_runways AS (
+    SELECT * FROM {{ ref('src_runways') }}
+)
+
+SELECT
+    runway_id,
+    airport_ident,
+    runway_length_ft,
+    runway_width_ft,
+    CASE
+        WHEN runway_surface IS NULL OR runway_surface = '' THEN '__UNKNOWN__' ELSE runway_surface
+    END AS runway_surface,
+    runway_lighted,
+    runway_closed
+FROM src_runways

airstats/models/sources.yml

@@ -0,0 +1,10 @@
+sources:
+  - name: airstats
+    schema: raw
+    tables:
+      - name: airports
+        identifier: airports
+      - name: runways
+        identifier: runways
+      - name: comments
+        identifier: airport_comments