From c347ec36287d84999e135f2a6ce79ee202a3d7d8 Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 12:05:52 -0500 Subject: [PATCH 1/7] feat: add dockerized ingestion stack --- Dockerfile | 21 ++++++++ data_sources/gsom_sample_csv.csv | 5 ++ data_sources/open_meteo_sample.json | 75 ++++++++++++++++++++++++++++ data_sources/small_sample.csv | 27 ++++++++++ dev-requirements.txt | 16 ++++++ docker-compose.yml | 76 +++++++++++++++++++++++++++++ docker/pgadmin/servers.json | 13 +++++ 7 files changed, 233 insertions(+) create mode 100644 Dockerfile create mode 100644 data_sources/gsom_sample_csv.csv create mode 100644 data_sources/open_meteo_sample.json create mode 100644 data_sources/small_sample.csv create mode 100644 dev-requirements.txt create mode 100644 docker-compose.yml create mode 100644 docker/pgadmin/servers.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bf4785d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + gcc \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src +COPY sql/ ./sql +COPY config/ ./config +COPY data_sources/ ./data_sources + +ENV PYTHONPATH=/app/src + +CMD ["python", "-m", "src.main"] \ No newline at end of file diff --git a/data_sources/gsom_sample_csv.csv b/data_sources/gsom_sample_csv.csv new file mode 100644 index 0000000..ddcaf5b --- /dev/null +++ b/data_sources/gsom_sample_csv.csv @@ -0,0 +1,5 @@ +STATION,NAME,LATITUDE,LONGITUDE,ELEVATION,DATE,CLDD,CLDD_ATTRIBUTES,HTDD,HTDD_ATTRIBUTES,DX32,DX32_ATTRIBUTES,DX70,DX70_ATTRIBUTES,DX90,DX90_ATTRIBUTES,DT00,DT00_ATTRIBUTES,DT32,DT32_ATTRIBUTES,DP01,DP01_ATTRIBUTES,DP05,DP05_ATTRIBUTES,DP10,DP10_ATTRIBUTES,EMXP,EMXP_ATTRIBUTES,EMSD,EMSD_ATTRIBUTES,EMSN,EMSN_ATTRIBUTES,DSND,DSND_ATTRIBUTES,DSNW,DSNW_ATTRIBUTES,PRCP,PRCP_ATTRIBUTES,SNOW,SNOW_ATTRIBUTES,CDSD,CDSD_ATTRIBUTES,EMXT,EMXT_ATTRIBUTES,EMNT,EMNT_ATTRIBUTES,HDSD,HDSD_ATTRIBUTES,TMAX,TMAX_ATTRIBUTES,TMIN,TMIN_ATTRIBUTES,AWND,AWND_ATTRIBUTES,WDF2,WDF2_ATTRIBUTES,WDF5,WDF5_ATTRIBUTES,WSF2,WSF2_ATTRIBUTES,WSF5,WSF5_ATTRIBUTES +USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-01,0,",W",517,",W",3,",W",0,",W",0,",W",0,",W",28,",W",8,",W",2,",W",2,",W",1.53,",,W,22,",11.8,",,W,23,",13.4,",,W,22,",8,",W",1,",W",3.29,",,,W",13.8,",,,W",0,,61,",W,31,",11,",W,19,",1137,,45,",,,W",25,",,,W",8.1,",W",330,",W",330,",W",36,",W",47,",W" +USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-02,0,",W",403,",W",4,",W",0,",W",0,",W",0,",W",17,",W",9,",W",5,",W",2,",W",2.15,",,W,03,",0,",T,W,26,+",0.1,",,W,13,",0,",W",0,",W",5.69,",,,W",0.1,",,,W",0,,67,",W,29,+",16,",W,10,",1540,,49,",,,W",31,",,,W",8.7,",W",190,",W",180,",W",38,",W",55,",W" +USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-03,0,",W",204,",W",0,",W",12,",W",0,",W",0,",W",6,",W",10,",W",1,",W",0,",W",0.59,",,W,13,",0,",,W,31,+",0,",T,W,03,",0,",W",0,",W",1.56,",,,W",0,",T,,W",0,,80,",W,16,",26,",W,03,",1744,,65,",,,W",41,",,,W",7.8,",W",330,",W",330,",W",32,",W",40.9,",W" +USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-04,9,",W",144,",W",0,",W",14,",W",0,",W",0,",W",2,",W",7,",W",2,",W",1,",W",1.07,",,W,30,",0,",,W,30,+",0,",,W,30,+",0,",W",0,",W",2.5,",,,W",0,",,,W",9,,84,",W,28,",27,",W,10,",1888,,69,",,,W",44,",,,W",7.4,",W",340,",W",340,",W",36,",W",48.1,",W" diff --git a/data_sources/open_meteo_sample.json b/data_sources/open_meteo_sample.json new file mode 100644 index 0000000..3aa38aa --- /dev/null +++ b/data_sources/open_meteo_sample.json @@ -0,0 +1,75 @@ +{ + "latitude": 35.5951, + "longitude": -82.5515, + "generationtime_ms": 0.42, + "utc_offset_seconds": 0, + "timezone": "GMT", + "timezone_abbreviation": "GMT", + "elevation": 650, + + "hourly_units": { + "time": "iso8601", + "temperature_2m": "°C", + "apparent_temperature": "°C", + "relative_humidity_2m": "%", + "precipitation": "mm", + "snowfall": "cm", + "wind_speed_10m": "km/h", + "wind_gusts_10m": "km/h", + "cloud_cover": "%", + "pressure_msl": "hPa" + }, + + "hourly": { + "time": [ + "2026-06-05T00:00", + "2026-06-05T01:00", + "2026-06-05T02:00" + ], + "temperature_2m": [ + 18.2, + 17.8, + 17.4 + ], + "apparent_temperature": [ + 18.0, + 17.5, + 17.1 + ], + "relative_humidity_2m": [ + 72, + 74, + 76 + ], + "precipitation": [ + 0.0, + 0.2, + 0.0 + ], + "snowfall": [ + 0.0, + 0.0, + 0.0 + ], + "wind_speed_10m": [ + 6.1, + 5.8, + 5.4 + ], + "wind_gusts_10m": [ + 10.2, + 9.8, + 9.1 + ], + "cloud_cover": [ + 40, + 55, + 70 + ], + "pressure_msl": [ + 1015.2, + 1015.0, + 1014.8 + ] + } +} \ No newline at end of file diff --git a/data_sources/small_sample.csv b/data_sources/small_sample.csv new file mode 100644 index 0000000..3fa43c8 --- /dev/null +++ b/data_sources/small_sample.csv @@ -0,0 +1,27 @@ +STATION,NAME,DATE,PRCP,SNOW,TMAX,TMIN,AWND +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-01,3.29,13.8,45,25,8.1 +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-02,5.69,0.1,49,31,8.7 +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-03,1.56,0.0,65,41,7.8 +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-04,2.50,0.0,69,44,7.4 + +,"ASHEVILLE AIRPORT, NC US",2016-05,4.12,0.0,75,52,6.9 +USW00003812,"ASHEVILLE AIRPORT, NC US",,3.85,0.0,78,55,7.2 +,,2016-07,5.21,0.0,82,61,6.8 +,,,,0.0,80,60,7.0 + +USW00003812,,2016-08,4.87,0.0,84,63,6.5 +USW00003812,,2016-09,2.91,0.0,81,58,6.3 + +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-10,-1.00,0.0,70,45,5.8 +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-11,2.10,0.0,40,44,5.5 + +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-12,1.75,0.0,60,35,5.2 +USW00003812,"ASHEVILLE AIRPORT, NC US",2016-12,1.75,0.0,60,35,5.2 + +USW00003812,"ASHEVILLE AIRPORT, NC US",2017-01,2.34,1.2,55,30,6.1 +USW00003812,"ASHEVILLE AIRPORT, NC US",2017-01,2.50,1.0,56,31,6.0 + +USW00003812,"ASHEVILLE AIRPORT, NC US",2017-02,3.10,0.0,61,38,5.9 +USW00013874,"ATLANTA HARTSFIELD, GA US",2017-02,4.75,0.0,68,45,7.4 +USW00094846,"CHICAGO OHARE, IL US",2017-02,1.90,8.4,39,21,11.2 +USW00023174,"LOS ANGELES AIRPORT, CA US",2017-02,0.15,0.0,72,54,5.3 \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..f307bc9 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,16 @@ +-r requirements.txt + +pytest==9.0.3 +pytest-cov==7.1.0 +coverage==7.14.1 + +black==26.5.1 +ruff==0.15.14 + +pre_commit==4.6.0 + +debugpy==1.8.21 + +ipykernel==7.3.0 +jupyterlab==4.5.8 +notebook==7.5.7 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6429d5e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,76 @@ +services: + + postgres: + image: postgres:16 + container_name: weather_postgres + + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + + ports: + - "5432:5432" + + volumes: + - postgres_data:/var/lib/postgresql/data + - ./sql/init.sql:/docker-entrypoint-initdb.d/init.sql + + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] + interval: 5s + timeout: 5s + retries: 5 + + + ingestion_app: + build: + context: . + dockerfile: Dockerfile + + container_name: weather_ingestion + + depends_on: + postgres: + condition: service_healthy + + env_file: + - .env + + environment: + RUNNING_IN_DOCKER: "true" + CONFIG_PATH: config/sources.yml + + working_dir: /app + + command: python -m src.main + + + pgadmin: + image: dpage/pgadmin4 + + container_name: weather_admin + + environment: + PGADMIN_DEFAULT_EMAIL: admin@example.com + PGADMIN_DEFAULT_PASSWORD: admin + + PGADMIN_CONFIG_SERVER_MODE: "False" + PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: "False" + + PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION: "False" + PGADMIN_CONFIG_LOGIN_BANNER: '"Weather ingestion local"' + + ports: + - "8080:80" + + depends_on: + - postgres + + volumes: + - pgadmin_data:/var/lib/pgadmin + - ./docker/pgadmin/servers.json:/pgadmin4/servers.json + +volumes: + postgres_data: + pgadmin_data: \ No newline at end of file diff --git a/docker/pgadmin/servers.json b/docker/pgadmin/servers.json new file mode 100644 index 0000000..798dda6 --- /dev/null +++ b/docker/pgadmin/servers.json @@ -0,0 +1,13 @@ +{ + "Servers": { + "1": { + "Name": "Weather DB", + "Group": "Servers", + "Host": "postgres", + "Port": 5432, + "MaintenanceDB": "weather_ingestion", + "Username": "weather_user", + "SSLMode": "prefer" + } + } +} \ No newline at end of file From 155f3c12faa2bfb8a6d877816edb7e191f5e92d1 Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 12:06:45 -0500 Subject: [PATCH 2/7] refactor: move sample data to data sources --- sample_data/gsom_sample_csv.csv | 5 -- sample_data/open_meteo_sample.json | 75 ------------------------------ sample_data/small_sample.csv | 27 ----------- 3 files changed, 107 deletions(-) delete mode 100644 sample_data/gsom_sample_csv.csv delete mode 100644 sample_data/open_meteo_sample.json delete mode 100644 sample_data/small_sample.csv diff --git a/sample_data/gsom_sample_csv.csv b/sample_data/gsom_sample_csv.csv deleted file mode 100644 index ddcaf5b..0000000 --- a/sample_data/gsom_sample_csv.csv +++ /dev/null @@ -1,5 +0,0 @@ -STATION,NAME,LATITUDE,LONGITUDE,ELEVATION,DATE,CLDD,CLDD_ATTRIBUTES,HTDD,HTDD_ATTRIBUTES,DX32,DX32_ATTRIBUTES,DX70,DX70_ATTRIBUTES,DX90,DX90_ATTRIBUTES,DT00,DT00_ATTRIBUTES,DT32,DT32_ATTRIBUTES,DP01,DP01_ATTRIBUTES,DP05,DP05_ATTRIBUTES,DP10,DP10_ATTRIBUTES,EMXP,EMXP_ATTRIBUTES,EMSD,EMSD_ATTRIBUTES,EMSN,EMSN_ATTRIBUTES,DSND,DSND_ATTRIBUTES,DSNW,DSNW_ATTRIBUTES,PRCP,PRCP_ATTRIBUTES,SNOW,SNOW_ATTRIBUTES,CDSD,CDSD_ATTRIBUTES,EMXT,EMXT_ATTRIBUTES,EMNT,EMNT_ATTRIBUTES,HDSD,HDSD_ATTRIBUTES,TMAX,TMAX_ATTRIBUTES,TMIN,TMIN_ATTRIBUTES,AWND,AWND_ATTRIBUTES,WDF2,WDF2_ATTRIBUTES,WDF5,WDF5_ATTRIBUTES,WSF2,WSF2_ATTRIBUTES,WSF5,WSF5_ATTRIBUTES -USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-01,0,",W",517,",W",3,",W",0,",W",0,",W",0,",W",28,",W",8,",W",2,",W",2,",W",1.53,",,W,22,",11.8,",,W,23,",13.4,",,W,22,",8,",W",1,",W",3.29,",,,W",13.8,",,,W",0,,61,",W,31,",11,",W,19,",1137,,45,",,,W",25,",,,W",8.1,",W",330,",W",330,",W",36,",W",47,",W" -USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-02,0,",W",403,",W",4,",W",0,",W",0,",W",0,",W",17,",W",9,",W",5,",W",2,",W",2.15,",,W,03,",0,",T,W,26,+",0.1,",,W,13,",0,",W",0,",W",5.69,",,,W",0.1,",,,W",0,,67,",W,29,+",16,",W,10,",1540,,49,",,,W",31,",,,W",8.7,",W",190,",W",180,",W",38,",W",55,",W" -USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-03,0,",W",204,",W",0,",W",12,",W",0,",W",0,",W",6,",W",10,",W",1,",W",0,",W",0.59,",,W,13,",0,",,W,31,+",0,",T,W,03,",0,",W",0,",W",1.56,",,,W",0,",T,,W",0,,80,",W,16,",26,",W,03,",1744,,65,",,,W",41,",,,W",7.8,",W",330,",W",330,",W",32,",W",40.9,",W" -USW00003812,"ASHEVILLE AIRPORT, NC US",35.4319,-82.5375,645.3,2016-04,9,",W",144,",W",0,",W",14,",W",0,",W",0,",W",2,",W",7,",W",2,",W",1,",W",1.07,",,W,30,",0,",,W,30,+",0,",,W,30,+",0,",W",0,",W",2.5,",,,W",0,",,,W",9,,84,",W,28,",27,",W,10,",1888,,69,",,,W",44,",,,W",7.4,",W",340,",W",340,",W",36,",W",48.1,",W" diff --git a/sample_data/open_meteo_sample.json b/sample_data/open_meteo_sample.json deleted file mode 100644 index 3aa38aa..0000000 --- a/sample_data/open_meteo_sample.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "latitude": 35.5951, - "longitude": -82.5515, - "generationtime_ms": 0.42, - "utc_offset_seconds": 0, - "timezone": "GMT", - "timezone_abbreviation": "GMT", - "elevation": 650, - - "hourly_units": { - "time": "iso8601", - "temperature_2m": "°C", - "apparent_temperature": "°C", - "relative_humidity_2m": "%", - "precipitation": "mm", - "snowfall": "cm", - "wind_speed_10m": "km/h", - "wind_gusts_10m": "km/h", - "cloud_cover": "%", - "pressure_msl": "hPa" - }, - - "hourly": { - "time": [ - "2026-06-05T00:00", - "2026-06-05T01:00", - "2026-06-05T02:00" - ], - "temperature_2m": [ - 18.2, - 17.8, - 17.4 - ], - "apparent_temperature": [ - 18.0, - 17.5, - 17.1 - ], - "relative_humidity_2m": [ - 72, - 74, - 76 - ], - "precipitation": [ - 0.0, - 0.2, - 0.0 - ], - "snowfall": [ - 0.0, - 0.0, - 0.0 - ], - "wind_speed_10m": [ - 6.1, - 5.8, - 5.4 - ], - "wind_gusts_10m": [ - 10.2, - 9.8, - 9.1 - ], - "cloud_cover": [ - 40, - 55, - 70 - ], - "pressure_msl": [ - 1015.2, - 1015.0, - 1014.8 - ] - } -} \ No newline at end of file diff --git a/sample_data/small_sample.csv b/sample_data/small_sample.csv deleted file mode 100644 index 3fa43c8..0000000 --- a/sample_data/small_sample.csv +++ /dev/null @@ -1,27 +0,0 @@ -STATION,NAME,DATE,PRCP,SNOW,TMAX,TMIN,AWND -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-01,3.29,13.8,45,25,8.1 -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-02,5.69,0.1,49,31,8.7 -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-03,1.56,0.0,65,41,7.8 -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-04,2.50,0.0,69,44,7.4 - -,"ASHEVILLE AIRPORT, NC US",2016-05,4.12,0.0,75,52,6.9 -USW00003812,"ASHEVILLE AIRPORT, NC US",,3.85,0.0,78,55,7.2 -,,2016-07,5.21,0.0,82,61,6.8 -,,,,0.0,80,60,7.0 - -USW00003812,,2016-08,4.87,0.0,84,63,6.5 -USW00003812,,2016-09,2.91,0.0,81,58,6.3 - -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-10,-1.00,0.0,70,45,5.8 -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-11,2.10,0.0,40,44,5.5 - -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-12,1.75,0.0,60,35,5.2 -USW00003812,"ASHEVILLE AIRPORT, NC US",2016-12,1.75,0.0,60,35,5.2 - -USW00003812,"ASHEVILLE AIRPORT, NC US",2017-01,2.34,1.2,55,30,6.1 -USW00003812,"ASHEVILLE AIRPORT, NC US",2017-01,2.50,1.0,56,31,6.0 - -USW00003812,"ASHEVILLE AIRPORT, NC US",2017-02,3.10,0.0,61,38,5.9 -USW00013874,"ATLANTA HARTSFIELD, GA US",2017-02,4.75,0.0,68,45,7.4 -USW00094846,"CHICAGO OHARE, IL US",2017-02,1.90,8.4,39,21,11.2 -USW00023174,"LOS ANGELES AIRPORT, CA US",2017-02,0.15,0.0,72,54,5.3 \ No newline at end of file From 313c55629f0fbc8d9c22d9fb01326416882ca420 Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 12:07:32 -0500 Subject: [PATCH 3/7] refactor: reduce verbosity in ETL logging output --- src/loggers/logging_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loggers/logging_config.py b/src/loggers/logging_config.py index bfc72a6..26766b1 100644 --- a/src/loggers/logging_config.py +++ b/src/loggers/logging_config.py @@ -18,7 +18,7 @@ def log_source_start(source_name): def log_source_complete(source_name, rows_loaded, rejects): logger.info( "Completed source '%s' (rows_loaded=%s rejects=%s)", - source_name, + str(source_name), rows_loaded, rejects, ) From 978250439131be3dcf5bdafe7c9c4f041d57d7f3 Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 12:08:59 -0500 Subject: [PATCH 4/7] refactor: improve pipeline robustness for Dockerization --- config/sources.yml | 6 +++--- requirements.txt | 39 +++------------------------------------ src/cleaners/cleaners.py | 9 ++++++++- src/database/database.py | 11 +++++++++-- src/main.py | 6 ++++-- src/readers/readers.py | 5 +++-- src/utils/utils.py | 9 +++++++++ 7 files changed, 39 insertions(+), 46 deletions(-) diff --git a/config/sources.yml b/config/sources.yml index 09c6040..4b11bf6 100644 --- a/config/sources.yml +++ b/config/sources.yml @@ -1,5 +1,5 @@ defaults: - db_url: DATABASE_URL + db_url_env: DATABASE_URL batch_size: 500 on_conflict: upsert @@ -98,7 +98,7 @@ sources: - name: noaa_monthly_sample type: csv - path: sample_data/small_sample.csv + path: data_sources/small_sample.csv target_table: stg_noaa_monthly pk: [station, date] @@ -120,7 +120,7 @@ sources: - name: open_meteo_sample type: json json_root: hourly - path: sample_data/open_meteo_sample.json + path: data_sources/open_meteo_sample.json target_table: stg_open_meteo pk: [latitude, longitude, time] diff --git a/requirements.txt b/requirements.txt index e0ad836..344ada9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,7 @@ -annotated-types==0.7.0 -black==26.5.1 -certifi==2026.5.20 -cfgv==3.5.0 -charset-normalizer==3.4.7 -click==8.4.1 -colorama==0.4.6 -coverage==7.14.1 -distlib==0.4.0 -filelock==3.29.0 -greenlet==3.5.1 -identify==2.6.19 -idna==3.16 -iniconfig==2.3.0 -mypy_extensions==1.1.0 -nodeenv==1.10.0 -numpy==2.4.4 -packaging==26.2 pandas==3.0.2 -pathspec==1.1.1 -platformdirs==4.10.0 -pluggy==1.6.0 -pre_commit==4.6.0 +numpy==2.4.4 +requests==2.34.2 psycopg==3.3.4 psycopg-binary==3.3.4 -pydantic_core==2.46.4 -Pygments==2.20.0 -pytest==9.0.3 -pytest-cov==7.1.0 -python-dateutil==2.9.0.post0 -python-discovery==1.4.0 -python-dotenv==1.2.2 -pytokens==0.4.1 PyYAML==6.0.3 -ruff==0.15.14 -six==1.17.0 -typing-inspection==0.4.2 -typing_extensions==4.15.0 -tzdata==2026.2 -urllib3==2.7.0 +python-dotenv==1.2.2 \ No newline at end of file diff --git a/src/cleaners/cleaners.py b/src/cleaners/cleaners.py index d7ac3c7..98b42e4 100644 --- a/src/cleaners/cleaners.py +++ b/src/cleaners/cleaners.py @@ -13,8 +13,15 @@ def normalize_columns(df): df.columns = df.columns.str.strip() df.columns = df.columns.str.lower() df.columns = df.columns.str.replace(" ", "_") - logger.info(f"Normalized columns: {original_data} to: {df.columns.tolist()}") + logger.info( + "Normalized columns: %d → %d (added=%d removed=%d)", + len(original_data), + len(df.columns), + len(set(df.columns) - set(original_data)), + len(set(original_data) - set(df.columns)), + ) + return df diff --git a/src/database/database.py b/src/database/database.py index 74f39eb..d0a2864 100644 --- a/src/database/database.py +++ b/src/database/database.py @@ -1,6 +1,8 @@ import os import psycopg import logging +from pathlib import Path +from utils.utils import project_path logger = logging.getLogger(__name__) @@ -12,7 +14,7 @@ def database_setup(defaults): Returns the connection """ - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(defaults["db_url_env"]) if not db_url: raise ValueError(f"DATABASE_URL env value not set or invalid: {db_url}") @@ -40,10 +42,15 @@ def database_setup(defaults): return conn -def init_sql(conn, path="../sql/init.sql"): +def init_sql(conn, path=None): """ Reads and initializes the sql database from the init.sql file """ + if path is None: + path = project_path("sql", "init.sql") + else: + path = Path(path) + try: with open(path, "r", encoding="utf-8") as file: sql = file.read() diff --git a/src/main.py b/src/main.py index dc6bff5..33a6e01 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,5 @@ from dotenv import load_dotenv +from pathlib import Path from config.config_loader import load_config from database.database import database_setup, create_schema @@ -27,6 +28,7 @@ log_source_complete, ) +from utils.utils import project_path def run_source(connection, source, defaults): """ @@ -54,13 +56,13 @@ def run_source(connection, source, defaults): batch_size=defaults["batch_size"], ) write_rejects(connection, rejects, defaults["batch_size"]) - log_source_complete(str(source), len(df), len(rejects)) + log_source_complete(str(source["name"]), len(df), len(rejects)) def main(): load_dotenv() - path: str = "../config/sources.yml" + path = project_path("config", "sources.yml") config = load_config(path) defaults = config["defaults"] diff --git a/src/readers/readers.py b/src/readers/readers.py index 356a869..36fbf55 100644 --- a/src/readers/readers.py +++ b/src/readers/readers.py @@ -6,6 +6,7 @@ from io import StringIO from loggers.logging_config import logger +from utils.utils import project_path def read_input(source): @@ -31,7 +32,7 @@ def read_csv(source): Reads a .csv file based on the path provided in the config Returns a pandas dataframe """ - path = os.path.join("..", source["path"]) + path = project_path(source["path"]) logger.info(f"CSV file read from: {path}") try: @@ -47,7 +48,7 @@ def read_json(source): Reads a .json file based on the path provided in the config Returns a pandas dataframe """ - path = os.path.join("..", source["path"]) + path = project_path(source["path"]) # Load the entire json file try: diff --git a/src/utils/utils.py b/src/utils/utils.py index afe270a..42fcebd 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -1,5 +1,14 @@ import pandas as pd +import os +from pathlib import Path +if os.getenv("RUNNING_IN_DOCKER") == "true": + BASE_DIR = Path("/app") +else: + BASE_DIR = Path(__file__).resolve().parents[2] + +def project_path(*parts): + return BASE_DIR.joinpath(*parts) def emit_reject(source_name, reason, row): """ From b94e887f799bf706404169d92484fb09f73a3dad Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 12:11:52 -0500 Subject: [PATCH 5/7] chore: lint code for consistency --- src/cleaners/cleaners.py | 2 +- src/main.py | 2 +- src/readers/readers.py | 1 - src/utils/utils.py | 2 ++ 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cleaners/cleaners.py b/src/cleaners/cleaners.py index 98b42e4..8284ab9 100644 --- a/src/cleaners/cleaners.py +++ b/src/cleaners/cleaners.py @@ -21,7 +21,7 @@ def normalize_columns(df): len(set(df.columns) - set(original_data)), len(set(original_data) - set(df.columns)), ) - + return df diff --git a/src/main.py b/src/main.py index 33a6e01..f6683fb 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,4 @@ from dotenv import load_dotenv -from pathlib import Path from config.config_loader import load_config from database.database import database_setup, create_schema @@ -30,6 +29,7 @@ from utils.utils import project_path + def run_source(connection, source, defaults): """ Runs the ETL pipeline for a given source. diff --git a/src/readers/readers.py b/src/readers/readers.py index 36fbf55..6e80305 100644 --- a/src/readers/readers.py +++ b/src/readers/readers.py @@ -1,5 +1,4 @@ import json -import os import pandas as pd import requests diff --git a/src/utils/utils.py b/src/utils/utils.py index 42fcebd..a8c4073 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -7,9 +7,11 @@ else: BASE_DIR = Path(__file__).resolve().parents[2] + def project_path(*parts): return BASE_DIR.joinpath(*parts) + def emit_reject(source_name, reason, row): """ Emits a rejected row based on the source and reason From 643f4b21f36de42b7a66d7acc8795f5fbb14ba7f Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 12:18:33 -0500 Subject: [PATCH 6/7] chore: ignore coverage artifacts --- .gitignore | 4 +++- src/.coverage | Bin 53248 -> 0 bytes 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 src/.coverage diff --git a/.gitignore b/.gitignore index a2f4d49..2eb9f13 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ __pycache__/ .venv/ .env .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db +etl.log +.coverage \ No newline at end of file diff --git a/src/.coverage b/src/.coverage deleted file mode 100644 index a629b268f0c3f9e6017d2427037bc3155ddfa063..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53248 zcmeI4Z*1IV9mnmn?~m{7d(x%p5u&Yr(J2i}G9s!$iw2}!`Y#={qYV>1c6IKY#O>WV zwS8%$X<}D84c^iTgodU{FjeccNQezyY!XAf0Ktn2iE2}e5Yhyj1Vus<5@LJ7@A02I zU)oH&%c^zJ*UI^^pFdy!exK)`pL2X)KXSxb!uatW8k-T+)T07>O#3t&Q|Hxx=YFhyK{=MQ(xb{aC13*q5C8!X z*f0c|N7Qs-a8Nw=glEiFP0umPrW5S7J&#OH9-Y#UPVPQ5r3Z2P_N-3Z*qA=4JN6lU z(RB1Vt7__2y<(LO&#KSs-hvtBZn$QJ&sh{lvlh=d?w99BC96WEJae8T79Fc*ILrDe zb9vOy5MIqQ&w4=wiI~=Wo#)WEhs8&9$DA`AvtBmcz>4iwWh6WDoo{E;h5ml=Y18+~ zaLV+T>+psm+O8a!BcZbGRJirCj!`c!nC_@xYsKia3*Vw-m^~~!N5<9d2y43_zcdy0 zZLT+JvxNL)qqfK^he`w&_UyGvu2t5vLuWs9-Q1!s+x~V`XM~*BXi%zAw@x?AR^^ZC z?UJLu8+ChB=T1-yxHp$7?Cci>YL(_$_jHwRok~W-v;FfDReouLcIx>}nRH?6R`Kc6 zena7SO0$;dwhokzM$?L3qv`PUwc8EvC7q3g_o8Z5x99vW!%OQy2_; znr?>~$m-|q*mrTc)H^;-&U|9IL z8xl302_;pS+PYTLX*Ge;MC&ly989GPLqp;@#cxJoZG}O)qX|Vdd5d8*N~)QL+i;?R zIBli|b3RN;#QYZ=Z>C{6Hv>U>|qH~G7gPrpWkKL1D^Ci9rK$^f0 z7EP;s+H!YAvHI|W;qp<)9oA=UyJ{NsU~Zri2VAuGlr{|9g`Bnxzsm|h%KzPDv;Sv zDbW}jcr!9AeNfxsRa~0jd%jA}@LRd&Rxw?;?>@2F2zziaIF($w6Q~)Hui5C8!X009sH0T2KI5C8!X009sHflnj> zH6h*2qJIndY@a9%4Tk>%@cHqH9pe*O@+!gJ5!gHIKc7ekVKfMU00@8p2!H?xfB*=9 z00@8p2!KF~fGXWBMqdLYveJ+eeha{#|M%t23+xxH#;<%&EwlDTQ=cmpb(3dWBx1FNPsGwKvm)Igw1vt$k_I>F?5Vg+&KkD6t-)m}~n*5*p~e%0ipSB>w!r zFZYzd>g?Y9Gx>*ePie1d4ff0YpYyUdrar2^t^QQia<6|R106&_00ck)1V8`;KmY_l z00abKm$WQ)c{3lb|N9Qf&7p3~wief`*8l9F+$^j^EVe+&Ps`20Zlz*t%9e?=MA3H3 z&2+a~o$G&fN^WYsNyMDW?US3Ex>boe6|MiXd*vqURwr7xx7YtlFXlutF}E`NSFMdTbF~5@Q^sBZ0mwOS4mi_VdzqBr)m}?0V8c@0%GGbNM zUjO6w|1bf800@8p2!H?xfB*=900@8p2!OzbCLoEDpz-JbB70w;KWrcX0w4eaAOHd& z00JNY0w4eaAOHdzkbop<3V;63*k*zKm)&50V^`R3*~{#C_I>s^JH-yO{p?HZ0ZPUO z0w4eaAOHd&00JNY0w4eaAOHdzj)2Z&QQ6H_UcdCGD-$P=oV&bFF(15Z@AO6axhH@2 z!;8;eNT%|XBcB&D8ijYM*(aow%42fkV2;8a+LkA;&K#%h`p>_408>S3`K8~SKhw1bYto9mFG^q zDr#w2QSR9?^WxQAFJ8Tt-IU@vl*fKO{pOzEJ)63)?Nbk=k`$HCh?`{!Ka={wi9Z}X z{_YEB-s=0_jW^!fnUZ*FQdANYZkA@=&mK#OJSrjb=l_iTNML_t|6}j6-?M+xw*bFk ze_?O1*VqcX#9m<+DH$6GfB*=900@8p2!H?xfB*=900@AKuYj$43q?a{x9_14hNtK0w4ea zAOHd&00JNY0w4eaAOHd&aHkO9&;POhzf)=hc7Ol~fB*=900@8p2!H?xfB*=9!0jP` h=l{2dD^x)M1V8`;KmY_l00ck)1V8`;K;TXy@P9ht`?>%C From 2f61a4dcd35df6f45a2a2f60692cec7ec3fe66fe Mon Sep 17 00:00:00 2001 From: ZeroQuest96 Date: Mon, 22 Jun 2026 14:08:30 -0500 Subject: [PATCH 7/7] chore: update README to current repo status --- README.md | 134 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 82 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index b241cb7..7883785 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,75 @@ -# data-ingestion +# Data Ingestion Sub-System -A configuration-driven ETL pipeline for ingesting structured data into PostgreSQL. +Configuration-driven ETL pipeline for ingesting structured data into PostgreSQL. -The project demonstrates modular ingestion, schema validation, reject handling, dynamic table creation, UPSERT loading, logging, and automated testing. +The system implements modular ingestion with validation, transformation, reject handling, and structured loading into staging (bronze) tables. + +--- ## Features -* YAML-driven ingestion workflows -* CSV and JSON source support -* Automatic PostgreSQL table creation -* Schema-based type casting -* Primary key enforcement -* Rule-based validation -* Reject auditing via `stg_rejects` -* UPSERT loading strategy -* Pytest unit testing and coverage reporting -* Structured logging +- YAML-driven ingestion workflows +- CSV and JSON source support +- PostgreSQL staging table creation +- Schema-based type casting +- Primary key enforcement +- Rule-based validation +- Reject capture via `stg_rejects` +- Upsert-based loading strategy +- Structured logging +- Unit tests with pytest and coverage reporting + +--- ## Architecture +### Pipeline Flow + +```text +CSV / JSON / API Sources + ↓ +Reader (extract raw data) + ↓ +Validator (schema + rule checks) + ↓ +Cleaner (type casting + normalization) + ↓ +Loader (insert into PostgreSQL staging tables) + ↓ +stg_* tables (clean dataset) +``` + +--- + +### Reject Flow + ```text -Reader - ↓ -Validator - ↓ -Cleaner - ↓ -Loader - ↓ -PostgreSQL - -Rejected Records +Invalid Records + ↓ +Validation Failure Reason Attached ↓ - stg_rejects +stg_rejects (audit table for debugging and replay) ``` +--- + ## Repository Structure ```text src/ -├── cleaners/ -├── database/ -├── loaders/ -├── loggers/ -├── readers/ -├── tests/ -├── utils/ -├── validators/ -└── main.py +├── cleaners/ # Data normalization and type casting +├── database/ # PostgreSQL connection + query execution +├── loaders/ # Insert and upsert logic +├── loggers/ # Structured logging utilities +├── readers/ # CSV, JSON, API ingestion +├── tests/ # Unit and integration tests +├── utils/ # Shared helpers +├── validators/ # Schema + rule validation engine +└── main.py # Pipeline entry point ``` +--- + ## Configuration Example ```yaml @@ -69,10 +89,14 @@ sources: - temperature >= -100 ``` +--- + ## Requirements -* Python 3.11+ -* PostgreSQL +- Python 3.11+ +- PostgreSQL + +--- ## Setup @@ -81,18 +105,22 @@ python -m venv .venv pip install -r requirements.txt ``` -Create a `.env` file: +Environment variables: -```env +```bash DATABASE_URL=postgresql://user:password@localhost:5432/database ``` +--- + ## Running ```bash python src/main.py ``` +--- + ## Testing ```bash @@ -100,22 +128,24 @@ pytest pytest --cov --cov-report=term-missing ``` +--- + ## Future Enhancements -* Open-Meteo API ingestion -* NOAA GSOM remote CSV ingestion -* Integration testing -* Docker support -* GitHub Actions CI/CD -* Incremental loading -* ETL audit tracking +- Open-Meteo API ingestion +- NOAA GSOM ingestion +- Integration testing +- Docker support +- CI/CD pipeline +- Incremental loading +- Audit tracking -## Design Principles +--- -* Thin orchestration layer -* Configuration over hardcoding -* Modular processing stages -* Testable components -* Fail-fast validation -* Explicit reject handling +## Design Principles +- Modular pipeline stages (extract → validate → transform → load) +- Configuration-driven design +- Explicit reject handling (no silent failures) +- Reproducibility and idempotency +- Testable components \ No newline at end of file