docs: improve README for ingestion pipeline clarity and presentation readiness

.gitignore

@@ -3,4 +3,6 @@ __pycache__/
 .venv/
 .env
 .DS_Store
-Thumbs.db
+Thumbs.db
+etl.log
+.coverage

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY src/ ./src
+COPY sql/ ./sql
+COPY config/ ./config
+COPY data_sources/ ./data_sources
+
+ENV PYTHONPATH=/app/src
+
+CMD ["python", "-m", "src.main"]

README.md

@@ -1,55 +1,75 @@
-# data-ingestion
+# Data Ingestion Sub-System
 
-A configuration-driven ETL pipeline for ingesting structured data into PostgreSQL.
+Configuration-driven ETL pipeline for ingesting structured data into PostgreSQL.
 
-The project demonstrates modular ingestion, schema validation, reject handling, dynamic table creation, UPSERT loading, logging, and automated testing.
+The system implements modular ingestion with validation, transformation, reject handling, and structured loading into staging (bronze) tables.
+
+---
 
 ## Features
 
-* YAML-driven ingestion workflows
-* CSV and JSON source support
-* Automatic PostgreSQL table creation
-* Schema-based type casting
-* Primary key enforcement
-* Rule-based validation
-* Reject auditing via `stg_rejects`
-* UPSERT loading strategy
-* Pytest unit testing and coverage reporting
-* Structured logging
+- YAML-driven ingestion workflows  
+- CSV and JSON source support  
+- PostgreSQL staging table creation  
+- Schema-based type casting  
+- Primary key enforcement  
+- Rule-based validation  
+- Reject capture via `stg_rejects`  
+- Upsert-based loading strategy  
+- Structured logging  
+- Unit tests with pytest and coverage reporting  
+
+---
 
 ## Architecture
 
+### Pipeline Flow
+
+```text
+CSV / JSON / API Sources
+        ↓
+Reader (extract raw data)
+        ↓
+Validator (schema + rule checks)
+        ↓
+Cleaner (type casting + normalization)
+        ↓
+Loader (insert into PostgreSQL staging tables)
+        ↓
+stg_* tables (clean dataset)
+```
+
+---
+
+### Reject Flow
+
 ```text
-Reader
-  ↓
-Validator
-  ↓
-Cleaner
-  ↓
-Loader
-  ↓
-PostgreSQL
-
-Rejected Records
+Invalid Records
+        ↓
+Validation Failure Reason Attached
         ↓
-   stg_rejects
+stg_rejects (audit table for debugging and replay)
 ```
 
+---
+
 ## Repository Structure
 
 ```text
 src/
-├── cleaners/
-├── database/
-├── loaders/
-├── loggers/
-├── readers/
-├── tests/
-├── utils/
-├── validators/
-└── main.py
+├── cleaners/        # Data normalization and type casting
+├── database/        # PostgreSQL connection + query execution
+├── loaders/         # Insert and upsert logic
+├── loggers/         # Structured logging utilities
+├── readers/         # CSV, JSON, API ingestion
+├── tests/           # Unit and integration tests
+├── utils/           # Shared helpers
+├── validators/      # Schema + rule validation engine
+└── main.py          # Pipeline entry point
 ```
 
+---
+
 ## Configuration Example
 
 ```yaml
@@ -69,10 +89,14 @@ sources:
       - temperature >= -100
 ```
 
+---
+
 ## Requirements
 
-* Python 3.11+
-* PostgreSQL
+- Python 3.11+
+- PostgreSQL
+
+---
 
 ## Setup
 
@@ -81,41 +105,47 @@ python -m venv .venv
 pip install -r requirements.txt
 ```
 
-Create a `.env` file:
+Environment variables:
 
-```env
+```bash
 DATABASE_URL=postgresql://user:password@localhost:5432/database
 ```
 
+---
+
 ## Running
 
 ```bash
 python src/main.py
 ```
 
+---
+
 ## Testing
 
 ```bash
 pytest
 pytest --cov --cov-report=term-missing
 ```
 
+---
+
 ## Future Enhancements
 
-* Open-Meteo API ingestion
-* NOAA GSOM remote CSV ingestion
-* Integration testing
-* Docker support
-* GitHub Actions CI/CD
-* Incremental loading
-* ETL audit tracking
+- Open-Meteo API ingestion  
+- NOAA GSOM ingestion  
+- Integration testing  
+- Docker support  
+- CI/CD pipeline  
+- Incremental loading  
+- Audit tracking  
 
-## Design Principles
+---
 
-* Thin orchestration layer
-* Configuration over hardcoding
-* Modular processing stages
-* Testable components
-* Fail-fast validation
-* Explicit reject handling
+## Design Principles
 
+- Modular pipeline stages (extract → validate → transform → load)  
+- Configuration-driven design  
+- Explicit reject handling (no silent failures)  
+- Reproducibility and idempotency  
+- Testable components  

config/sources.yml

@@ -1,5 +1,5 @@
 defaults:
-  db_url: DATABASE_URL
+  db_url_env: DATABASE_URL
   batch_size: 500
   on_conflict: upsert
 
@@ -98,7 +98,7 @@ sources:
 
   - name: noaa_monthly_sample
     type: csv
-    path: sample_data/small_sample.csv
+    path: data_sources/small_sample.csv
     target_table: stg_noaa_monthly
     pk: [station, date]
 
@@ -120,7 +120,7 @@ sources:
   - name: open_meteo_sample
     type: json
     json_root: hourly
-    path: sample_data/open_meteo_sample.json
+    path: data_sources/open_meteo_sample.json
     target_table: stg_open_meteo
     pk: [latitude, longitude, time]
 

dev-requirements.txt

@@ -0,0 +1,16 @@
+-r requirements.txt
+
+pytest==9.0.3
+pytest-cov==7.1.0
+coverage==7.14.1
+
+black==26.5.1
+ruff==0.15.14
+
+pre_commit==4.6.0
+
+debugpy==1.8.21
+
+ipykernel==7.3.0
+jupyterlab==4.5.8
+notebook==7.5.7

docker-compose.yml

@@ -0,0 +1,76 @@
+services:
+
+  postgres:
+    image: postgres:16
+    container_name: weather_postgres
+
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+
+    ports:
+      - "5432:5432"
+
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./sql/init.sql:/docker-entrypoint-initdb.d/init.sql
+
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+
+  ingestion_app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+    container_name: weather_ingestion
+
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+    env_file:
+      - .env
+
+    environment:
+      RUNNING_IN_DOCKER: "true"
+      CONFIG_PATH: config/sources.yml
+
+    working_dir: /app
+
+    command: python -m src.main
+
+
+  pgadmin:
+    image: dpage/pgadmin4
+
+    container_name: weather_admin
+
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@example.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+
+      PGADMIN_CONFIG_SERVER_MODE: "False"
+      PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: "False"
+
+      PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION: "False"
+      PGADMIN_CONFIG_LOGIN_BANNER: '"Weather ingestion local"'
+
+    ports:
+      - "8080:80"
+
+    depends_on:
+      - postgres
+
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+      - ./docker/pgadmin/servers.json:/pgadmin4/servers.json
+
+volumes:
+  postgres_data:
+  pgadmin_data:

docker/pgadmin/servers.json

@@ -0,0 +1,13 @@
+{
+  "Servers": {
+    "1": {
+      "Name": "Weather DB",
+      "Group": "Servers",
+      "Host": "postgres",
+      "Port": 5432,
+      "MaintenanceDB": "weather_ingestion",
+      "Username": "weather_user",
+      "SSLMode": "prefer"
+    }
+  }
+}