diff --git a/configs/data/heat_krakow.yaml b/configs/data/heat_krakow.yaml new file mode 100644 index 0000000..1644b12 --- /dev/null +++ b/configs/data/heat_krakow.yaml @@ -0,0 +1,23 @@ +_target_: src.data.base_datamodule.BaseDataModule + +dataset: + _target_: src.data.heat_krakow_dataset.HeatKrakowDataset + data_dir: ${paths.data_dir} + modalities: + coords: {} + use_target_data: true + use_features: true + use_aux_data: none + seed: ${seed} + cache_dir: ${paths.cache_dir} + +batch_size: 64 +num_workers: 8 +pin_memory: true + +split_mode: "from_file" +save_split: false +saved_split_file_name: "heat_krakow_split_20260522_1120.pth" + +train_val_test_split: [0.7, 0.15, 0.15] +seed: ${seed} diff --git a/configs/experiment/heat_krakow_full_fusion_avg.yaml b/configs/experiment/heat_krakow_full_fusion_avg.yaml new file mode 100644 index 0000000..e33c9d5 --- /dev/null +++ b/configs/experiment/heat_krakow_full_fusion_avg.yaml @@ -0,0 +1,35 @@ +# @package _global_ +# Guatemala performance: R2=0.672, RMSE=1.104, MAE=0.896 +# Kraków performance: R2=0.935, RMSE=0.523, MAE = 0.397 +defaults: + - override /model: heat_krakow_full_fusion_avg + - override /data: heat_krakow + - override /metrics: krakow_regression +tags: ["heat_island", "krakow", "full_fusion_avg", "best", "regression"] +seed: 12345 +trainer: + min_epochs: 1 + max_epochs: 100 +data: + batch_size: 64 + dataset: + modalities: + coords: {} + tessera: + year: 2024 + size: 10 #9 resulted in slightly worse performance + format: npy +callbacks: + model_checkpoint: + monitor: val_r2 + mode: max + early_stopping: + monitor: val_r2 + mode: max + patience: 20 +logger: + wandb: + tags: ${tags} + group: "heat_island" + aim: + experiment: "heat_island" diff --git a/configs/experiment/heat_krakow_fusion.yaml b/configs/experiment/heat_krakow_fusion.yaml new file mode 100644 index 0000000..7f9fdac --- /dev/null +++ b/configs/experiment/heat_krakow_fusion.yaml @@ -0,0 +1,25 @@ +# @package _global_ +# Variant C: GeoClip + tabular fusion +# Guatemala performance: R2=0.555, RMSE=1.285, MAE=1.039 +# Kraków performance: R2=0.804, RMSE=0.902, MAE=0.688 +defaults: + - override /model: heat_fusion_reg + - override /data: heat_krakow + - override /metrics: krakow_regression + +tags: ["heat_island", "krakow", "fusion", "regression"] +seed: 12345 + +trainer: + min_epochs: 1 + max_epochs: 50 + +data: + batch_size: 64 + +logger: + wandb: + tags: ${tags} + group: "heat_island" + aim: + experiment: "heat_island" diff --git a/configs/experiment/heat_krakow_geoclip.yaml b/configs/experiment/heat_krakow_geoclip.yaml new file mode 100644 index 0000000..148cc30 --- /dev/null +++ b/configs/experiment/heat_krakow_geoclip.yaml @@ -0,0 +1,28 @@ +# @package _global_ +# Guatemala performance: R2=0.323, RMSE=1.607, MAE=1.344 +# Kraków performance: R2=0.450, RMSE=1.519, MAE=1.183 +defaults: + - override /model: heat_geoclip_reg + - override /data: heat_krakow + - override /metrics: krakow_regression +tags: ["heat_island", "krakow", "coords", "regression"] +seed: 12345 +trainer: + min_epochs: 1 + max_epochs: 100 +data: + batch_size: 64 +callbacks: + model_checkpoint: + monitor: val_r2 + mode: max + early_stopping: + monitor: val_r2 + mode: max + patience: 20 +logger: + wandb: + tags: ${tags} + group: "heat_island" + aim: + experiment: "heat_island" diff --git a/configs/experiment/heat_krakow_tabular.yaml b/configs/experiment/heat_krakow_tabular.yaml new file mode 100644 index 0000000..59a4e42 --- /dev/null +++ b/configs/experiment/heat_krakow_tabular.yaml @@ -0,0 +1,28 @@ +# @package _global_ +# Guatemala performance: R2=0.562, RMSE=1.282, MAE=1.040 +# Kraków performance: R2=0.621, RMSE=1.257, MAE=0.949 +defaults: + - override /model: heat_tabular_reg + - override /data: heat_krakow + - override /metrics: krakow_regression +tags: ["heat_island", "krakow", "tabular", "regression"] +seed: 12345 +trainer: + min_epochs: 1 + max_epochs: 100 +data: + batch_size: 64 +callbacks: + model_checkpoint: + monitor: val_r2 + mode: max + early_stopping: + monitor: val_r2 + mode: max + patience: 20 +logger: + wandb: + tags: ${tags} + group: "heat_island" + aim: + experiment: "heat_island" diff --git a/configs/experiment/heat_krakow_tessera_avg.yaml b/configs/experiment/heat_krakow_tessera_avg.yaml new file mode 100644 index 0000000..223dfe3 --- /dev/null +++ b/configs/experiment/heat_krakow_tessera_avg.yaml @@ -0,0 +1,35 @@ +# @package _global_ +# Guatemala performance: R2=0.733, RMSE=1.011, MAE=0.814 +# Kraków performance: R2=0.924, RMSE=0.568, MAE=0.441 +defaults: + - override /model: heat_krakow_tessera_avg + - override /data: heat_krakow + - override /metrics: krakow_regression +tags: ["heat_island", "krakow", "tessera_avg", "best", "regression"] +seed: 12345 +trainer: + min_epochs: 1 + max_epochs: 100 +data: + batch_size: 64 + dataset: + modalities: + coords: {} + tessera: + year: 2024 + size: 10 + format: npy +callbacks: + model_checkpoint: + monitor: val_r2 + mode: max + early_stopping: + monitor: val_r2 + mode: max + patience: 20 +logger: + wandb: + tags: ${tags} + group: "heat_island" + aim: + experiment: "heat_island" diff --git a/configs/metrics/krakow_regression.yaml b/configs/metrics/krakow_regression.yaml new file mode 100644 index 0000000..79c441d --- /dev/null +++ b/configs/metrics/krakow_regression.yaml @@ -0,0 +1,7 @@ +_target_: src.models.components.metrics.metrics_wrapper.MetricsWrapper + +metrics: + - _target_: src.models.components.loss_fns.mse_loss.MSELoss + - _target_: src.models.components.loss_fns.rmse_loss.RMSELoss + - _target_: src.models.components.loss_fns.mae_loss.MAELoss + - _target_: src.models.components.metrics.r2.RSquared diff --git a/configs/model/heat_krakow_full_fusion_avg.yaml b/configs/model/heat_krakow_full_fusion_avg.yaml new file mode 100644 index 0000000..df5799d --- /dev/null +++ b/configs/model/heat_krakow_full_fusion_avg.yaml @@ -0,0 +1,34 @@ +_target_: src.models.predictive_model.PredictiveModel +geo_encoder: + _target_: src.models.components.geo_encoders.encoder_wrapper.EncoderWrapper + encoder_branches: + - encoder: + _target_: src.models.components.geo_encoders.geoclip.GeoClipCoordinateEncoder + - encoder: + _target_: src.models.components.geo_encoders.tabular_encoder.TabularEncoder + output_dim: 64 + geo_data_name: tabular + - encoder: + _target_: src.models.components.geo_encoders.average_encoder.AverageEncoder + geo_data_name: tessera + fusion_strategy: concat +prediction_head: + _target_: src.models.components.pred_heads.mlp_regression_head.MLPRegressionPredictionHead + nn_layers: 3 + hidden_dim: 512 +trainable_modules: [geo_encoder.encoder_branches.1, prediction_head] +normalize_features: false +metrics: ${metrics} +optimizer: + _target_: torch.optim.Adam + _partial_: true + lr: 0.0001 + weight_decay: 0.0 +scheduler: + _target_: torch.optim.lr_scheduler.ReduceLROnPlateau + _partial_: true + mode: min + factor: 0.1 + patience: 10 +loss_fn: + _target_: torch.nn.MSELoss diff --git a/configs/model/heat_krakow_tessera_avg.yaml b/configs/model/heat_krakow_tessera_avg.yaml new file mode 100644 index 0000000..b68ac43 --- /dev/null +++ b/configs/model/heat_krakow_tessera_avg.yaml @@ -0,0 +1,24 @@ +_target_: src.models.predictive_model.PredictiveModel +geo_encoder: + _target_: src.models.components.geo_encoders.average_encoder.AverageEncoder + geo_data_name: tessera +prediction_head: + _target_: src.models.components.pred_heads.mlp_regression_head.MLPRegressionPredictionHead + nn_layers: 2 + hidden_dim: 512 +trainable_modules: [prediction_head] +normalize_features: true +metrics: ${metrics} +optimizer: + _target_: torch.optim.Adam + _partial_: true + lr: 0.001 + weight_decay: 0.0 +scheduler: + _target_: torch.optim.lr_scheduler.ReduceLROnPlateau + _partial_: true + mode: min + factor: 0.1 + patience: 10 +loss_fn: + _target_: torch.nn.MSELoss diff --git a/data/heat_krakow/splits/heat_krakow_split_20260522_1120.pth b/data/heat_krakow/splits/heat_krakow_split_20260522_1120.pth new file mode 100644 index 0000000..5d13fbe Binary files /dev/null and b/data/heat_krakow/splits/heat_krakow_split_20260522_1120.pth differ diff --git a/src/data/heat_krakow_dataset.py b/src/data/heat_krakow_dataset.py new file mode 100644 index 0000000..6499ca6 --- /dev/null +++ b/src/data/heat_krakow_dataset.py @@ -0,0 +1,203 @@ +"""Heat Krakow LST dataset. + +Location: src/data/heat_krakow_dataset.py + +Based on Heat Guatemala dataset class, follows the same changes described below: +Changes vs original: + - tabular_dim property added so the datamodule (and model) can read it + without hardcoding anything. + - implemented_mod stays {"coords"} because tabular data arrives + automatically through feat_* CSV columns, not through the modalities dict. + This is documented explicitly below. + - Implemented an override for `setup_tessera` in `HeatKrakowDataset`: + setup_tessera inherited from `BaseDataset` where `self.records.pop(i)` + was mutating the list mid-iteration, caused it to skip every second + missing file and eventually trigger a PyTorch DataLoader `IndexError: list index out of range`. + - Minor: __getitem__ guard tightened (tabular only added when feat_names exist + and modality logic is cleaner). +""" + +import os +from typing import Any, Dict, override + +import numpy as np +import torch + +from src.data.base_dataset import BaseDataset + + +class HeatKrakowDataset(BaseDataset): + """Dataset for the urban heat island use case (Kraków, LST regression). + + CSV layout expected (produced by scripts/make_model_ready_heat_krakow.py): + - name_loc : unique location identifier + - lat, lon : WGS84 coordinates + - target_lst : Land Surface Temperature [°C] + - feat_* : tabular features (numeric + one-hot categorical) + + Modality design note + -------------------- + `implemented_mod = {"coords"}` because in this framework a "modality" refers + to data loaded from a separate file (e.g. a GeoTIFF or .npy embedding). + Tabular features live directly in the model-ready CSV and are picked up + automatically by BaseDataset.get_records() via the `feat_` column prefix. + They do NOT need to be listed in `modalities`. + """ + + def __init__( + self, + data_dir: str, + modalities: dict, + use_target_data: bool = True, + use_aux_data: Dict[str, Any] | str = "all", + seed: int = 12345, + cache_dir: str = None, + mock: bool = False, + use_features: bool = True, + ) -> None: + super().__init__( + data_dir=data_dir, + modalities=modalities, + use_target_data=use_target_data, + use_aux_data=use_aux_data, + dataset_name="heat_krakow", + seed=seed, + cache_dir=cache_dir, + implemented_mod={"coords", "tessera"}, + mock=mock, + use_features=use_features, + ) + + # ------------------------------------------------------------------ + # Required overrides + # ------------------------------------------------------------------ + + def setup(self) -> None: + """No files to download / prepare for this dataset.""" + # Set up each requested modality + for mod in self.modalities.keys(): + if mod == "coords" and len(self.modalities.keys()) == 1: + return + elif mod == "tessera": + self.setup_tessera() + # elif mod == "aef": + # self.setup_aef() + return + + @override + def __getitem__(self, idx: int) -> Dict[str, Any]: + row = self.records[idx] + sample: Dict[str, Any] = {"eo": {}} + + # --- EO modalities --- + for modality in self.modalities: + if modality in ["coords"]: + sample["eo"][modality] = torch.tensor([row["lat"], row["lon"]]) + elif modality == "tessera": + sample["eo"][modality] = self.load_tessera(row["tessera_path"]) + elif modality == "aef": + sample["eo"][modality] = self.load_aef(row["aef_path"]) + + # --- Tabular features (always included if present in CSV) --- + if self.use_features and self.feat_names: + sample["eo"]["tabular"] = torch.tensor( + [row[k] for k in self.feat_names], dtype=torch.float32 + ) + + # --- Target --- + if self.use_target_data: + sample["target"] = torch.tensor( + [row[k] for k in self.target_names], dtype=torch.float32 + ) + + # --- Auxiliary data --- + if self.use_aux_data: + sample["aux"] = {} + for aux_cat, vals in self.use_aux_data.items(): + if aux_cat == "aux": + sample["aux"][aux_cat] = torch.tensor( + [row[v] for v in vals], dtype=torch.float32 + ) + else: + sample["aux"][aux_cat] = [row[v] for v in vals] + + return sample + + @override + def setup_tessera(self) -> None: + """Overridden setup_tessera to fix the list mutation bug.""" + from src.data_preprocessing.tessera_embeds import ( + get_tessera_embeds, + tessera_from_df, + ) + + print("\n\nSetting up Tessera data (Using Krakow Overridden Method)...\n\n") + download_missing_tiles = False + + # Check if data is already available + dst_dir = os.path.join(self.data_dir, "eo/tessera") + + year = self.modalities["tessera"].get( + "year", KeyError('Missing parameter "year" for Tessera modality') + ) + size = self.modalities["tessera"].get( + "size", KeyError('Missing parameter "size" for Tessera modality') + ) + + # If data does not exist or is empty → full download + if not os.path.exists(dst_dir) or len(os.listdir(dst_dir)) == 0: + os.makedirs(dst_dir, exist_ok=True) + + tessera_from_df( + self.df, + data_dir=dst_dir, + year=year, + tile_size=size, + cache_dir=self.cache_dir, + ) + + # Download missing rows (if any) + else: + from geotessera import GeoTessera + + print("Downloading missing Tessera tiles...") + print("[Warning]: it may download tessera tiles filled with 0a") + + avail_files = os.listdir(dst_dir) + gt = None + + # Create a safe list to collect valid records + valid_records = [] + + for rec in self.records: + fname = os.path.basename(rec["tessera_path"]) + + if fname in avail_files: + # File exists locally, keep it + valid_records.append(rec) + else: + # File is missing + if download_missing_tiles: + print(f"Retrieving missing Tessera data: {fname}") + gt = gt or GeoTessera(cache_dir=self.cache_dir) + row = self.df[self.df["name_loc"] == rec["name_loc"]] + lon, lat = row.lon.item(), row.lat.item() + try: + get_tessera_embeds( + lon, + lat, + rec["name_loc"], + year=year, + save_dir=dst_dir, + tile_size=size, + tessera_con=gt, + ) + valid_records.append(rec) + continue + except Exception as e: + print(f"Tile for {fname} could not be retrieved. Error: {e}") + + print(f"No tile found for {fname} thus it will not be used.") + + # Safely swap the filtered list in place + self.records = valid_records diff --git a/src/data_preprocessing/heat_krakow_spatial_splits.py b/src/data_preprocessing/heat_krakow_spatial_splits.py new file mode 100644 index 0000000..7746d64 --- /dev/null +++ b/src/data_preprocessing/heat_krakow_spatial_splits.py @@ -0,0 +1,120 @@ +import os + +import numpy as np +import pandas as pd +import torch +from sklearn.cluster import KMeans +from sklearn.model_selection import train_test_split + + +def generate_spatial_stratified_split( + csv_path: str, + output_pth_path: str, + train_ratio: float = 0.70, + val_ratio: float = 0.15, + test_ratio: float = 0.15, + seed: int = 12345, +): + """Generate and save a spatially and target-stratified train/val/test split. + + Executes a two-stage stratification: + 1. Clusters coordinates via KMeans into 8 geographical spatial zones. + 2. Quantizes the 'target_lst' variable into 8 uniform distribution buckets. + 3. Merges these into a composite key, cleans singleton strata, and splits the location identifiers into train/val/test subsets. + + The final splits are stored as a mapped PyTorch serialized pandas Series. + + Parameters + ---------- + csv_path : str + Path to the source CSV containing 'lat', 'lon', 'target_lst', and 'name_loc'. + output_pth_path : str + Destination file path to save the serialized PyTorch (.pth) object. + train_ratio : float, default 0.70 + Proportion of the dataset allocated to the training set. + val_ratio : float, default 0.15 + Proportion of the dataset allocated to the validation set. + test_ratio : float, default 0.15 + Proportion of the dataset allocated to the test set. + seed : int, default 12345 + Random seed for KMeans initialization and stratified shuffling stability. + + Returns + ------- + None + Saves a pd.Series to `output_pth_path` mapping split tags to location IDs. + """ + # Load Kraków CSV + print(f"Reading dataset from: {csv_path}") + df = pd.read_csv(csv_path) + + # Spatial Binning: Group coordinates into 8 geographic clusters + print("Clustering spatial locations...") + coords = df[["lat", "lon"]].values + kmeans = KMeans(n_clusters=8, random_state=seed, n_init="auto") + df["spatial_zone"] = kmeans.fit_predict(coords) + + # Target Binning: Segment LST temperatures into 8 quantile buckets + print("Binning target variable distributions...") + df["target_bucket"] = pd.qcut(df["target_lst"], q=8, labels=False, duplicates="drop") + + # Combine into a joint Strata Key + df["composite_strata"] = df["spatial_zone"].astype(str) + "_" + df["target_bucket"].astype(str) + + # Clean single-sample strata to prevent train_test_split from crashing + strata_counts = df["composite_strata"].value_counts() + singletons = strata_counts[strata_counts < 2].index + if len(singletons) > 0: + # Merge singletons into the largest stratum as a safety fallback + largest_stratum = strata_counts.idxmax() + df["composite_strata"] = df["composite_strata"].replace(singletons, largest_stratum) + + # Execute First Split: Train vs Temp (Val + Test) + temp_ratio = val_ratio + test_ratio + train_idx, temp_idx = train_test_split( + df.index.tolist(), test_size=temp_ratio, stratify=df["composite_strata"], random_state=seed + ) + + # Execute Second Split: Split Temp into Validation and Test + relative_test_ratio = test_ratio / temp_ratio + val_idx, test_idx = train_test_split( + temp_idx, + test_size=relative_test_ratio, + stratify=df.loc[temp_idx, "composite_strata"], + random_state=seed, + ) + + # Map indices to unique location identifiers (name_loc) + train_locs = df.loc[train_idx, "name_loc"].tolist() + val_locs = df.loc[val_idx, "name_loc"].tolist() + test_locs = df.loc[test_idx, "name_loc"].tolist() + + # Build the Pandas Series + print("Converting dictionary structures to an indexed pd.Series...") + index_labels = ( + ["train_indices"] * len(train_locs) + + ["val_indices"] * len(val_locs) + + ["test_indices"] * len(test_locs) + ) + name_loc_values = train_locs + val_locs + test_locs + split_series = pd.Series(data=name_loc_values, index=index_labels) + + # Save to a .pth file + os.makedirs(os.path.dirname(output_pth_path), exist_ok=True) + torch.save(split_series, output_pth_path) + + # Print Diagnostics to verify distribution percentages + print("\n--- Split Generation Complete ---") + print(f"Train samples : {len(train_locs)} ({len(train_locs)/len(df):.1%})") + print(f"Val samples : {len(val_locs)} ({len(val_locs)/len(df):.1%})") + print(f"Test samples : {len(test_locs)} ({len(test_locs)/len(df):.1%})") + print(f"Saved split object type: {type(split_series)}") + print(f"File successfully written to -> {output_pth_path}") + + +# --- Execution Example --- +if __name__ == "__main__": + CSV_INPUT = "/aether/data/heat_krakow/model_ready_heat_krakow.csv" + PTH_OUTPUT = "/aether/data/heat_krakow/splits.pth" + + generate_spatial_stratified_split(CSV_INPUT, PTH_OUTPUT) diff --git a/src/data_preprocessing/make_model_ready_heat_krakow.py b/src/data_preprocessing/make_model_ready_heat_krakow.py new file mode 100644 index 0000000..7bb44e3 --- /dev/null +++ b/src/data_preprocessing/make_model_ready_heat_krakow.py @@ -0,0 +1,183 @@ +"""Build model-ready CSV for the UHI use case - Kraków dataset +(data/heat_krakow/model_ready_heat_krakow.csv).""" + +import argparse +import re + +import numpy as np +import pandas as pd + +# ----------------------------------------------------------------------- +# Continuous / numeric columns → kept as float feat_* columns +# ----------------------------------------------------------------------- +NUMERIC_COLS = [ + "AreaM2", + "PerimeterMeter", + # Vegetation & water indices + "NDVI_20240829", + "NDWI_20240829", + "ARI_20240829", + "NBR_20240829", + # Urban structure + "MeanBuildingsHeight", + "BuildingsCoverHa", + "BuildingsCoverPercent", + "CopernicusMeanImperviousDensity2021", + # Socio-demographic + "RES_per_1ha", + # Terrain + "dem_5m", + "slope_5m", + "aspect_5m", + "tpi_5m", + "tri_5m", + "AnnualSolarRadiation", + # Forest cover + "MeanCanopyHeight", + "ForestCoverHa", + "ForestCoverPercent", + "VegetationVoxelNumber", + "VegetationLiDARPtsNumber", + "HVV2BV", + "CopernicusMeanTreeCoverDensity2023", +] + +# ----------------------------------------------------------------------- +# Genuinely categorical / nominal columns → one-hot encoded feat_* columns +# ----------------------------------------------------------------------- +CATEGORICAL_COLS = [ + "DistrictID", + "DistrictName", + "GridType", + "GridDominantLanduse", + "CopernicusCLCPlusBackbone2023Class", + "CopernicusHerbaceousVegPresence2023", +] + +# Columns where >this fraction of values are NaN → drop column entirely +NAN_DROP_THRESHOLD = 0.30 + + +def clean_token(x: str) -> str: + """Make a string safe for use as a column name.""" + x = str(x).strip() + x = re.sub(r"[^\w]+", "_", x) + x = re.sub(r"_+", "_", x).strip("_") + return x if x else "NA" + + +def main(source_csv: str, out_csv: str, drop_zero_lst: bool = True) -> None: + df = pd.read_csv( + source_csv, encoding="windows-1250" + ) # enforce windows-1250 encoding to handle special Polish characters in district names + print(f"Loaded: {source_csv} → {df.shape[0]} rows, {df.shape[1]} cols") + + # ------------------------------------------------------------------ # + # 1. Clean target: drop zero and NaN LST rows # + # ------------------------------------------------------------------ # + target_col = "LST_20240829" + + if drop_zero_lst: + n_before = len(df) + df = df[df[target_col] != 0].copy().reset_index(drop=True) + print(f"Dropped {n_before - len(df)} rows with LST == 0") + + n_before = len(df) + df = df.dropna(subset=[target_col]).reset_index(drop=True) + if len(df) < n_before: + print(f"Dropped {n_before - len(df)} rows with NaN LST target") + else: + print("No NaN LST targets found — good.") + + # ------------------------------------------------------------------ # + # 2. Build output skeleton # + # ------------------------------------------------------------------ # + out = pd.DataFrame( + { + "name_loc": [f"heat_{i:06d}" for i in range(len(df))], + "lat": df["LAT"].astype(float), + "lon": df["LONG"].astype(float), + "target_lst": df[target_col].astype(float), + } + ) + + # Verify target is clean + assert out["target_lst"].isna().sum() == 0, "BUG: NaN in target after cleaning" + print( + f"Target stats: mean={out['target_lst'].mean():.2f} " + f"std={out['target_lst'].std():.2f} " + f"min={out['target_lst'].min():.2f} " + f"max={out['target_lst'].max():.2f}" + ) + + # ------------------------------------------------------------------ # + # 3. Numeric features — impute or drop # + # ------------------------------------------------------------------ # + numeric_feat_cols = [] + dropped_cols = [] + + for c in NUMERIC_COLS: + if c not in df.columns: + print(f" [WARN] numeric column not found, skipping: {c}") + continue + col_name = f"feat_{clean_token(c).lower()}" + series = pd.to_numeric(df[c], errors="coerce").astype(float) + nan_frac = series.isna().sum() / len(series) + + if nan_frac > NAN_DROP_THRESHOLD: + print(f" [DROP] {col_name}: {nan_frac:.0%} NaN — exceeds threshold, dropped") + dropped_cols.append(col_name) + continue + + if nan_frac > 0: + col_mean = series.mean() + print(f" [IMPUTE] {col_name}: {nan_frac:.1%} NaN → filled with mean ({col_mean:.4f})") + series = series.fillna(col_mean) + + out[col_name] = series + numeric_feat_cols.append(col_name) + + # ------------------------------------------------------------------ # + # 4. Categorical features (one-hot) # + # ------------------------------------------------------------------ # + for c in CATEGORICAL_COLS: + if c not in df.columns: + print(f" [WARN] categorical column not found, skipping: {c}") + continue + cats = df[c].astype(str).fillna("NA").map(clean_token) + prefix = f"feat_{clean_token(c).lower()}" + dummies = pd.get_dummies(cats, prefix=prefix, prefix_sep="__") + out = pd.concat([out, dummies.astype(np.float32)], axis=1) + + # ------------------------------------------------------------------ # + # 5. Final NaN check — should be zero # + # ------------------------------------------------------------------ # + total_nan = out.isna().sum().sum() + if total_nan > 0: + print("\n[ERROR] NaN values remain in output:") + print(out.isna().sum()[out.isna().sum() > 0]) + raise ValueError(f"{total_nan} NaN values remain — fix before training.") + else: + print("\nNaN check passed — output is clean.") + + # ------------------------------------------------------------------ # + # 6. Save and report # + # ------------------------------------------------------------------ # + out.to_csv(out_csv, index=False) + + feat_cols = [c for c in out.columns if c.startswith("feat_")] + print(f"\nWrote: {out_csv}") + print(f"Shape: {out.shape}") + print(f"tabular_dim (feat_* columns): {len(feat_cols)}") + print(f" numeric features kept: {len(numeric_feat_cols)}") + print(f" numeric features dropped:{len(dropped_cols)}") + print(f" one-hot features: {len(feat_cols) - len(numeric_feat_cols)}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--source_csv", required=True) + ap.add_argument("--out_csv", required=True) + ap.add_argument("--drop_zero_lst", type=lambda x: x.lower() != "false", default=True) + args = ap.parse_args() + main(args.source_csv, args.out_csv, args.drop_zero_lst)