clean_data.py

# -*- coding: utf-8 -*-
"""
clean_data.py
--------------
Cleans the raw serial-killer incident dataset and writes a clean copy to
data/Data_Analytics_clean.csv. This script documents every transformation
so the cleaning step is fully reproducible and auditable (see README ->
"Data Preprocessing" for the before/after summary this script produces).

Issues found in the raw file and how each is handled:

1. Inconsistent / misspelled categorical labels
   - Method: "Stragulation" -> "Strangulation"; "Sexual Assualt" -> "Sexual Assault"
   - Relationship: trailing whitespace on some "Stranger " values created a
     duplicate category ("Stranger" vs "Stranger ") -> stripped.
   - MentalIllness: "ASPD" and "Antisocial PD" are the same diagnosis
     (Antisocial Personality Disorder) recorded under two different labels.
     They are merged into a single canonical label, "ASPD", matching the
     terminology used in the presentation.
   - "Paranoid Schizophrenia (contested)" is kept distinct from
     "Paranoid Schizophrenia" on purpose: it flags a real diagnosis that is
     disputed in the historical record (David Berkowitz), which is
     analytically meaningful and shouldn't be silently merged away.

2. Missing values
   - None found in this dataset (checked with df.isna().sum()).

3. Duplicate rows
   - None found (checked with df.duplicated().sum()).

4. Multi-valued column
   - MentalIllness holds a comma-separated list of disorders per row. A
     `Disorders` list column is added, plus one binary 0/1 indicator column
     per distinct disorder, to support co-occurrence analysis.
"""

import pandas as pd
from pathlib import Path

RAW_PATH = Path(__file__).resolve().parent.parent / "data" / "raw" / "Data_Analytics_raw.csv"
CLEAN_PATH = Path(__file__).resolve().parent.parent / "data" / "Data_Analytics_clean.csv"

METHOD_FIXES = {
    "Stragulation": "Strangulation",
}

DISORDER_MERGE = {
    "Antisocial PD": "ASPD",
}


def clean(df: pd.DataFrame) -> pd.DataFrame:
    df = df.copy()

    # --- Method: fix typos (including inside comma-joined compound values) ---
    df["Method"] = df["Method"].replace(METHOD_FIXES)
    df["Method"] = df["Method"].str.replace("Sexual Assualt", "Sexual Assault", regex=False)

    # --- Relationship: strip stray whitespace that created a duplicate category ---
    df["Relationship"] = df["Relationship"].str.strip()

    # --- MentalIllness: merge synonym labels, then normalize whitespace ---
    def normalize_disorders(cell: str) -> str:
        parts = [p.strip() for p in cell.split(",")]
        parts = [DISORDER_MERGE.get(p, p) for p in parts]
        return ", ".join(parts)

    df["MentalIllness"] = df["MentalIllness"].apply(normalize_disorders)

    return df


def main():
    df_raw = pd.read_csv(RAW_PATH)
    n_before_methods = df_raw["Method"].nunique()
    n_before_rel = df_raw["Relationship"].nunique()

    df_clean = clean(df_raw)

    n_after_methods = df_clean["Method"].nunique()
    n_after_rel = df_clean["Relationship"].nunique()

    CLEAN_PATH.parent.mkdir(parents=True, exist_ok=True)
    df_clean.to_csv(CLEAN_PATH, index=False)

    print("Cleaning summary")
    print("-----------------")
    print(f"Rows: {len(df_raw)} -> {len(df_clean)} (no rows dropped)")
    print(f"Distinct Method values: {n_before_methods} -> {n_after_methods}")
    print(f"Distinct Relationship values: {n_before_rel} -> {n_after_rel}")
    print(f"Clean file written to: {CLEAN_PATH}")


if __name__ == "__main__":
    main()