📝 Add docstrings to feat/default-models

build_combined.py

@@ -0,0 +1,375 @@
+"""
+Merge notebooks 01-05 into one combined notebook.
+Output: colab_package/sarcasm_classification.ipynb
+"""
+import json
+from pathlib import Path
+
+ROOT     = Path(".")
+NB_DIR   = ROOT / "notebooks"
+OUT_DIR  = ROOT / "colab_package"
+OUT_DIR.mkdir(exist_ok=True)
+
+# ── Load all notebooks ────────────────────────────────────────────────────────
+def load_nb(name):
+    """
+    Load a Jupyter notebook file from NB_DIR and return its cells in a simplified form.
+
+    Parameters:
+        name (str): Notebook filename located inside NB_DIR (e.g., "01_data_preparation.ipynb").
+
+    Returns:
+        list[dict]: A list of dictionaries for each cell with keys:
+            - "type" (str): the cell type, e.g., "code" or "markdown".
+            - "src" (str): the cell source as a single normalized string.
+    """
+    path = NB_DIR / name
+    nb = json.loads(path.read_bytes().decode("utf-8"))
+    cells = []
+    for c in nb["cells"]:
+        src = c["source"]
+        if isinstance(src, list):
+            src = "".join(src)
+        cells.append({"type": c["cell_type"], "src": src})
+    return cells
+
+nb01 = load_nb("01_data_preparation.ipynb")
+nb02 = load_nb("02_tfidf_lr_baseline.ipynb")
+nb03 = load_nb("03_naive_bayes_baseline.ipynb")
+nb04 = load_nb("04_bert_classification.ipynb")
+nb05 = load_nb("05_error_analysis.ipynb")
+
+
+# ── Single combined setup cell ────────────────────────────────────────────────
+# Merges all imports + file detection + all output-dir definitions
+
+SETUP_SRC = "\n".join([
+    "# ============================================================",
+    "# SETUP — imports, file upload, paths",
+    "# ============================================================",
+    "from __future__ import annotations",
+    "import json, hashlib, random, os, warnings, shutil",
+    "from dataclasses import dataclass",
+    "from pathlib import Path",
+    "from collections import Counter",
+    "from urllib.parse import urlparse",
+    "from typing import Optional",
+    "",
+    "import numpy as np",
+    "import pandas as pd",
+    "import matplotlib.pyplot as plt",
+    "import matplotlib.gridspec as gridspec",
+    "import seaborn as sns",
+    "",
+    "from sklearn.pipeline import Pipeline",
+    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer",
+    "from sklearn.linear_model import LogisticRegression",
+    "from sklearn.naive_bayes import MultinomialNB, ComplementNB",
+    "from sklearn.model_selection import GridSearchCV, GroupKFold",
+    "from sklearn.metrics import (",
+    "    accuracy_score, precision_score, recall_score,",
+    "    f1_score, classification_report, confusion_matrix,",
+    ")",
+    "",
+    "warnings.filterwarnings('ignore')",
+    "",
+    "SEED = 42",
+    "random.seed(SEED)",
+    "np.random.seed(SEED)",
+    "",
+    "# ── Locate or upload the JSONL data file ─────────────────────────────────",
+    'FILENAME = "sarcasm_pairs_step35_clean.jsonl"',
+    "",
+    "def _locate_file(filename):",
+    "    candidates = []",
+    "    for root in [Path.cwd()] + list(Path.cwd().parents):",
+    "        for sub in [",
+    '            Path("data") / "processed" / filename,',
+    '            Path("data") / filename,',
+    "            Path(filename),",
+    "        ]:",
+    "            candidates.append(root / sub)",
+    "    for p in [",
+    '        Path("/content") / filename,',
+    '        Path("/mnt/data") / filename,',
+    "    ]:",
+    "        candidates.append(p)",
+    "    _c = Path('/content')",
+    "    for p in (_c.rglob(filename) if _c.exists() else []):",
+    "        candidates.append(p)",
+    "    for p in candidates:",
+    "        if p.is_file():",
+    "            return p",
+    "    return None",
+    "",
+    "print(f'cwd: {Path.cwd()}')",
+    "print(f'files in cwd: {[p.name for p in Path.cwd().iterdir()][:10]}')",
+    "",
+    "DATA_FILE = _locate_file(FILENAME)",
+    "if DATA_FILE is None:",
+    "    try:",
+    "        from google.colab import files as _cf",
+    '        print(f"Upload {FILENAME!r}:")',
+    "        _up = _cf.upload()",
+    "        if not _up:",
+    '            raise RuntimeError("No file uploaded.")',
+    "        _name = list(_up.keys())[0]",
+    '        DATA_FILE = Path("/content") / FILENAME',
+    "        if Path(_name) != DATA_FILE:",
+    "            shutil.move(_name, str(DATA_FILE))",
+    '        print(f"Saved to {DATA_FILE}")',
+    "    except ImportError:",
+    "        raise FileNotFoundError(",
+    '            f"Cannot find {FILENAME!r}. Place it in the same folder as this notebook."',
+    "        )",
+    "",
+    "# ── Project root + all output directories ────────────────────────────────",
+    "def _find_root(data_file):",
+    '    for parent in [data_file.parent] + list(data_file.parents):',
+    '        if any((parent / m).exists() for m in ["outputs","notebooks","data"]):',
+    "            return parent",
+    "    return data_file.parent",
+    "",
+    "ROOT = _find_root(DATA_FILE)",
+    "",
+    "OUT_DATASETS  = ROOT / 'outputs' / 'datasets'",
+    "OUT_SPLITS    = ROOT / 'outputs' / 'splits'",
+    "OUT_TFIDF     = ROOT / 'outputs' / 'classical' / 'tfidf_lr'",
+    "OUT_NB        = ROOT / 'outputs' / 'classical' / 'naive_bayes'",
+    "BERT_OUT      = ROOT / 'outputs' / 'bert'",
+    "REPORTS_DIR   = ROOT / 'outputs' / 'reports'",
+    "SPLITS        = OUT_SPLITS",
+    "",
+    "for d in [OUT_DATASETS, OUT_SPLITS, OUT_TFIDF, OUT_NB, BERT_OUT, REPORTS_DIR]:",
+    "    d.mkdir(parents=True, exist_ok=True)",
+    "",
+    'print(f"Data   : {DATA_FILE}")',
+    'print(f"Root   : {ROOT}")',
+    'print(f"Output : {ROOT / \'outputs\'}")',
+]) + "\n"
+
+# Validate setup cell
+try:
+    compile(SETUP_SRC, "<setup>", "exec")
+    print("Setup cell: OK")
+except SyntaxError as e:
+    lines = SETUP_SRC.split("\n")
+    print(f"SyntaxError line {e.lineno}: {lines[e.lineno-1]!r} — {e}")
+    raise
+
+
+# ── Helper: make a code cell ──────────────────────────────────────────────────
+def code_cell(src):
+    """
+    Create a Jupyter code cell representation dictionary.
+
+    Parameters:
+        src (str): Source code for the cell as a single string.
+
+    Returns:
+        dict: A notebook cell mapping with keys:
+            - "cell_type": "code"
+            - "execution_count": None
+            - "metadata": {}
+            - "outputs": []
+            - "source": the provided `src`
+    """
+    return {
+        "cell_type": "code",
+        "execution_count": None,
+        "metadata": {},
+        "outputs": [],
+        "source": src,
+    }
+
+def md_cell(src):
+    """
+    Create a dictionary representing a Jupyter notebook markdown cell.
+
+    Parameters:
+        src (str | list[str]): Markdown source for the cell; either a single string or a list of source lines.
+
+    Returns:
+        dict: A notebook cell dictionary with keys "cell_type" set to "markdown", empty "metadata", and "source" set to `src`.
+    """
+    return {
+        "cell_type": "markdown",
+        "metadata": {},
+        "source": src,
+    }
+
+
+# ── Determine which cells are "setup" cells to skip ──────────────────────────
+# A setup cell is: first code cell in each notebook that defines ROOT/_find_
+def is_setup_cell(src):
+    """
+    Determines whether a notebook cell's source contains any setup-related trigger strings.
+
+    Parameters:
+        src (str): The cell source text to inspect.
+
+    Returns:
+        `True` if the source contains any known setup trigger substrings, `False` otherwise.
+    """
+    triggers = [
+        "_find_project_root",
+        "_find_data_file",
+        "_locate_file",
+        "Colab / environment setup",
+        "import json, hashlib, random",
+        "SEED = 42\nrandom.seed",
+    ]
+    return any(t in src for t in triggers)
+
+
+# ── Collect content cells from each notebook (skip setup cells) ───────────────
+def content_cells(cells):
+    """
+    Filter notebook cells to remove setup code and redirect TF-IDF output references to OUT_TFIDF.
+
+    Parameters:
+        cells (iterable): An iterable of cell dictionaries, each with keys "type" (e.g., "code" or "markdown") and "src" (the cell source string).
+
+    Returns:
+        list: A list of cell dictionaries with keys "type" and "src". Code cells that are identified as setup are omitted; remaining code cells have occurrences of OUT_DIR and the specific TF-IDF output path replaced with `OUT_TFIDF`. Markdown cells are returned unchanged.
+    """
+    out = []
+    for c in cells:
+        if c["type"] == "code" and is_setup_cell(c["src"]):
+            continue
+        # Patch nb02's OUT_DIR references to use OUT_TFIDF
+        src = c["src"].replace(
+            'ROOT / "outputs" / "classical" / "tfidf_lr"',
+            "OUT_TFIDF"
+        ).replace(
+            "OUT_DIR", "OUT_TFIDF"
+        ) if c["type"] == "code" else c["src"]
+
+        out.append({"type": c["type"], "src": src})
+    return out
+
+def content_cells_nb(cells, out_dir_var, out_dir_val):
+    """
+    Create a list of notebook cells where code cells have the token "OUT_DIR" replaced by a provided output-directory variable and setup cells are omitted.
+
+    Parameters:
+        cells (list[dict]): Sequence of cell objects each with "type" and "src" keys.
+        out_dir_var (str): Variable name to substitute for occurrences of "OUT_DIR" inside code cell sources.
+        out_dir_val (str): Unused parameter retained for compatibility; it does not affect processing.
+
+    Returns:
+        list[dict]: Filtered and transformed cells where setup code cells are removed and code cell sources have the substitution applied.
+    """
+    result = []
+    for c in cells:
+        if c["type"] == "code" and is_setup_cell(c["src"]):
+            continue
+        src = c["src"]
+        if c["type"] == "code":
+            src = src.replace("OUT_DIR", out_dir_var)
+        result.append({"type": c["type"], "src": src})
+    return result
+
+
+# ── Build combined cell list ──────────────────────────────────────────────────
+all_cells = []
+
+# Title
+all_cells.append(md_cell(
+    "# Sarcasm Classification — Complete Pipeline\n\n"
+    "**Sections**:\n"
+    "1. Setup & Data Preparation\n"
+    "2. TF-IDF + Logistic Regression Baseline\n"
+    "3. Naive Bayes Baseline\n"
+    "4. BERT / DistilBERT Classification\n"
+    "5. Error Analysis & Model Comparison\n\n"
+    "**Run all cells in order (Runtime → Run all).**"
+))
+
+# Setup cell
+all_cells.append(code_cell(SETUP_SRC))
+
+# ── Section 1: Data Preparation ───────────────────────────────────────────────
+all_cells.append(md_cell("---\n# Part 1 — Data Preparation"))
+for c in content_cells(nb01):
+    if c["type"] == "markdown":
+        all_cells.append(md_cell(c["src"]))
+    else:
+        all_cells.append(code_cell(c["src"]))
+
+# ── Section 2: TF-IDF + LR ───────────────────────────────────────────────────
+all_cells.append(md_cell("---\n# Part 2 — TF-IDF + Logistic Regression Baseline"))
+for c in content_cells_nb(nb02, "OUT_TFIDF", "OUT_TFIDF"):
+    if c["type"] == "markdown":
+        all_cells.append(md_cell(c["src"]))
+    else:
+        all_cells.append(code_cell(c["src"]))
+
+# ── Section 3: Naive Bayes ────────────────────────────────────────────────────
+all_cells.append(md_cell("---\n# Part 3 — Naive Bayes Baseline"))
+for c in content_cells_nb(nb03, "OUT_NB", "OUT_NB"):
+    if c["type"] == "markdown":
+        all_cells.append(md_cell(c["src"]))
+    else:
+        all_cells.append(code_cell(c["src"]))
+
+# ── Section 4: BERT ───────────────────────────────────────────────────────────
+all_cells.append(md_cell("---\n# Part 4 — BERT / DistilBERT Classification"))
+for c in content_cells_nb(nb04, "BERT_OUT", "BERT_OUT"):
+    if c["type"] == "markdown":
+        all_cells.append(md_cell(c["src"]))
+    else:
+        all_cells.append(code_cell(c["src"]))
+
+# ── Section 5: Error Analysis ─────────────────────────────────────────────────
+all_cells.append(md_cell("---\n# Part 5 — Error Analysis & Model Comparison"))
+for c in content_cells_nb(nb05, "REPORTS_DIR", "REPORTS_DIR"):
+    if c["type"] == "markdown":
+        all_cells.append(md_cell(c["src"]))
+    else:
+        # Also patch CLASSICAL and BERT_OUT references (already correct var names)
+        all_cells.append(code_cell(c["src"]))
+
+# ── Compile-check all code cells ──────────────────────────────────────────────
+errors = []
+for i, c in enumerate(all_cells):
+    if c["cell_type"] == "code":
+        src = c["source"]
+        try:
+            compile(src, f"cell{i}", "exec")
+        except SyntaxError as e:
+            errors.append((i, e.lineno, str(e), src.split("\n")[e.lineno-1] if e.lineno else ""))
+
+if errors:
+    print(f"\n{len(errors)} syntax error(s):")
+    for i, ln, msg, line in errors:
+        print(f"  Cell {i:3d} line {ln}: {line!r}")
+        print(f"           {msg}")
+else:
+    print(f"All {len(all_cells)} cells: syntax OK")
+
+# ── Write combined notebook ───────────────────────────────────────────────────
+combined = {
+    "nbformat": 4,
+    "nbformat_minor": 4,
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {"name": "python", "version": "3.11.0"},
+        "colab": {"provenance": []},
+    },
+    "cells": all_cells,
+}
+
+out_nb = OUT_DIR / "sarcasm_classification.ipynb"
+out_nb.write_bytes(json.dumps(combined, indent=1).encode("utf-8"))
+print(f"\nSaved: {out_nb}")
+print(f"Total cells: {len(all_cells)}")
+print(f"\ncolab_package/ contents:")
+for p in sorted(OUT_DIR.iterdir()):
+    print(f"  {p.name}")
+print(f"\nAlso copy into colab_package/:")
+print(f"  sarcasm_pairs_step35_clean.jsonl  (from data/processed/)")