diff --git a/.gitignore b/.gitignore index 8b84345..cd6b4b9 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,7 @@ checkpoints/* # pretrained weights (large binaries: mgfn/vadclip ckpts, i3d caffe2 pkls) pretrained/ + +# run logs (keep result.json / epoch_log.json; drop verbose logs) +experiments/runs/**/*.log +experiments/runs/*.log diff --git a/configs/data/default.yaml b/configs/data/default.yaml index 5c0d485..f9166e2 100644 --- a/configs/data/default.yaml +++ b/configs/data/default.yaml @@ -9,8 +9,8 @@ cache_dir: null root: ${oc.env:WSAD_DATA,"~/data/wsad"} source: auto # local | hub | auto (local if present, else HF) backbone: i3d # i3d | clip -dataset_dir: ucf_crime # subdir under root holding {train,test}.zip + ground_truth.json + UCFClipFeatures (zip-direct I3D path) -ground_truth: null # explicit frame-level gt json; null -> //ground_truth.json, else HF +dataset_dir: ucf_crime # per-dataset subdir: features/{i3d,clip}/, annotations/, manifest.* (see docs/DATA_LOCAL.md) +ground_truth: null # explicit gt json; null -> /annotations/ground_truth.json (legacy /ground_truth.json), else HF clip_length: 256 # CLIP train segment length (vadclip 256) segment: null # null=keep; int=uniform mean-pool to N segments (clip_tsa/tpwng 32) single_crop: true # CLIP: use only crop __0 (official single-crop) diff --git a/configs/runner/bn_wvad.yaml b/configs/runner/bn_wvad.yaml index 8823e65..76fa940 100644 --- a/configs/runner/bn_wvad.yaml +++ b/configs/runner/bn_wvad.yaml @@ -21,6 +21,8 @@ model_config: attn_impl: eager # "sdpa" for FlashAttention/fused kernels (speed; not bit-exact) # Paper: Adam, lr 1e-4, weight_decay 5e-5, batch 64 (32+32), num_segments 200. +# Official train.py:16 applies clip_grad_norm_(net.parameters(), 1.) every step. optimizer: learning_rate: 1e-4 weight_decay: 0.00005 + grad_clip_norm: 1.0 diff --git a/configs/runner/s3r.yaml b/configs/runner/s3r.yaml index 3bbec43..7ca0197 100644 --- a/configs/runner/s3r.yaml +++ b/configs/runner/s3r.yaml @@ -21,4 +21,4 @@ model_config: optimizer: learning_rate: 1e-3 - weight_decay: 0.0005 + weight_decay: 0.005 diff --git a/configs/runner/ur_dmu.yaml b/configs/runner/ur_dmu.yaml index 877c57f..ad1a915 100644 --- a/configs/runner/ur_dmu.yaml +++ b/configs/runner/ur_dmu.yaml @@ -12,7 +12,9 @@ model_config: mem_size: 60 topk_ratio: 16 dropout_rate: 0.5 - attn_impl: eager # "sdpa" for FlashAttention/fused kernels (speed; not bit-exact) + attn_impl: eager # exact default. NB: SDPA alone doesn't fit 8GB full-length test — + # the distance-decay branch materializes a T×T adjacency regardless (O(T²) is structural). + # -> efficient temporal modeling (sliding-window / bounded distance) is a Spec2 task. margin: 1.0 w_mem: 1.0 w_triplet: 0.1 @@ -20,4 +22,4 @@ model_config: optimizer: learning_rate: 1e-4 - weight_decay: 0.0005 + weight_decay: 0.00005 diff --git a/docs/DATA_LOCAL.md b/docs/DATA_LOCAL.md index 8df393e..d46c225 100644 --- a/docs/DATA_LOCAL.md +++ b/docs/DATA_LOCAL.md @@ -1,35 +1,86 @@ # Local data layout (`~/data/wsad`) -The data layer is **local-first with HF fallback** (`data.source: auto`). Drop the -files you download into the tree below and the loader picks them up; if a backbone -dir is missing it falls back to the Hugging Face cache. +The data layer is **local-first with HF fallback** (`data.source: auto`). Features +are organized **per dataset** so a second dataset (e.g. AIHub) slots in as a sibling +without ambiguity. If a backbone is missing locally the loader falls back to the +Hugging Face cache. ``` -~/data/wsad/ # = data.root (override: WSAD_DATA env or data.root=...) -├── i3d/ # I3D 10-crop features (RTFM-style; rtfm/mgfn/ur_dmu/s3r/bn_wvad/mil/gs_moe) -│ ├── train/ _i3d.npy # (10, 32, 2048) 10-crop, 32-segment -│ └── test/ _i3d.npy # (T, 10, 2048) full-length, 10-crop -│ # ground_truth.json comes from the HF dataset (jinmang2/ucf_crime_tencrop_i3d_seg32) -└── clip/ # CLIP ViT-B/16 features (vadclip/clip_tsa/tpwng) - ├── train/ _x264__.npy # (Tframes, 512) per-crop, snippet-level (1 snippet = 16 frames) - └── test/ _x264__.npy - # split = official VadCLIP test list; ground_truth.json reused from the i3d HF dataset (backbone-agnostic) +~/data/wsad/ # = data.root (override: WSAD_DATA env or data.root=...) +└── ucf_crime/ # = data.dataset_dir (one dir per dataset) + ├── raw/ # source videos + official splits (Anomaly-Videos-*.zip, ...) + ├── features/ + │ ├── i3d/ + │ │ ├── train.zip # MGFN canonical: 1610 × (10, 32, 2048) seg32, crop-first + │ │ └── test.zip # 290 × (T, 10, 2048) full-length + │ └── clip/ + │ ├── _byclass//*.npy # VadCLIP UCFClipFeatures dump (per-crop (T,512) fp16) + │ ├── train/ _x264__.npy # symlinks into _byclass (official VadCLIP split) + │ └── test/ _x264__.npy + ├── annotations/ + │ └── ground_truth.json # frame-level labels, keys _i3d.npy (290, backbone-agnostic) + ├── manifest.parquet / manifest.jsonl # unified catalog (built by scripts/build_manifest.py) + └── _archive/ # incompatible/superseded features (NOT used by the pipeline) + ├── UCF_{Train,Test}_ten_crop_i3d/ # RTFM extraction — ~10x scale (L2~22), only 109/290 test + └── UCF_test_feature.zip ``` +The loader resolves this new layout first and falls back to the **legacy** layout +(top-level `i3d/`, `clip/`, dataset-root `{train,test}.zip` / `ground_truth.json`), +so older trees keep working. Path resolution lives in `src/data/local.py` +(`_features_root`, `_clip_dir`, `_i3d_npy_dir`, `_i3d_zip_path`, gt resolver). + +## Feature provenance & scale (do NOT mix sources) + +Magnitude-based methods (RTFM / MGFN / BN-WVAD) depend on feature L2 scale, so +mixing extractions silently breaks them. Verified per-snippet L2 norms: + +| Source | I3D L2/snip | Notes | +|--------|-------------|-------| +| **`features/i3d/*.zip`** (canonical, DeepMIL/Roc-Ng) | **~2.5** | the working pair, 1610/290, what every runner expects (its PROVENANCE says DeepMIL; the "MGFN" label here was loose) | +| **`features/i3d_mgfn/{train,test}/`** (MGFN authors' OneDrive) | **~22** | full-length `(T,10,2048)`, **COMPLETE 1610/290** verified 2026-06-22, raw pre-seg32. RTFM-family scale. See its PROVENANCE.md | +| RTFM `_archive/...` | ~22 | different extraction, 109/290 test — quarantined, ignore (i3d_mgfn supersedes it) | +| our tushar-n extractor (`scripts/validate_extraction.py`) | ~22 | self-consistent w/ RTFM, **NOT** with the ~2.5 i3d/ | + +> Two distinct lineages share the "MGFN" name loosely. The L2~2.5 pair in +> `features/i3d/` is DeepMIL (what the default runners use). The genuine +> MGFN-author distribution (HKU OneDrive) is L2~22 and lives in +> `features/i3d_mgfn/` — select with `data.feature_variant: i3d_mgfn`. Never mix +> the two scales across train/test. + +→ A fresh extraction must re-do **both** train+test with one model; you cannot +reuse MGFN's test against tushar-n train. CLIP similarly: the provided +`UCFClipFeatures` are **OpenAI** CLIP ViT-B/16; our extractor defaults to +`laion2b` — a different space, so fresh CLIP features need a retrain, not a swap. + ## Where each download goes | Source | What | Put under | |--------|------|-----------| -| RTFM Google Drive (`ucf_crime_tencrop_i3d`) | I3D 10-crop `.npy` per video | `i3d/train`, `i3d/test` (or skip — HF `ucf_crime_tencrop_i3d_seg32` is the fallback) | -| VadCLIP (Baidu/OneDrive `UCFClipFeatures`) | CLIP per-crop `.npy` (`_x264__0..9.npy`, each `(T,512)`) | `clip/train`, `clip/test` | +| DeepMIL UCF features (canonical) | I3D seg32 `train.zip` + full-length `test.zip` | `ucf_crime/features/i3d/` | +| MGFN authors' OneDrive 10-crop I3D | full-length `(T,10,2048)` per-video `.npy`, 1610/290 | `ucf_crime/features/i3d_mgfn/{train,test}/` (fetch via `scripts/onedrive_fetch/`) | +| VadCLIP `UCFClipFeatures` | CLIP per-crop `.npy` (`_x264__0..9.npy`, `(T,512)`) | `ucf_crime/features/clip/_byclass/` then run prepare | -The VadCLIP download is **not folder-split** — it ships `list/ucf_CLIP_rgb.csv` -(train) and `list/ucf_CLIP_rgbtest.csv` (test). Run the prepare script to sort the -flat `UCFClipFeatures/` dump into `clip/train` and `clip/test`: +The VadCLIP dump is **not folder-split** — it ships `list/ucf_CLIP_rgbtest.csv`. +Sort the flat dump into `clip/{train,test}` symlinks: ```bash python scripts/prepare_clip_features.py \ - --src /path/to/UCFClipFeatures --dst ~/data/wsad/clip # symlinks by official test list + --src ~/data/wsad/ucf_crime/features/clip/_byclass \ + --dst ~/data/wsad/ucf_crime/features/clip +``` + +## Unified manifest (the catalog) + +`scripts/build_manifest.py` writes `manifest.parquet` + `manifest.jsonl` — one row +per `(backbone, split, video)` with `video_id, label, path, zip_member, n_crops, +shape, dtype` (read from `.npy` headers only, ~24 s over the 9 GB zips). Use it to +query/verify the data without loading arrays; it's the integration point for new +tooling and the AIHub dataset. + +```bash +python scripts/build_manifest.py # dataset=ucf_crime (default) +python scripts/build_manifest.py --dataset aihub # future, same shape ``` ## Per-runner CLIP post-processing (one download, three contracts) @@ -39,11 +90,11 @@ per runner (set by the `data=` config you select): | Runner | crops | train length | test | data config | |--------|-------|--------------|------|-------------| -| `vadclip` | 1 (`__0`) | `process_feat` → 256 | `process_split` → (n,256,512) | `data=clip_vadclip` | +| `vadclip` | 1 (`__0`) | `process_feat` → 256 | full-length (T,1,512) | `data=clip_vadclip` | | `tpwng` | 1 (`__0`) | uniform → 32 seg | full-length (T,1,512) | `data=clip_seg` | -| `clip_tsa` | 1 (`__0`) or 10 | uniform → 32 seg | full-length (T,10/1,512) | `data=clip_seg` | +| `clip_tsa` | 1 (`__0`) | uniform → 32 seg | full-length (T,1,512) | `data=clip_seg` | -I3D models need no `data=` override (default `data=i3d`). Example: +I3D models need no `data=` override (default `data=i3d`, zip-direct). Example: ```bash python train.py runner=rtfm # I3D, local→HF auto @@ -51,11 +102,31 @@ python train.py runner=vadclip data=clip_vadclip # CLIP, single-crop, len 256 python train.py runner=tpwng data=clip_seg # CLIP, 32-seg ``` +## Extracting features from raw video + +`scripts/validate_extraction.py` runs one raw video end-to-end (decord → I3D / +CLIP) and reports shape, scale, and wall time. Validated on RTX2070S (2026-06-14): + +| Line | time / video | full UCF (~1900) | output | +|------|-------------|------------------|--------| +| I3D tushar-n (`I3Res50`, self-contained, no pytorchvideo) | ~37 s | ~19.5 h | `(T,10,2048)` → seg32 `(10,32,2048)` | +| CLIP ViT-B/16 | ~7 s | ~3.7 h | `(T,512)` | + +```bash +python scripts/validate_extraction.py # i3d baseline, 1 sample video +python scripts/validate_extraction.py --nonlocal # I3Res50(use_nl=True) +python scripts/validate_extraction.py --clip # also time the CLIP line +``` + +`pytorchvideo` is only needed for the alternative `i3d_8x8_r50` SlowFast variant +(imported lazily); the default tushar-n / nonlocal `I3Res50` path needs only +`decord` + `torch`. + ## Notes -- Class label is parsed free from the filename (`Abuse001_…` → class 7-style id), - so VadCLIP's CLASM and TPWNG's prompts work without extra annotation. -- `ground_truth.json` (frame-level labels) is backbone-agnostic and reused across - i3d/clip; the loader pulls it from the HF i3d dataset. +- Class label is parsed free from the filename (`Abuse001_…`), so VadCLIP CLASM / + TPWNG prompts work without extra annotation. +- `ground_truth.json` is backbone-agnostic; `_align_gt`/`_bare_vid` re-key it from + `_i3d.npy` onto CLIP per-crop names. - Faithful CLIP text branches (vadclip/tpwng) download OpenAI CLIP ViT-B/16 text - weights once (open_clip `pretrained="openai"`) to match the extracted features. + weights once (open_clip `pretrained="openai"`) to match the **provided** features. diff --git a/docs/FEATURE_EXTRACTOR_RESEARCH.md b/docs/FEATURE_EXTRACTOR_RESEARCH.md new file mode 100644 index 0000000..fdea560 --- /dev/null +++ b/docs/FEATURE_EXTRACTOR_RESEARCH.md @@ -0,0 +1,139 @@ +# Feature Extractor Research: architectures, training data, and pipeline cost + +Companion to `RESEARCH_ROADMAP.md` and `FEATURE_EXTRACTORS.md`. Answers four +questions for the WSVAD feature-extractor → anomaly-detection pipeline: +1. What architecture does each backbone use? +2. What data was each trained on, and how much? +3. If we *re-build* this, how much training data do we actually need? +4. How far can the `extractor → detector` pipeline be optimized for production? + +Grounded in this repo's extractors (`src/features/*.py`) + the 2024–25 literature. + +--- + +## 1 + 2. Backbone architectures and their training data + +| Backbone (repo) | Architecture | Pretrain data | Scale | Supervision | Output | +|---|---|---|---|---|---| +| **I3D** (`i3d_gowtham.py`) | Inflated 3D ConvNet — here the **I3D-ResNet50** (`I3Res50`) variant, 2D ResNet inflated to 3D + optional Non-Local blocks | **ImageNet** (2D inflate) → **Kinetics-400** | K400 ≈ **240k** train clips, 400 action classes, ~10s each | Supervised (action labels) | `(T, 10crop, 2048)` per 16-frame snippet | +| **CLIP ViT-B/16** (`clip.py`) | ViT-B/16 image encoder (12-layer transformer, 16×16 patches) contrastively aligned to a text encoder | OpenAI: **WIT-400M**; this repo uses open_clip **`laion2b_s34b_b88k`** = **LAION-2B** (2.3B EN image-text pairs, 34B samples seen) | 400M – 2B **image-text pairs** | Weak/contrastive (web alt-text) | `(frame, 512)` → mean-pool to 32 seg | +| **VideoMAE v1** (`videomae.py`, `MCG-NJU/videomae-base`) | ViT-B with **tubelet** embedding, masked-autoencoder pretrain (90% masking) | **Kinetics-400** (frames only, **no labels**) | ~240k videos, 800–1600 epochs | **Self-supervised** (reconstruction) | `(T, 768)` per 16-frame clip | +| **VideoMAEv2** (loader TODO) | Same MAE + **dual masking**, scaled to ViT-g (**1.0B params**) | **UnlabeledHybrid** = Kinetics + Something-Something + AVA + WebVid2M + self-collected IG | **~1.35M** videos | Self-supervised + label distill | `(T, 768/1024/1408)` | +| **InternVideo2** (frontier) | Progressive: masked modeling + crossmodal contrastive + next-token; video encoder up to **6B params** | Multimodal video-audio-speech caption corpus | billion-scale; 6B model = **256×A100 for ~18+14+3 days** | Multi-stage multimodal | text-aligned video features | +| **VGGish** (`vggish.py`, AV phase) | VGG-style CNN on log-mel spectrogram | AudioSet | ~2M audio clips | Supervised (audio tags) | `(T_audio, 128)` | + +**The structural split that drives method choice** (from `FEATURE_EXTRACTORS.md`): +`text_aligned` is the single most important property. CLIP and InternVideo2 live +in a shared image/text space → they unlock the VLM/text-branch heads (CLIP-TSA, +VadCLIP, TPWNG, WSVAD-CLIP). I3D / VideoMAE are visually strong but have **no +text space** → visual-only heads only (Sultani, RTFM, MGFN, UR-DMU, GS-MoE). + +**Performance context** (UCF-Crime frame AUC, from roadmap): I3D-only ceiling +~86; CLIP + multi-backbone fusion → ~87–88; the lever is the *backbone*, not the +head. One independent study even reports I3D (90%) > plain ViT (86%) on the same +LSTM head — i.e. a generic ViT is **not** automatically better than I3D; the win +comes from *video-pretrained* or *text-aligned* backbones, not transformers per se. + +--- + +## 3. If we re-build this — how much data do we actually need? + +**Key reframing: there are two completely different "training" budgets, and people +conflate them.** + +### 3a. Training the *feature extractor* (the backbone) — you almost never do this +The backbones above cost **240k–1.35M videos** (or 400M–2B image-text pairs) and +**hundreds of GPU-days**. Reproducing one from scratch is a foundation-model +project, not a VAD project. **Don't.** The correct move (and what this repo does) +is to **download pretrained frozen weights** and run inference-only extraction. + +If you *must* adapt a backbone to a new domain (e.g. CCTV/AIHub), the options in +ascending cost: +- **Zero training** — use frozen I3D/CLIP/VideoMAE as-is. Default. Works because + Kinetics/LAION already cover human-activity + object semantics. +- **Linear probe / LoRA** on the backbone — hundreds–few thousand labeled clips. +- **Self-supervised continued pretrain** (VideoMAE-style, no labels) on your raw + domain footage — tens of thousands of *unlabeled* clips is enough to shift the + representation, since MAE needs no annotation. This is the realistic "rebuild" + path for a new camera domain. +- **Full re-pretrain** — only if you are publishing a new foundation model. + +### 3b. Training the *anomaly-detection head* (what WSVAD actually trains) +This is tiny by comparison and is **the entire point of the offline-feature +design**. The standard benchmark training sets: + +| Dataset | Train videos | Labels | Notes | +|---|---|---|---| +| **UCF-Crime** | ~1,610 (800 normal + 810 anomaly) | **video-level only** (weak) | the standard WSVAD benchmark | +| **XD-Violence** | ~3,954 | video-level + audio | largest; AV | +| **ShanghaiTech (weakly)** | 437 | video-level | smaller | + +So to stand up a working detector on a **new** domain you need on the order of +**~1–4k weakly-labeled videos** (just "this video contains an anomaly somewhere" +vs "normal") — *not* frame-level annotation. The features are cached once; the MIL +head trains from scratch in minutes-to-hours on a single 6 GB GPU (this repo +trains the light heads at batch 32 in AMP on an RTX 2060). + +**Practical recipe for a new deployment:** +1. Frozen pretrained backbone (CLIP for text-branch methods, or I3D for the + proven visual baseline). Optional unlabeled MAE continued-pretrain if the + domain is very off-distribution. +2. Collect ~1–3k videos with **video-level** normal/anomaly tags only. +3. Extract features once → cache `.npy`. +4. Train a RTFM/UR-DMU/VadCLIP head on the cache. + +--- + +## 4. Pipeline optimization: `extractor → detector` for production + +### Where the cost actually is +The detector head is **negligible** (a few-layer MLP/attention over cached +features — runs at thousands of FPS). **~95%+ of wall-clock and FLOPs is the +feature extractor.** Optimization = optimize extraction + I/O; the head is free. + +Worst offender in this repo's faithful I3D path (`i3d_gowtham.py`): it does +**ffmpeg → per-frame JPG → PIL → 10-crop** = 10× forward passes per snippet, plus +disk round-trips. Bit-exact for reproduction, **terrible** for production. + +### Optimization levers (ROI-ordered, production) + +| Lever | Effect | Cost / caveat | +|---|---|---| +| **Drop 10-crop → 1 center crop** | **~10× extraction speedup** | tiny AUC drop; the single biggest win | +| **In-memory decode (decord), no JPG dump** | removes ffmpeg+disk round-trip | already how `clip.py`/`videomae.py` read | +| **fp16 / AMP inference** | ~2× throughput, ½ VRAM | repo already does `.half()` | +| **Batch snippets across the clip** | saturates GPU | bounded by VRAM | +| **`torch.compile` + SDPA/Flash attention** | fuses kernels (`WEIGHTS_AND_OPTIMIZATION.md`) | ~1e-3 numeric drift — fine for prod, not for the bit-exact gate | +| **Sparse/strided sampling** (skip frames, larger stride) | linear speedup | coarser temporal resolution | +| **Smaller/faster backbone** (X3D, MobileNet-3D, CLIP ViT-B over ViT-L) | big | accuracy trade | +| **Cache features once, reuse across heads** | amortizes extraction to ~0 for experiments | the repo's whole design | +| **TensorRT / ONNX export, INT8** | edge-deployment throughput | engineering + calibration | + +### Real-time numbers from the literature +- Offline SOTA methods ignore latency entirely; they assume features are + pre-extracted. That is fine for batch/forensic use, **not** for live CCTV. +- The **Real-Time WSVAD** line (WACV 2024) reports **~23 FPS** end-to-end + (drops to ~21 FPS adding frame-level inference) and an **86.9% AUC with a ~6.4s + decision window** — i.e. real-time is achievable but you pay ~1–3 AUC points and + must design the extractor + temporal window for streaming, not 10-crop offline. + +### Practical deployment guidance +- **Forensic / batch** (search archived footage): keep accuracy-max config + (10-crop, ViT-L/InternVideo2, fusion). Throughput doesn't matter; cache once. +- **Live / edge** (alerting): single-crop, fp16, lightweight backbone (X3D / + CLIP-B / VideoMAE-B), strided sampling, sliding decision window. Target the + ~20–30 FPS regime; accept ~86–87 AUC. +- **Hybrid (recommended)**: cheap streaming detector for first-pass alerts → on + trigger, re-run the heavy accurate config on the flagged clip for confirmation. + Gets live latency *and* offline accuracy where it counts. + +--- + +## Sources +- RTFM (arXiv 2101.10030); DAKD (2406.02831); VAD survey (2405.10347); + "Evolution of VAD: DNN→MLLM" (2507.21649) — per `RESEARCH_ROADMAP.md`. +- VideoMAE V2 (CVPR 2023, arXiv 2303.16727) — UnlabeledHybrid ~1.35M videos, ViT-g 1B. +- InternVideo2 (ECCV 2024, arXiv 2403.15377) — 6B encoder, 256×A100 multi-stage. +- CLIP/WIT-400M; LAION-2B (arXiv 2210.08402 / 2212.07143). +- Real-Time WSVAD (WACV 2024) — ~23 FPS, 86.9% AUC / 6.4s window. +- Comparative I3D vs ViT on LSTM head (IJISAE) — I3D 90% > ViT 86%. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md new file mode 100644 index 0000000..70d2434 --- /dev/null +++ b/docs/HANDOFF.md @@ -0,0 +1,77 @@ +# Session handoff — WSVAD comparison framework + +Pick-up notes for continuing on another machine. **Single source of truth = PR #22**; +roadmap = Epic #16. No per-spec PRs, no new issues (track in PR #22). Delete this +file once the work lands. + +- **Branch:** `feat/eval-matrix-serving` (based on `feat/data-pipeline`, PR base `dev`) +- **PR:** #22 (draft) · **Epic:** #16 · sub-issues #17–21 were **closed/consolidated** into PR #22 +- **Last commit:** `857af70` (6 commits ahead of the data-pipeline base) + +## Environment (reproduce on the new PC) +- Interpreter: **`conda run -n balaenoptera python`** (not bare python). +- Run anything importing `src` with **`PYTHONPATH=.`** (e.g. the scripts). +- Installed THIS session into `balaenoptera`: `open_clip_torch`, `timm`. Still missing + / needed for runtime work: **`decord`** (video reading for clip/videomae/raw-video + path), GPU + model weights for VideoMAE/InternVideo2/VideoMAEv2. +- Tests: `conda run -n balaenoptera python -m pytest -q` → **74 passed** (scaffold only). + +## What is DONE (Spec 1 scaffold — code only, runtime-UNVERIFIED) +- `src/compat.py` — backbone↔head text-align guard (`assert_compatible`); flags + `requires_text_aligned=True` on VadCLIP + TPWNG only. +- `src/models/mgfn/modeling_mgfn.py` — removed hardcoded `2048`; split now from + `config.channels` (dim-agnostic). 2048 path verified byte-identical. +- `src/pipeline.py` — `AnomalyDetectionPipeline(backbone, head)`: extractor(processor) + + head + postprocess; `from_features()` (extraction-free) + `visualize()`; auto-sets + head feature-dim to backbone dim. **`mode="offline"` only.** +- `src/viz.py` — `plot_anomaly_scores()`: blue curve + light-orange GT band + dashed + red boundaries (matches paper Fig.10); `legend` toggle. +- `src/features/extract.py` + `scripts/extract_features.py --backbone` — registry + dispatcher caching `_.npy`; `import decord` made lazy; legacy I3D + path intact when `--backbone` unset. +- Tests: `tests/test_compat.py`, `tests/test_pipeline.py`, `tests/test_viz.py`, + `tests/test_extract.py`. + +## What is NOT done (next, in PR #22 checklist) +**Spec 1 finish (GPU):** real clip/videomae extraction on ≥1 video → raw-video +end-to-end through the pipeline (needs decord + a sample) → `{backbone×head}` matrix +with **real AUC** → InternVideo2/VideoMAEv2 loaders (OpenGVLab weights aren't +transformers-native; need a custom loader). +**Spec 2 (the real weight, NOT STARTED):** inference adapter with offline/sliding- +window/causal modes (same pipeline API); causal-finetune recipe on cached features; +LLM-inference-technique research track (StreamingLLM attention-sink, KV-cache, chunked +prefill → causal-VAD); real-time benchmark (FPS/AUC/latency). +**Spec 3 (NOT STARTED):** full qualitative suite + localization metrics + cross-matrix +dashboard. + +## Design + research (read first) +- `docs/superpowers/specs/2026-06-16-wsvad-matrix-serving-design.md` (architecture, + 3-spec roadmap, invariant: original training/model defs unchanged; causal is additive) +- `docs/FEATURE_EXTRACTOR_RESEARCH.md` (backbones, data, optimization, Real-Time WSVAD) + +## Open issues found in self-review (address during Spec 1 finish / Spec 2) +1. **[Med]** Pipeline defaults (`with_magnitude=True`, `segment_to=None`) suit visual + magnitude heads; text heads (tpwng/clip_tsa) trained with 32-seg & no magnitude → + serving/matrix config should carry **per-head preprocessing**. (Visual path OK.) +2. **[Med]** `pipeline._set_feature_dim` sets `visual_width` to backbone dim → VadCLIP + on a non-512 text-aligned backbone (InternVideo2) would mismatch its CLIP-512 text + tower. Fine for CLIP-512 today; handle when InternVideo2 lands. +3. **[Low]** `viz._coerce_gt` mask-vs-intervals heuristic can misread tiny inputs; + prefer explicit `[(start,end)]` intervals. +4. **[Low]** `--backbone` path doesn't segment (legacy path does); add `--seg` if a + head needs seg32 caches. + +## Gotchas +- `gh pr edit --body` fails on this repo (Projects-classic GraphQL bug). Edit the PR + body via REST: `gh api repos/jinmang2/wsad/pulls/22 -X PATCH -F body=@file.md`. +- Pre-existing: `scripts/extract_features.py` must run with `PYTHONPATH=.`; + `load_dataset(..., config_name=...)` kwarg kept as-was (verify against current `datasets`). + +## Quick commands +```bash +# tests +PYTHONPATH=. conda run -n balaenoptera python -m pytest -q +# backbone extraction (needs decord + GPU for clip/videomae) +PYTHONPATH=. conda run -n balaenoptera python scripts/extract_features.py --backbone clip +# figure smoke (see /tmp/gen_fig.py pattern): plot_anomaly_scores(scores, gt=[(s,e)], legend=False, save_path=...) +``` diff --git a/docs/NEXT_PHASE.md b/docs/NEXT_PHASE.md new file mode 100644 index 0000000..6b466f0 --- /dev/null +++ b/docs/NEXT_PHASE.md @@ -0,0 +1,65 @@ +# Next phase — feature extractor (primary, GPU) + Spec 2 efficiency (parallel, design) + +Motivation (proven this session, see `REPRO_FROMSCRATCH.md` VERDICT): the accuracy gap to +paper is a **feature-extraction ceiling**, not training/recipe/eval. The 2017 I3D features +carry the anomaly signal only weakly and atemporally — `feature_forensics.py` (content/probe +screen): `i3d_mgfn` content-AUC 0.56 / linear-probe 0.51, while a VideoMAE gate sample reaches +0.71 / 0.62 on the *same* videos. So the highest-leverage move is **better features**. + +⚠️ Gate caveat (corrected 2026-06-24): the forensic screen is a *relative* atemporal proxy, NOT +a ceiling — every per-snippet score (magnitude, content, probe) under-predicts the temporal +heads (i3d_mgfn probe 0.51 → head 0.83). Do NOT accept/reject a feature set on the oracle alone. +Use it to *rank* candidates cheaply; the real gate is a short head train (`run_matrix.py`). +The old "raw-magnitude AUC" headline was unsound (i3d_mgfn magnitude 0.46 yet head 0.83). + +## Track A — modern feature extractor (PRIMARY, GPU-serial) + +### Candidates (research 2026-06-24) +| extractor | why | access | note | +|---|---|---|---| +| **VideoMAE-v2 / VideoMAE-large** | strong self-sup video features, beats I3D | HF `MCG-NJU/videomae-large`; in Transformers | easiest pipeline | +| **OPear/videomae-large-finetuned-UCF-Crime** | already UCF-anomaly-adapted | HF | anomaly-tuned features = stronger magnitude signal? | +| **InternVideo2** | current SOTA video backbone | InternVideo repo | heaviest; check 8GB feasibility | +| **X-CLIP** | CLIP-aligned video → suits VLM heads | HF `microsoft/xclip-*` | pairs with vadclip/clip_tsa | +| pre-extracted UCF features | skips extraction compute | not found for these on UCF-Crime (only THUMOS/ANet via InternVideo) | likely must self-extract | + +Recent SOTA WSVAD (RefineVAD'25, TRACES, GV-VAD, AnomalyCLIP) ~0.87+ — confirms modern +features + better heads are where gains are. + +### Plan (each step gated) +1. **Pick + access** — start with VideoMAE-large (HF, lowest friction). Decide finetuned + (OPear) vs raw based on the forensic gate. +2. **Forensic screen (NEW, our edge)** — extract a SMALL balanced sample, run `scripts/diag/ + feature_forensics.py --dir ` on it. Does the set rank ABOVE `i3d_mgfn` on + content/probe-AUC? If it can't beat I3D even on the cheap atemporal screen, skip it. + If it ranks higher, proceed — but confirm with a short `run_matrix.py` head train before + committing to the full 1900-video extraction (the screen ranks, it does not certify). +3. **Extract** — adapt `scripts/extract_i3d_gowtham.py` pattern to a VideoMAE extractor + (10-crop or single-crop, snippet=16). Write `features/videomae_*/{train,test}` + PROVENANCE. +4. **Re-run matrix** (`run_matrix.py --variant videomae_*`) — measure lift vs i3d_mgfn. +5. **Head × feature mapping** — magnitude heads need magnitude-preserving features; VLM heads + need CLIP-align; memory heads need temporal richness. Tabulate which extractor wins which + head (novel contribution). + +### Open questions to resolve first +- VideoMAE output dim/snippet protocol (vs I3D 2048-d/16-frame) → head `feature_size` config. +- 8 GB extraction feasibility (VideoMAE-large inference, batch 1) + time for 1900 videos. +- Does anomaly-finetuned (OPear) leak test labels? (UCF test videos must NOT be in its + finetune set — verify before using, else it's leakage.) + +## Track B — Spec 2 efficiency / causality / inference (PARALLEL, design+CPU) + +Already have the toolkit (gradient checkpointing, `WSAD_ATTN=mem` SDPA attention, AMP). Spec 2 +makes the memory-heavy heads (UR-DMU/BN-WVAD/CLIP-TSA) run on 8 GB + real-time: +- **Efficient temporal attention** — sliding-window / linear / StreamingLLM attention-sink to + kill the O(T²) (already motivated by UR-DMU OOM); extend `src/modules/translayer.py` mem path. +- **Causal / streaming inference** — online scoring (no full-clip lookahead), KV-cache, chunked + prefill → causal-VAD for live surveillance. +- **Benchmark** — FPS / latency / peak-mem / AUC trade-off table across heads. + +Sequencing: GPU is 8 GB serial, so feature experiments take the GPU; Spec 2 runs as design + +CPU prototyping + the benchmark harness, then validates on whichever feature baseline wins. + +## Immediate next step (proposed) +Resolve Track A "open questions" (VideoMAE protocol + OPear leakage check + 8 GB feasibility), +then run the **forensic gate** on a VideoMAE sample before committing to full extraction. diff --git a/docs/NEXT_SESSION.md b/docs/NEXT_SESSION.md new file mode 100644 index 0000000..8357962 --- /dev/null +++ b/docs/NEXT_SESSION.md @@ -0,0 +1,47 @@ +# Next-session plan (kickoff) + +Trigger: say **"핸드오프대로 시작"** (or "NEXT_SESSION 진행"). I auto-load this via the +`next-session-plan` memory; this file is the human-readable copy. + +## Where we are (reproduction baseline, all committed) +| model | from-scratch | paper | port/eval verified | +|---|---|---|---| +| VadCLIP | 0.8654 | 0.8801 | ✅ | +| RTFM | 0.834 | 0.843 | ✅ | +| MGFN | 0.833 | 0.8667 | ✅ | +| S3R | 0.829 | 0.8599 | ✅ (learnable-dict cap) | +| UR-DMU | 0.797 | 0.8697 | ✅ official ckpt = 0.8697 EXACT on i3d_1024 | + +Infra done: lazy i3d load (RAM-OOM fix), per-crop eval (8GB GPU-OOM fix), DeepMIL +**1024-d** features (`i3d_1024_seg200`, UR-DMU/BN-WVAD) + `--feature-dim`, dual-report +(best + mean±std), cosine LR decay, `--eval-start/--eval-every`. 2048-d = `i3d_mgfn_seg200`. + +## Kick off 3 tracks at once — RESOURCE-AWARE (the "터질 수도 있으니 알아서 계획") +**Only ONE GPU-heavy run at a time** (8GB; 3 parallel GPU = OOM). CPU/doc track runs +in parallel. Use nohup + lazy load + stream logs (3 past session-crashes were 25GB +eager-load RAM-OOM — now fixed; keep it that way). Serialize the GPU queue: + +1. **[GPU, serial] Finish reproducing remaining heads** on the right features: + BN-WVAD (i3d_1024_seg200 + per-crop, --feature-dim 1024), Sultani / GS-MoE (2048 + i3d_mgfn_seg32), CLIP-TSA / TPWNG (CLIP). dual-report each. +2. **[GPU, serial] UR-DMU/BN-WVAD from-scratch training tuning** — close ~0.80→0.87. + The gap is the TRAINING loop (ckpt = 0.8697 exact proves data/eval/port faithful): + attack overfitting/early-peak — eval-every-10 (official) to catch the early peak, + early-stop, lr/wd, official 3000-step recipe. (NOT features — already correct.) +3. **[CPU/doc, parallel] Consolidation + README results table**: collapse + experiments/repro_vadclip.py + repro_i3d.py + build_* into ONE parametrized + reproduction harness (head/variant/recipe/dim-driven) — the user's anti-bloat + ask; delete superseded probes; write the paper-comparison results table into README. + +## Then Spec 2 (causality / real-time / efficiency) — INCLUDED, not deferred +Motivated by the UR-DMU O(T²) OOM: efficient temporal attention (sliding-window / +linear / streaming) so memory-heavy methods (UR-DMU/BN-WVAD/CLIP-TSA) run on commodity +8GB + real-time. See HANDOFF.md Spec-2 notes (StreamingLLM attention-sink, KV-cache, +chunked prefill, causal-VAD, FPS/AUC/latency benchmark). Spec 1-2 (new extractors like +VideoMAE) stays DEFERRED. + +## Don't re-derive (already settled this session) +- UR-DMU/BN-WVAD = 1024-d I3D (Carreira/pytorch-i3d, DeepMIL) ≠ 2048-d ResNet50-I3D + (RTFM/MGFN/S3R). Confirmed from ckpt (Conv1d 1024→512). +- random_perturb = np.linspace (no augmentation; no-op). +- The UR-DMU gap is training, not feature/eval/port (official ckpt = 0.8697 exact). diff --git a/docs/REPRO_FROMSCRATCH.md b/docs/REPRO_FROMSCRATCH.md new file mode 100644 index 0000000..bc2801c --- /dev/null +++ b/docs/REPRO_FROMSCRATCH.md @@ -0,0 +1,118 @@ +# From-scratch reproduction — results, levers, and the 8 GB ceiling + +Companion to `REPRO_STATUS.md` (which tracks **port fidelity** — every model loads its +official checkpoint and matches outputs). This doc tracks **from-scratch training** on +the clean cached features, on a single **RTX 2070 SUPER (8 GB, Turing sm_75)**. + +Regenerate the table any time: `PYTHONPATH=. python experiments/results_table.py`. + +## Results (UCF-Crime frame-level ROC-AUC, 290-video test) + +| head | feature | ours (from-scratch) | paper | port verified | +|------|---------|--------------------:|------:|:--:| +| sultani | i3d_mgfn_seg32 | 0.8071 | 0.7541 (C3D) | arch | +| rtfm | i3d_mgfn_seg32 | 0.8341 | 0.8430 | ✅ bit-exact | +| mgfn | i3d_mgfn_seg32 | 0.8332 | 0.8667 | ✅ ckpt | +| s3r | i3d_mgfn_seg32 | _queue_ | 0.8599 | ✅ bit-exact | +| ur_dmu | i3d_1024_seg200 | 0.8145 | 0.8697 | ✅ ckpt 0.8697 exact | +| bn_wvad | i3d_1024_seg200 | 0.8233 | 0.8724 | ✅ ckpt | +| gs_moe | i3d_mgfn_seg32 | _queue_ | 0.9160 | paper-only | +| clip_tsa | clip | _queue_ | 0.8758 | ✅ bit-exact | +| vadclip | clip | 0.8654 | 0.8801 | ✅ AUC 0.8802 exact | +| tpwng | clip | _queue_ | 0.8779 | paper-only | + +`_queue_` = serial GPU run in progress (`scripts/gpu_queue_pc.sh` + chained CLIP heads). +sultani > paper because the paper number is C3D-era; modern I3D features lift the +old MIL baseline. + +## Feature-provenance facts (don't re-derive — see the VERDICT below) +- Two distinct 2048-d I3D extractions exist: `features/i3d` = **DeepMIL/Roc-Ng** (L2~2.5) + and `i3d_mgfn` = **MGFN authors'** HKU OneDrive (L2~22, byte-verified). They are NOT + interchangeable: DeepMIL has **inverted** anomaly-magnitude (breaks RTFM/MGFN/S3R → + chance), so the magnitude heads must use **`i3d_mgfn`** (`i3d_mgfn_seg32` for seg32). +- The old `outputs/matrix/results.json` (feature=`None`, scoring ~0.85) trained on + now-DELETED `features/i3d/{train,test}/` npy dirs that mixed train=`_archive` (L2~21, + magnitude-correct) with test=DeepMIL (L2~2.6). It scored high because the TRAIN half had + the correct magnitude direction (content features transfer) — an artifact, not a better + set. Those numbers are not used here. +- UR-DMU/BN-WVAD use **1024-d** DeepMIL/Carreira I3D (`i3d_1024_seg200`, L2~8). + +## The memory-head ceiling (UR-DMU / BN-WVAD): ~0.82 from scratch +The port is faithful (official ckpt = 0.8697 exact). From-scratch nonetheless caps +~0.82 after exhausting every training lever: + +| lever | ur_dmu | bn_wvad | note | +|-------|-------:|--------:|------| +| batch 4 (old `HEAD_BATCH` cap) | 0.807 | 0.800 | batch 4 wrecks BN-WVAD's BatchNorm stats | +| batch 32 + toolkit + cosine | 0.8145 | **0.8233** | 8 GB batch ceiling | +| per-crop batch 64 (official layout) | 0.8114 | 0.8003 | faithful, but per-crop ↓ BN samples ⇒ hurts bn_wvad | + +- **batch** (4→32) is the biggest single lever — BN-WVAD is literally BatchNorm-based. + batch 64 (official) OOMs at seg200 even with the full toolkit (model body, not just + attention). 8 GB caps batch at 32. +- **per-crop training** (official lists each of the 10 crops as a separate sample, + 16100 rows; we crop-average to 1610) was the prime suspect but **did not help** — + ur_dmu flat, bn_wvad worse (its BatchNorm wants more flattened-crop samples, which the + stacked `(10,200,1024)` layout gives). Built as `i3d_1024_seg200_pc` for the record. +- Remaining ~0.05 gap is **training dynamics / seed** (runs oscillate 0.77–0.82; the + field reports best-of-N), not a structural bug. A seed sweep (official seed 2022) is + the only untried lever and would likely add ≤0.02 — still short of 0.87. + +## Training-scale toolkit (8 GB), all opt-in — `src/` +Verified the eager/fp32 path is unchanged (74 tests green; bit-exact/Δ checks): +- **gradient checkpointing** — `Transformer.gradient_checkpointing_enable()`, fwd/grad Δ=0. +- **mem-efficient attention** — `WSAD_ATTN=mem`: branch-1 SDPA + branch-2 einsum (drops + the `(b,h,T,T)` decay/`dots` materialization), Δ=7e-8 vs eager. +- **fp16/bf16 AMP** — `--mixed-precision`; `src/modules/amp.safe_bce` makes the BCE-on- + probability losses autocast-safe (no-op in fp32) across all 5 BCE heads. +- **gradient accumulation** — `--grad-accum` (note: does not enlarge BatchNorm's batch). + +CLI: `scripts/run_matrix.py --batch N --grad-checkpoint --mixed-precision fp16 --grad-accum K` +(+ `WSAD_ATTN=mem`). To reproduce the paper memory-head numbers, run the official +`--batch 64` on a ≥24 GB GPU (no toolkit needed). + +## VERDICT — the gap is a feature-extraction ceiling (ultragoal 2026-06-24) + +A thorough investigation (`.omc/ultragoal/`) settled why every head sits ~0.02-0.05 below +paper. It is the **I3D feature-extraction provenance ceiling**, not training/recipe/eval: + +1. **Eval sound** — UR-DMU ckpt → 0.8697 and VadCLIP ckpt → 0.8802 reproduce paper EXACTLY. +2. **Recipe ruled out** — RTFM (matching recipe) reproduces (−0.009). MGFN at the FULL + official recipe (15000 steps, batch 16, cosine) = 0.8197 and peaked at **step 200** then + overfit — 3× more training does not help. +3. **Feature ceiling proven** — the authors' OWN MGFN ckpt on the byte-verified MGFN features + (`i3d_mgfn`) scores only **0.8346** (paper 0.8667); our from-scratch MGFN **0.8332 = 99.8% + of that ckpt**. We reach what the features allow. +4. **Feature forensics** (`scripts/diag/feature_forensics.py`, rebuilt 2026-06-24) — the + feature cache carries the anomaly signal only *weakly* and *atemporally*. Three relative + proxies on the 290-vid test set (all crop-averaged, snippet-level): + + | set | MAGNITUDE-AUC | CONTENT-AUC | linear-PROBE-AUC | + |-----|--------------:|------------:|-----------------:| + | `i3d_mgfn` | 0.46 (near-chance) | 0.56 | 0.51 | + | `i3d_1024_seg200` | 0.48 | 0.62 | 0.55 | + | `videomae` (40-vid gate) | 0.50 | 0.71 | 0.62 | + + **Correction to the earlier verdict:** raw *magnitude*-AUC is NOT a feature-quality gate. + `i3d_mgfn` magnitude-AUC is 0.46 (mildly inverted, near chance) yet RTFM/MGFN train to 0.83 + on it — magnitude neither predicts nor bounds trainable performance. The real distinction + between `i3d_mgfn` and DeepMIL `features/i3d` for the magnitude heads is *scale/content*, + not a clean "magnitude direction". Likewise every atemporal proxy here (incl. the supervised + linear probe) under-predicts the temporal heads (probe 0.51 → head 0.83), because WSVAD is + dominated by temporal modeling these per-snippet scores can't see. + + What the proxies DO show, consistently and apples-to-apples (same 40 videos): **modern + features are more linearly separable** — `videomae` content/probe (0.71/0.62) clearly beats + `i3d_mgfn` (0.56/0.51). So "better features" remains the right lever, but for the + content-separability reason, and gated by a short head train, not by magnitude. + +Bottom line: the reproductions are faithful and reach the ceiling the cached I3D features +permit. The forensic screen says modern backbones (VideoMAE) carry a stronger, more separable +anomaly signal — the next lever — but the only honest accept/reject gate is a short +temporal-head train on the new set, NOT a training-free oracle. + +## Harness +- `experiments/reproduce.py` — single parametrized from-scratch harness (epoch recipes). +- `scripts/run_matrix.py` — {variant × head} matrix (step recipes, toolkit flags). +- `scripts/gpu_queue*.sh` — serial 8 GB queues (one heavy run at a time). +- `experiments/results_table.py` — regenerates the table above from result JSONs. diff --git a/docs/REPRO_STATUS.md b/docs/REPRO_STATUS.md index 31c9e1c..9bfe4aa 100644 --- a/docs/REPRO_STATUS.md +++ b/docs/REPRO_STATUS.md @@ -121,20 +121,41 @@ WebDataset/manifest path is still the scale plan for extraction outputs + AIHub. ## 3. I3D extraction pipeline -Two distinct I3D lineages — **do not mix their features** (scale differs ~10×): - -1. **tushar-n** (`src/i3d.py`, `src/features/i3d.py`) — torch-native, HF weights. - Already wired into `scripts/extract_features.py`. This is the repo default. -2. **GowthamGottimukkala / RTFM** (`pretrained/i3d/*.pkl`) — **Caffe2 blobs** - (`res_conv1_bn_*`, `nonlocal_conv*`; baseline 430 / nonlocal 540 blobs from - facebookresearch/video-nonlocal-net). Needs **Caffe2→PyTorch conversion** - before use. This is the lineage RTFM used for `UCF_*_ten_crop_i3d`. - -Plan: add a Caffe2→torch converter for the two pkls, build a ResNet-50 I3D -(+nonlocal) torch module, and reproduce the RTFM extract path; cross-check feature -stats against `UCF_Train_ten_crop_i3d` to confirm fidelity. Separately, the -`open_clip` ViT-B/16 extractor (FEATURE_EXTRACTORS.md Step B) reproduces the -`UCFClipFeatures` CLIP cache. +Three I3D lineages exist; **do not mix their features** (scale + content differ): + +1. **pytorchvideo `i3d_8x8_r50`** (`src/features/i3d.py` default) — SlowFast-lib + I3D, Kinetics 0.45/0.225 norm + torchvision TenCrop. A separate branch (needs + the `pytorchvideo` lib, now imported lazily). +2. **tushar-n baseline** (HF `converted_ref_i3d.pt`) — the baseline Caffe2 weights + pre-converted; loadable into `src.i3d.I3Res50`. +3. **GowthamGottimukkala / RTFM lineage** — facebookresearch/video-nonlocal-net + Caffe2 blobs (`pretrained/i3d/i3d_{baseline,nonlocal}_32x2_IN_pretrain_400k.pkl`, + 430 / 540 blobs). **The faithful extractor for the standard WSVAD I3D features.** + +**DONE 2026-06-14 — Gowtham-faithful pipeline (bit-exact, verified):** +- `scripts/convert_i3d_caffe2.py` converts both Caffe2 pkls → `pretrained/i3d/ + i3d_{baseline,nonlocal}_r50_kinetics.pth` (regex blob-rename ported verbatim from + Gowtham `utils/convert_weights.py`; baseline 265/318, nonlocal 325/383 params, + fc/num_batches_tracked legitimately skipped). +- `src/features/i3d_gowtham.py` mirrors Gowtham `extract_features.py` exactly + (resize **340×256** LANCZOS, **`(x*2/255)−1`**, exact 10-crop coords, ffmpeg→jpg, + snippet 16) → `(T,10,2048)`. `scripts/extract_i3d_gowtham.py` is the batch entry. +- `scripts/verify_i3d_extract.py`: our extractor == Gowtham's *original* code on the + same frames is **BIT-EXACT (max|Δ|=0)**. ~37 s + ffmpeg I/O per video. + +**RTFM's released `UCF_*_ten_crop_i3d` are NOT reproducible from this pipeline.** +Cross-check vs RTFM `_archive/Abuse001` (`scripts/diag_i3d_rtfm.py`, +`scripts/grid_i3d_rtfm.py`): nonlocal crop-avg cosine **0.74** (baseline 0.51). +Ruled out: 10-crop order (crop-avg also 0.74), temporal offset (shift 0 optimal, +flat per-snippet cos — no drift), BGR channel (0.67, worse), and the interpolation +sweep at the geometry-forced 340×256 (lanczos 0.74 / bicubic 0.75 / bilinear 0.76 — +resize is NOT a free variable: the 10-crop coords `16:240,58:282,…` require a +256-tall frame). Everything plateaus at ~0.74–0.76. The flat consistent gap ⇒ +**RTFM used a different / unpublished I3D checkpoint or extractor** (closest to +nonlocal + bilinear). Since the MGFN `features/i3d/*.zip` (L2~2.5) are +our canonical set and RTFM's are archived/incompatible, exact RTFM-byte repro is +moot — the deliverable is the verified Gowtham-faithful extractor (use **nonlocal** +for new data / AIHub). Separately, `open_clip` ViT-B/16 reproduces the CLIP cache. ## 4. Dataset strategy (open decision — needs your call) diff --git a/docs/RESEARCH_ROADMAP.md b/docs/RESEARCH_ROADMAP.md new file mode 100644 index 0000000..b43072d --- /dev/null +++ b/docs/RESEARCH_ROADMAP.md @@ -0,0 +1,55 @@ +# WSVAD research roadmap (performance track) + +Where we are and where the gains are, for UCF-Crime weakly-supervised video anomaly +detection. Grounded in the 2024–25 landscape + this repo's assets. + +## Landscape (UCF-Crime frame-level AUC) + +| Era | Approach | AUC | Note | +|-----|----------|-----|------| +| 2018 | Sultani MIL (C3D/I3D) | ~75–77 | the original baseline | +| 2021 | RTFM (I3D) | 84.3 | feature-magnitude MIL | +| 2022–23 | MGFN / S3R / UR-DMU (I3D) | 85–86 | I3D-feature ceiling | +| 2024 | DAKD, fusion (I3D+S3D+**CLIP**) | ~87 | **CLIP backbone + fusion is the lever** | +| 2023+ | VadCLIP / text-aligned (CLIP) | 86–88 | VLM branch, prompts | +| 2025 | MLLM / zero-shot reasoning | — | frontier; heavier, explainable | + +**Takeaways:** (1) the I3D-only ceiling is ~86; (2) **CLIP features + multi-backbone +fusion** is the proven, cheap step up; (3) stronger visual backbones (VideoMAEv2, +InternVideo2) raise the visual ceiling; (4) the research frontier is VLM/MLLM. + +## Assets in this repo (verified) + +- **I3D (canonical)**: MGFN `features/i3d/*.zip` (L2~2.5). MGFN head local ROC-AUC + ~0.82 (paper 0.867). 7 heads output-verified, 3 paper-faithful. +- **I3D (faithful extractor)**: `src/features/i3d_gowtham.py` bit-exact to Gowtham, + baseline+nonlocal converted weights — re-extract any data (AIHub) self-consistently. +- **CLIP**: VadCLIP `features/clip/` (OpenAI ViT-B/16), VadCLIP head trains. +- **VideoMAE**: `src/features/videomae.py` (v1 ViT-B, (T,768)) — implemented, validated. + +## Experimental ladder (ROI-ordered) + +1. **Backbone ablation on existing heads (I3D vs CLIP)** — *no new extraction*. + Train/eval the same head (Sultani/RTFM) on `data.backbone=i3d` vs `clip`, compare + test AUC. Establishes the in-repo ablation harness + a baseline table. ← START HERE +2. **VideoMAE-B features for UCF + ablation** — needs a batch extraction entry from + the raw zips (~3.7h on RTX2070S) then retrain a visual head. Measures v1 ViT-B vs I3D. +3. **VideoMAEv2 loader** — OpenGVLab checkpoints are NOT transformers-native; needs a + custom loader (or a converted HF export). Higher visual ceiling than v1. +4. **Multi-backbone fusion (I3D + CLIP)** — concat/late-fuse features into one head; + the landscape's best single step. Reuses assets we already have. +5. **VLM/MLLM frontier** — prompt/reasoning heads on CLIP-aligned features; longer-term. + +## Tracked gaps / TODO + +- [ ] Batch VideoMAE extraction entry from raw `Anomaly-Videos-*.zip` (mirror + `extract_i3d_gowtham.py`; needs a zip→video read or an unzip step). +- [ ] VideoMAEv2 (OpenGVLab) loader — transformers `VideoMAEModel` won't load it as-is. +- [ ] Ablation harness: one entry that trains+evals a head on a chosen `data.backbone` + and prints test ROC-AUC, for an apples-to-apples backbone table. + +## Reference + +See `docs/REPRO_STATUS.md` (faithful-port status, I3D lineages), `docs/DATA_LOCAL.md` +(layout, manifest). Landscape sources: RTFM (arXiv 2101.10030), DAKD (2406.02831), +VAD survey (2405.10347), "Evolution of VAD: DNN→MLLM" (2507.21649). diff --git a/docs/RESULTS_TABLE.md b/docs/RESULTS_TABLE.md new file mode 100644 index 0000000..1354ead --- /dev/null +++ b/docs/RESULTS_TABLE.md @@ -0,0 +1,16 @@ +# UCF-Crime reproduction — ours vs paper (clean features) + +Frame-level ROC-AUC on the 290-video test split. `*` = official checkpoint. + +| head | feature | ours | paper | Δ | source | +|---|---|---|---|---|---| +| sultani | i3d_mgfn_seg32 | 0.8071 | 0.7541 | +0.0530 | `outputs/matrix_sultani_clean/results.json` | +| rtfm | i3d_mgfn_seg32 | 0.8341 | 0.8430 | -0.0089 | `outputs/matrix_rtfm_cosine/results.json` | +| mgfn | i3d_mgfn_seg32 | 0.8332 | 0.8667 | -0.0335 | `outputs/matrix_mgfn_freq/results.json` | +| s3r | i3d_mgfn_seg32 | _pending_ | 0.8599 | — | — | +| ur_dmu | i3d_1024_seg200 | 0.8145 | 0.8697 | -0.0552 | `outputs/matrix_urdmu_1024_b32cos/results.json` | +| bn_wvad | i3d_1024_seg200 | 0.8233 | 0.8724 | -0.0491 | `outputs/matrix_bnwvad_1024_b32full/results.json` | +| gs_moe | i3d_mgfn_seg32 | 0.7838 | 0.9160 | -0.1322 | `outputs/matrix_gs_moe_clean/results.json` | +| clip_tsa | clip | 0.8123 | 0.8758 | -0.0635 | `outputs/matrix_clip_tsa_clip/results.json` | +| vadclip | clip | 0.8654 | 0.8801 | -0.0147 | `experiments/runs/vadclip_scratch/seed234/result.json` | +| tpwng | clip | _pending_ | 0.8779 | — | — | diff --git a/docs/SPEC2_EFFICIENCY.md b/docs/SPEC2_EFFICIENCY.md new file mode 100644 index 0000000..1c15fc3 --- /dev/null +++ b/docs/SPEC2_EFFICIENCY.md @@ -0,0 +1,68 @@ +# Spec 2 — efficient / causal / streaming WSVAD (design) + +Parallel design track (the user's "feature 먼저, 효율 병렬"). Goal: make the memory-heavy +heads (UR-DMU / BN-WVAD / CLIP-TSA) run on 8 GB **and in real time / causally**, with a +measured AUC-vs-efficiency trade-off. Builds on the toolkit already shipped this effort: +`WSAD_ATTN=mem` (SDPA + einsum, `src/modules/translayer.py:83-89`), gradient checkpointing +(`Transformer.gradient_checkpointing_enable`), fp16 AMP, `src/modules/amp.safe_bce`. + +## Motivation +- The dual-branch temporal Transformer is **O(T²)** (`translayer.py:91` `q·kᵀ`; the decay + `attn2` is `(n,n)`). At seg200 / full-length test it OOMs 8 GB (the whole reason `mem` + and per-crop eval exist). For live surveillance T is effectively unbounded. +- Current scoring is **bidirectional + full-sequence** (`src/eval_matrix.py` scores the + whole clip at once) → no online/causal operation, no bounded latency. + +## Key structural lever (why this head is unusually amenable) +`translayer.Attention` is **dual-branch**: +- **branch-2** = a FIXED distance-decay `exp(-|i-j|/e)`, `e=e¹≈2.718` → it is already + ~0 beyond `|i-j|≳15`. So restricting branch-2 to a local **window** is *near-lossless* + (the tail it drops is numerically negligible). Only **branch-1** (learned `qkᵀ`) is + approximated by windowing. This makes a sliding-window variant far cheaper in accuracy + than for a generic full-attention Transformer. + +## Design — opt-in attention variants (extend the `WSAD_ATTN` flag) +Add to `translayer.py` alongside `eager`/`mem` (same env-flag pattern, default unchanged +so verified ckpts stay bit-exact): +1. **`WSAD_ATTN=window` (sliding-window, band W≈64)** — O(T·W). + - branch-1: `F.scaled_dot_product_attention(q,k,v, attn_mask=band)` (band/local mask). + - branch-2: compute the decay only within the band (near-exact, see above). + - Recommended default for long sequences; smallest accuracy hit. +2. **`WSAD_ATTN=linear` (linear attention, O(T))** — feature-map `φ(q)(φ(k)ᵀv)` for + branch-1; branch-2 stays the (cheap, local) decay. Most aggressive; validate AUC drop. +3. **`WSAD_ATTN=sink` (StreamingLLM attention-sink)** — keep the first `s` "sink" tokens + + a recent window; enables *unbounded* streaming without re-encoding history. + +## Design — causal & streaming inference +- **Causal mask**: score each frame from past-only (triangular mask on branch-1; branch-2 + decay restricted to `j≤i`). Needed because surveillance is online. Expect a small AUC + drop vs bidirectional; quantify it. +- **`StreamingScorer`** (new, `src/inference/streaming.py`): process a video in chunks of + `C` frames, keep a KV-cache of the last `W` keys/values (+ `s` sink tokens), emit + per-frame anomaly scores online. Peak memory = `O(W)`, latency = `C` frames. Wraps any + head whose backbone is the translayer (UR-DMU/BN-WVAD); CLIP-TSA's TSA gets the same + windowing. + +## Benchmark (`scripts/diag/bench_efficiency.py`) +For each `(head ∈ {ur_dmu, bn_wvad, clip_tsa}) × (attn ∈ {eager, mem, window, linear, sink}) +× (mode ∈ {bidir, causal-stream})`, measure on the UCF-Crime test set: +- **AUC** (frame-level ROC) — the quality axis. +- **FPS** (frames/sec), **per-chunk latency** (ms), **peak GPU mem** (MB), max T before OOM. +Emit a markdown table: the efficiency frontier (AUC vs FPS/mem). Headline target: UR-DMU/ +BN-WVAD running full-length on 8 GB at real-time FPS with ≤1 pt AUC drop vs `eager`. + +## Implementation order (small, verifiable steps) +1. `WSAD_ATTN=window` in `translayer.Attention.forward` (band SDPA + banded decay). Verify + max|Δ| vs `eager` is tiny (branch-2 near-exact) and AUC drop ≤ ~0.5 pt on UR-DMU eval. +2. `bench_efficiency.py` (AUC + FPS + peak-mem table) — quantify `window`/`linear` vs + `eager`/`mem`. +3. `causal` mask variant + AUC delta. +4. `StreamingScorer` + the streaming FPS/latency numbers. + +## Validation +- Numerics: `window` vs `eager` max|Δ| (expect branch-2 ≈0, branch-1 = the approximation). +- Quality: AUC drop per variant on the existing UR-DMU/BN-WVAD test eval (reuse + `src.eval_matrix.evaluate`). +- Efficiency: the FPS / peak-mem / max-T table; confirm full-length on 8 GB without per-crop + splitting (which `mem`/per-crop currently work around). No change to the `eager` default, + so all verified checkpoints and reproduced numbers are untouched. diff --git a/docs/WSVAD_REPRO_RECIPES.md b/docs/WSVAD_REPRO_RECIPES.md new file mode 100644 index 0000000..d2f5cdd --- /dev/null +++ b/docs/WSVAD_REPRO_RECIPES.md @@ -0,0 +1,54 @@ +# WSVAD Reproduction — verified per-method recipes & findings + +Source of truth for training/eval reproduction. Recipes verified against vendored +official code in `.reference/` (sciomc multi-agent audit, 2026-06-17). Feed +`scripts/run_matrix.py` (`HEADS` dict) + `src/trainer.py` (`WSVADTrainer.fit_steps`). + +## Key findings (corrected understanding) + +1. **"step-based" is NOT a property of MIL.** It is the iteration-counting convention + in the official repos: `--lr '[0.001]*15000'` is an `eval()`-d list of 15000 LRs → + 15000 gradient steps. MIL's only essential is a balanced normal+abnormal pairing per + update. 15000 steps ≈ ~600 reshuffled epochs on UCF (~810 abn + 800 nor, batch 32). +2. **No Gaussian smoothing anywhere.** RTFM/S3R/UR-DMU/BN-WVAD/CLIP-TSA/VadCLIP all do + only `np.repeat(score, 16)` (snippet→frame) then sklearn AUC. Do NOT add smoothing. +3. **Best-checkpoint on test AUC is the standard** (every repo evals during training and + keeps the best). Not leakage — it's the published protocol. (BN-WVAD selects on AP.) +4. **The reproduction gap is FEATURE PROVENANCE, not recipe.** On consistent Tushar-N + I3D features, RTFM tops ~0.80–0.82 regardless of weight_decay (0.0005 vs official + 0.005 made ~no difference). The paper's 0.843 is on the authors' exact I3D (`mix_5c`) + extraction. → faithful reproduction requires controlling feature extraction + end-to-end (re-extract train+test with one verified extractor). [confirmed empirically] +5. **weight_decay paper-vs-code mismatches** (use the CODE value — it produced the + reported number): RTFM/S3R/CLIP-TSA code = 0.005 (paper text often says 0.0005); + UR-DMU/BN-WVAD = 5e-5. Our configs originally inherited a flat 0.0005 (wrong). + +## Per-method recipe (UCF-Crime, I3D unless noted) + +| method | steps/epochs | batch (nor+abn) | num_seg train | lr | wd | schedule | best-ckpt | reported | source | +|---|---|---|---|---|---|---|---|---|---| +| RTFM | 15000 steps | 32+32 | 32 | 1e-3 const | **0.005** | none | AUC (eval from step 200) | 0.8430 | .reference/RTFM main.py:41 | +| S3R | 15000 steps | 32+32 | 32 | 1e-3 const | **0.005** | none, **eval warmup 5000** | AUC | 0.8599 | .reference/S3R trainval:135 | +| MGFN | ~5000 steps | 8+8 (×10crop) | 32 | 1e-3 | 5e-4 | none | AUC | 0.8667 (ours ckpt 0.8208) | paper (no .ref) | +| UR-DMU | 3000 steps | 64+64 | **200** (test 32) | 1e-4 const | **5e-5** | none | AUC | 0.8697 | .reference/UR-DMU ucf_main:55 | +| BN-WVAD | 1000 steps | 64+64 | **200** | 1e-4 const | **5e-5** | **grad_clip 1.0** | **AP (not AUC)** | 0.8275 | .reference/BN-WVAD train.py:16 | +| Sultani (MIL) | ~30 ep | 30+30 | 32 | 1e-3 (Adam; paper Adagrad 0.01) | 5e-4 | none | AUC | 0.75 (C3D)/~0.83 (I3D) | paper (no .ref) | +| CLIP-TSA | 4000 steps | 16+16 | 32 | 1e-3 const | **0.005** | none | AUC (eval from 150) | ~0.87 | .reference/CLIP-TSA main:122 | +| VadCLIP | 10 epochs | 64+64 | 256 (window) | 2e-5 | 0.01 (AdamW) | MultiStepLR [4,8] γ0.1 | **ROC1 = sigmoid(logits1)** | 0.8801 | .reference/VadCLIP ucf_train:47 | +| GS-MoE | ~30–50 ep [est] | 64+64 | 200(paper)/32(ours) | 1e-4 [est] | 5e-4 | none | AUC | 0.916 | paper (no .ref) | + +Notes: UR-DMU/BN-WVAD num_seg=200 (our i3d features are 32-seg → mismatch; also i3d-OOM +on 8 GB GPU). VadCLIP eval uses `sigmoid(logits1)`=ROC1 (0.880); `1-softmax(logits2)[:,0]` +=ROC2 is ~0.857 (don't use). S3R uses an offline OMP dictionary (ours is learnable → caps +AUC). MGFN/Sultani/GS-MoE have no vendored reference (recipe from paper/estimate). + +## Eval protocol (all methods) +Per-snippet score → (10-crop mean for I3D test) → `np.repeat(score, 16)` to frame level → +concatenate all test frames → `sklearn.roc_auc_score(gt, preds)`. No post-processing. +Our impl: `src/eval_matrix.py` (per-head shaping; vadclip windowed; visual heads 4-D crop layout). + +## Hardware constraints (this box: RTX 2070 8 GB GPU + 15 GB WSL RAM) +- i3d features: lazy zip load + `num_workers=0` (zip handle not shareable across workers → + BadZipFile). Eager-loading 1610 npy = ~12.8 GB RAM → WSL crash. ONE i3d job at a time. +- NEVER CPU-eval-fallback for big heads (clip_tsa full-length test attention >20 GB → WSL OOM kill). +- i3d UR-DMU/BN-WVAD/CLIP-TSA eval OOMs on 8 GB (full-length test attention) → CLIP versions OK. diff --git a/docs/superpowers/specs/2026-06-16-wsvad-matrix-serving-design.md b/docs/superpowers/specs/2026-06-16-wsvad-matrix-serving-design.md new file mode 100644 index 0000000..3f212c1 --- /dev/null +++ b/docs/superpowers/specs/2026-06-16-wsvad-matrix-serving-design.md @@ -0,0 +1,154 @@ +# WSVAD comparison framework — multi-backbone matrix, HF-style serving, streaming inference + +**Status:** approved design (Spec 1 detailed; Specs 2–3 scoped) +**Date:** 2026-06-16 +**Branch:** `feat/eval-matrix-serving` + +## Goal (decisive) + +Turn this repo from a *paper-reproduction* codebase into a **WSVAD comparison + +serving framework**: + +1. Implement every WSVAD method as an HF-style, easy-to-serve model. +2. Run them on **real video input** end-to-end (no separate offline extraction + step at serving time — the extractor is absorbed like an HF *processor*). +3. Produce **paper-style test figures** (where an anomaly was detected vs ground + truth) and **real-time inference**. + +AIHub CCTV is *one input source*, not the goal. The goal is the framework. + +## Why this is a separate effort + +Distinct role from `feat/data-pipeline` (which builds the extractors + data +layout this work consumes). Managed as its own branch + epic + per-component +issues + PR. + +--- + +## Spec roadmap (dependency order) + +| Spec | Scope | Deliverable | +|---|---|---| +| **Spec 1 — Foundation** (this doc, detailed) | backbone-agnostic extraction · dim-agnostic heads · matrix harness · HF-style pipeline (offline mode) | `{backbone × head}` AUC table + `raw video → scores + basic figure` | +| **Spec 2 — Streaming/Causal** | inference adapter (window/causal) · causal-finetune recipe · **LLM-inference-technique research track** · real-time benchmark | real-time inference + causal checkpoints | +| **Spec 3 — Qualitative suite** | full paper-style figures · localization metrics · cross-matrix comparison dashboard | qualitative-eval module | + +Architectural invariant across all specs: **original bidirectional training + +offline eval is never modified** (source of truth for the matrix + accurate +figures). Streaming/causal is an *additive inference layer*, never a rewrite of +model definitions. + +--- + +## Spec 1 — Foundation (detailed) + +### Architecture + +Three decoupled layers + a bundling layer on top: + +``` +[backbone registry] ─┐ + ├─► pipeline(backbone, head) ──► raw video → per-frame scores → figure +[head registry] ──┘ (HF-style bundle factory) + ▲ +[matrix harness] ── crosses both registries → {backbone × head} train/eval table +``` + +Internals stay **fully decoupled** (matrix needs swappability); the surface is a +**bundle** (HF-style serving). Both requirements satisfied by one factory. + +### Components + +**① Backbone-agnostic extraction** +- `scripts/extract_features.py` → `--backbone {i3d,clip,videomae,videomaev2,internvideo2}` + dispatcher (roadmap step A). Reuse `build_extractor()`; cache `*_.npy`; + keep `segment_features` unchanged (dim-agnostic). Legacy I3D path stays byte-identical. +- **New loaders:** `internvideo2.py`, `videomaev2.py`. Both ship as OpenGVLab + native weights (not transformers-native) → custom loader (prefer an HF export + if one validates). `dim` read from model config so downstream adapts. +- Reuse existing dataset-keyed layout + unified manifest from `feat/data-pipeline`. + +**② Dim-agnostic heads + compatibility guard** +- **Fix MGFN hardcoded 2048** (`src/models/mgfn/modeling_mgfn.py:83`, + `x[:, :2048] / x[:, 2048:]`) → derive the magnitude split from + `config.feature_size`. Other heads already read `feature_size` (verified by grep: + Sultani, GS-MoE, BN-WVAD, VadCLIP). +- **text-align guard:** each head declares `requires_text_aligned: bool`. The + pipeline factory checks it against `extractor.text_aligned` and rejects + incompatible pairs (e.g. VadCLIP + VideoMAE) with a clear error. CLIP and + InternVideo2 are text-aligned → pass both visual-only and VLM heads. + +**③ Matrix harness** +- One entry (`scripts/run_matrix.py` or `run.py` extension): `(backbone, head, + dataset)` → train + eval → log test ROC-AUC + artifacts. +- Auto-selects only compatible pairs (derived from ② guard) → emits the + `{backbone × head}` table. This is the body of "compare all methods". + +**④ HF-style pipeline (processor absorbed)** +- `AnomalyDetectionPipeline(backbone, head_ckpt)`: holds extractor (= HF + processor role) + trained head + post-process. Input: raw video → output: + per-frame anomaly scores (+ figure). No separate extraction step at serve time. +- Spec 1 implements **`mode="offline"`** only (bidirectional, exact = current). + `window`/`causal` extend the *same* interface in Spec 2. + +**⑤ Minimal figure (pipeline liveness)** +- One per-frame score curve + GT anomaly-region shading. Full suite is Spec 3; + this is just visual proof the pipeline runs end-to-end. + +### Data flow + +`raw video → extractor(processor) → [T, dim] features → head(mode=offline) → +[T] per-frame scores → postprocess → score curve + figure`. +Training flow unchanged: `cached *_.npy → head → ROC-AUC`. + +### Error handling +- Incompatible `{backbone, head}` (text-align): explicit `ValueError` at factory + time, never a silent shape error. +- Dim mismatch: `feature_size` is the single source; heads slice the appended + magnitude channel relative to it; assert on load. +- Missing weights / loader: `strict=True` load after key remap (existing + convention in `WEIGHTS_AND_OPTIMIZATION.md`). + +### Testing +- Dim-agnostic forward smoke test per head with dummy features (2048/512/768). +- **MGFN numerical-equivalence regression** after the 2048 fix + (`verify_numerical_equivalence`, eager/fp32/TF32-off) so reproduction stays + bit-exact. +- Compatibility guard unit test (incompatible pair raises). +- Pipeline test: short sample video → assert score length == frame/snippet count. + +### Repo changes +- **Modify:** `scripts/extract_features.py`, `src/models/mgfn/*` (2048), + per-head `requires_text_aligned`. +- **New:** `src/features/internvideo2.py`, `src/features/videomaev2.py`, + `src/pipeline.py` (bundle factory), `scripts/run_matrix.py`, minimal figure util. +- **Unchanged:** all original training entries + model definitions (additive only). + +--- + +## Spec 2 preview — streaming/causal (research-driven) + +Non-causality has two sources: **(A) training-time loss construction** (top-k, +32-seg norm, MIL max — exists only at train time, *not* an inference problem) and +**(B) inference-time temporal context** (RTFM MTN / MGFN glance-focus / UR-DMU / +GS-MoE global attention — each snippet score depends on future). The score head +is per-snippet; only context is bidirectional → conversion is *not* fatal. + +Mitigation spectrum (zero-retrain → clean): +1. **Sliding window** (no retrain, all heads) — window W, emit last/center, slide. +2. **Causal mask + cheap finetune on cached features** — removes discrepancy; + nearly free because features are cached. +3. **Stateful streaming** (KV-cache-like state + dilated-conv ring buffer) — true + online, constant per-step cost; needs (2). + +**Research track:** map LLM inference-optimization techniques (StreamingLLM +attention sinks, sliding-window attention, KV-cache, chunked prefill, +quantization) onto causal WSVAD — an under-explored space; documenting + +validating these is a genuine contribution. Train/inference discrepancy is real +and measured, not assumed. + +--- + +## Sources +See `docs/FEATURE_EXTRACTOR_RESEARCH.md` for backbone/data/optimization research +and citations (RTFM, VideoMAEv2, InternVideo2, LAION, Real-Time WSVAD WACV 2024). diff --git a/experiments/repro_i3d.py b/experiments/repro_i3d.py new file mode 100644 index 0000000..84293d6 --- /dev/null +++ b/experiments/repro_i3d.py @@ -0,0 +1,90 @@ +"""From-scratch reproduction for seg32 I3D heads (MGFN/RTFM/S3R/Sultani). + +Uses the PROVEN epoch-based recipe (scripts/diag/train_archive_probe.py reached +MGFN 0.8667 / RTFM 0.8448): Adam lr 5e-5, batch 16, ~12-15 epochs, best-checkpoint +on test AUC — NOT run_matrix's unverified lr 1e-3 step-based recipe, which diverges +to chance (0.52). Trains on the clean MGFN-lineage features (norm ~22): + + features/i3d_mgfn_seg32/ (train seg32 (10,32,2048) + test full-length (T,10,2048), + built from the byte-verified MGFN-authors' distribution by build_mgfn_seg32.py). + + PYTHONPATH=. WSAD_DATA=~/data/wsad conda run -n balaenoptera \ + python experiments/repro_i3d.py # HEAD=mgfn default + HEAD=rtfm LR=5e-5 EPOCHS=15 python experiments/repro_i3d.py +""" + +import json +import os +import random +import time + +import numpy as np +import torch +from omegaconf import OmegaConf + +import src.models # noqa: F401 (register heads) +from scripts.run_matrix import _build_model +from src.eval_matrix import evaluate +from src.trainer import WSVADTrainer, build_datasets + +HEAD = os.environ.get("HEAD", "mgfn") +VARIANT = os.environ.get("VARIANT", "i3d_mgfn_seg32") +LR = float(os.environ.get("LR", 5e-5)) +EPOCHS = int(os.environ.get("EPOCHS", 15)) +BATCH = int(os.environ.get("BATCH", 16)) +SEED = int(os.environ.get("SEED", 0)) +OUT = f"experiments/runs/{HEAD}_{VARIANT}/seed{SEED}" + +PAPER = {"mgfn": 0.8667, "rtfm": 0.8430, "s3r": 0.8599, "sultani": 0.83} + + +def set_seed(s): + random.seed(s); np.random.seed(s); torch.manual_seed(s); torch.cuda.manual_seed_all(s) + + +def main(): + os.makedirs(OUT, exist_ok=True) + set_seed(SEED) + device = "cuda" if torch.cuda.is_available() else "cpu" + t0 = time.time() + + data_cfg = OmegaConf.create({ + "root": os.environ.get("WSAD_DATA", "~/data/wsad"), "source": "local", + "backbone": "i3d", "dataset_dir": "ucf_crime", "feature_variant": VARIANT, + "ground_truth": None, "dynamic_load": False, "batch_size": 1, + "frames_per_clip": 16, "num_workers": 4, "clip_length": None, "segment": None, + "single_crop": True, "with_magnitude": True, + }) + train_sets, test_set = build_datasets(data_cfg) + model, _, wd = _build_model(HEAD, "i3d", device) + n_train = len(train_sets["normal"]) + len(train_sets["abnormal"]) + print(f"[data] variant={VARIANT} train={n_train} test={len(test_set)}", flush=True) + print(f"[recipe] {HEAD} Adam lr={LR} wd={wd} batch={BATCH}(+{BATCH}) epochs={EPOCHS} seed={SEED}", flush=True) + + log = [] + + def eval_fn(m): + res = evaluate(m, test_set, backbone="i3d", head=HEAD, device=device) + log.append({"roc_auc": res["roc_auc"], "pr_auc": res["pr_auc"]}) + json.dump(log, open(f"{OUT}/epoch_log.json", "w"), indent=2) + return res + + trainer = WSVADTrainer(model=model, learning_rate=LR, weight_decay=wd, batch_size=BATCH, + num_workers=4, frames_per_clip=16, mixed_precision="no") + best = trainer.fit(train_sets, epochs=EPOCHS, eval_fn=eval_fn, select_metric="roc_auc") + + result = { + "head": HEAD, "variant": VARIANT, "source": "from-scratch", + "recipe": {"optimizer": "adam", "lr": LR, "wd": wd, "batch_total": 2 * BATCH, + "epochs": EPOCHS, "seed": SEED}, + "best_roc_auc": round(best["roc_auc"], 4), "best_pr_auc": round(best["pr_auc"], 4), + "best_epoch": best["best_epoch"], "last_roc_auc": round(best["last"], 4), + "paper_roc_auc": PAPER.get(HEAD), "n_test": len(test_set), + "seconds": round(time.time() - t0, 1), + } + json.dump(result, open(f"{OUT}/result.json", "w"), indent=2) + print("\n=== result ===\n" + json.dumps(result, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/repro_vadclip.py b/experiments/repro_vadclip.py new file mode 100644 index 0000000..108e839 --- /dev/null +++ b/experiments/repro_vadclip.py @@ -0,0 +1,115 @@ +"""Faithful VadCLIP (AAAI'24) from-scratch reproduction on UCF-Crime CLIP features. + +Goal: confirm our verified VadCLIP *port* (bit-exact to official under weight load, +ROC1=0.8801) also reproduces the paper number when trained FROM SCRATCH with the +official recipe — distinct from the eval-only `--official` matrix cell. + +Official recipe (.reference/VadCLIP/src/ucf_train.py + ucf_option.py, verified): + AdamW(lr=2e-5) [AdamW default weight_decay=0.01] + MultiStepLR(milestones=[4, 8], gamma=0.1) + max_epoch=10, batch_size=64 (per normal/abnormal loader -> 128 total), seed=234 + windowed visual_length=256, per-window feat_lengths, eval = sigmoid(logits1)=ROC1. + +This drives the canonical `src.trainer.WSVADTrainer` (now with optimizer_name/ +scheduler knobs) + the per-head windowed eval in `src.eval_matrix.evaluate`. +Reproducible: fixed seed, recipe in-code, results + per-epoch log written under +`experiments/runs/vadclip_scratch/`. + + PYTHONPATH=. WSAD_DATA=~/data/wsad conda run -n balaenoptera \ + python experiments/repro_vadclip.py + +KNOWN RISK (open): our training loss path does not yet thread per-window +`feat_lengths` into CLAS2/CLASM masking + length-based top-k. If from-scratch +ROC1 lands well below 0.88, that masking is the prime suspect to fix next. +""" + +import json +import os +import random +import time + +import numpy as np +import torch + +import src.models # noqa: F401 (register heads) +from scripts.run_matrix import _build_model, _data_cfg, HEADS +from src.eval_matrix import evaluate +from src.models.vadclip.modeling_vadclip import load_pretrained_clip_text +from src.trainer import WSVADTrainer, build_datasets + +SEED = int(os.environ.get("REPRO_SEED", 234)) # official default 234; sweep for variance +MAX_EPOCH = int(os.environ.get("REPRO_EPOCHS", 10)) # override for smoke (REPRO_EPOCHS=1) +BATCH = 64 # per loader (normal / abnormal); official batch_size +LR = 2e-5 +WEIGHT_DECAY = 0.01 # AdamW default (official passes no weight_decay) +MILESTONES = [4, 8] +GAMMA = 0.1 +OUT = f"experiments/runs/vadclip_scratch/seed{os.environ.get('REPRO_SEED', 234)}" + + +def set_seed(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + set_seed(SEED) + device = "cuda" if torch.cuda.is_available() else "cpu" + t0 = time.time() + + spec = HEADS["vadclip"] + data_cfg = _data_cfg("clip", spec) + # official trains on all 10 crops as separate samples (16100 rows); test single-crop + data_cfg.train_all_crops = True + train_sets, test_set = build_datasets(data_cfg) + model, _, _ = _build_model("vadclip", "clip", device) + # from-scratch == random head + FROZEN pretrained CLIP text tower (official freezes + # CLIP). Without this the text-alignment branch is random -> ROC stuck at chance. + n_clip = load_pretrained_clip_text(model) + print(f"[init] loaded + froze {n_clip} pretrained CLIP text-tower tensors", flush=True) + + n_train = len(train_sets["normal"]) + len(train_sets["abnormal"]) + print(f"[data] train={n_train} (normal {len(train_sets['normal'])} + " + f"abnormal {len(train_sets['abnormal'])}) test={len(test_set)}", flush=True) + print(f"[recipe] AdamW lr={LR} wd={WEIGHT_DECAY} MultiStepLR{MILESTONES}x{GAMMA} " + f"epochs={MAX_EPOCH} batch={BATCH}(+{BATCH}) seed={SEED}", flush=True) + + log = [] + + def eval_fn(m): + res = evaluate(m, test_set, backbone="clip", head="vadclip", device=device) + log.append({"roc_auc": res["roc_auc"], "pr_auc": res["pr_auc"]}) + json.dump(log, open(f"{OUT}/epoch_log.json", "w"), indent=2) + return res + + trainer = WSVADTrainer( + model=model, learning_rate=LR, weight_decay=WEIGHT_DECAY, batch_size=BATCH, + num_workers=int(data_cfg.num_workers), frames_per_clip=16, mixed_precision="no", + optimizer_name="adamw", scheduler_milestones=MILESTONES, scheduler_gamma=GAMMA, + ) + # official evals at step % 1280 == 0 (step = iter * batch * 2 = iter * 128) -> every + # 10 optimizer iters, keeping the best checkpoint across all eval points. + best = trainer.fit(train_sets, epochs=MAX_EPOCH, eval_fn=eval_fn, + select_metric="roc_auc", eval_every=10) + + result = { + "head": "vadclip", "backbone": "clip", "source": "from-scratch", + "recipe": {"optimizer": "adamw", "lr": LR, "weight_decay": WEIGHT_DECAY, + "milestones": MILESTONES, "gamma": GAMMA, "epochs": MAX_EPOCH, + "batch_total": 2 * BATCH, "seed": SEED}, + "best_roc_auc": round(best["roc_auc"], 4), "best_pr_auc": round(best["pr_auc"], 4), + "best_epoch": best["best_epoch"], "last_roc_auc": round(best["last"], 4), + "paper_roc_auc": 0.8801, "official_ckpt_roc_auc": 0.8770, + "n_test": len(test_set), "seconds": round(time.time() - t0, 1), + } + json.dump(result, open(f"{OUT}/result.json", "w"), indent=2) + print("\n=== VadCLIP from-scratch result ===", flush=True) + print(json.dumps(result, indent=2), flush=True) + print(f"\nwrote {OUT}/result.json", flush=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/reproduce.py b/experiments/reproduce.py new file mode 100644 index 0000000..30d0bc9 --- /dev/null +++ b/experiments/reproduce.py @@ -0,0 +1,149 @@ +"""Single parametrized from-scratch reproduction harness for the WSVAD heads. + +Replaces ``experiments/repro_i3d.py`` + ``experiments/repro_vadclip.py``: one +``RECIPES`` registry holds the VERIFIED from-scratch recipe per head and a single +``main()`` drives the canonical :class:`src.trainer.WSVADTrainer.fit` + the per-head +:func:`src.eval_matrix.evaluate`. The recipes are the ones that actually reproduce +(epoch-based, Adam lr 5e-5 for I3D; the official AdamW/MultiStepLR for VadCLIP) — +NOT ``run_matrix``'s generic step-based lr 1e-3, which diverges to chance (0.52) on +these heads. ``run_matrix.py`` stays the {variant × head} *matrix* orchestrator and +owns the step-based official recipes for the memory heads (UR-DMU / BN-WVAD). + + HEAD=mgfn python experiments/reproduce.py # I3D 2048-d seg32 + HEAD=rtfm python experiments/reproduce.py + HEAD=s3r python experiments/reproduce.py + HEAD=vadclip python experiments/reproduce.py # CLIP 512-d, AdamW MultiStepLR + +Every field is env-overridable for sweeps: VARIANT, DIM, LR, EPOCHS, BATCH, SEED. + +NOTE (feature provenance, open): the I3D recipe reaches MGFN ~0.81 on the +``i3d_mgfn_seg32`` train set (the byte-verified MGFN distribution, REPRO_STATUS §2), +while ``scripts/diag/train_archive_probe.py`` reached 0.8667 training on the +``_archive/UCF_Train_ten_crop_i3d`` set — i.e. the residual MGFN gap tracks the +*train-feature lineage*, not the recipe. Override VARIANT to compare. +""" + +import json +import os +import random +import time + +import numpy as np +import torch +from omegaconf import OmegaConf + +import src.models # noqa: F401 (register heads) +from scripts.run_matrix import _build_model +from src.eval_matrix import evaluate +from src.trainer import WSVADTrainer, build_datasets + +# Verified from-scratch recipe per head. ``wd=None`` -> use the runner-config weight +# decay. ``milestones`` -> MultiStepLR (VadCLIP); ``freeze_clip``/``all_crops`` are +# VadCLIP-only knobs. ``paper`` is the published UCF-Crime frame-level ROC-AUC. +RECIPES = { + "mgfn": dict(backbone="i3d", variant="i3d_mgfn_seg32", dim=2048, optimizer="adam", lr=5e-5, wd=None, epochs=15, batch=16, seed=0, paper=0.8667), + "rtfm": dict(backbone="i3d", variant="i3d_mgfn_seg32", dim=2048, optimizer="adam", lr=5e-5, wd=None, epochs=15, batch=16, seed=0, paper=0.8430), + "s3r": dict(backbone="i3d", variant="i3d_mgfn_seg32", dim=2048, optimizer="adam", lr=5e-5, wd=None, epochs=15, batch=16, seed=0, paper=0.8599), + "sultani": dict(backbone="i3d", variant="i3d_mgfn_seg32", dim=2048, optimizer="adam", lr=5e-5, wd=None, epochs=15, batch=16, seed=0, paper=0.8300), + "vadclip": dict(backbone="clip", variant=None, dim=512, optimizer="adamw", lr=2e-5, wd=0.01, epochs=10, batch=64, seed=234, + milestones=[4, 8], gamma=0.1, eval_every=10, freeze_clip=True, all_crops=True, paper=0.8801, official_ckpt=0.8770), +} + + +def _env(key, cast, default): + v = os.environ.get(key) + return cast(v) if v is not None else default + + +def set_seed(s: int) -> None: + random.seed(s); np.random.seed(s); torch.manual_seed(s); torch.cuda.manual_seed_all(s) + + +def _data_cfg(rec: dict): + """Loader config matching the two original scripts (eager npy dirs, single-crop).""" + backbone = rec["backbone"] + cfg = OmegaConf.create({ + "root": os.environ.get("WSAD_DATA", "~/data/wsad"), "source": "local", + "backbone": backbone, "dataset_dir": "ucf_crime", + "feature_variant": rec["variant"], "ground_truth": None, + "dynamic_load": False, "batch_size": 1, "frames_per_clip": 16, + "num_workers": 4, "clip_length": 256 if backbone == "clip" else None, + "segment": None, "single_crop": True, "with_magnitude": True, + }) + if backbone == "clip" and rec.get("all_crops"): + cfg.train_all_crops = True # VadCLIP trains all 10 crops as separate rows + return cfg + + +def main() -> None: + head = os.environ.get("HEAD", "mgfn") + if head not in RECIPES: + raise SystemExit(f"HEAD={head} not in RECIPES {list(RECIPES)}; UR-DMU/BN-WVAD use run_matrix.py") + rec = dict(RECIPES[head]) + # env overrides + rec["variant"] = os.environ.get("VARIANT", rec["variant"]) + rec["dim"] = _env("DIM", int, rec["dim"]) + rec["lr"] = _env("LR", float, rec["lr"]) + rec["epochs"] = _env("EPOCHS", int, rec["epochs"]) + rec["batch"] = _env("BATCH", int, rec["batch"]) + rec["seed"] = _env("SEED", int, rec["seed"]) + + out = f"experiments/runs/{head}_{rec['variant'] or rec['backbone']}/seed{rec['seed']}" + os.makedirs(out, exist_ok=True) + set_seed(rec["seed"]) + device = "cuda" if torch.cuda.is_available() else "cpu" + t0 = time.time() + + data_cfg = _data_cfg(rec) + train_sets, test_set = build_datasets(data_cfg) + model, _, cfg_wd = _build_model(head, rec["backbone"], device, dim=rec["dim"]) + wd = rec["wd"] if rec["wd"] is not None else cfg_wd + + n_clip = None + if rec.get("freeze_clip"): # VadCLIP: random head + FROZEN pretrained CLIP text tower + from src.models.vadclip.modeling_vadclip import load_pretrained_clip_text + n_clip = load_pretrained_clip_text(model) + + n_train = len(train_sets["normal"]) + len(train_sets["abnormal"]) + print(f"[data] head={head} backbone={rec['backbone']} variant={rec['variant']} " + f"dim={rec['dim']} train={n_train} test={len(test_set)}", flush=True) + sched = f" MultiStepLR{rec['milestones']}x{rec['gamma']}" if rec.get("milestones") else "" + clip_msg = f" +froze {n_clip} CLIP text tensors" if n_clip else "" + print(f"[recipe] {rec['optimizer']} lr={rec['lr']} wd={wd}{sched} " + f"epochs={rec['epochs']} batch={rec['batch']}(+{rec['batch']}) seed={rec['seed']}{clip_msg}", flush=True) + + log = [] + + def eval_fn(m): + res = evaluate(m, test_set, backbone=rec["backbone"], head=head, device=device) + log.append({"roc_auc": res["roc_auc"], "pr_auc": res["pr_auc"]}) + json.dump(log, open(f"{out}/epoch_log.json", "w"), indent=2) + return res + + trainer = WSVADTrainer( + model=model, learning_rate=rec["lr"], weight_decay=wd, batch_size=rec["batch"], + num_workers=int(data_cfg.num_workers), frames_per_clip=16, mixed_precision="no", + optimizer_name=rec["optimizer"], + scheduler_milestones=rec.get("milestones"), scheduler_gamma=rec.get("gamma", 0.1), + ) + best = trainer.fit(train_sets, epochs=rec["epochs"], eval_fn=eval_fn, + select_metric="roc_auc", eval_every=rec.get("eval_every")) + + result = { + "head": head, "backbone": rec["backbone"], "variant": rec["variant"], + "dim": rec["dim"], "source": "from-scratch", + "recipe": {"optimizer": rec["optimizer"], "lr": rec["lr"], "wd": wd, + "milestones": rec.get("milestones"), "gamma": rec.get("gamma"), + "epochs": rec["epochs"], "batch_total": 2 * rec["batch"], "seed": rec["seed"]}, + "best_roc_auc": round(best["roc_auc"], 4), "best_pr_auc": round(best["pr_auc"], 4), + "best_epoch": best["best_epoch"], "last_roc_auc": round(best["last"], 4), + "paper_roc_auc": rec["paper"], "official_ckpt_roc_auc": rec.get("official_ckpt"), + "n_test": len(test_set), "seconds": round(time.time() - t0, 1), + } + json.dump(result, open(f"{out}/result.json", "w"), indent=2) + print("\n=== result ===\n" + json.dumps(result, indent=2), flush=True) + print(f"\nwrote {out}/result.json", flush=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/results_table.py b/experiments/results_table.py new file mode 100644 index 0000000..b454de2 --- /dev/null +++ b/experiments/results_table.py @@ -0,0 +1,102 @@ +"""Aggregate every reproduction result into one paper-comparison table. + +Scans the matrix JSONs under ``outputs/`` and the from-scratch ``result.json`` files +under ``experiments/runs/`` for the BEST clean-feature number per head, and prints a +markdown table (ours vs paper). Clean = the byte-verified MGFN-lineage I3D +(``i3d_mgfn_seg32``, 2048-d) / DeepMIL (``i3d_1024_seg200``, 1024-d) / CLIP — NOT the +contaminated ``features/i3d`` the old ``outputs/matrix/results.json`` (feature=None) used. + + PYTHONPATH=. python experiments/results_table.py # print + PYTHONPATH=. python experiments/results_table.py --write README_TABLE.md +""" + +import argparse +import glob +import json +import os + +PAPER = { # published UCF-Crime frame-level ROC-AUC + "sultani": 0.7541, "rtfm": 0.8430, "mgfn": 0.8667, "s3r": 0.8599, + "ur_dmu": 0.8697, "bn_wvad": 0.8724, "gs_moe": 0.9160, + "clip_tsa": 0.8758, "vadclip": 0.8801, "tpwng": 0.8779, +} +ORDER = ["sultani", "rtfm", "mgfn", "s3r", "ur_dmu", "bn_wvad", "gs_moe", "clip_tsa", "vadclip", "tpwng"] +# feature lineage each head reproduces on (clean sets only) +FEATURE = {h: "i3d_1024_seg200" for h in ("ur_dmu", "bn_wvad")} +FEATURE.update({h: "clip" for h in ("clip_tsa", "vadclip", "tpwng")}) +FEATURE.update({h: "i3d_mgfn_seg32" for h in ("sultani", "rtfm", "mgfn", "s3r", "gs_moe")}) + +# treat these matrix dirs as authoritative clean results; skip the contaminated default. +CONTAMINATED = {"outputs/matrix"} # feature=None == features/i3d (MISMATCHED with test) + + +# the ONLY feature lineage that counts as a faithful reproduction per head; anything +# else (contaminated features/i3d=None, the 2048-d i3d_mgfn_seg200 for memory heads, the +# i3d_pyvideo/tushar probes) is rejected so a stray weak run can't masquerade as the result. +ACCEPT = {h: {"i3d_1024_seg200"} for h in ("ur_dmu", "bn_wvad")} +ACCEPT.update({h: {"clip", None} for h in ("clip_tsa", "vadclip", "tpwng")}) +ACCEPT.update({h: {"i3d_mgfn_seg32"} for h in ("sultani", "rtfm", "mgfn", "s3r", "gs_moe")}) + + +def collect() -> dict: + """head -> best record on the head's canonical clean feature (max roc_auc).""" + best = {} + def consider(rec, src): + h = rec.get("head") + roc = rec.get("roc_auc", rec.get("best_roc_auc")) + if not h or roc is None: + return + feat = rec.get("feature") or rec.get("variant") + if feat not in ACCEPT.get(h, set()): # reject non-canonical / contaminated features + return + if h not in best or roc > best[h]["roc"]: + best[h] = {"roc": roc, "pr": rec.get("pr_auc", rec.get("best_pr_auc")), + "feature": feat or "clip", "source": rec.get("source", "trained"), "path": src} + + for jp in glob.glob("outputs/**/results.json", recursive=True): + if os.path.dirname(jp) in CONTAMINATED: + continue + try: + for rec in json.load(open(jp)): + consider(rec, jp) + except Exception: + continue + for jp in glob.glob("experiments/runs/**/result.json", recursive=True): + try: + consider(json.load(open(jp)), jp) + except Exception: + continue + return best + + +def render(best: dict) -> str: + lines = ["# UCF-Crime reproduction — ours vs paper (clean features)", "", + "Frame-level ROC-AUC on the 290-video test split. `*` = official checkpoint.", "", + "| head | feature | ours | paper | Δ | source |", + "|---|---|---|---|---|---|"] + for h in ORDER: + paper = PAPER.get(h) + r = best.get(h) + if r is None: + lines.append(f"| {h} | {FEATURE.get(h,'?')} | _pending_ | {paper:.4f} | — | — |") + continue + tag = "*" if r["source"] == "official-ckpt" else "" + delta = f"{r['roc'] - paper:+.4f}" if paper else "—" + lines.append(f"| {h} | {r['feature']} | {r['roc']:.4f}{tag} | {paper:.4f} | {delta} | `{r['path']}` |") + return "\n".join(lines) + "\n" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--write", default=None, help="write the table to this markdown file") + args = ap.parse_args() + table = render(collect()) + print(table) + if args.write: + with open(args.write, "w") as f: + f.write(table) + print(f"wrote {args.write}") + + +if __name__ == "__main__": + main() diff --git a/experiments/runs/mgfn_i3d_mgfn_seg32/seed0/epoch_log.json b/experiments/runs/mgfn_i3d_mgfn_seg32/seed0/epoch_log.json new file mode 100644 index 0000000..b520767 --- /dev/null +++ b/experiments/runs/mgfn_i3d_mgfn_seg32/seed0/epoch_log.json @@ -0,0 +1,62 @@ +[ + { + "roc_auc": 0.7871533569839708, + "pr_auc": 0.19160503494309236 + }, + { + "roc_auc": 0.8003130409733549, + "pr_auc": 0.19361579691127448 + }, + { + "roc_auc": 0.7898225076021868, + "pr_auc": 0.19406928273541363 + }, + { + "roc_auc": 0.8068274320734781, + "pr_auc": 0.19537151745339712 + }, + { + "roc_auc": 0.8072677743561065, + "pr_auc": 0.20984782896291734 + }, + { + "roc_auc": 0.7929827331909549, + "pr_auc": 0.2002400905916079 + }, + { + "roc_auc": 0.8079470259043391, + "pr_auc": 0.2168852375076838 + }, + { + "roc_auc": 0.807484753013477, + "pr_auc": 0.1936988084202161 + }, + { + "roc_auc": 0.8042400413181449, + "pr_auc": 0.21198470016440712 + }, + { + "roc_auc": 0.8052757231387578, + "pr_auc": 0.21326115447479385 + }, + { + "roc_auc": 0.8113000185386701, + "pr_auc": 0.22044620102533086 + }, + { + "roc_auc": 0.796357982028013, + "pr_auc": 0.2272241206000149 + }, + { + "roc_auc": 0.7971980779929129, + "pr_auc": 0.19184839236273188 + }, + { + "roc_auc": 0.8021988712931316, + "pr_auc": 0.20495937856608323 + }, + { + "roc_auc": 0.7888075210649566, + "pr_auc": 0.1902587586746201 + } +] \ No newline at end of file diff --git a/experiments/runs/mgfn_i3d_mgfn_seg32/seed0/result.json b/experiments/runs/mgfn_i3d_mgfn_seg32/seed0/result.json new file mode 100644 index 0000000..6e946e8 --- /dev/null +++ b/experiments/runs/mgfn_i3d_mgfn_seg32/seed0/result.json @@ -0,0 +1,20 @@ +{ + "head": "mgfn", + "variant": "i3d_mgfn_seg32", + "source": "from-scratch", + "recipe": { + "optimizer": "adam", + "lr": 5e-05, + "wd": 0.0005, + "batch_total": 32, + "epochs": 15, + "seed": 0 + }, + "best_roc_auc": 0.8113, + "best_pr_auc": 0.2204, + "best_epoch": 10, + "last_roc_auc": 0.7888, + "paper_roc_auc": 0.8667, + "n_test": 290, + "seconds": 884.4 +} \ No newline at end of file diff --git a/experiments/runs/rtfm_i3d_mgfn_seg32/seed0/epoch_log.json b/experiments/runs/rtfm_i3d_mgfn_seg32/seed0/epoch_log.json new file mode 100644 index 0000000..4551634 --- /dev/null +++ b/experiments/runs/rtfm_i3d_mgfn_seg32/seed0/epoch_log.json @@ -0,0 +1,62 @@ +[ + { + "roc_auc": 0.7792723671358942, + "pr_auc": 0.18022400284180215 + }, + { + "roc_auc": 0.7957255027965489, + "pr_auc": 0.18425960308660275 + }, + { + "roc_auc": 0.8014758021866466, + "pr_auc": 0.19711601845177576 + }, + { + "roc_auc": 0.7755867123508949, + "pr_auc": 0.23501986307795508 + }, + { + "roc_auc": 0.8080766498156822, + "pr_auc": 0.2489466332599928 + }, + { + "roc_auc": 0.8100975633500048, + "pr_auc": 0.2216422567880845 + }, + { + "roc_auc": 0.7957781490774565, + "pr_auc": 0.20945709441206672 + }, + { + "roc_auc": 0.8075269475729823, + "pr_auc": 0.23903402817208308 + }, + { + "roc_auc": 0.8001784237227532, + "pr_auc": 0.21311474703581612 + }, + { + "roc_auc": 0.7950492396254998, + "pr_auc": 0.241511932238353 + }, + { + "roc_auc": 0.8039508693812226, + "pr_auc": 0.239252643585809 + }, + { + "roc_auc": 0.8037476134955641, + "pr_auc": 0.2156345420736435 + }, + { + "roc_auc": 0.8024264534624914, + "pr_auc": 0.2147827354101347 + }, + { + "roc_auc": 0.8007534901909782, + "pr_auc": 0.2159090634112334 + }, + { + "roc_auc": 0.8046545854641703, + "pr_auc": 0.20320943556032853 + } +] \ No newline at end of file diff --git a/experiments/runs/rtfm_i3d_mgfn_seg32/seed0/result.json b/experiments/runs/rtfm_i3d_mgfn_seg32/seed0/result.json new file mode 100644 index 0000000..ba0b179 --- /dev/null +++ b/experiments/runs/rtfm_i3d_mgfn_seg32/seed0/result.json @@ -0,0 +1,20 @@ +{ + "head": "rtfm", + "variant": "i3d_mgfn_seg32", + "source": "from-scratch", + "recipe": { + "optimizer": "adam", + "lr": 5e-05, + "wd": 0.0005, + "batch_total": 32, + "epochs": 15, + "seed": 0 + }, + "best_roc_auc": 0.8101, + "best_pr_auc": 0.2216, + "best_epoch": 5, + "last_roc_auc": 0.8047, + "paper_roc_auc": 0.843, + "n_test": 290, + "seconds": 699.7 +} \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/epoch_log.json b/experiments/runs/vadclip_scratch/epoch_log.json new file mode 100644 index 0000000..ecbddf8 --- /dev/null +++ b/experiments/runs/vadclip_scratch/epoch_log.json @@ -0,0 +1,42 @@ +[ + { + "roc_auc": 0.7830149789753043, + "pr_auc": 0.22782243102063332 + }, + { + "roc_auc": 0.8186403173837691, + "pr_auc": 0.25355759688560714 + }, + { + "roc_auc": 0.8388579750749069, + "pr_auc": 0.26289049964857447 + }, + { + "roc_auc": 0.8382064291288804, + "pr_auc": 0.2594132059448334 + }, + { + "roc_auc": 0.8396123235941798, + "pr_auc": 0.26083534610631826 + }, + { + "roc_auc": 0.8426686625735237, + "pr_auc": 0.2662964343917057 + }, + { + "roc_auc": 0.8435162018831195, + "pr_auc": 0.2691795444961866 + }, + { + "roc_auc": 0.8454936004187209, + "pr_auc": 0.2763778931932007 + }, + { + "roc_auc": 0.845673312984166, + "pr_auc": 0.2767227632972033 + }, + { + "roc_auc": 0.8456674846275685, + "pr_auc": 0.2767256315344619 + } +] \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/result.json b/experiments/runs/vadclip_scratch/result.json new file mode 100644 index 0000000..ccb3f38 --- /dev/null +++ b/experiments/runs/vadclip_scratch/result.json @@ -0,0 +1,26 @@ +{ + "head": "vadclip", + "backbone": "clip", + "source": "from-scratch", + "recipe": { + "optimizer": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "milestones": [ + 4, + 8 + ], + "gamma": 0.1, + "epochs": 10, + "batch_total": 128, + "seed": 234 + }, + "best_roc_auc": 0.8457, + "best_pr_auc": 0.2767, + "best_epoch": 8, + "last_roc_auc": 0.8457, + "paper_roc_auc": 0.8801, + "official_ckpt_roc_auc": 0.877, + "n_test": 290, + "seconds": 219.5 +} \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/seed0/epoch_log.json b/experiments/runs/vadclip_scratch/seed0/epoch_log.json new file mode 100644 index 0000000..4d7ead3 --- /dev/null +++ b/experiments/runs/vadclip_scratch/seed0/epoch_log.json @@ -0,0 +1,42 @@ +[ + { + "roc_auc": 0.7963609706523098, + "pr_auc": 0.20252438730591799 + }, + { + "roc_auc": 0.8118747279408288, + "pr_auc": 0.2217185791563233 + }, + { + "roc_auc": 0.8306304080611318, + "pr_auc": 0.24553394894719477 + }, + { + "roc_auc": 0.8415394811666822, + "pr_auc": 0.2745686415047165 + }, + { + "roc_auc": 0.8421950015978082, + "pr_auc": 0.27687265717765713 + }, + { + "roc_auc": 0.8428804885566707, + "pr_auc": 0.27881603906348046 + }, + { + "roc_auc": 0.8436458316062615, + "pr_auc": 0.2793446100242116 + }, + { + "roc_auc": 0.8443968468207256, + "pr_auc": 0.27953746524021444 + }, + { + "roc_auc": 0.8445436635989827, + "pr_auc": 0.27977654246249134 + }, + { + "roc_auc": 0.8446794741890986, + "pr_auc": 0.2800609392894736 + } +] \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/seed0/result.json b/experiments/runs/vadclip_scratch/seed0/result.json new file mode 100644 index 0000000..decd4f0 --- /dev/null +++ b/experiments/runs/vadclip_scratch/seed0/result.json @@ -0,0 +1,26 @@ +{ + "head": "vadclip", + "backbone": "clip", + "source": "from-scratch", + "recipe": { + "optimizer": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "milestones": [ + 4, + 8 + ], + "gamma": 0.1, + "epochs": 10, + "batch_total": 128, + "seed": 0 + }, + "best_roc_auc": 0.8447, + "best_pr_auc": 0.2801, + "best_epoch": 9, + "last_roc_auc": 0.8447, + "paper_roc_auc": 0.8801, + "official_ckpt_roc_auc": 0.877, + "n_test": 290, + "seconds": 209.9 +} \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/seed234/epoch_log.json b/experiments/runs/vadclip_scratch/seed234/epoch_log.json new file mode 100644 index 0000000..fe9b61d --- /dev/null +++ b/experiments/runs/vadclip_scratch/seed234/epoch_log.json @@ -0,0 +1,542 @@ +[ + { + "roc_auc": 0.7936066898573814, + "pr_auc": 0.23068880225997265 + }, + { + "roc_auc": 0.819969512067782, + "pr_auc": 0.2563383303889104 + }, + { + "roc_auc": 0.8350868883698955, + "pr_auc": 0.27334029269975013 + }, + { + "roc_auc": 0.8347774125714728, + "pr_auc": 0.26448867154954 + }, + { + "roc_auc": 0.8389450666030202, + "pr_auc": 0.25752647439483645 + }, + { + "roc_auc": 0.8505015882997393, + "pr_auc": 0.26948849117559903 + }, + { + "roc_auc": 0.8520372750500526, + "pr_auc": 0.27927056514616816 + }, + { + "roc_auc": 0.8476372599643709, + "pr_auc": 0.28489875540368087 + }, + { + "roc_auc": 0.8493148561969199, + "pr_auc": 0.28214036848418017 + }, + { + "roc_auc": 0.8421649010976242, + "pr_auc": 0.2780690136657768 + }, + { + "roc_auc": 0.8475600571387497, + "pr_auc": 0.29287800667516994 + }, + { + "roc_auc": 0.8441690671061679, + "pr_auc": 0.28005782194680584 + }, + { + "roc_auc": 0.8415447214508518, + "pr_auc": 0.2687075129720164 + }, + { + "roc_auc": 0.842796939526532, + "pr_auc": 0.26644536792394014 + }, + { + "roc_auc": 0.8459985308227286, + "pr_auc": 0.2745973739779653 + }, + { + "roc_auc": 0.8525550962311391, + "pr_auc": 0.2769424866957293 + }, + { + "roc_auc": 0.8401611758658345, + "pr_auc": 0.2541359449440782 + }, + { + "roc_auc": 0.8438470623858058, + "pr_auc": 0.2640852473942858 + }, + { + "roc_auc": 0.841358128549025, + "pr_auc": 0.2549235947424525 + }, + { + "roc_auc": 0.8468494228668391, + "pr_auc": 0.271571831980934 + }, + { + "roc_auc": 0.8575525075055586, + "pr_auc": 0.27921834862007633 + }, + { + "roc_auc": 0.8545187794299574, + "pr_auc": 0.27411746300904977 + }, + { + "roc_auc": 0.8471442460764812, + "pr_auc": 0.25955932941332516 + }, + { + "roc_auc": 0.8521292390857619, + "pr_auc": 0.27437566909620464 + }, + { + "roc_auc": 0.8494538385477195, + "pr_auc": 0.26608903493790353 + }, + { + "roc_auc": 0.8522490728138652, + "pr_auc": 0.2649403419351394 + }, + { + "roc_auc": 0.8522490728138652, + "pr_auc": 0.2649403419351394 + }, + { + "roc_auc": 0.8505215498243164, + "pr_auc": 0.26299601604329503 + }, + { + "roc_auc": 0.8512763006310212, + "pr_auc": 0.2792109031734019 + }, + { + "roc_auc": 0.8547956269234751, + "pr_auc": 0.2804347770946103 + }, + { + "roc_auc": 0.8529676582765124, + "pr_auc": 0.29424204258567005 + }, + { + "roc_auc": 0.8579969719250203, + "pr_auc": 0.2778343381702896 + }, + { + "roc_auc": 0.8504936361837785, + "pr_auc": 0.2781055746046931 + }, + { + "roc_auc": 0.847518173169671, + "pr_auc": 0.28329744324983025 + }, + { + "roc_auc": 0.8508060698310765, + "pr_auc": 0.277332743143526 + }, + { + "roc_auc": 0.8640342460637966, + "pr_auc": 0.290272607980098 + }, + { + "roc_auc": 0.8625248949843454, + "pr_auc": 0.2901757636388032 + }, + { + "roc_auc": 0.8632818095200941, + "pr_auc": 0.3048960466032717 + }, + { + "roc_auc": 0.8654274255445932, + "pr_auc": 0.31971281272877483 + }, + { + "roc_auc": 0.8643205553614136, + "pr_auc": 0.3261059261677303 + }, + { + "roc_auc": 0.8617280051242818, + "pr_auc": 0.31926716552290085 + }, + { + "roc_auc": 0.8569945938667881, + "pr_auc": 0.3021040399076695 + }, + { + "roc_auc": 0.8567548614598507, + "pr_auc": 0.3027270297624157 + }, + { + "roc_auc": 0.8559432396267802, + "pr_auc": 0.30945838802871484 + }, + { + "roc_auc": 0.8571218111606526, + "pr_auc": 0.3138712698355847 + }, + { + "roc_auc": 0.8543870281679897, + "pr_auc": 0.3092129844136663 + }, + { + "roc_auc": 0.8480128207257267, + "pr_auc": 0.3020865542087414 + }, + { + "roc_auc": 0.849286599853381, + "pr_auc": 0.2796381609312095 + }, + { + "roc_auc": 0.8486329297779016, + "pr_auc": 0.2667548237559599 + }, + { + "roc_auc": 0.8542516823179316, + "pr_auc": 0.2734444997734933 + }, + { + "roc_auc": 0.8596705455608173, + "pr_auc": 0.29671419080705314 + }, + { + "roc_auc": 0.8551160236969009, + "pr_auc": 0.2813364424753595 + }, + { + "roc_auc": 0.8564028764259132, + "pr_auc": 0.2976045647988843 + }, + { + "roc_auc": 0.8564028764259132, + "pr_auc": 0.2976045647988843 + }, + { + "roc_auc": 0.8571498270384518, + "pr_auc": 0.2977383694323745 + }, + { + "roc_auc": 0.8578569335234505, + "pr_auc": 0.2971851852627538 + }, + { + "roc_auc": 0.8588062094556255, + "pr_auc": 0.29923323800809354 + }, + { + "roc_auc": 0.8591090815596678, + "pr_auc": 0.30047933288724377 + }, + { + "roc_auc": 0.859169549949796, + "pr_auc": 0.3001036719800327 + }, + { + "roc_auc": 0.8598126670118909, + "pr_auc": 0.300017180684248 + }, + { + "roc_auc": 0.8601082749966015, + "pr_auc": 0.29948801253047996 + }, + { + "roc_auc": 0.8598198438825915, + "pr_auc": 0.29735889365059537 + }, + { + "roc_auc": 0.8596125341032451, + "pr_auc": 0.29757482780969274 + }, + { + "roc_auc": 0.8599825833972522, + "pr_auc": 0.2989395794262086 + }, + { + "roc_auc": 0.8600567962687224, + "pr_auc": 0.2997601439166527 + }, + { + "roc_auc": 0.8599962056564923, + "pr_auc": 0.3008702600624142 + }, + { + "roc_auc": 0.8599337807875149, + "pr_auc": 0.30142190844061195 + }, + { + "roc_auc": 0.8599625373997163, + "pr_auc": 0.30096641622449827 + }, + { + "roc_auc": 0.8598828255514227, + "pr_auc": 0.2997427204408697 + }, + { + "roc_auc": 0.8596042214274586, + "pr_auc": 0.2994045512157043 + }, + { + "roc_auc": 0.8598883153685278, + "pr_auc": 0.29975668384915505 + }, + { + "roc_auc": 0.8602801045603247, + "pr_auc": 0.29981906363624394 + }, + { + "roc_auc": 0.8604068545235041, + "pr_auc": 0.2993499033730825 + }, + { + "roc_auc": 0.860385116846252, + "pr_auc": 0.29886827781429265 + }, + { + "roc_auc": 0.8602693688151831, + "pr_auc": 0.29886538722990075 + }, + { + "roc_auc": 0.8604631260046629, + "pr_auc": 0.299159438810961 + }, + { + "roc_auc": 0.8607656377339242, + "pr_auc": 0.29987942182806204 + }, + { + "roc_auc": 0.8604459162815578, + "pr_auc": 0.29900017367500653 + }, + { + "roc_auc": 0.8603429731667729, + "pr_auc": 0.2990581936442486 + }, + { + "roc_auc": 0.8602061275284738, + "pr_auc": 0.2998568699757777 + }, + { + "roc_auc": 0.8602061275284738, + "pr_auc": 0.2998568699757777 + }, + { + "roc_auc": 0.8597670726166663, + "pr_auc": 0.30003794003175177 + }, + { + "roc_auc": 0.8596158146702815, + "pr_auc": 0.29908899033021735 + }, + { + "roc_auc": 0.859896005461028, + "pr_auc": 0.2993879743179751 + }, + { + "roc_auc": 0.8601039104370279, + "pr_auc": 0.29881603225651027 + }, + { + "roc_auc": 0.8598458811686861, + "pr_auc": 0.2990026254671434 + }, + { + "roc_auc": 0.859731742287345, + "pr_auc": 0.29871056671633084 + }, + { + "roc_auc": 0.8590240977845411, + "pr_auc": 0.2958677800823479 + }, + { + "roc_auc": 0.8586240044290192, + "pr_auc": 0.29516162585296984 + }, + { + "roc_auc": 0.8587644264284945, + "pr_auc": 0.29613521702204737 + }, + { + "roc_auc": 0.8588137005324252, + "pr_auc": 0.2958636841426335 + }, + { + "roc_auc": 0.8592637091721681, + "pr_auc": 0.29599259457141386 + }, + { + "roc_auc": 0.8598278444802944, + "pr_auc": 0.2972810399988661 + }, + { + "roc_auc": 0.8600570466343596, + "pr_auc": 0.2986858632603385 + }, + { + "roc_auc": 0.860226785469219, + "pr_auc": 0.29970379198452346 + }, + { + "roc_auc": 0.860695605950092, + "pr_auc": 0.3019502981587366 + }, + { + "roc_auc": 0.8608814822658652, + "pr_auc": 0.3029396197864177 + }, + { + "roc_auc": 0.8608714812411693, + "pr_auc": 0.3032202765539093 + }, + { + "roc_auc": 0.860347022317456, + "pr_auc": 0.302200372746141 + }, + { + "roc_auc": 0.8605591080110037, + "pr_auc": 0.30334577426665044 + }, + { + "roc_auc": 0.8606061590790268, + "pr_auc": 0.30501187187477247 + }, + { + "roc_auc": 0.8607438158612807, + "pr_auc": 0.3053286424779782 + }, + { + "roc_auc": 0.8608590249493203, + "pr_auc": 0.30343020837522355 + }, + { + "roc_auc": 0.8606860293256835, + "pr_auc": 0.30088983295011684 + }, + { + "roc_auc": 0.8603911454213414, + "pr_auc": 0.3002951312701141 + }, + { + "roc_auc": 0.8603785430366097, + "pr_auc": 0.3008315681351294 + }, + { + "roc_auc": 0.8606570707370644, + "pr_auc": 0.301225709104402 + }, + { + "roc_auc": 0.8606570707370644, + "pr_auc": 0.301225709104402 + }, + { + "roc_auc": 0.860605604592233, + "pr_auc": 0.30106663544974405 + }, + { + "roc_auc": 0.8605472525596753, + "pr_auc": 0.30090880879071585 + }, + { + "roc_auc": 0.8604822785133004, + "pr_auc": 0.300851447205271 + }, + { + "roc_auc": 0.8604109664043801, + "pr_auc": 0.3008387155469873 + }, + { + "roc_auc": 0.8603898712952547, + "pr_auc": 0.3008219181220686 + }, + { + "roc_auc": 0.860379696143386, + "pr_auc": 0.3008342108475162 + }, + { + "roc_auc": 0.8603979116775934, + "pr_auc": 0.30089226761743143 + }, + { + "roc_auc": 0.8603911780817517, + "pr_auc": 0.3009189722514265 + }, + { + "roc_auc": 0.8603917618981488, + "pr_auc": 0.3009226514566127 + }, + { + "roc_auc": 0.8604038842766256, + "pr_auc": 0.3010583001997094 + }, + { + "roc_auc": 0.8604146093816902, + "pr_auc": 0.301148390846502 + }, + { + "roc_auc": 0.8604158878563299, + "pr_auc": 0.30122289931307783 + }, + { + "roc_auc": 0.8604124465778702, + "pr_auc": 0.3011994196299533 + }, + { + "roc_auc": 0.8603969690592961, + "pr_auc": 0.3011553263859578 + }, + { + "roc_auc": 0.8603775290835355, + "pr_auc": 0.3011829096457214 + }, + { + "roc_auc": 0.8603496320045084, + "pr_auc": 0.3011748541772303 + }, + { + "roc_auc": 0.8603823629167646, + "pr_auc": 0.3010583975983275 + }, + { + "roc_auc": 0.8604436835307971, + "pr_auc": 0.30107388922304806 + }, + { + "roc_auc": 0.8604703168462471, + "pr_auc": 0.3010347436528097 + }, + { + "roc_auc": 0.8604717248522589, + "pr_auc": 0.3010264286632542 + }, + { + "roc_auc": 0.860475576004988, + "pr_auc": 0.3009960117035966 + }, + { + "roc_auc": 0.860426810866826, + "pr_auc": 0.30095068248116824 + }, + { + "roc_auc": 0.8604411643306601, + "pr_auc": 0.30103592864503614 + }, + { + "roc_auc": 0.860446391384125, + "pr_auc": 0.3011646916768893 + }, + { + "roc_auc": 0.8604445279828191, + "pr_auc": 0.30143766483649487 + }, + { + "roc_auc": 0.8604453997289514, + "pr_auc": 0.3016349526712991 + }, + { + "roc_auc": 0.8604453997289514, + "pr_auc": 0.3016349526712991 + } +] \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/seed234/result.json b/experiments/runs/vadclip_scratch/seed234/result.json new file mode 100644 index 0000000..c1f2603 --- /dev/null +++ b/experiments/runs/vadclip_scratch/seed234/result.json @@ -0,0 +1,26 @@ +{ + "head": "vadclip", + "backbone": "clip", + "source": "from-scratch", + "recipe": { + "optimizer": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "milestones": [ + 4, + 8 + ], + "gamma": 0.1, + "epochs": 10, + "batch_total": 128, + "seed": 234 + }, + "best_roc_auc": 0.8654, + "best_pr_auc": 0.3197, + "best_epoch": 2, + "last_roc_auc": 0.8604, + "paper_roc_auc": 0.8801, + "official_ckpt_roc_auc": 0.877, + "n_test": 290, + "seconds": 2501.1 +} \ No newline at end of file diff --git a/experiments/runs/vadclip_scratch/seed42/epoch_log.json b/experiments/runs/vadclip_scratch/seed42/epoch_log.json new file mode 100644 index 0000000..5b78d63 --- /dev/null +++ b/experiments/runs/vadclip_scratch/seed42/epoch_log.json @@ -0,0 +1,34 @@ +[ + { + "roc_auc": 0.7892275433859691, + "pr_auc": 0.1896662991801766 + }, + { + "roc_auc": 0.8150765327787441, + "pr_auc": 0.21542821159785658 + }, + { + "roc_auc": 0.8251619997161808, + "pr_auc": 0.22443925207154972 + }, + { + "roc_auc": 0.8346919585974665, + "pr_auc": 0.23135235678937052 + }, + { + "roc_auc": 0.8367755645309453, + "pr_auc": 0.23330555325496996 + }, + { + "roc_auc": 0.8379083705031374, + "pr_auc": 0.23431797396987283 + }, + { + "roc_auc": 0.8389389644726162, + "pr_auc": 0.23595240060404257 + }, + { + "roc_auc": 0.8406734682778819, + "pr_auc": 0.23874766484974888 + } +] \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..f577909 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,26 @@ +# scripts/ + +Run everything with `PYTHONPATH=. WSAD_DATA=~/data/wsad conda run -n balaenoptera python scripts/<...>.py`. + +## Top-level (core pipeline) +| script | purpose | +|---|---| +| `run_matrix.py` | `{feature-variant × head}` comparison matrix. Per-paper recipe in `HEADS`; trains via `WSVADTrainer.fit_steps`; per-head eval via `src.eval_matrix`; wandb-logged; resumable. See `docs/WSVAD_REPRO_RECIPES.md`. | +| `gpu_queue.sh` | Serial GPU reproduction queue (8 GB → one heavy run at a time): memory heads on 1024-d DeepMIL + the i3d 2048-d heads on **clean** `i3d_mgfn_seg32` + cosine. Run detached (`nohup setsid`). | +| `build_correct_i3d.py` | Build consistent I3D features (`features/i3d/{train,test}`) from a source extraction. | +| `extract_features.py` | Backbone-agnostic extraction dispatcher (`--backbone clip/i3d/...`). | +| `extract_i3d_gowtham.py` | Faithful Gowtham/Tushar-N I3D 10-crop extractor (raw video → `(T,10,2048)`). | +| `convert_official_to_hf.py` | Official ckpt → repo state_dict converter (imported by run_matrix + verify/). | +| `convert_i3d_caffe2.py` | I3D Caffe2 `.pkl` → torch `.pth`. | +| `prepare_clip_features.py` | Sort VadCLIP `UCFClipFeatures` into `clip/{train,test}`. | +| `build_manifest.py` | Build the unified dataset manifest. | +| `train_archive_probe.py` | Standalone trainer probe (train=_archive seg32, test=test.zip). Reproduced MGFN 0.8667 (lr 5e-5, batch 16, 12 ep, seed 0). | + +## verify/ — fidelity checks (output-match vs official) +`verify_{rtfm,mgfn,vadclip,ur_dmu,bn_wvad,s3r,clip_tsa,checkpoints,i3d_extract}.py`, +`eval_{mgfn_local,vadclip_ucf}.py`. Confirm ported models match official weights/outputs. + +## diag/ — diagnostics & superseded probes (kept for reference) +`stabilize.py` (per-step AUC probe), `smoke_train_{mgfn,vadclip}.py` (train-loop smoke), +`diag_i3d_rtfm.py`, `grid_i3d_rtfm.py`, `vadclip_oracle.py`, `validate_extraction.py`. +Function now largely covered by `run_matrix.py` + `WSVADTrainer.fit_steps`. diff --git a/scripts/build_correct_i3d.py b/scripts/build_correct_i3d.py new file mode 100644 index 0000000..033f56a --- /dev/null +++ b/scripts/build_correct_i3d.py @@ -0,0 +1,83 @@ +"""Rebuild `features/i3d/{train,test}` with the CORRECT I3D features. + +Root cause (verified, see memory eval-matrix-serving §5): `features/i3d/train.zip` +(10,32,2048) is a mismatched/corrupted extraction — the official MGFN ckpt scores +its abnormal videos LOWER than normal (5% vs 100% on test). The consistent train +features live in `_archive/UCF_Train_ten_crop_i3d` (T,10,2048, official MGFN 100%); +RTFM/MGFN train from scratch to paper AUC on them (0.8448 / 0.816). + +This streams (low RAM): + - train: `_archive/UCF_Train_ten_crop_i3d/*.npy` (T,10,2048) -> mean-pool to + 32 segments -> `features/i3d/train/.npy` (10,32,2048) + - test : `features/i3d/test.zip` (already correct) -> `features/i3d/test/.npy` + +Non-destructive: leaves the bad `train.zip` in place; the loader prefers the +populated npy dir (src/data/local.py:_load_i3d). Idempotent (skips existing). + + PYTHONPATH=. WSAD_DATA=~/data/wsad python scripts/build_correct_i3d.py +""" + +import io +import os +import zipfile + +import numpy as np + +ROOT = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) +DS = os.path.join(ROOT, "ucf_crime") +ARCH_TRAIN = os.path.join(DS, "_archive", "UCF_Train_ten_crop_i3d") +TEST_ZIP = os.path.join(DS, "features", "i3d", "test.zip") +OUT_TRAIN = os.path.join(DS, "features", "i3d", "train") +OUT_TEST = os.path.join(DS, "features", "i3d", "test") + + +def seg32(a): # (10, T, 2048) -> (10, 32, 2048) uniform mean-pool over T + T = a.shape[1] + out = np.zeros((a.shape[0], 32, a.shape[2]), np.float32) + e = np.linspace(0, T, 33, dtype=int) + for i in range(32): + lo, hi = e[i], e[i + 1] + out[:, i] = a[:, lo:hi].mean(1) if hi > lo else a[:, min(lo, T - 1)] + return out + + +def build_train(): + os.makedirs(OUT_TRAIN, exist_ok=True) + files = sorted(f for f in os.listdir(ARCH_TRAIN) if f.endswith(".npy")) + n = 0 + for f in files: + dst = os.path.join(OUT_TRAIN, f) + if os.path.exists(dst): + continue + a = np.load(os.path.join(ARCH_TRAIN, f)).astype(np.float32) # (T, 10, 2048) + a = np.transpose(a, (1, 0, 2)) # (10, T, 2048) + np.save(dst, seg32(a)) # (10, 32, 2048) + n += 1 + if n % 200 == 0: + print(f" train {n}/{len(files)}", flush=True) + print(f"train: wrote {n}, total {len(files)} -> {OUT_TRAIN}", flush=True) + + +def build_test(): + os.makedirs(OUT_TEST, exist_ok=True) + z = zipfile.ZipFile(TEST_ZIP) + infos = [i for i in z.infolist() if i.filename.endswith(".npy")] + n = 0 + for info in infos: + name = info.filename.split("/")[-1] + dst = os.path.join(OUT_TEST, name) + if os.path.exists(dst): + continue + a = np.load(io.BytesIO(z.read(info))) # (T, 10, 2048) — keep as-is (test layout) + np.save(dst, a) + n += 1 + if n % 100 == 0: + print(f" test {n}/{len(infos)}", flush=True) + print(f"test: wrote {n}, total {len(infos)} -> {OUT_TEST}", flush=True) + + +if __name__ == "__main__": + print("Building CORRECT i3d features (non-destructive; bad train.zip kept)...") + build_train() + build_test() + print("DONE. Loader will now use features/i3d/{train,test}/*.npy (correct).") diff --git a/scripts/build_deepmil_1024.py b/scripts/build_deepmil_1024.py new file mode 100644 index 0000000..54873a1 --- /dev/null +++ b/scripts/build_deepmil_1024.py @@ -0,0 +1,68 @@ +"""Stack DeepMIL UCF per-crop 1024-d I3D into per-video (10, seg, 1024) for UR-DMU/BN-WVAD. + +The DeepMIL release (Roc-Ng, the 1024-d Carreira/pytorch-i3d features UR-DMU/BN-WVAD +actually use — NOT the 2048-d ResNet50-I3D of RTFM/MGFN) ships per-crop full-length +files: ``_x264.npy`` (crop 0) + ``_x264__1..9.npy`` (crops 1-9), each +``(T, 1024)``. This stacks the 10 crops -> ``(10, T, 1024)``, segments train to 200 +(uniform, = UR-DMU's np.linspace "random_perturb"), keeps test full-length, and writes +``features/i3d_1024_seg200/{train,test}/_x264_i3d.npy``. + + PYTHONPATH=. WSAD_DATA=~/data/wsad python scripts/build_deepmil_1024.py +""" + +import glob +import os + +import numpy as np + +from src.data.local import segment + +ROOT = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) +RAW = os.path.join(ROOT, "ucf_crime", "features", "i3d_1024_raw") +DST = os.path.join(ROOT, "ucf_crime", "features", "i3d_1024_seg200") +SEG = 200 + + +def crop_files(stem_dir: str, stem: str): + """The 10 per-crop files for one video, in crop order 0..9.""" + return [os.path.join(stem_dir, stem + sfx) for sfx in + [".npy"] + [f"__{c}.npy" for c in range(1, 10)]] + + +def stack_video(stem_dir: str, stem: str) -> np.ndarray: + """10 per-crop ``(T, 1024)`` -> ``(10, T, 1024)`` (min-clip T for safety).""" + arrs = [np.load(f).astype(np.float32) for f in crop_files(stem_dir, stem)] + t = min(a.shape[0] for a in arrs) + return np.stack([a[:t] for a in arrs]) # (10, T, 1024) + + +def build(split: str, src_dir: str, do_seg: bool) -> None: + out = os.path.join(DST, split) + os.makedirs(out, exist_ok=True) + # crop-0 files (no __c suffix) enumerate the videos; index stem->dir in ONE walk + stem_dir = {os.path.basename(f)[:-4]: os.path.dirname(f) + for f in glob.glob(os.path.join(src_dir, "**", "*_x264.npy"), recursive=True)} + stems = sorted(stem_dir) + n = 0 + for stem in stems: + d = stem_dir[stem] + dst = os.path.join(out, stem + "_i3d.npy") + if os.path.exists(dst): + continue + v = stack_video(d, stem) # (10, T, 1024) + if do_seg: # train: (10, 200, 1024) == i3d_mgfn train layout (crops, seg, D) + v = np.stack([segment(v[c], SEG) for c in range(v.shape[0])]) + else: # test: full-length, transpose to (T, 10, 1024) == i3d test layout + v = np.transpose(v, (1, 0, 2)) + np.save(dst, v) + n += 1 + if n % 200 == 0: + print(f" {split} {n}/{len(stems)}", flush=True) + print(f"{split}: wrote {n}, total {len(stems)} -> {out}", flush=True) + + +if __name__ == "__main__": + print(f"Building i3d_1024_seg200 from {RAW} (DeepMIL 1024-d, UR-DMU/BN-WVAD)...", flush=True) + build("train", os.path.join(RAW, "train_npy"), do_seg=True) + build("test", os.path.join(RAW, "test_npy"), do_seg=False) + print("DONE. Use data.feature_variant=i3d_1024_seg200 + feature dim 1024.", flush=True) diff --git a/scripts/build_deepmil_1024_percrop.py b/scripts/build_deepmil_1024_percrop.py new file mode 100644 index 0000000..fd1e706 --- /dev/null +++ b/scripts/build_deepmil_1024_percrop.py @@ -0,0 +1,60 @@ +"""Per-crop DeepMIL 1024-d train set — the FAITHFUL UR-DMU/BN-WVAD training layout. + +The official UR-DMU/BN-WVAD ``dataset_loader`` lists each 10-crop file as a SEPARATE +training sample (UCF_Train.list has 1610 vids x 10 crops = 16100 rows; each ``np.load`` +is one ``(T, 1024)`` crop, segmented to 200). Our stacked ``i3d_1024_seg200`` instead +keeps ``(10, 200, 1024)`` per video and the model crop-AVERAGES — i.e. 1610 samples +with a weaker per-video loss instead of 16100 independent per-crop losses. That 10x data ++ per-crop gradient is the main remaining from-scratch lever (the 8 GB batch cap only got +BN-WVAD to ~0.82 / UR-DMU ~0.81). + +This writes ``features/i3d_1024_seg200_pc/`` with: + train/ = 16100 per-crop ``___i3d.npy`` of shape ``(200, 1024)`` (model sees + dim==3 -> n=1, one loss per crop, exactly official) + test/ = symlink to the stacked ``i3d_1024_seg200/test`` (eval stays per-video 10-crop) + + PYTHONPATH=. WSAD_DATA=~/data/wsad python scripts/build_deepmil_1024_percrop.py +""" + +import glob +import os + +import numpy as np + +from src.data.local import segment + +ROOT = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) +RAW = os.path.join(ROOT, "ucf_crime", "features", "i3d_1024_raw", "train_npy") +STACKED = os.path.join(ROOT, "ucf_crime", "features", "i3d_1024_seg200") +DST = os.path.join(ROOT, "ucf_crime", "features", "i3d_1024_seg200_pc") +SEG = 200 + + +def main() -> None: + out_tr = os.path.join(DST, "train") + os.makedirs(out_tr, exist_ok=True) + files = sorted(glob.glob(os.path.join(RAW, "**", "*.npy"), recursive=True)) + n = 0 + for f in files: + stem = os.path.basename(f)[:-4] # _x264 or _x264__ + dst = os.path.join(out_tr, stem + "_i3d.npy") + if os.path.exists(dst): + n += 1 + continue + a = np.load(f).astype(np.float32) # (T, 1024) one crop + np.save(dst, segment(a, SEG)) # (200, 1024) + n += 1 + if n % 2000 == 0: + print(f" {n}/{len(files)}", flush=True) + print(f"train: wrote {n} per-crop files -> {out_tr}", flush=True) + + # test: reuse the stacked per-video 10-crop set (eval crop-averages) + test_link = os.path.join(DST, "test") + if not os.path.exists(test_link): + os.symlink(os.path.join(STACKED, "test"), test_link) + print(f"test: symlinked {test_link} -> {STACKED}/test", flush=True) + print("DONE. feature_variant=i3d_1024_seg200_pc --feature-dim 1024 (single_crop train=per-crop sample)", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_manifest.py b/scripts/build_manifest.py new file mode 100644 index 0000000..25ad7bc --- /dev/null +++ b/scripts/build_manifest.py @@ -0,0 +1,130 @@ +"""Build the unified feature manifest for a dataset (the catalog every loader/tool reads). + +One row per (backbone, split, video) with provenance + shape, read from the +``.npy`` headers only (no full array load), so it runs in seconds over the 9 GB +I3D zips. Writes both ``manifest.parquet`` (columnar, mmap-friendly) and +``manifest.jsonl`` (grep-able) under ``/``. + +Layout assumed (see docs/DATA_LOCAL.md): + /features/i3d/{train,test}.zip # per-video _i3d.npy + /features/clip/{train,test}/*.npy # per-crop __.npy (symlinks ok) + + python scripts/build_manifest.py # dataset=ucf_crime + python scripts/build_manifest.py --dataset aihub # future dataset, same shape + +Columns: dataset, backbone, split, video_id, label(normal|abnormal), path, +zip_member, n_crops, shape, dtype. +""" + +import argparse +import json +import os +import zipfile +from collections import defaultdict + +import numpy as np +from numpy.lib import format as npfmt + +DATA_ROOT = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) + + +def _npy_header(fobj): + """(shape, dtype) from an open .npy file object, reading only the header.""" + major, minor = npfmt.read_magic(fobj) + reader = getattr(npfmt, f"read_array_header_{major}_{minor}", None) + if reader is None: # very old/new .npy version + reader = npfmt.read_array_header_1_0 + shape, _fortran, dtype = reader(fobj) + return list(shape), str(dtype) + + +def _bare_vid(name: str) -> str: + b = name[:-4] if name.endswith(".npy") else name + if b.endswith("_i3d"): + b = b[: -len("_i3d")] + return b.split("__")[0] + + +def _label(video_id: str) -> str: + return "normal" if "Normal" in video_id else "abnormal" + + +def _i3d_rows(dataset: str, ds_dir: str): + rows = [] + for split in ("train", "test"): + zp = os.path.join(ds_dir, "features", "i3d", f"{split}.zip") + if not os.path.exists(zp): + continue + with zipfile.ZipFile(zp) as z: + for info in z.infolist(): + if info.is_dir() or not info.filename.endswith(".npy"): + continue + with z.open(info) as f: + shape, dtype = _npy_header(f) + vid = _bare_vid(info.filename.split("/")[-1]) + rows.append(dict( + dataset=dataset, backbone="i3d", split=split, video_id=vid, + label=_label(vid), path=os.path.relpath(zp, DATA_ROOT), + zip_member=info.filename, n_crops=shape[0] if len(shape) == 3 else 1, + shape=shape, dtype=dtype, + )) + return rows + + +def _clip_rows(dataset: str, ds_dir: str): + rows = [] + for split in ("train", "test"): + d = os.path.join(ds_dir, "features", "clip", split) + if not os.path.isdir(d): + continue + by_vid = defaultdict(list) + for f in os.listdir(d): + if f.endswith(".npy"): + by_vid[_bare_vid(f)].append(f) + for vid, files in by_vid.items(): + crop0 = sorted(files)[0] + with open(os.path.join(d, crop0), "rb") as fobj: + shape, dtype = _npy_header(fobj) + rows.append(dict( + dataset=dataset, backbone="clip", split=split, video_id=vid, + label=_label(vid), path=os.path.relpath(d, DATA_ROOT), + zip_member=None, n_crops=len(files), shape=shape, dtype=dtype, + )) + return rows + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--dataset", default="ucf_crime") + ap.add_argument("--data-root", default=DATA_ROOT) + args = ap.parse_args() + + ds_dir = os.path.join(os.path.expanduser(args.data_root), args.dataset) + rows = _i3d_rows(args.dataset, ds_dir) + _clip_rows(args.dataset, ds_dir) + if not rows: + raise SystemExit(f"no features found under {ds_dir}/features (see docs/DATA_LOCAL.md)") + + jsonl = os.path.join(ds_dir, "manifest.jsonl") + with open(jsonl, "w") as f: + for r in rows: + f.write(json.dumps(r) + "\n") + + import pandas as pd + + df = pd.DataFrame(rows) + df["shape"] = df["shape"].apply(lambda s: ",".join(map(str, s))) # parquet-friendly + parquet = os.path.join(ds_dir, "manifest.parquet") + df.to_parquet(parquet, index=False) + + # summary + g = df.groupby(["backbone", "split"]).agg( + n=("video_id", "count"), + normal=("label", lambda s: (s == "normal").sum()), + abnormal=("label", lambda s: (s == "abnormal").sum()), + ) + print(g.to_string()) + print(f"\nwrote {parquet}\n {jsonl} ({len(rows)} rows)") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_mgfn_seg32.py b/scripts/build_mgfn_seg32.py new file mode 100644 index 0000000..88e4860 --- /dev/null +++ b/scripts/build_mgfn_seg32.py @@ -0,0 +1,65 @@ +"""Build a seg32 cache from the verified MGFN-authors' full-length I3D features. + +`features/i3d_mgfn/` is the MGFN/RTFM-lineage 10-crop I3D (L2~22), full-length +`(T, 10, 2048)`, COMPLETE 1610/290 (byte-verified). Seg32 models (MGFN/RTFM/S3R/ +Sultani) expect `(10, 32, 2048)` train; eval stays full-length `(T, 10, 2048)`. + +This streams (low RAM) train -> 32-segment mean-pool -> `features/i3d_mgfn_seg32/ +train/.npy` (10, 32, 2048); test is symlinked to the full-length source. +Idempotent. Unlike the bad `build_correct_i3d.py` (mixed scales), train AND test +here are the SAME lineage (L2~22) -> a clean consistent pair for paper reproduction. + + PYTHONPATH=. WSAD_DATA=~/data/wsad python scripts/build_mgfn_seg32.py +""" + +import os +import glob + +import numpy as np + +from src.data.local import segment + +ROOT = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) +SEG = int(os.environ.get("SEG", 32)) # 32 = MGFN/RTFM/S3R; 200 = UR-DMU/BN-WVAD +SRC = os.path.join(ROOT, "ucf_crime", "features", "i3d_mgfn") +DST = os.path.join(ROOT, "ucf_crime", "features", f"i3d_mgfn_seg{SEG}") + + +def seg32_crops(a: np.ndarray) -> np.ndarray: + """``(T, 10, 2048)`` full-length -> ``(10, SEG, 2048)`` per-crop SEG-seg pool.""" + ac = np.transpose(a.astype(np.float32), (1, 0, 2)) # (10, T, 2048) + return np.stack([segment(ac[c], SEG) for c in range(ac.shape[0])]) # (10, SEG, 2048) + + +def build_train() -> None: + out = os.path.join(DST, "train") + os.makedirs(out, exist_ok=True) + files = sorted(glob.glob(os.path.join(SRC, "train", "*.npy"))) + n = 0 + for f in files: + dst = os.path.join(out, os.path.basename(f)) + if os.path.exists(dst): + continue + np.save(dst, seg32_crops(np.load(f))) + n += 1 + if n % 200 == 0: + print(f" train {n}/{len(files)}", flush=True) + print(f"train: wrote {n}, total {len(files)} -> {out}", flush=True) + + +def link_test() -> None: + """Eval is full-length; point seg32 test at the full-length source (no copy).""" + dst = os.path.join(DST, "test") + src = os.path.join(SRC, "test") + if os.path.islink(dst) or os.path.exists(dst): + print(f"test: {dst} already present", flush=True) + return + os.symlink(src, dst) + print(f"test: symlinked {dst} -> {src} (full-length, {len(os.listdir(src))} files)", flush=True) + + +if __name__ == "__main__": + print(f"Building seg32 cache from {SRC} (L2~22 MGFN lineage)...", flush=True) + build_train() + link_test() + print("DONE. Use with data.feature_variant=i3d_mgfn_seg32 (seg32 train, full test).", flush=True) diff --git a/scripts/convert_i3d_caffe2.py b/scripts/convert_i3d_caffe2.py new file mode 100644 index 0000000..f5aafc9 --- /dev/null +++ b/scripts/convert_i3d_caffe2.py @@ -0,0 +1,123 @@ +"""Convert facebookresearch/video-nonlocal-net Caffe2 I3D weights -> PyTorch. + +Faithful port of GowthamGottimukkala/I3D_Feature_Extraction_resnet +`utils/convert_weights.py` (which itself follows Tushar-N/pytorch-resnet3d), but +targets THIS repo's ``src.i3d.I3Res50`` (architecturally identical to Gowtham's +model minus the unused ``fc``/``drop`` classifier head, so the feature path is +unchanged). The blob->param regex mapping is reproduced verbatim. + + python scripts/convert_i3d_caffe2.py \ + pretrained/i3d/i3d_baseline_32x2_IN_pretrain_400k.pkl \ + pretrained/i3d/i3d_baseline_r50_kinetics.pth --no-nl + python scripts/convert_i3d_caffe2.py \ + pretrained/i3d/i3d_nonlocal_32x2_IN_pretrain_400k.pkl \ + pretrained/i3d/i3d_nonlocal_r50_kinetics.pth --nl + +The .pth is a plain state_dict loadable into ``I3Res50(use_nl=...)`` with +``strict=False`` (our model has no ``fc``; every conv/bn/nl key is filled). +""" + +import argparse +import os +import pickle +import re +import sys + +import torch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.i3d import I3Res50 + +# ---- blob -> pytorch-param mapping (verbatim from Gowtham convert_weights.py) ---- +_DS_PAT = re.compile("res(.)_(.)_branch1_.*") +_CONV_PAT = re.compile("res(.)_(.)_branch2(.)_.*") +_NL_PAT = re.compile("nonlocal_conv(.)_(.)_(.*)_.*") +_M2NUM = dict(zip("abc", [1, 2, 3])) +_SUFFIX = {"b": "bias", "w": "weight", "s": "weight", "rm": "running_mean", "riv": "running_var"} + + +def _key_map(blob_keys): + km = { + "conv1.weight": "conv1_w", + "bn1.weight": "res_conv1_bn_s", + "bn1.bias": "res_conv1_bn_b", + "bn1.running_mean": "res_conv1_bn_rm", + "bn1.running_var": "res_conv1_bn_riv", + "fc.weight": "pred_w", + "fc.bias": "pred_b", + } + for key in blob_keys: + m = _CONV_PAT.match(key) + if m: + layer, block, module = int(m.group(1)), int(m.group(2)), _M2NUM[m.group(3)] + name = "bn" if "bn_" in key else "conv" + nk = "layer%d.%d.%s%d.%s" % (layer - 1, block, name, module, _SUFFIX[key.split("_")[-1]]) + km[nk] = key + m = _DS_PAT.match(key) + if m: + layer, block = int(m.group(1)), int(m.group(2)) + module = 0 if key[-1] == "w" else 1 + nk = "layer%d.%d.downsample.%d.%s" % (layer - 1, block, module, _SUFFIX[key.split("_")[-1]]) + km[nk] = key + m = _NL_PAT.match(key) + if m: + layer, block, module = int(m.group(1)), int(m.group(2)), m.group(3) + nk = "layer%d.%d.nl.%s.%s" % (layer - 1, block, module, _SUFFIX[key.split("_")[-1]]) + km[nk] = key + return km + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("pkl") + ap.add_argument("out") + ap.add_argument("--nl", dest="nl", action="store_true", help="non-local model") + ap.add_argument("--no-nl", dest="nl", action="store_false") + ap.set_defaults(nl=False) + args = ap.parse_args() + + blobs = pickle.load(open(args.pkl, "rb"), encoding="latin")["blobs"] + blobs = {k: v for k, v in blobs.items() if "momentum" not in k} + km = _key_map(blobs.keys()) + + model = I3Res50(use_nl=args.nl) + sd = model.state_dict() + + new_sd, missing, mism = {}, [], [] + for key in sd: + if key not in km: + missing.append(key) + continue + c2 = torch.from_numpy(blobs[km[key]]) + if tuple(c2.shape) != tuple(sd[key].shape): + mism.append((key, tuple(c2.shape), tuple(sd[key].shape))) + continue + new_sd[key] = c2 + + if mism: + for k, a, b in mism: + print(f" SHAPE MISMATCH {k}: caffe2 {a} != model {b}") + raise SystemExit("aborting on shape mismatch") + if missing: + # legitimately absent from caffe2: classifier head (our model has none) and + # num_batches_tracked (a non-learnable BN counter, unused in eval mode). + bad = [m for m in missing + if not (m.startswith(("fc.", "drop.")) or m.endswith("num_batches_tracked"))] + if bad: + raise SystemExit(f"unmapped model params (would init randomly!): {bad}") + + # confirm a clean strict=False load (no leftover-random conv/bn/nl weights/buffers) + miss, unexp = model.load_state_dict(new_sd, strict=False) + leftover = [k for k in miss + if not (k.startswith(("fc.", "drop.")) or k.endswith("num_batches_tracked"))] + assert not leftover, f"params not loaded: {leftover}" + + os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) + torch.save(new_sd, args.out) + print(f"use_nl={args.nl}: mapped {len(new_sd)}/{len(sd)} params " + f"(skipped fc/drop), {len(blobs)} blobs -> {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/diag/diag_i3d_rtfm.py b/scripts/diag/diag_i3d_rtfm.py new file mode 100644 index 0000000..d2b0d54 --- /dev/null +++ b/scripts/diag/diag_i3d_rtfm.py @@ -0,0 +1,65 @@ +"""Diagnose why our Gowtham-faithful extraction != RTFM's released Abuse001. + +Hypotheses: (A) 10-crop ORDER differs, (B) RTFM used nonlocal weights, +(C) different extractor entirely. Extracts once per model (cached to /tmp) and +reports crop-matched cosine, crop-AVERAGED cosine (order-invariant), and the +snippet-0 crop permutation (best rtfm-crop per mine-crop). +""" + +import os +import sys + +import numpy as np +import torch + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +from src.features.i3d_gowtham import build_model, extract_from_video # noqa: E402 + +VIDEO = os.path.join(ROOT, ".reference", "I3D_Feature_Extraction_resnet", + "samplevideos", "Abuse001_x264.mp4") +RTFM = os.path.expanduser( + "~/data/wsad/ucf_crime/_archive/UCF_Train_ten_crop_i3d/Abuse001_x264_i3d.npy") + + +def _cos(a, b): + return float((a * b).sum() / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9)) + + +def _extract_cached(tag, use_nl): + cache = f"/tmp/diag_i3d_{tag}.npy" + if os.path.exists(cache): + return np.load(cache) + pth = os.path.join(ROOT, "pretrained", "i3d", f"i3d_{tag}_r50_kinetics.pth") + m = build_model(pth, use_nl=use_nl, device="cuda" if torch.cuda.is_available() else "cpu") + feat = extract_from_video(m, VIDEO) + np.save(cache, feat) + return feat + + +def analyze(tag, mine, rtfm): + T = min(mine.shape[0], rtfm.shape[0]) + m, r = mine[:T], rtfm[:T] + # crop-matched (order-sensitive) + cm = np.mean([_cos(m[t].ravel(), r[t].ravel()) for t in range(T)]) + # crop-averaged (order-invariant) + ma, ra = m.mean(1), r.mean(1) + ca = np.mean([_cos(ma[t], ra[t]) for t in range(T)]) + print(f"[{tag}] T mine={mine.shape[0]} rtfm={rtfm.shape[0]} | " + f"crop-matched cos={cm:.4f} | crop-AVG cos={ca:.4f}") + # snippet-0 permutation: best rtfm crop for each mine crop + M = np.array([[_cos(m[0, i], r[0, j]) for j in range(10)] for i in range(10)]) + best = M.argmax(1) + print(f" snippet0 mine-crop -> best rtfm-crop: {list(best)} " + f"(diag={'identity' if list(best)==list(range(10)) else 'PERMUTED'}), " + f"max-per-row mean={M.max(1).mean():.3f}") + + +def main(): + rtfm = np.load(RTFM) + analyze("baseline", _extract_cached("baseline", False), rtfm) + analyze("nonlocal", _extract_cached("nonlocal", True), rtfm) + + +if __name__ == "__main__": + main() diff --git a/scripts/diag/feature_forensics.py b/scripts/diag/feature_forensics.py new file mode 100644 index 0000000..12778ce --- /dev/null +++ b/scripts/diag/feature_forensics.py @@ -0,0 +1,181 @@ +"""Feature-forensics — *relative* discriminability screen for a feature set. + +WHY THIS EXISTS (and what the old ``magnitude_forensics.py`` got wrong) +---------------------------------------------------------------------- +The repro VERDICT once leaned on a single proxy — the frame-level ROC-AUC of the +**raw feature L2 magnitude** — as if it were the feature-quality *ceiling* and a +go/no-go *gate* for new extractors. That premise is unsound, and this module +exists to replace it. Measured facts (``scripts/diag``, 290-vid UCF test): + +* ``i3d_mgfn`` raw-magnitude AUC = **0.45** (mildly *inverted*, near chance) yet + RTFM/MGFN train to **0.83** on it. So magnitude-AUC does NOT predict, let alone + upper-bound, trainable performance — it must not be used as an accept/reject gate. +* Every *atemporal* proxy here (magnitude, content-distance, even a supervised + linear probe) UNDER-predicts the temporal heads, because WSVAD performance is + dominated by temporal modeling (RTFM multi-scale, MGFN attention, MIL ranking) + that no per-snippet score captures. ``i3d_mgfn`` linear-probe ≈ 0.51 → head 0.83. + +So treat the numbers below as a **fast relative screen**, not a ceiling: they tell +you whether a set carries *any* linearly-accessible anomaly signal and how two sets +*rank*, cheaply, before spending a GPU. They do NOT tell you the final head AUC. +The only faithful gate is a short temporal-head train (e.g. ``run_matrix.py`` on a +sample). Validated ranking that DID hold downstream-ward (same 40-vid subset): +``videomae`` probe 0.615 > ``i3d_1024`` 0.554 > ``i3d_mgfn`` 0.539. + +Three proxies, all snippet-level, crop-averaged, reported as ``|AUC-0.5|`` too +(an inverted 0.45 still carries 0.05 of signal — sign is learnable): + - **MAGNITUDE** snippet L2-norm (norm-then-crop-mean = RTFM-faithful). + - **CONTENT** L2-distance to the NORMAL-video centroid (content separability). + - **PROBE** 5-fold GroupKFold(by video) logistic-regression AUC — the + strongest atemporal proxy; "how linearly separable is anomaly". + +Run in the conda env (needs scikit-learn). +""" + +import glob +import io +import json +import os +import zipfile +from typing import Iterator, Tuple + +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import GroupKFold + +ROOT = os.path.expanduser("~/data/wsad/ucf_crime") +GT = json.load(open(os.path.join(ROOT, "annotations", "ground_truth.json"))) +FPC = 16 # frames per snippet (I3D/VideoMAE stride 16; GT is per-frame) + + +def _bare(name: str) -> str: + """Bare UCF video id: everything before ``_x264`` (robust to any backbone suffix).""" + return os.path.basename(name).split("_x264")[0] + + +def _gt_for(name: str): + b = _bare(name) + for k in GT: + if _bare(k) == b: + return np.asarray(GT[k], dtype=np.int8) + return None + + +def _snip(feat: np.ndarray) -> np.ndarray: + """(T, ncrops, D) or (T, D) -> per-snippet (T, D) by crop-mean.""" + return feat.mean(axis=1) if feat.ndim == 3 else feat + + +def _mag(feat: np.ndarray) -> np.ndarray: + """RTFM-faithful snippet magnitude: per-crop L2 norm, then crop-mean -> (T,). + + NB: norm-then-mean (not crop-mean-then-norm); the old module did the weaker + crop-mean-then-norm which understates the magnitude signal by Jensen. + """ + pc = np.linalg.norm(feat, axis=-1) # (T, ncrops) or (T,) + return pc.mean(axis=1) if pc.ndim == 2 else pc + + +def _snip_labels(gt: np.ndarray, T: int) -> np.ndarray: + """Snippet label = any anomalous frame in its 16-frame window.""" + return np.array([gt[i * FPC:(i + 1) * FPC].max() if i * FPC < len(gt) else 0 + for i in range(T)], dtype=np.int8) + + +def _iter_zip(path) -> Iterator[Tuple[str, np.ndarray]]: + z = zipfile.ZipFile(path) + for i in z.infolist(): + if i.filename.endswith(".npy"): + yield i.filename.split("/")[-1], np.load(io.BytesIO(z.read(i))) + + +def _iter_dir(d) -> Iterator[Tuple[str, np.ndarray]]: + for f in sorted(glob.glob(os.path.join(d, "**", "*.npy"), recursive=True)): + yield os.path.basename(f), np.load(f) + + +def _fmt(auc: float) -> str: + return f"{auc:.4f}(|dev|{abs(auc - 0.5):.3f})" + + +def screen(name, it, run_probe=True): + """Compute the three relative-discriminability proxies for one feature set. + + Returns a dict (also prints a line). ``it`` yields ``(filename, feat)`` where + ``feat`` is ``(T, ncrops, D)`` or ``(T, D)``. + """ + vids = [] # (vid, snip_feats (T,D), mag (T,), gt (frames,)) + for vid, feat in it: + g = _gt_for(vid) + if g is None: + continue + feat = feat.astype(np.float32) + vids.append((vid, _snip(feat), _mag(feat), g)) + if not vids: + print(f" {name}: no aligned videos"); return None + + # normal-video centroid (content reference); fall back to all if no Normal in sample + nor = [f for v, f, _, _ in vids if "Normal" in v] + centroid = (np.concatenate(nor, 0) if nor + else np.concatenate([f for _, f, _, _ in vids], 0)).mean(0) + + mag_s, con_s, allg = [], [], [] + X, y, grp = [], [], [] + for gi, (vid, f, mag, g) in enumerate(vids): + L = len(np.repeat(mag, FPC)) + mag_s.append(np.repeat(mag, FPC)[:len(g)]) + con_s.append(np.repeat(np.linalg.norm(f - centroid, axis=-1), FPC)[:len(g)]) + allg.append(g[:L]) + X.append(f); y.append(_snip_labels(g, f.shape[0])); grp.append(np.full(f.shape[0], gi)) + g = np.concatenate(allg) + if len(set(g.tolist())) < 2: + print(f" {name}: single-class GT"); return None + mauc = roc_auc_score(g, np.concatenate(mag_s)) + cauc = roc_auc_score(g, np.concatenate(con_s)) + + probe = float("nan") + if run_probe: + X = np.concatenate(X); y = np.concatenate(y); grp = np.concatenate(grp) + Xn = (X - X.mean(0)) / (X.std(0) + 1e-6) + aucs = [] + for tr, te in GroupKFold(5).split(Xn, y, grp): + if y[tr].sum() == 0 or y[te].sum() in (0, len(y[te])): + continue + clf = LogisticRegression(max_iter=500).fit(Xn[tr], y[tr]) + aucs.append(roc_auc_score(y[te], clf.decision_function(Xn[te]))) + probe = float(np.mean(aucs)) if aucs else float("nan") + + print(f" {name:30} n={len(vids):3} | MAG={_fmt(mauc)} | CONTENT={_fmt(cauc)} " + f"| PROBE={probe:.4f} (atemporal — NOT the head ceiling)") + return {"name": name, "n": len(vids), "magnitude_auc": mauc, + "content_auc": cauc, "probe_auc": probe} + + +def main(): + import argparse + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--dir", default=None, help="screen an arbitrary feature dir (gate use)") + ap.add_argument("--name", default=None, help="label for --dir") + ap.add_argument("--no-probe", action="store_true", help="skip the (slower) linear probe") + args = ap.parse_args() + print(f"GT: {len(GT)} videos") + print("RELATIVE atemporal screen (higher = more linearly-accessible signal). " + "These do NOT predict temporal-head AUC — gate with a short head train.\n") + if args.dir: + screen(args.name or os.path.basename(args.dir.rstrip('/')), + _iter_dir(args.dir), run_probe=not args.no_probe) + return + F = os.path.join(ROOT, "features") + tz = os.path.join(F, "i3d", "test.zip") + if os.path.exists(tz): + screen("i3d test.zip (DeepMIL)", _iter_zip(tz), run_probe=not args.no_probe) + for v in ("i3d_mgfn", "i3d_1024_seg200"): + d = os.path.join(F, v, "test") + if os.path.isdir(d): + screen(f"{v}/test", _iter_dir(d), run_probe=not args.no_probe) + + +if __name__ == "__main__": + main() diff --git a/scripts/diag/grid_i3d_rtfm.py b/scripts/diag/grid_i3d_rtfm.py new file mode 100644 index 0000000..865955d --- /dev/null +++ b/scripts/diag/grid_i3d_rtfm.py @@ -0,0 +1,107 @@ +"""Grid over I3D preprocessing hypotheses vs RTFM's released Abuse001 feature. + +After verify_i3d_extract.py proved our extractor is bit-exact to Gowtham yet only +hits crop-avg cosine 0.74 (nonlocal) / 0.51 (baseline) vs RTFM — and crop-order, +temporal offset, and BGR were ruled out — this sweeps the remaining cheap +preprocessing knobs (resize dims x interpolation) to see if any closes the gap. +If none beats ~0.74, the evidence-based conclusion is that RTFM used a different +(unpublished) I3D checkpoint / extractor, not a preprocessing variant of Gowtham. + + conda run -n balaenoptera python scripts/grid_i3d_rtfm.py + +Uses the non-local model (closest to RTFM). Each config re-extracts Abuse001 +(~90 s); results cached to /tmp/grid_i3d_.npy. +""" + +import os +import sys + +import numpy as np +import torch +from PIL import Image + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +import src.features.i3d_gowtham as g # noqa: E402 + +VIDEO = os.path.join(ROOT, ".reference", "I3D_Feature_Extraction_resnet", + "samplevideos", "Abuse001_x264.mp4") +RTFM = os.path.expanduser( + "~/data/wsad/ucf_crime/_archive/UCF_Train_ten_crop_i3d/Abuse001_x264_i3d.npy") +PTH = os.path.join(ROOT, "pretrained", "i3d", "i3d_nonlocal_r50_kinetics.pth") + +_INTERP = { + "lanczos": getattr(Image, "Resampling", Image).LANCZOS, + "bilinear": getattr(Image, "Resampling", Image).BILINEAR, + "bicubic": getattr(Image, "Resampling", Image).BICUBIC, +} + +# (tag, W, H, interp). The frame MUST be 256-tall: Gowtham's 10-crop coords +# (16:240, 58:282, -224:, ...) are hardcoded for a 256x340 frame, so resize is NOT +# a free variable — only the interpolation filter is. (340,256,lanczos) = faithful default. +CONFIGS = [ + ("w340h256_lanczos", 340, 256, "lanczos"), + ("w340h256_bilinear", 340, 256, "bilinear"), + ("w340h256_bicubic", 340, 256, "bicubic"), +] + + +def _make_load_frame(W, H, interp): + rs = _INTERP[interp] + + def load_frame(frame_file): + data = np.array(Image.open(frame_file).resize((W, H), rs)).astype(float) + data = (data * 2 / 255) - 1 + return data + + return load_frame + + +def _cos(a, b): + return float((a * b).sum() / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9)) + + +def crop_avg_cos(feat, rtfm): + T = min(len(feat), len(rtfm)) + a, b = feat[:T].mean(1), rtfm[:T].mean(1) + return np.mean([_cos(a[t], b[t]) for t in range(T)]) + + +def main(): + rtfm = np.load(RTFM) + dev = "cuda" if torch.cuda.is_available() else "cpu" + model = g.build_model(PTH, use_nl=True, device=dev) + orig = g.load_frame + results = [] + for tag, W, H, interp in CONFIGS: + cache = f"/tmp/grid_i3d_{tag}.npy" + if os.path.exists(cache): + feat = np.load(cache) + else: + # H!=256 means the oversample crop coords (designed for 256-tall) shift; + # keep Gowtham's crop logic, vary only the resize filter (the one free + # knob — see CONFIGS: the 10-crop coords pin the frame to 256-tall). + g.load_frame = _make_load_frame(W, H, interp) + try: + feat = g.extract_from_video(model, VIDEO, device=dev) + except Exception as e: + print(f" [{tag}] extract failed: {e}") + continue + finally: + g.load_frame = orig + np.save(cache, feat) + c = crop_avg_cos(feat, rtfm) + results.append((tag, c)) + print(f" [{tag}] crop-avg cos vs RTFM = {c:.4f} shape={feat.shape}") + + print("\n=== summary (vs RTFM Abuse001, nonlocal) ===") + for tag, c in sorted(results, key=lambda x: -x[1]): + print(f" {c:.4f} {tag}") + best = max(results, key=lambda x: x[1]) if results else ("none", 0) + verdict = ("CLOSES gap" if best[1] > 0.9 else + "does NOT close gap -> RTFM used a different/unpublished I3D checkpoint") + print(f"\nbest = {best[0]} ({best[1]:.4f}) -> {verdict}") + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke_train_mgfn.py b/scripts/diag/smoke_train_mgfn.py similarity index 84% rename from scripts/smoke_train_mgfn.py rename to scripts/diag/smoke_train_mgfn.py index 2e1ff0b..b73b8b9 100644 --- a/scripts/smoke_train_mgfn.py +++ b/scripts/diag/smoke_train_mgfn.py @@ -19,7 +19,13 @@ ROOT = os.path.join(os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")), "ucf_crime") N_PER = 8 # videos per class for the smoke subset -z = zipfile.ZipFile(f"{ROOT}/train.zip") +# resolve new features/i3d layout (legacy dataset-root fallback) via loader resolver +from types import SimpleNamespace + +from src.data.local import _i3d_zip_path + +_cfg = SimpleNamespace(root=os.path.dirname(ROOT), dataset_dir="ucf_crime") +z = zipfile.ZipFile(_i3d_zip_path(os.path.dirname(ROOT), _cfg, "train")) infos = [i for i in z.infolist() if not i.is_dir() and i.filename.endswith(".npy")] names = [i.filename.split("/")[-1] for i in infos] values = {n: i for n, i in zip(names, infos)} diff --git a/scripts/smoke_train_vadclip.py b/scripts/diag/smoke_train_vadclip.py similarity index 93% rename from scripts/smoke_train_vadclip.py rename to scripts/diag/smoke_train_vadclip.py index b7c79f1..9bd5ffc 100644 --- a/scripts/smoke_train_vadclip.py +++ b/scripts/diag/smoke_train_vadclip.py @@ -16,7 +16,10 @@ from src.models.vadclip.modeling_vadclip import VadCLIPForVideoAnomalyDetection from src.trainer import WSVADTrainer -CLIP_TRAIN = os.path.join(os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")), "clip/train") +CLIP_TRAIN = os.path.join( + os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")), + "ucf_crime/features/clip/train", +) N_PER = 8 diff --git a/scripts/diag/stabilize.py b/scripts/diag/stabilize.py new file mode 100644 index 0000000..6afed03 --- /dev/null +++ b/scripts/diag/stabilize.py @@ -0,0 +1,132 @@ +"""Training-reproduction probe: train one head, log per-epoch ROC-AUC to a FLUSHED +file so progress is monitorable despite ``conda run`` stdout buffering (issue #6). + +Uses the correct per-head eval adapter (:mod:`src.eval_matrix`) every epoch, on +``test_dataset=None`` training (no double eval). Writes ```` as JSONL, one +line per epoch, flushed + fsynced, plus a final summary line. + + PYTHONPATH=. WSAD_DATA=~/data/wsad python scripts/stabilize.py \ + --head rtfm --backbone i3d --lr 5e-5 --epochs 20 --out /tmp/rtfm.jsonl +""" + +import argparse +import json +import os +import time + +import numpy as np +import torch +from hydra import compose, initialize +from hydra.utils import _locate, instantiate +from omegaconf import OmegaConf + +import src.models # noqa: F401 +from src.eval_matrix import evaluate +from src.trainer import WSVADTrainer, build_datasets + +BACKBONE_DIM = {"i3d": 2048, "clip": 512} +_DIM_FIELDS = ("feature_size", "channels", "visual_width") + +# per-head CLIP preprocessing (I3D zip is pre-segmented to 32 + magnitude already) +_SPEC = { + "rtfm": dict(runner="rtfm", mag=True, seg=32, length=None), + "mgfn": dict(runner="mgfn", mag=True, seg=32, length=None), + "sultani": dict(runner="mil", mag=True, seg=32, length=None), + "mil": dict(runner="mil", mag=True, seg=32, length=None), +} + + +def _set_dim(cfg, dim): + for f in _DIM_FIELDS: + if hasattr(cfg, f): + setattr(cfg, f, dim) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--head", required=True) + ap.add_argument("--backbone", default="i3d") + ap.add_argument("--lr", type=float, default=None, help="override runner lr") + ap.add_argument("--epochs", type=int, default=20) + ap.add_argument("--batch-size", type=int, default=16) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--out", default="/tmp/stabilize.jsonl") + args = ap.parse_args() + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + device = "cuda" if torch.cuda.is_available() else "cpu" + spec = _SPEC[args.head] + + data_cfg = OmegaConf.create({ + "root": os.environ.get("WSAD_DATA", "~/data/wsad"), + "source": "local", "backbone": args.backbone, "dataset_dir": "ucf_crime", + "ground_truth": None, "dynamic_load": False, "batch_size": 1, + "frames_per_clip": 16, "num_workers": 4, + "clip_length": spec["length"] if args.backbone == "clip" else None, + "segment": spec["seg"] if args.backbone == "clip" else None, + "single_crop": True, + "with_magnitude": spec["mag"] if args.backbone == "clip" else True, + }) + train_sets, test_set = build_datasets(data_cfg) + + with initialize(version_base=None, config_path="../configs"): + cfg = compose(config_name="default", overrides=[f"runner={spec['runner']}"]) + model_cfg = instantiate(cfg.runner.model_config) + _set_dim(model_cfg, BACKBONE_DIM[args.backbone]) + model = _locate(cfg.runner.model_class)(model_cfg).to(device) + lr = args.lr if args.lr is not None else float(cfg.runner.optimizer.learning_rate) + wd = float(cfg.runner.optimizer.weight_decay) + + # prepare ONCE, then loop epochs manually (mirrors WSVADTrainer.fit body) + import inspect + from torch.utils.data import DataLoader + from src.trainer import _collate_train + + accepts_class = "class_labels" in inspect.signature(model.forward).parameters + optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd) + loaders = {k: DataLoader(train_sets[k], batch_size=args.batch_size, shuffle=True, + drop_last=True, num_workers=4, collate_fn=_collate_train) + for k in ("normal", "abnormal")} + + log = open(args.out, "w") + def emit(d): + log.write(json.dumps(d) + "\n"); log.flush(); os.fsync(log.fileno()) + print(d, flush=True) + + emit({"event": "start", "head": args.head, "backbone": args.backbone, + "lr": lr, "wd": wd, "epochs": args.epochs, "batch": args.batch_size, + "n_test": len(test_set)}) + + aucs = [] + for ep in range(args.epochs): + t0 = time.time() + model.train() + total, nb_steps = 0.0, 0 + for nb, ab in zip(loaders["normal"], loaders["abnormal"]): + feature = torch.cat([nb["feature"], ab["feature"]], dim=0).to(device) + kwargs = {} + if accepts_class: + kwargs["class_labels"] = torch.cat([nb["class_id"], ab["class_id"]], dim=0).to(device) + out = model(video=feature, abnormal_labels=ab["anomaly"].to(device), + normal_labels=nb["anomaly"].to(device), **kwargs) + optimizer.zero_grad() + out.loss.backward() + optimizer.step() + total += out.loss.item(); nb_steps += 1 + m = evaluate(model, test_set, backbone=args.backbone, head=args.head, device=device) + aucs.append(m["roc_auc"]) + emit({"epoch": ep, "train_loss": round(total / max(nb_steps, 1), 4), + "roc_auc": round(m["roc_auc"], 4), "pr_auc": round(m["pr_auc"], 4), + "sec": round(time.time() - t0, 1)}) + + a = np.array(aucs) + emit({"event": "summary", "max": round(float(a.max()), 4), + "last": round(float(a[-1]), 4), + "last5_mean": round(float(a[-5:].mean()), 4), + "last5_std": round(float(a[-5:].std()), 4)}) + log.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/diag/train_archive_probe.py b/scripts/diag/train_archive_probe.py new file mode 100644 index 0000000..27604e0 --- /dev/null +++ b/scripts/diag/train_archive_probe.py @@ -0,0 +1,144 @@ +"""Validate from-scratch training on the CORRECT train features (_archive). + +Root cause (verified): `features/i3d/train.zip` (10,32,2048) is MISMATCHED with +`test.zip` — the official MGFN ckpt scores its abnormal videos LOWER than normal +(5% vs 100% on test). The `_archive/UCF_Train_ten_crop_i3d` (T,10,2048) features +ARE consistent (official MGFN 100%). This script trains a head from scratch on the +_archive train features (pooled to 32-seg) and evals on test.zip, logging per-epoch +AUC to a flushed JSONL — to prove training reproduces once the features are right. + + PYTHONPATH=. WSAD_DATA=~/data/wsad python scripts/train_archive_probe.py \ + --head rtfm --lr 5e-5 --epochs 15 --out /tmp/rtfm_archive.jsonl +""" + +import argparse +import glob +import io +import json +import os +import time +import zipfile + +import numpy as np +import torch +from hydra import compose, initialize +from hydra.utils import _locate, instantiate +from torch.utils.data import DataLoader + +import src.models # noqa: F401 +from src.data.features import FeatureDataset +from src.data.local import _align_gt, _load_ground_truth +from src.eval_matrix import evaluate +from src.trainer import _collate_train + +BACKBONE_DIM = 2048 +_DIM_FIELDS = ("feature_size", "channels", "visual_width") +_SPEC = {"rtfm": "rtfm", "mgfn": "mgfn", "sultani": "mil", "mil": "mil"} + + +def _seg32(a): # (10, T, 2048) -> (10, 32, 2048) uniform mean-pool over T + T = a.shape[1] + out = np.zeros((a.shape[0], 32, a.shape[2]), np.float32) + e = np.linspace(0, T, 33, dtype=int) + for i in range(32): + lo, hi = e[i], e[i + 1] + out[:, i] = a[:, lo:hi].mean(1) if hi > lo else a[:, min(lo, T - 1)] + return out + + +def _load_archive_train(root): + d = os.path.join(root, "ucf_crime", "_archive", "UCF_Train_ten_crop_i3d") + fs = sorted(glob.glob(d + "/*.npy")) + vals = {} + for f in fs: + a = np.load(f).astype(np.float32) # (T, 10, 2048) + a = np.transpose(a, (1, 0, 2)) # (10, T, 2048) + vals[os.path.basename(f)] = _seg32(a) # (10, 32, 2048) + return vals + + +def _load_test_zip(root): + zp = os.path.join(root, "ucf_crime", "features", "i3d", "test.zip") + z = zipfile.ZipFile(zp) + infos = [i for i in z.infolist() if i.filename.endswith(".npy")] + names = [i.filename.split("/")[-1] for i in infos] + vals = {n: np.load(io.BytesIO(z.read(i))) for n, i in zip(names, infos)} + return names, vals + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--head", default="rtfm") + ap.add_argument("--lr", type=float, default=5e-5) + ap.add_argument("--epochs", type=int, default=15) + ap.add_argument("--batch-size", type=int, default=16) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--out", default="/tmp/train_archive.jsonl") + args = ap.parse_args() + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + device = "cuda" if torch.cuda.is_available() else "cpu" + root = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) + + log = open(args.out, "w") + def emit(d): + log.write(json.dumps(d) + "\n"); log.flush(); os.fsync(log.fileno()) + print(d, flush=True) + + tr = _load_archive_train(root) + names = list(tr) + normal = [n for n in names if "Normal" in n] + abnormal = [n for n in names if "Normal" not in n] + train_sets = { + "normal": FeatureDataset(normal, {n: tr[n] for n in normal}, with_magnitude=True), + "abnormal": FeatureDataset(abnormal, {n: tr[n] for n in abnormal}, with_magnitude=True), + } + te_names, te_vals = _load_test_zip(root) + from types import SimpleNamespace + gt = _align_gt(te_names, _load_ground_truth(SimpleNamespace(root=root, dataset_dir="ucf_crime", ground_truth=None))) + test_set = FeatureDataset(te_names, te_vals, labels=gt, with_magnitude=True) + emit({"event": "start", "head": args.head, "lr": args.lr, "n_train_nor": len(normal), + "n_train_abn": len(abnormal), "n_test": len(test_set)}) + + with initialize(version_base=None, config_path="../configs"): + cfg = compose(config_name="default", overrides=[f"runner={_SPEC[args.head]}"]) + model_cfg = instantiate(cfg.runner.model_config) + for f in _DIM_FIELDS: + if hasattr(model_cfg, f): + setattr(model_cfg, f, BACKBONE_DIM) + model = _locate(cfg.runner.model_class)(model_cfg).to(device) + wd = float(cfg.runner.optimizer.weight_decay) + + import inspect + accepts_class = "class_labels" in inspect.signature(model.forward).parameters + opt = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=wd) + loaders = {k: DataLoader(train_sets[k], batch_size=args.batch_size, shuffle=True, + drop_last=True, num_workers=4, collate_fn=_collate_train) + for k in ("normal", "abnormal")} + + aucs = [] + for ep in range(args.epochs): + t0 = time.time(); model.train(); total = 0.0; n = 0 + for nb, ab in zip(loaders["normal"], loaders["abnormal"]): + feat = torch.cat([nb["feature"], ab["feature"]], 0).to(device) + kw = {} + if accepts_class: + kw["class_labels"] = torch.cat([nb["class_id"], ab["class_id"]], 0).to(device) + out = model(video=feat, abnormal_labels=ab["anomaly"].to(device), + normal_labels=nb["anomaly"].to(device), **kw) + opt.zero_grad(); out.loss.backward(); opt.step() + total += out.loss.item(); n += 1 + m = evaluate(model, test_set, backbone="i3d", head=args.head, device=device) + aucs.append(m["roc_auc"]) + emit({"epoch": ep, "train_loss": round(total / max(n, 1), 4), + "roc_auc": round(m["roc_auc"], 4), "pr_auc": round(m["pr_auc"], 4), + "sec": round(time.time() - t0, 1)}) + a = np.array(aucs) + emit({"event": "summary", "max": round(float(a.max()), 4), "last": round(float(a[-1]), 4), + "last5_mean": round(float(a[-5:].mean()), 4), "last5_std": round(float(a[-5:].std()), 4)}) + log.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/vadclip_oracle.py b/scripts/diag/vadclip_oracle.py similarity index 100% rename from scripts/vadclip_oracle.py rename to scripts/diag/vadclip_oracle.py diff --git a/scripts/diag/validate_extraction.py b/scripts/diag/validate_extraction.py new file mode 100644 index 0000000..14235a0 --- /dev/null +++ b/scripts/diag/validate_extraction.py @@ -0,0 +1,126 @@ +"""Validate the feature-extraction lines (I3D tushar-n / nonlocal, CLIP) end-to-end. + +Pulls one raw UCF-Crime video out of the ``raw/Anomaly-Videos-*.zip`` shards, +runs it through each extractor, and reports shape, per-snippet L2 scale, and wall +time so we know (a) the line is correct and (b) roughly how long a full extraction +takes. It also seg32-pools the I3D output to confirm it matches the MGFN train +layout ``(10, 32, 2048)`` and prints the scale gap vs the MGFN canonical (~2.5). + + python scripts/validate_extraction.py # i3d baseline, 1 video + python scripts/validate_extraction.py --nonlocal # i3d use_nl=True + python scripts/validate_extraction.py --clip # also time the CLIP line + python scripts/validate_extraction.py --video /path/to.mp4 + +Findings (RTX2070S, 2026-06-14): tushar-n I3D ~39 s / 1795-frame video, L2/snip +~22 (= RTFM scale, ~10x the MGFN ~2.5 canonical). => fresh extractions are +self-consistent but NOT mixable with the MGFN zips; re-extract train+test together. +""" + +import argparse +import os +import sys +import time +import zipfile + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +RAW = os.path.expanduser( + os.path.join(os.environ.get("WSAD_DATA", "~/data/wsad"), "ucf_crime", "raw") +) + + +def _extract_sample_video(dst="/tmp/wsad_extract_test") -> str: + """Pull a small Abuse video out of Anomaly-Videos-Part-1.zip (once).""" + os.makedirs(dst, exist_ok=True) + member = "Anomaly-Videos-Part-1/Abuse/Abuse037_x264.mp4" + out = os.path.join(dst, os.path.basename(member)) + if not os.path.exists(out): + with zipfile.ZipFile(os.path.join(RAW, "Anomaly-Videos-Part-1.zip")) as z: + with z.open(member) as src, open(out, "wb") as f: + f.write(src.read()) + return out + + +def _scale(feat: np.ndarray) -> float: + return float(np.linalg.norm(feat.reshape(-1, feat.shape[-1]), axis=1).mean()) + + +def validate_i3d(video: str, use_nl: bool) -> None: + import decord + + from src.data.local import segment + from src.features.i3d import I3DFeatureExtractor + + name = "tushar-n-nonlocal" if use_nl else "tushar-n-baseline" + # I3Res50(use_nl=...) shares the tushar-n checkpoint; nonlocal adds NL blocks. + ext = I3DFeatureExtractor(model_name="tushar-n-baseline", device="cuda", batch_size=16) + if use_nl: + # NB: this loads the BASELINE checkpoint into a use_nl=True model, so the + # non-local blocks stay randomly initialized. It is a timing/shape SMOKE + # test only — NOT fidelity-grade. For real non-local features use the + # converted weights via scripts/extract_i3d_gowtham.py --nonlocal. + from src.i3d import I3Res50 + + print(" [warn] nonlocal smoke: NL blocks random-init (not fidelity)", file=sys.stderr) + sd = ext.model.state_dict() + ext.model = I3Res50(use_nl=True).eval().to("cuda") + ext.model.load_state_dict(sd, strict=False) + + n = len(decord.VideoReader(video)) + t = time.time() + feat = ext.extract(video) # (T, 10, 2048) + dt = time.time() - t + T, nc, dim = feat.shape + seg = np.stack([segment(feat[:, c, :], 32) for c in range(nc)], 0) + print(f"[I3D {name}] frames={n} -> {feat.shape} in {dt:.1f}s") + print(f" L2/snip={_scale(feat):.2f} (MGFN canonical ~2.5; RTFM ~22)") + print(f" seg32 crop-first {seg.shape} == MGFN train (10,32,2048)") + + +def validate_videomae(video: str) -> None: + from src.features.videomae import VideoMAEFeatureExtractor + + ext = VideoMAEFeatureExtractor(device="cuda", segment_to=None) # per-snippet (T,dim) + t = time.time() + feat = ext.extract(video) + dt = time.time() - t + print(f"[VideoMAE {ext.model.config._name_or_path if hasattr(ext.model.config,'_name_or_path') else 'base'}] " + f"-> {feat.shape} {feat.dtype} in {dt:.1f}s | dim={ext.dim} " + f"finite={np.isfinite(feat).all()} L2/snip={_scale(feat):.2f}") + + +def validate_clip(video: str) -> None: + from src.features.clip import CLIPFeatureExtractor + + ext = CLIPFeatureExtractor(device="cuda", segment_to=None) # frame-level (T,512) + t = time.time() + feat = ext.extract(video) + dt = time.time() - t + print(f"[CLIP ViT-B-16 laion2b] -> {feat.shape} {feat.dtype} in {dt:.1f}s") + print(f" L2/frame={_scale(feat):.2f} (NOTE: laion2b space != VadCLIP's") + print(" OpenAI-CLIP UCFClipFeatures; fresh != provided, retrain on it)") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--video", default=None, help="raw .mp4 (default: sample from zip)") + ap.add_argument("--nonlocal", dest="nonlocal_", action="store_true") + ap.add_argument("--clip", action="store_true", help="also time the CLIP line") + ap.add_argument("--videomae", action="store_true", help="also time the VideoMAE line") + ap.add_argument("--skip-i3d", action="store_true", help="skip the I3D line") + args = ap.parse_args() + + video = args.video or _extract_sample_video() + print(f"video: {video}\n") + if not args.skip_i3d: + validate_i3d(video, use_nl=args.nonlocal_) + if args.clip: + validate_clip(video) + if args.videomae: + validate_videomae(video) + + +if __name__ == "__main__": + main() diff --git a/scripts/extract_features.py b/scripts/extract_features.py index d43282f..642e7a7 100644 --- a/scripts/extract_features.py +++ b/scripts/extract_features.py @@ -12,8 +12,6 @@ from torch.utils.data import DataLoader from tqdm.auto import tqdm -import decord - def load_feature_extractor( model_name: str = "i3d_8x8_r50", device: str = "cpu" @@ -90,6 +88,8 @@ def extract_features_from_video( # Among the transforms of `TenCropVideoFrameDataset`, `LoopPad` forcibly pads clips lower than # `frames_per_clip`(default: 16), so segment_length is designated as a multiple of 16. seg_len = 16 * 188 # 3,008 + import decord # lazy: only the legacy big-video path needs it + # read video frames vr = decord.VideoReader(uri=sample["video_path"]) segments = [] @@ -160,6 +160,28 @@ def segment_features(feat_output_dir: str, seg_output_dir: str, seg_length: int def main(args: argparse.Namespace): + # Backbone-agnostic path (issue #17): dispatch to any registered extractor and + # cache `_.npy`. The legacy i3d_8x8_r50 path below is kept for + # `--backbone` unset so existing reproduction stays byte-identical. + if args.backbone: + from src.features import build_extractor + from src.features.extract import run_extraction + + extractor = build_extractor(args.backbone, device=args.device) + dset = load_dataset(args.repo_id, config_name=args.config_name) + feat_output_dir = os.path.join(args.output_dir, f"features_{args.backbone}") + subsets = ( + dset.items() + if isinstance(dset, datasets.DatasetDict) + else [(None, dset)] + ) + for mode, dsub in subsets: + out_dir = ( + feat_output_dir if mode is None else os.path.join(feat_output_dir, mode) + ) + run_extraction(dsub, extractor, out_dir, args.backbone) + return + feat_output_dir = os.path.join(args.output_dir, "anomaly_features") # anomaly = load_ucf_crime_dataset(args.repo_id, args.cache_dir, args.config_name) ucf_crime_anomaly_dset = load_dataset("jinmang2/ucf_crime", config_name="anomaly") @@ -203,7 +225,15 @@ def main(args: argparse.Namespace): "--model_name", type=str, default="i3d_8x8_r50", - help="Feature extraction model name.", + help="Legacy pytorchvideo I3D model name (used only when --backbone is unset).", + ) + parser.add_argument( + "--backbone", + type=str, + default=None, + help="Registered backbone to dispatch on (i3d, clip, videomae, ...). " + "When set, uses the backbone-agnostic extractor and caches " + "_.npy; when unset, runs the legacy I3D path.", ) parser.add_argument( "--output_dir", diff --git a/scripts/extract_i3d_gowtham.py b/scripts/extract_i3d_gowtham.py new file mode 100644 index 0000000..b7bf01a --- /dev/null +++ b/scripts/extract_i3d_gowtham.py @@ -0,0 +1,80 @@ +"""Extract Gowtham/RTFM-faithful I3D features for a set of raw videos. + +Wraps :mod:`src.features.i3d_gowtham` (bit-exact to GowthamGottimukkala's +extractor, verified by ``scripts/verify_i3d_extract.py``). Produces one +``_i3d.npy`` of shape ``(T, 10, 2048)`` per video. + + # baseline weights + python scripts/extract_i3d_gowtham.py --videos '/path/*.mp4' --out OUTDIR + # non-local weights (recommended; closer to RTFM-style features) + python scripts/extract_i3d_gowtham.py --videos '/path/*.mp4' --out OUTDIR --nonlocal + +Weights default to the converted ``pretrained/i3d/i3d_{baseline,nonlocal}_r50_kinetics.pth`` +(run ``scripts/convert_i3d_caffe2.py`` first). Note: this faithful path writes per-frame +JPGs via ffmpeg then reads them (~37 s + I/O per video on an RTX2070S), so a full +UCF-Crime pass (~1900 videos) is an overnight job; parallelize across GPUs if available. +""" + +import argparse +import glob +import os +import sys +import time + +import numpy as np +import torch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.features.i3d_gowtham import build_model, extract_from_video + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--videos", required=True, help="glob or dir of .mp4") + ap.add_argument("--out", required=True, help="output dir for _i3d.npy") + ap.add_argument("--nonlocal", dest="nl", action="store_true") + ap.add_argument("--weights", default=None, help="override .pth path") + ap.add_argument("--frequency", type=int, default=16) + ap.add_argument("--batch-size", type=int, default=20) + ap.add_argument("--sample-mode", default="oversample", choices=["oversample", "center_crop"]) + ap.add_argument("--overwrite", action="store_true") + args = ap.parse_args() + + variant = "nonlocal" if args.nl else "baseline" + weights = args.weights or os.path.join( + ROOT, "pretrained", "i3d", f"i3d_{variant}_r50_kinetics.pth" + ) + if not os.path.exists(weights): + raise SystemExit(f"missing weights {weights} (run scripts/convert_i3d_caffe2.py)") + + pat = args.videos + videos = sorted(glob.glob(os.path.join(pat, "*.mp4") if os.path.isdir(pat) else pat)) + if not videos: + raise SystemExit(f"no videos matched {pat}") + os.makedirs(args.out, exist_ok=True) + + dev = "cuda" if torch.cuda.is_available() else "cpu" + model = build_model(weights, use_nl=args.nl, device=dev) + print(f"i3d-gowtham {variant} | {len(videos)} videos -> {args.out} (dev={dev})") + + for i, v in enumerate(videos): + name = os.path.splitext(os.path.basename(v))[0] + out = os.path.join(args.out, f"{name}_i3d.npy") + if os.path.exists(out) and not args.overwrite: + print(f" [{i + 1}/{len(videos)}] skip {name} (exists)") + continue + t = time.time() + feat = extract_from_video( + model, v, args.frequency, args.batch_size, args.sample_mode, dev + ) + np.save(out, feat) + l2 = np.linalg.norm(feat.reshape(-1, feat.shape[-1]), axis=1).mean() # per-2048-vec + print(f" [{i + 1}/{len(videos)}] {name} {feat.shape} " + f"L2/snip={l2:.1f} {time.time() - t:.1f}s") + + +if __name__ == "__main__": + main() diff --git a/scripts/extract_modern.py b/scripts/extract_modern.py new file mode 100644 index 0000000..89bf29f --- /dev/null +++ b/scripts/extract_modern.py @@ -0,0 +1,123 @@ +"""Disk/VRAM-conscious modern-backbone extraction from the UCF-Crime raw ZIPs. + +The raw videos (~128 GB) live inside `raw/*.zip`, so we NEVER unzip them all. For +each video we stream ONE member out of its zip to a temp file, run a registered +`src.features` extractor (which batches snippets to bound VRAM), save the `.npy`, +and delete the temp — disk peak ≈ one video. Backbone-agnostic via the registry, +so `--backbone videomae|xclip|internvideo|clip` all dispatch here. + + # forensic GATE first: a small test sample, then run feature_forensics on it + # (relative content/probe screen — confirm the set ranks above i3d before full extract) + PYTHONPATH=. WSAD_DATA=~/data/wsad conda run -n balaenoptera python scripts/extract_modern.py \ + --backbone videomae --model-name MCG-NJU/videomae-base --split test --limit 20 \ + --out ~/data/wsad/ucf_crime/features/videomae_base_GATE + # full run (all 1900): drop --limit, --split train then test +""" + +import argparse +import os +import tempfile +import zipfile + +import numpy as np + +ROOT = os.path.expanduser(os.environ.get("WSAD_DATA", "~/data/wsad")) +RAW = os.path.join(ROOT, "ucf_crime", "raw") +SPLIT = os.path.join(RAW, "UCF_Crimes-Train-Test-Split") + + +def build_zip_index() -> dict: + """{bare_video_id: (zip_path, member_name)} over every raw video zip (filelist only).""" + idx = {} + for z in sorted(os.listdir(RAW)): + if not z.endswith(".zip"): + continue + zp = os.path.join(RAW, z) + try: + with zipfile.ZipFile(zp) as zf: + for m in zf.namelist(): + if m.endswith(".mp4"): + vid = os.path.splitext(os.path.basename(m))[0] # _x264 + idx.setdefault(vid, (zp, m)) + except zipfile.BadZipFile: + continue + return idx + + +def split_video_ids(split: str) -> list: + """Test ids from the temporal-annotation file; train ids = everything else in the zips.""" + test_txt = os.path.join(SPLIT, "Temporal_Anomaly_Annotation_for_Testing_Videos.txt") + test_ids = [] + if os.path.exists(test_txt): + for line in open(test_txt): + parts = line.split() + if parts: + test_ids.append(os.path.splitext(parts[0])[0]) # _x264 + if split == "test": + return test_ids + # train = all videos in zips minus the test set + allids = set(build_zip_index()) + return sorted(allids - set(test_ids)) + + +def extract_one(extractor, zip_path: str, member: str, backbone: str) -> np.ndarray: + """Stream one video out of its zip to a temp file, extract, delete temp.""" + with zipfile.ZipFile(zip_path) as zf, tempfile.NamedTemporaryFile( + suffix=".mp4", delete=False + ) as tmp: + tmp.write(zf.read(member)) + tmp_path = tmp.name + try: + return np.asarray(extractor.extract(tmp_path)) + finally: + os.unlink(tmp_path) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--backbone", required=True, help="videomae | xclip | internvideo | clip") + ap.add_argument("--model-name", default=None, help="HF id (e.g. MCG-NJU/videomae-large)") + ap.add_argument("--split", default="test", choices=["test", "train"]) + ap.add_argument("--limit", type=int, default=None, help="extract only the first N (forensic gate)") + ap.add_argument("--segment-to", type=int, default=None, help="mean-pool to N segments (train) or keep full (test)") + ap.add_argument("--sample-to", type=int, default=None, help="FAST: extract only N uniformly-spaced snippets/video (~10x fewer forwards); output is (N, dim)") + ap.add_argument("--device", default="cuda") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + from src.features import build_extractor + + kw = {"device": args.device} + if args.model_name: + kw["model_name"] = args.model_name + if args.segment_to is not None: + kw["segment_to"] = args.segment_to + if args.sample_to is not None: + kw["sample_to"] = args.sample_to + extractor = build_extractor(args.backbone, **kw) + + idx = build_zip_index() + ids = split_video_ids(args.split) + ids = [v for v in ids if v in idx] + if args.limit: + # balanced gate sample: interleave anomaly + Normal + anom = [v for v in ids if "Normal" not in v][: args.limit // 2] + norm = [v for v in ids if "Normal" in v][: args.limit - len(anom)] + ids = anom + norm + os.makedirs(args.out, exist_ok=True) + print(f"[extract] backbone={args.backbone} model={args.model_name} split={args.split} " + f"n={len(ids)} dim={extractor.dim} -> {args.out}", flush=True) + + for i, vid in enumerate(ids): + dst = os.path.join(args.out, f"{vid}_{args.backbone}.npy") + if os.path.exists(dst): + continue + zp, member = idx[vid] + feats = extract_one(extractor, zp, member, args.backbone) + np.save(dst, feats) + print(f" [{i+1}/{len(ids)}] {vid} {feats.shape}", flush=True) + print("DONE", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/gpu_queue.sh b/scripts/gpu_queue.sh new file mode 100755 index 0000000..64f41b3 --- /dev/null +++ b/scripts/gpu_queue.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Serial GPU queue (8 GB -> ONE heavy run at a time). MEMORY HEADS FIRST (the open +# 0.80->0.87 question), then the clean i3d 2048-d heads. Memory heads use the full +# 8 GB toolkit (mem attention + gradient checkpointing + fp16) at batch 32 (the 8 GB +# ceiling; b64 OOMs) PLUS the two remaining 8 GB levers: cosine LR decay (the rtfm +# 0.81->0.834 lever) and EXTENDED iters (bn_wvad's peak was still rising at 1000). +# +# nohup setsid bash scripts/gpu_queue.sh > outputs/gpu_queue.log 2>&1 < /dev/null & disown +# +set -u +cd "$(dirname "$0")/.." +export PYTHONPATH=. +export WSAD_DATA="${WSAD_DATA:-$HOME/data/wsad}" +export PYTHONUNBUFFERED=1 +RUN="conda run --no-capture-output -n balaenoptera python -u scripts/run_matrix.py" + +run_job () { # tag ATTN + local tag="$1"; local attn="$2"; shift 2 + echo "===== [$(date +%H:%M:%S)] START $tag =====" + WSAD_ATTN="$attn" $RUN "$@" > "outputs/${tag}.log" 2>&1 + echo "===== [$(date +%H:%M:%S)] END $tag (exit $?) =====" + tail -3 "outputs/${tag}.log" +} + +MEM="--batch 32 --grad-checkpoint --mixed-precision fp16 --lr-decay cosine --eval-every 50 --eval-start 0" + +# --- Track 2: memory heads, batch 32 full-toolkit + cosine + extended iters --- +run_job urdmu_1024_b32cos mem --backbones i3d --heads ur_dmu --variant i3d_1024_seg200 --feature-dim 1024 $MEM --steps 3000 --force --out outputs/matrix_urdmu_1024_b32cos +run_job bnwvad_1024_b32cos mem --backbones i3d --heads bn_wvad --variant i3d_1024_seg200 --feature-dim 1024 $MEM --steps 3000 --force --out outputs/matrix_bnwvad_1024_b32cos + +# --- Track 1: clean i3d 2048-d heads + cosine (mgfn already 0.8332, sultani 0.8071 done) --- +for h in s3r gs_moe; do + run_job "${h}_clean_cosine" eager --backbones i3d --heads "$h" --variant i3d_mgfn_seg32 --lr-decay cosine --force --out "outputs/matrix_${h}_clean" +done + +echo "===== [$(date +%H:%M:%S)] QUEUE DONE =====" diff --git a/scripts/gpu_queue_pc.sh b/scripts/gpu_queue_pc.sh new file mode 100644 index 0000000..92c52f0 --- /dev/null +++ b/scripts/gpu_queue_pc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Per-crop FAITHFUL memory-head queue (the official training layout: each 10-crop file +# is a separate sample, 16100 rows). batch 64 (official) fits 8 GB because per-crop +# samples are n=1 (~10x lighter than the stacked (10,200,1024)); attention still needs +# WSAD_ATTN=mem (eager dots/repeat is (b,h,200,200)=8GB at b64). fp32, no checkpoint — +# the faithful config. Then the clean i3d 2048-d heads. +# +# nohup setsid bash scripts/gpu_queue_pc.sh > outputs/gpu_queue.log 2>&1 < /dev/null & disown +# +set -u +cd "$(dirname "$0")/.." +export PYTHONPATH=. +export WSAD_DATA="${WSAD_DATA:-$HOME/data/wsad}" +export PYTHONUNBUFFERED=1 +RUN="conda run --no-capture-output -n balaenoptera python -u scripts/run_matrix.py" + +run_job () { # tag ATTN + local tag="$1"; local attn="$2"; shift 2 + echo "===== [$(date +%H:%M:%S)] START $tag =====" + WSAD_ATTN="$attn" $RUN "$@" > "outputs/${tag}.log" 2>&1 + echo "===== [$(date +%H:%M:%S)] END $tag (exit $?) =====" + tail -3 "outputs/${tag}.log" +} + +PC="--variant i3d_1024_seg200_pc --feature-dim 1024 --batch 64 --eval-every 100 --eval-start 0" + +# memory heads, per-crop, official batch 64, fp32 (faithful), mem attention for the b64 dots +run_job urdmu_pc_b64 mem --backbones i3d --heads ur_dmu $PC --steps 3000 --force --out outputs/matrix_urdmu_pc_b64 +run_job bnwvad_pc_b64 mem --backbones i3d --heads bn_wvad $PC --steps 1000 --force --out outputs/matrix_bnwvad_pc_b64 + +# clean i3d 2048-d heads + cosine (mgfn 0.8332 / sultani 0.8071 already done) +for h in s3r gs_moe; do + run_job "${h}_clean_cosine" eager --backbones i3d --heads "$h" --variant i3d_mgfn_seg32 --lr-decay cosine --force --out "outputs/matrix_${h}_clean" +done + +echo "===== [$(date +%H:%M:%S)] QUEUE DONE =====" diff --git a/scripts/onedrive_fetch/README.md b/scripts/onedrive_fetch/README.md new file mode 100644 index 0000000..ce0b77d --- /dev/null +++ b/scripts/onedrive_fetch/README.md @@ -0,0 +1,62 @@ +# onedrive_fetch — download a SharePoint/OneDrive folder file-by-file + +Built to recover the **MGFN pre-seg32 ten-crop UCF-Crime I3D features** from the HKU +OneDrive share (`connecthkuhk-my.sharepoint.com/.../UCF-Crime`). + +## Why this exists +The data is a **folder of individual `.npy` files** (Train = 1610 files / 64.87 GB, +Test = 290 files). Clicking **Download** on the whole folder makes OneDrive zip it +on the fly, which hits SharePoint's size limit and fails: + +> The file size exceeds the allowed limit. CorrelationId: ... + +The share is an **anonymous link** (no interactive login possible), but the +`_layouts/15/download.aspx?SourceUrl=` endpoint works **per-file** as long as +you send the browser's session **cookie**. So we list the folder via REST, then pull +each file individually. No zip, no size limit. + +## One thing only you can do: grab the cookie +1. Open the share folder in the browser (you're already authenticated there). +2. F12 → **Network** tab → filter **Fetch/XHR** → refresh (F5). +3. Right-click a `RenderListDataAsStream` (or any `connecthkuhk-my.sharepoint.com`) + request with status **200** → **Copy → Copy as cURL (bash)**. +4. From the pasted command, take the value after `-b '...'` (or `-H 'cookie: ...'`) + — the `FedAuth=...; ...` string — and save it as a single line: + ```bash + printf '%s' 'FedAuth=...; rtFa=...; ...' > /tmp/sp_cookie.txt + chmod 600 /tmp/sp_cookie.txt + ``` + FedAuth is short-lived (~1 h). If a run starts failing with http=403, re-copy a + fresh cookie and re-run — completed files are skipped. + +## Run +```bash +SITE='https://connecthkuhk-my.sharepoint.com/personal/cyxcarol_connect_hku_hk' +COOKIE=/tmp/sp_cookie.txt + +# 1) list the folder -> manifest TSV (ServerRelativeUrl \t Name \t Length) +scripts/onedrive_fetch/sp_list.sh "$COOKIE" "$SITE" \ + '/personal/cyxcarol_connect_hku_hk/Documents/UCF-Crime/UCF_Train_ten_i3d' \ + /tmp/train_manifest.tsv + +# 2) download all files (parallel 4, resumable, size-verified) +scripts/onedrive_fetch/sp_download.sh "$COOKIE" "$SITE" \ + /tmp/train_manifest.tsv ~/data/UCF_Train_ten_i3d 4 + +# 3) verify count + that every file matches its manifest length +scripts/onedrive_fetch/sp_verify.sh /tmp/train_manifest.tsv ~/data/UCF_Train_ten_i3d +``` +Test set: same commands with `UCF_Test_ten_i3d` and a `test_manifest.tsv`. + +## Files +- `sp_list.sh` — REST folder listing → manifest TSV (uses `GetFolderByServerRelativeUrl/Files`, `$top=5000`). +- `sp_download.sh` — orchestrator: manifest → parallel resumable download. +- `dl_one.sh` — per-file worker (download.aspx?SourceUrl, size-check, 3 retries). +- `sp_verify.sh` — checks on-disk files against the manifest (count + per-file length). + +## Notes +- Resumability is by **exact byte length** match vs the manifest, so a truncated file + is re-fetched on the next run. +- `$top=5000` is a single page; folders >5000 files would need paging (none here). +- The captured cookie authorises whoever holds it for the share's lifetime — keep + `/tmp/sp_cookie.txt` local (mode 600) and let it expire; do not commit it. diff --git a/scripts/onedrive_fetch/_dl_deepmil_train.sh b/scripts/onedrive_fetch/_dl_deepmil_train.sh new file mode 100644 index 0000000..ade90d4 --- /dev/null +++ b/scripts/onedrive_fetch/_dl_deepmil_train.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# one-off: sequentially download the 3 DeepMIL UCF 1024-d I3D train zips. +# sequential = each gets full bandwidth -> finishes within dl_one's 30-min max-time. +export SP_SITE='https://stuxidianeducn-my.sharepoint.com/personal/pengwu_stu_xidian_edu_cn' +export SP_COOKIE_FILE=/tmp/sp_cookie.txt +export DEST_DIR="$HOME/data/wsad/ucf_crime/features/i3d_1024_raw/train" +cd "$HOME/wsad" || exit 1 +mkdir -p "$DEST_DIR" +while IFS=$'\t' read -r src name len; do + echo "=== $(date +%H:%M:%S) start $name ($len) ===" + bash scripts/onedrive_fetch/dl_one.sh "$src" "$name" "$len" + echo "=== $(date +%H:%M:%S) done $name ===" +done < /tmp/train_zips.tsv +echo "=== ALL TRAIN ZIPS DONE $(date +%H:%M:%S) ===" diff --git a/scripts/onedrive_fetch/_dl_test.sh b/scripts/onedrive_fetch/_dl_test.sh new file mode 100644 index 0000000..e1c5507 --- /dev/null +++ b/scripts/onedrive_fetch/_dl_test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +export SP_SITE='https://stuxidianeducn-my.sharepoint.com/personal/pengwu_stu_xidian_edu_cn' +export SP_COOKIE_FILE=/tmp/sp_cookie.txt +export DEST_DIR="$HOME/data/wsad/ucf_crime/features/i3d_1024_raw/test_npy" +cd "$HOME/wsad" || exit 1 +export -f 2>/dev/null +# parallel per-file (small files -> overhead-bound, 8-way) +cat /tmp/test_manifest.tsv | xargs -P 8 -d '\n' -I {} bash -c ' + IFS=$'"'"'\t'"'"' read -r src name len <<< "{}" + SP_SITE="'"$SP_SITE"'" SP_COOKIE_FILE=/tmp/sp_cookie.txt DEST_DIR="'"$DEST_DIR"'" bash scripts/onedrive_fetch/dl_one.sh "$src" "$name" "$len" >/dev/null 2>&1 +' +echo "=== TEST DOWNLOAD DONE $(date +%H:%M:%S); files: $(ls $DEST_DIR/*.npy 2>/dev/null | wc -l)/2901 ===" diff --git a/scripts/onedrive_fetch/_unzip_train.sh b/scripts/onedrive_fetch/_unzip_train.sh new file mode 100644 index 0000000..b3b581c --- /dev/null +++ b/scripts/onedrive_fetch/_unzip_train.sh @@ -0,0 +1,7 @@ +#!/bin/bash +D="$HOME/data/wsad/ucf_crime/features/i3d_1024_raw" +for z in RGB0 Training_Normal_Videos_Anomaly1 Training_Normal_Videos_Anomaly2; do + echo "=== unzip $z $(date +%H:%M:%S) ===" + unzip -oq "$D/train/$z.zip" -d "$D/train_npy/" +done +echo "=== UNZIP DONE $(date +%H:%M:%S); files: $(find $D/train_npy -name '*.npy' | wc -l) ===" diff --git a/scripts/onedrive_fetch/dl_one.sh b/scripts/onedrive_fetch/dl_one.sh new file mode 100755 index 0000000..0114781 --- /dev/null +++ b/scripts/onedrive_fetch/dl_one.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Per-file download worker for a SharePoint/OneDrive folder. +# Args: +# Env : SP_COOKIE_FILE SP_SITE DEST_DIR +# +# Downloads one file via the _layouts/15/download.aspx?SourceUrl= endpoint, +# verifies byte length, retries up to 3x, and skips files already complete +# (makes the whole run resumable). +set -u +SRC="$1"; NAME="$2"; LEN="$3" +OUT="$DEST_DIR/$NAME" +COOKIE=$(cat "$SP_COOKIE_FILE") +DL="${SP_SITE}/_layouts/15/download.aspx" + +# already complete? +if [ -f "$OUT" ]; then + cur=$(stat -c %s "$OUT" 2>/dev/null || echo 0) + [ "$cur" = "$LEN" ] && { echo "SKIP $NAME"; exit 0; } +fi + +code="" cur=0 +for try in 1 2 3; do + code=$(curl -sS -L --max-time 1800 -G -b "$COOKIE" -H 'User-Agent: Mozilla/5.0' \ + --data-urlencode "SourceUrl=$SRC" "$DL" -o "$OUT" -w '%{http_code}') + cur=$(stat -c %s "$OUT" 2>/dev/null || echo 0) + if [ "$code" = "200" ] && [ "$cur" = "$LEN" ]; then + echo "OK $NAME ($cur)"; exit 0 + fi + echo "RETRY($try) $NAME http=$code got=$cur want=$LEN" + sleep 3 +done +echo "FAIL $NAME http=$code got=$cur want=$LEN" +exit 1 diff --git a/scripts/onedrive_fetch/sp_download.sh b/scripts/onedrive_fetch/sp_download.sh new file mode 100755 index 0000000..eacc49a --- /dev/null +++ b/scripts/onedrive_fetch/sp_download.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Download every file listed in a manifest TSV from a SharePoint/OneDrive folder. +# Resumable: files already present with the correct byte length are skipped. +# +# Usage: sp_download.sh [parallel] +# parallel defaults to 4. +# +# On finish prints OK/SKIP/FAIL counts. Re-run with a fresh cookie to resume +# (e.g. after the FedAuth cookie expires mid-run) — completed files are skipped. +set -euo pipefail +export SP_COOKIE_FILE="$1" +export SP_SITE="$2" +MANIFEST="$3" +export DEST_DIR="$4" +P="${5:-4}" +HERE="$(cd "$(dirname "$0")" && pwd)" +mkdir -p "$DEST_DIR" + +# Convert BOTH tab and newline to NUL so each (SourceUrl,Name,Length) field is a +# separate NUL token; xargs -0 -n3 then feeds exactly 3 args per file. +tr '\t\n' '\0\0' < "$MANIFEST" \ + | xargs -0 -n3 -P"$P" "$HERE/dl_one.sh" + +echo "----" +echo "want : $(wc -l < "$MANIFEST")" +echo "have : $(ls -1 "$DEST_DIR" | wc -l)" diff --git a/scripts/onedrive_fetch/sp_list.sh b/scripts/onedrive_fetch/sp_list.sh new file mode 100755 index 0000000..47f8b16 --- /dev/null +++ b/scripts/onedrive_fetch/sp_list.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# List every file in a SharePoint/OneDrive folder via REST -> manifest TSV. +# Usage: sp_list.sh +# +# cookie_file : text file holding the raw Cookie header value (FedAuth=...; ...) +# site_base : https://-my.sharepoint.com/personal/ +# folder : server-relative URL, e.g. +# /personal//Documents/UCF-Crime/UCF_Train_ten_i3d +# out_tsv : output manifest, 3 cols: ServerRelativeUrl Name Length +# +# NOTE: $top=5000 returns up to 5000 files in one page; folders larger than that +# need paging (not implemented — none of the UCF folders need it). +set -euo pipefail +COOKIE_FILE="$1"; SITE="$2"; FOLDER="$3"; OUT="$4" +COOKIE=$(cat "$COOKIE_FILE") +TMP=$(mktemp) +curl -fsS --max-time 180 -b "$COOKIE" \ + -H 'Accept: application/json;odata=nometadata' \ + "${SITE}/_api/web/GetFolderByServerRelativeUrl('${FOLDER}')/Files?\$select=Name,ServerRelativeUrl,Length&\$top=5000" \ + -o "$TMP" +python3 - "$TMP" "$OUT" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])); v = d['value'] +with open(sys.argv[2], 'w') as f: + for x in v: + f.write(f"{x['ServerRelativeUrl']}\t{x['Name']}\t{x['Length']}\n") +gb = sum(int(x['Length']) for x in v) / 1e9 +print(f"{len(v)} files, {gb:.2f} GB -> {sys.argv[2]}") +PY +rm -f "$TMP" diff --git a/scripts/onedrive_fetch/sp_verify.sh b/scripts/onedrive_fetch/sp_verify.sh new file mode 100755 index 0000000..411d7dd --- /dev/null +++ b/scripts/onedrive_fetch/sp_verify.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Verify a downloaded folder against its manifest TSV. +# Usage: sp_verify.sh +# Reports missing files, wrong-size files, and confirms NumPy magic on a sample. +set -euo pipefail +MANIFEST="$1"; DEST="$2" +python3 - "$MANIFEST" "$DEST" <<'PY' +import os, sys +manifest, dest = sys.argv[1], sys.argv[2] +want = {} +for line in open(manifest): + src, name, length = line.rstrip('\n').split('\t') + want[name] = int(length) +missing, wrong = [], [] +for name, length in want.items(): + p = os.path.join(dest, name) + if not os.path.exists(p): + missing.append(name) + elif os.path.getsize(p) != length: + wrong.append((name, os.path.getsize(p), length)) +print(f"manifest : {len(want)} files") +print(f"on disk : {sum(1 for n in want if os.path.exists(os.path.join(dest,n)))}") +print(f"missing : {len(missing)}") +print(f"badsize : {len(wrong)}") +for n in missing[:20]: print(" MISSING", n) +for n,g,w in wrong[:20]: print(f" BADSIZE {n} got={g} want={w}") +# sample NumPy magic check +ok = True +for n in list(want)[:5]: + p = os.path.join(dest, n) + if os.path.exists(p): + with open(p, 'rb') as f: + if f.read(6) != b'\x93NUMPY': + ok = False; print(" NOT-NPY", n) +print("sample npy magic:", "OK" if ok else "FAIL") +sys.exit(1 if (missing or wrong) else 0) +PY diff --git a/scripts/prepare_clip_features.py b/scripts/prepare_clip_features.py index 1aa9438..26fe4a4 100644 --- a/scripts/prepare_clip_features.py +++ b/scripts/prepare_clip_features.py @@ -5,7 +5,9 @@ This symlinks each ``.npy`` into ``/train`` or ``/test`` so the loader in ``src/data/local.py`` can walk them. See ``docs/DATA_LOCAL.md``. - python scripts/prepare_clip_features.py --src /path/UCFClipFeatures --dst ~/data/wsad/clip + python scripts/prepare_clip_features.py \ + --src ~/data/wsad/ucf_crime/features/clip/_byclass \ + --dst ~/data/wsad/ucf_crime/features/clip # optional: --test-list list/ucf_CLIP_rgbtest.csv (else fetched from VadCLIP repo) """ diff --git a/scripts/run_matrix.py b/scripts/run_matrix.py new file mode 100644 index 0000000..37f80ac --- /dev/null +++ b/scripts/run_matrix.py @@ -0,0 +1,334 @@ +"""``{backbone × head}`` comparison matrix — train + eval, real ROC-AUC (Spec 1 ③). + +Crosses the backbone registry (I3D 2048-d, CLIP 512-d) with the head registry, +auto-selecting only compatible pairs (text-branch heads need a text-aligned +backbone — :mod:`src.compat`). Each pair is trained on the cached local features +and evaluated with the per-head eval adapter (:mod:`src.eval_matrix`), then logged +to a JSON + Markdown table. + +The original training loop (:class:`src.trainer.WSVADTrainer`) and model defs are +unchanged; per-epoch eval is skipped during training (``test_dataset=None``) and a +single correct eval is run at the end — the matrix only *orchestrates*. + +Examples +-------- + # quick smoke of two solid pairs (few epochs) + PYTHONPATH=. python scripts/run_matrix.py --heads rtfm,vadclip --quick + # full matrix + PYTHONPATH=. python scripts/run_matrix.py --epochs 50 + # paper-faithful cells from official checkpoints (no training) + PYTHONPATH=. python scripts/run_matrix.py --official-only +""" + +import argparse +import json +import os +import time +import traceback +from typing import Optional + +import torch +from hydra import compose, initialize +from hydra.utils import _locate, instantiate +from omegaconf import OmegaConf + +import src.models # noqa: F401 (register heads) +from src.compat import is_compatible +from src.eval_matrix import evaluate +from src.trainer import WSVADTrainer, build_datasets + +BACKBONE_DIM = {"i3d": 2048, "clip": 512} +_DIM_FIELDS = ("feature_size", "channels", "visual_width") + +# (backbone, head) -> train batch size override; i3d (2048d) attention / memory-bank +# heads OOM at the default batch on an 8GB GPU. Smaller batch is the only change. +HEAD_BATCH = {("i3d", "ur_dmu"): 4, ("i3d", "bn_wvad"): 4} + +# head -> training/data recipe. ``runner`` is the configs/runner/.yaml. +# ``text`` heads need a text-aligned backbone; ``magnitude``/``segment``/``length`` +# pick the per-head CLIP preprocessing (I3D zip is pre-segmented to 32 + magnitude). +# PER-PAPER TRAINING (these MIL methods train by ITERATIONS, not epochs): +# ``steps`` = training iterations, ``batch`` = videos per loader (total = 2*batch +# = normal+abnormal), ``lr`` = paper learning rate. +# [ref] = from vendored .reference/; [est] = best-estimate, verify vs paper. +# wd/eval_start/grad_clip/ckpt verified vs official code (sciomc audit 2026-06-17). +HEADS = { + "sultani": dict(runner="mil", text=False, magnitude=True, segment=32, length=None, steps=5000, batch=30, lr=1e-3, wd=5e-4), # [est] Sultani CVPR'18 + "rtfm": dict(runner="rtfm", text=False, magnitude=True, segment=32, length=None, steps=15000, batch=32, lr=1e-3, wd=5e-3, eval_start=200), # [ref] RTFM main.py:41 wd=0.005 + "mgfn": dict(runner="mgfn", text=False, magnitude=True, segment=32, length=None, steps=5000, batch=8, lr=1e-3, wd=5e-4), # [est] MGFN (paper 0.8667) + "ur_dmu": dict(runner="ur_dmu", text=False, magnitude=True, segment=32, length=None, steps=3000, batch=64, lr=1e-4, wd=5e-5), # [ref] UR-DMU wd=5e-5 (i3d-OOM 8GB) + "bn_wvad": dict(runner="bn_wvad", text=False, magnitude=True, segment=32, length=None, steps=1000, batch=64, lr=1e-4, wd=5e-5, grad_clip=1.0, ckpt="pr_auc"), # [ref] BN-WVAD best on AP + "s3r": dict(runner="s3r", text=False, magnitude=True, segment=32, length=None, steps=15000, batch=32, lr=1e-3, wd=5e-3, eval_start=5000), # [ref] S3R wd=0.005, 5000 warmup + "gs_moe": dict(runner="gs_moe", text=False, magnitude=True, segment=32, length=None, steps=5000, batch=64, lr=1e-4, wd=5e-4), # [est] GS-MoE + "clip_tsa": dict(runner="clip_tsa", text=False, magnitude=False, segment=32, length=None, steps=4000, batch=16, lr=1e-3, wd=5e-3), # [ref] CLIP-TSA main.py:122 wd=0.005 + "vadclip": dict(runner="vadclip", text=True, magnitude=False, segment=None, length=256, steps=3000, batch=64, lr=2e-5, wd=1e-2), # [ref] VadCLIP AdamW (official ckpt used) + "tpwng": dict(runner="tpwng", text=True, magnitude=False, segment=32, length=None, steps=3000, batch=16, lr=1e-3, wd=5e-3), # [est] TPWNG (paper-only) +} + +# paper-faithful official head checkpoints present locally (eval-only). +OFFICIAL_CKPT = { + ("clip", "vadclip"): "pretrained/vadclip/model_ucf.pth", + ("i3d", "mgfn"): "pretrained/mgfn/mgfn_ucf.pkl", +} + + +def _set_dim(cfg, dim: int) -> None: + for f in _DIM_FIELDS: + if hasattr(cfg, f): + setattr(cfg, f, dim) + + +def _data_cfg(backbone: str, spec: dict, variant: str = "i3d"): + dim_seg = spec["segment"] if backbone == "clip" else None # I3D zip already 32-seg + return OmegaConf.create( + { + "root": os.environ.get("WSAD_DATA", "~/data/wsad"), + "source": "local", + "backbone": backbone, + "dataset_dir": "ucf_crime", + "feature_variant": variant, # which I3D extraction (i3d_tushar/i3d_pyvideo/i3d_ours) + "ground_truth": None, + # i3d: lazy zip load (avoids eager-loading 1610 npy ~12.8 GB into the + # 15 GB WSL RAM, which crashes WSL). clip uses npy dirs (unaffected). + "dynamic_load": backbone == "i3d", + "batch_size": 1, + "frames_per_clip": 16, + # i3d lazy-zip: the ZipFile handle isn't shareable across DataLoader + # worker processes (BadZipFile), so load in-process. clip = picklable npy. + "num_workers": 0 if backbone == "i3d" else 4, + "clip_length": spec["length"], + "segment": dim_seg, + "single_crop": True, + "with_magnitude": spec["magnitude"] if backbone == "clip" else True, + } + ) + + +def _build_model(head: str, backbone: str, device: str, dim: Optional[int] = None): + spec = HEADS[head] + with initialize(version_base=None, config_path="../configs"): + cfg = compose(config_name="default", overrides=[f"runner={spec['runner']}"]) + model_cfg = instantiate(cfg.runner.model_config) + _set_dim(model_cfg, dim or BACKBONE_DIM[backbone]) # dim override for 1024-d DeepMIL (UR-DMU/BN-WVAD) + model = _locate(cfg.runner.model_class)(model_cfg) + lr = float(cfg.runner.optimizer.learning_rate) + wd = float(cfg.runner.optimizer.weight_decay) + return model.to(device), lr, wd + + +def _official_converter(head: str): + """Return the official-checkpoint -> repo-state converter for a head.""" + if head == "vadclip": + from src.models.vadclip.modeling_vadclip import convert_official_vadclip + + return convert_official_vadclip + if head == "mgfn": + from scripts.convert_official_to_hf import convert + + return convert + return None + + +def _load_official(model, path: str, head: str): + state = torch.load(path, map_location="cpu", weights_only=False) + state = state.get("state_dict", state) if isinstance(state, dict) else state + conv = _official_converter(head) + if conv is not None: + state = conv(state) + clean = {k[len("model.") :]: v for k, v in state.items() if k.startswith("model.")} + missing, unexpected = model.load_state_dict(clean or state, strict=False) + if missing or unexpected: + print(f" [official load] missing={len(missing)} unexpected={len(unexpected)}") + return model + + +def run_pair( + backbone: str, + head: str, + device: str, + official: bool, + variant: str = "i3d", + wandb_proj: Optional[str] = None, + steps_override: Optional[int] = None, + batch_override: Optional[int] = None, + lr_override: Optional[float] = None, + lr_decay: Optional[str] = None, + eval_every: Optional[int] = None, + eval_start_override: Optional[int] = None, + dim_override: Optional[int] = None, + mixed_precision: str = "no", + grad_checkpoint: bool = False, + grad_accum: int = 1, +) -> dict: + spec = HEADS[head] + t0 = time.time() + feat_tag = variant if backbone == "i3d" else backbone + data_cfg = _data_cfg(backbone, spec, variant) + train_sets, test_set = build_datasets(data_cfg) + model, cfg_lr, wd = _build_model(head, backbone, device, dim=dim_override) + # per-paper recipe (HEADS[head]) with optional CLI overrides + max_steps = steps_override or spec.get("steps", 3000) + lr = lr_override or spec.get("lr") or cfg_lr + eval_interval = eval_every or max(max_steps // 30, 50) + + if official and (backbone, head) in OFFICIAL_CKPT: + _load_official(model, OFFICIAL_CKPT[(backbone, head)], head) + metrics = evaluate(model, test_set, backbone=backbone, head=head, device=device) + return { + "backbone": backbone, "head": head, "feature": feat_tag, "dim": dim_override or BACKBONE_DIM[backbone], + "steps": 0, "source": "official-ckpt", + "roc_auc": round(metrics["roc_auc"], 4), "pr_auc": round(metrics["pr_auc"], 4), + "best_step": -1, "n_test": len(test_set), "seconds": round(time.time() - t0, 1), + } + + # thin orchestration: per-paper recipe (HEADS) -> the canonical WSVADTrainer.fit_steps. + from src.trainer import WSVADTrainer + + # explicit --batch wins; HEAD_BATCH is only a fallback memory cap for 2048-d OOM + # (it must NOT silently throttle 1024-d memory heads to 4 — that breaks BN-WVAD's + # BatchNorm stats; official batch=64). + bs = batch_override or HEAD_BATCH.get((backbone, head)) or spec.get("batch", 16) + wd_v = spec.get("wd", wd) + grad_clip = spec.get("grad_clip") + ckpt_metric = spec.get("ckpt", "roc_auc") # BN-WVAD selects on AP (pr_auc) + eval_start = eval_start_override if eval_start_override is not None else spec.get("eval_start", 0) + + run = None + if wandb_proj: + import wandb + run = wandb.init(project=wandb_proj, name=f"{feat_tag}-{head}", reinit=True, + config={"feature": feat_tag, "backbone": backbone, "head": head, + "lr": lr, "wd": wd_v, "batch": 2 * bs, "steps": max_steps, + "grad_clip": grad_clip, "ckpt_metric": ckpt_metric, + "n_train": len(train_sets["normal"]) + len(train_sets["abnormal"]), + "n_test": len(test_set)}) + + def eval_fn(m): # per-head eval; GPU-only (no CPU fallback -> clip_tsa >20GiB kills WSL) + try: + res = evaluate(m, test_set, backbone=backbone, head=head, device=device) + except torch.cuda.OutOfMemoryError: + torch.cuda.empty_cache() + raise + if run: + run.log({"roc_auc": res["roc_auc"], "pr_auc": res["pr_auc"]}) + return res + + trainer = WSVADTrainer(model=model, learning_rate=lr, weight_decay=wd_v, batch_size=bs, + num_workers=int(data_cfg.num_workers), frames_per_clip=16, + mixed_precision=mixed_precision, grad_clip_norm=grad_clip, + gradient_checkpointing=grad_checkpoint, + gradient_accumulation_steps=grad_accum) + best = trainer.fit_steps(train_sets, max_steps=max_steps, eval_fn=eval_fn, + eval_interval=eval_interval, eval_start=eval_start, + select_metric=ckpt_metric, lr_decay=lr_decay) + if run: + run.summary.update({"best_roc_auc": best["roc_auc"], "best_pr_auc": best["pr_auc"], + "best_step": best["best_step"], "last_auc": best["last"]}) + run.finish() + return { + "backbone": backbone, "head": head, "feature": feat_tag, "dim": dim_override or BACKBONE_DIM[backbone], + "steps": max_steps, "batch": 2 * bs, "lr": lr, "wd": wd_v, "source": "trained", + "roc_auc": round(best["roc_auc"], 4), "pr_auc": round(best["pr_auc"], 4), + "best_step": best["best_step"], "last_auc": round(best["last"], 4), + "auc_mean": best.get("auc_mean"), "auc_std": best.get("auc_std"), "lr_decay": lr_decay, + "n_test": len(test_set), "seconds": round(time.time() - t0, 1), + } + + +def write_table(results: list, out_md: str) -> None: + backbones = sorted({r["backbone"] for r in results if "roc_auc" in r}) + heads = [h for h in HEADS if any(r["head"] == h for r in results)] + cell = {(r["backbone"], r["head"]): r for r in results} + lines = ["# {backbone × head} ROC-AUC matrix", "", "Frame-level ROC-AUC on UCF-Crime test (290 videos).", ""] + lines.append("| head \\ backbone | " + " | ".join(f"{b} ({BACKBONE_DIM[b]}d)" for b in backbones) + " |") + lines.append("|" + "---|" * (len(backbones) + 1)) + for h in heads: + row = [h] + for b in backbones: + r = cell.get((b, h)) + if r is None: + row.append("·") + elif "error" in r: + row.append("err") + else: + tag = "*" if r["source"] == "official-ckpt" else "" + row.append(f"{r['roc_auc']:.4f}{tag}") + lines.append("| " + " | ".join(row) + " |") + lines += ["", "`*` = official checkpoint (eval-only, paper-faithful); others trained from scratch.", ""] + with open(out_md, "w") as f: + f.write("\n".join(lines)) + + +def main(): + ap = argparse.ArgumentParser(description="{backbone × head} comparison matrix") + ap.add_argument("--backbones", default="i3d,clip") + ap.add_argument("--heads", default=",".join(HEADS)) + ap.add_argument("--steps", type=int, default=None, help="override per-paper step count (HEADS recipe)") + ap.add_argument("--batch", type=int, default=None, help="override per-loader batch (total=2x)") + ap.add_argument("--quick", action="store_true", help="200 steps (smoke)") + ap.add_argument("--official", action="store_true", help="use official ckpt where available") + ap.add_argument("--official-only", action="store_true", help="only eval official-ckpt pairs") + ap.add_argument("--out", default="outputs/matrix") + ap.add_argument("--force", action="store_true", help="recompute pairs already in results") + ap.add_argument("--variant", default="i3d", help="i3d feature variant: i3d_tushar/i3d_pyvideo/i3d_ours") + ap.add_argument("--wandb", default=None, help="wandb project name (enables per-cell logging)") + ap.add_argument("--lr", type=float, default=None, help="override runner lr (e.g. 5e-5)") + ap.add_argument("--lr-decay", default=None, choices=[None, "cosine"], help="per-step LR decay (stabilizes constant-lr divergence)") + ap.add_argument("--eval-every", type=int, default=None, help="override eval interval in steps (MGFN overfits early -> eval often to catch the peak)") + ap.add_argument("--eval-start", type=int, default=None, help="override step to start evaluating (default = per-head recipe)") + ap.add_argument("--feature-dim", type=int, default=None, help="override feature dim (1024 for DeepMIL UR-DMU/BN-WVAD vs default 2048)") + ap.add_argument("--mixed-precision", default="no", choices=["no", "fp16", "bf16"], help="AMP — halves activation memory (helps fit batch 64 on 8GB)") + ap.add_argument("--grad-checkpoint", action="store_true", help="activation checkpointing — recompute in backward so the full batch-64 forward fits 8GB (BN heads need the real batch)") + ap.add_argument("--grad-accum", type=int, default=1, help="gradient accumulation steps (NB: does not enlarge BatchNorm's effective batch)") + args = ap.parse_args() + + device = "cuda" if torch.cuda.is_available() else "cpu" + steps_override = 200 if args.quick else args.steps + os.makedirs(args.out, exist_ok=True) + json_path, md_path = f"{args.out}/results.json", f"{args.out}/matrix.md" + + results = [] + done = set() + if os.path.exists(json_path) and not args.force: + results = json.load(open(json_path)) + # re-run failed cells; only treat successful pairs as done + done = {(r["backbone"], r["head"]) for r in results if "error" not in r} + + backbones = args.backbones.split(",") + heads = args.heads.split(",") + for backbone in backbones: + for head in heads: + if head not in HEADS: + continue + if not is_compatible(backbone, head): + continue + if args.official_only and (backbone, head) not in OFFICIAL_CKPT: + continue + if (backbone, head) in done: + print(f"[skip] {backbone}×{head} (already in results)") + continue + print(f"\n=== {backbone} × {head} (feature={args.variant if backbone=='i3d' else backbone}, steps={steps_override or HEADS[head].get('steps')}) ===") + try: + r = run_pair(backbone, head, device, + args.official or args.official_only, variant=args.variant, + wandb_proj=args.wandb, steps_override=steps_override, + batch_override=args.batch, lr_override=args.lr, + lr_decay=args.lr_decay, eval_every=args.eval_every, + eval_start_override=args.eval_start, dim_override=args.feature_dim, + mixed_precision=args.mixed_precision, grad_checkpoint=args.grad_checkpoint, + grad_accum=args.grad_accum) + print(" ->", r) + except Exception as e: + r = {"backbone": backbone, "head": head, "error": str(e)[:300]} + print(" !! FAILED:", str(e)[:200]) + traceback.print_exc() + results = [x for x in results if not (x["backbone"] == backbone and x["head"] == head)] + results.append(r) + json.dump(results, open(json_path, "w"), indent=2) + write_table(results, md_path) + + print(f"\nwrote {json_path} and {md_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_phase1_videomae.sh b/scripts/run_phase1_videomae.sh new file mode 100755 index 0000000..aff4a2f --- /dev/null +++ b/scripts/run_phase1_videomae.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Robust, RESUMABLE Phase-1 runner: VideoMAE features -> content heads, vs I3D. +# The WSL host reboots intermittently (kills background jobs + /tmp scripts), so this +# lives in the repo and every stage is IDEMPOTENT — just re-run it after any reboot and +# it skips finished work and continues. Extraction skips existing .npy; head training +# skips a stage whose results.json already exists. +# +# nohup setsid bash scripts/run_phase1_videomae.sh > outputs/phase1_videomae.log 2>&1 < /dev/null & disown +# +set -u +cd "$(dirname "$0")/.." +export PYTHONPATH=. WSAD_DATA="${WSAD_DATA:-$HOME/data/wsad}" PYTHONUNBUFFERED=1 +RUN="conda run --no-capture-output -n balaenoptera python -u" +FEAT="$WSAD_DATA/ucf_crime/features/videomae_seg32" +MODEL="MCG-NJU/videomae-base" +TRAIN_N=1610; TEST_N=290 + +ts() { date +%H:%M:%S; } + +# ---- stage 1: extract train (sample_to 32, fast ~1.3s/vid). resumable. ---- +have=$(ls "$FEAT/train" 2>/dev/null | wc -l) +if [ "$have" -lt "$TRAIN_N" ]; then + echo "[$(ts)] EXTRACT train (have $have/$TRAIN_N)" + $RUN scripts/extract_modern.py --backbone videomae --model-name $MODEL --split train \ + --sample-to 32 --device cuda --out "$FEAT/train" >> outputs/p1_ex_train.log 2>&1 +fi +echo "[$(ts)] train ready: $(ls "$FEAT/train" 2>/dev/null | wc -l)/$TRAIN_N" + +# ---- stage 2: extract test (full-length, needed for frame-eval). resumable. ---- +have=$(ls "$FEAT/test" 2>/dev/null | wc -l) +if [ "$have" -lt "$TEST_N" ]; then + echo "[$(ts)] EXTRACT test (have $have/$TEST_N)" + $RUN scripts/extract_modern.py --backbone videomae --model-name $MODEL --split test \ + --device cuda --out "$FEAT/test" >> outputs/p1_ex_test.log 2>&1 +fi +echo "[$(ts)] test ready: $(ls "$FEAT/test" 2>/dev/null | wc -l)/$TEST_N" + +# only train heads once BOTH splits are complete +if [ "$(ls "$FEAT/train" 2>/dev/null | wc -l)" -lt "$TRAIN_N" ] || \ + [ "$(ls "$FEAT/test" 2>/dev/null | wc -l)" -lt "$TEST_N" ]; then + echo "[$(ts)] extraction incomplete — re-run this script after the next reboot to resume." + exit 0 +fi + +# ---- stage 3: content-head training (skip if result exists). ---- +train_head () { # head extra-args... + local h="$1"; shift + local out="outputs/matrix_${h}_videomae" + if [ -f "$out/results.json" ] && ! grep -q '"error"' "$out/results.json"; then + echo "[$(ts)] $h @ videomae already done: $($RUN -c "import json;print(json.load(open('$out/results.json'))[0].get('roc_auc'))" 2>/dev/null)" + return + fi + echo "[$(ts)] TRAIN $h @ videomae" + $RUN scripts/run_matrix.py --backbones i3d --heads "$h" --variant videomae_seg32 \ + --feature-dim 768 --lr-decay cosine "$@" --force --out "$out" > "outputs/p1_${h}.log" 2>&1 + $RUN -c "import json;r=json.load(open('$out/results.json'))[0];print('[$(ts)] $h @ videomae roc=%s'%r.get('roc_auc'))" 2>/dev/null +} +train_head sultani --eval-every 200 +train_head ur_dmu --batch 64 --eval-every 100 --eval-start 0 --steps 3000 + +echo "[$(ts)] PHASE1 DONE — sultani vs i3d 0.807, ur_dmu vs i3d 0.815" diff --git a/scripts/eval_mgfn_local.py b/scripts/verify/eval_mgfn_local.py similarity index 82% rename from scripts/eval_mgfn_local.py rename to scripts/verify/eval_mgfn_local.py index 19e1b2a..06a0393 100644 --- a/scripts/eval_mgfn_local.py +++ b/scripts/verify/eval_mgfn_local.py @@ -29,12 +29,18 @@ assert not m and not u, (len(m), len(u)) model = model.to(DEVICE).eval() -# --- data (local, dynamic load) --- -z = zipfile.ZipFile(f"{ROOT}/test.zip") +# --- data (local, dynamic load) — resolve the new features/i3d + annotations +# layout (legacy dataset-root fallback) via the loader's own resolvers --- +from types import SimpleNamespace + +from src.data.local import _i3d_zip_path, _local_ground_truth_path + +_cfg = SimpleNamespace(root=os.path.dirname(ROOT), dataset_dir="ucf_crime", ground_truth=None) +z = zipfile.ZipFile(_i3d_zip_path(os.path.dirname(ROOT), _cfg, "test")) infos = [i for i in z.infolist() if not i.is_dir()] filenames = [i.filename.split("/")[-1] for i in infos] values = {fn: info for fn, info in zip(filenames, infos)} -gt = json.load(open(f"{ROOT}/ground_truth.json")) +gt = json.load(open(_local_ground_truth_path(_cfg))) ds = FeatureDataset(filenames, values, labels=gt, open_func=z.open, with_magnitude=True) print("test videos:", len(ds)) diff --git a/scripts/eval_vadclip_ucf.py b/scripts/verify/eval_vadclip_ucf.py similarity index 98% rename from scripts/eval_vadclip_ucf.py rename to scripts/verify/eval_vadclip_ucf.py index 4e6e213..5760d90 100644 --- a/scripts/eval_vadclip_ucf.py +++ b/scripts/verify/eval_vadclip_ucf.py @@ -35,7 +35,7 @@ CKPT = f"{ROOT}/pretrained/vadclip/model_ucf.pth" TESTCSV = f"{REF}/../list/ucf_CLIP_rgbtest.csv" GT = f"{REF}/../list/gt_ucf.npy" -CLIPDIR = f"{DATA}/ucf_crime/UCFClipFeatures" +CLIPDIR = f"{DATA}/ucf_crime/features/clip/_byclass" MAXLEN = 256 device = "cuda" diff --git a/scripts/verify_bn_wvad.py b/scripts/verify/verify_bn_wvad.py similarity index 100% rename from scripts/verify_bn_wvad.py rename to scripts/verify/verify_bn_wvad.py diff --git a/scripts/verify_checkpoints.py b/scripts/verify/verify_checkpoints.py similarity index 100% rename from scripts/verify_checkpoints.py rename to scripts/verify/verify_checkpoints.py diff --git a/scripts/verify_clip_tsa.py b/scripts/verify/verify_clip_tsa.py similarity index 100% rename from scripts/verify_clip_tsa.py rename to scripts/verify/verify_clip_tsa.py diff --git a/scripts/verify/verify_i3d_extract.py b/scripts/verify/verify_i3d_extract.py new file mode 100644 index 0000000..5b427b4 --- /dev/null +++ b/scripts/verify/verify_i3d_extract.py @@ -0,0 +1,89 @@ +"""Verify the Gowtham/RTFM-faithful I3D extractor (src.features.i3d_gowtham). + +Two checks, on Gowtham's bundled ``samplevideos/Abuse001_x264.mp4``: + + (1) PORT CORRECTNESS — our ``extract_from_frames_dir`` vs Gowtham's *original* + ``extract_features.run()`` on the *same* ffmpeg frames + same model. + Expect bit-exact (max|Δ| ~ 0): identical math, identical inputs. + + (2) LINEAGE / FAITHFULNESS — our output vs RTFM's released + ``_archive/UCF_Train_ten_crop_i3d/Abuse001_x264_i3d.npy``. Expect a tiny + gap only (different ffmpeg version / JPG bytes), cosine ~1.0, L2 scale ~22. + + conda run -n balaenoptera python scripts/verify_i3d_extract.py +""" + +import os +import sys +import tempfile + +import numpy as np +import torch +import torch.nn as nn + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, ROOT) +GOWTHAM = os.path.join(ROOT, ".reference", "I3D_Feature_Extraction_resnet") +sys.path.insert(0, GOWTHAM) + +from src.features.i3d_gowtham import ( # noqa: E402 + build_model, extract_from_frames_dir, ffmpeg_extract_frames, +) + +VIDEO = os.path.join(GOWTHAM, "samplevideos", "Abuse001_x264.mp4") +PTH = os.path.join(ROOT, "pretrained", "i3d", "i3d_baseline_r50_kinetics.pth") +RTFM = os.path.expanduser( + "~/data/wsad/ucf_crime/_archive/UCF_Train_ten_crop_i3d/Abuse001_x264_i3d.npy" +) + + +class _DictAdapter(nn.Module): + """Gowtham's run() calls ``i3d({'frames': tensor})``; our model takes a tensor.""" + + def __init__(self, m): + super().__init__() + self.m = m + + def forward(self, batch): + return self.m(batch["frames"]) + + +def main(): + # Gowtham's original code uses Image.ANTIALIAS (removed in Pillow >=10); it was + # a verbatim alias for LANCZOS, so restoring it runs the official code unchanged. + from PIL import Image + if not hasattr(Image, "ANTIALIAS"): + Image.ANTIALIAS = Image.Resampling.LANCZOS + + dev = "cuda" if torch.cuda.is_available() else "cpu" + model = build_model(PTH, use_nl=False, device=dev) + + with tempfile.TemporaryDirectory() as tmp: + ffmpeg_extract_frames(VIDEO, tmp) + n_frames = len(os.listdir(tmp)) + print(f"frames: {n_frames}") + + mine = extract_from_frames_dir(model, tmp, frequency=16, batch_size=20, + sample_mode="oversample", device=dev) + + from extract_features import run as gowtham_run # original code, unmodified + ref = gowtham_run(_DictAdapter(model), 16, tmp, 20, "oversample") + + print(f"mine {mine.shape} | gowtham {ref.shape}") + d = np.abs(mine - ref) + print(f"(1) PORT max|Δ|={d.max():.3e} mean|Δ|={d.mean():.3e} " + f"-> {'BIT-EXACT ✓' if d.max() < 1e-4 else 'MISMATCH ✗'}") + + rtfm = np.load(RTFM) + T = min(mine.shape[0], rtfm.shape[0]) + a, b = mine[:T].reshape(T, -1), rtfm[:T].reshape(T, -1) + cos = (a * b).sum(1) / (np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + 1e-9) + rel = np.abs(mine[:T] - rtfm[:T]).mean() / (np.abs(rtfm[:T]).mean() + 1e-9) + print(f"(2) RTFM mine T={mine.shape[0]} vs rtfm T={rtfm.shape[0]} (cmp {T})") + print(f" cosine mean={cos.mean():.4f} min={cos.min():.4f} | " + f"rel-mean-diff={rel:.4f} | L2/snip mine={np.linalg.norm(a,axis=1).mean()/np.sqrt(10):.2f} " + f"rtfm={np.linalg.norm(b,axis=1).mean()/np.sqrt(10):.2f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_rtfm.py b/scripts/verify/verify_rtfm.py similarity index 100% rename from scripts/verify_rtfm.py rename to scripts/verify/verify_rtfm.py diff --git a/scripts/verify_s3r.py b/scripts/verify/verify_s3r.py similarity index 100% rename from scripts/verify_s3r.py rename to scripts/verify/verify_s3r.py diff --git a/scripts/verify_ur_dmu.py b/scripts/verify/verify_ur_dmu.py similarity index 100% rename from scripts/verify_ur_dmu.py rename to scripts/verify/verify_ur_dmu.py diff --git a/scripts/verify_vadclip.py b/scripts/verify/verify_vadclip.py similarity index 100% rename from scripts/verify_vadclip.py rename to scripts/verify/verify_vadclip.py diff --git a/src/compat.py b/src/compat.py new file mode 100644 index 0000000..0503c51 --- /dev/null +++ b/src/compat.py @@ -0,0 +1,40 @@ +"""Backbone <-> head compatibility guard (issue #18). + +Text-branch heads (VadCLIP, TPWNG) consume a CLIP-style text encoder, so they +only make sense on a **text-aligned** backbone (CLIP, InternVideo2). Visual-only +backbones (I3D, VideoMAE) have no matching text space. This guard turns an +otherwise-silent shape/space mismatch into a clear, early error — meant to be +called by the pipeline factory and the matrix harness *before* any training. + +Both predicates read class attributes via the registries without instantiating +anything, so no heavy backbone deps (open_clip, transformers) are imported. +""" + +import src.features # noqa: F401 (registers FEATURE_EXTRACTORS entries) +import src.models # noqa: F401 (registers MODELS entries) +from src.registry import FEATURE_EXTRACTORS, MODELS + + +def backbone_is_text_aligned(backbone: str) -> bool: + """Whether a registered backbone produces text-aligned features.""" + return bool(getattr(FEATURE_EXTRACTORS.get(backbone), "text_aligned", False)) + + +def head_requires_text_aligned(head: str) -> bool: + """Whether a registered head needs a text-aligned backbone.""" + return bool(getattr(MODELS.get(head), "requires_text_aligned", False)) + + +def is_compatible(backbone: str, head: str) -> bool: + """True unless a text-branch head is paired with a visual-only backbone.""" + return not (head_requires_text_aligned(head) and not backbone_is_text_aligned(backbone)) + + +def assert_compatible(backbone: str, head: str) -> None: + """Raise ``ValueError`` if the head needs a text space the backbone lacks.""" + if not is_compatible(backbone, head): + raise ValueError( + f"head '{head}' requires a text-aligned backbone (e.g. clip, " + f"internvideo2), but backbone '{backbone}' is visual-only. " + f"Use a text-aligned backbone or a visual-only head." + ) diff --git a/src/data/local.py b/src/data/local.py index 8e279f1..fc54773 100644 --- a/src/data/local.py +++ b/src/data/local.py @@ -72,10 +72,51 @@ def _list_npy(d: str) -> List[str]: return sorted(f for f in os.listdir(d) if f.endswith(".npy")) if os.path.isdir(d) else [] +# ---- layout resolvers: dataset-keyed `features/{i3d,clip}` + `annotations/`, +# with legacy fallback (top-level `clip/`, dataset-root `*.zip`/`ground_truth.json`). +def _features_root(root: str, data_cfg) -> str: + return os.path.join(_dataset_root(root, data_cfg), "features") + + +def _clip_dir(root: str, data_cfg, mode: str) -> str: + new = os.path.join(_features_root(root, data_cfg), "clip", mode) + if os.path.isdir(new): + return new + return os.path.join(os.path.expanduser(root), "clip", mode) # legacy top-level + + +def _i3d_variant(data_cfg) -> str: + """Feature-extraction variant subdir under ``features/`` (default ``i3d``). + + Lets the comparison matrix select a specific I3D extraction — + ``i3d_tushar`` (HF tushar-n), ``i3d_pyvideo`` (HF main/pytorchvideo), + ``i3d_ours`` (our Gowtham re-extract) — each a self-consistent train+test set. + + ``i3d_mgfn`` = MGFN authors' 10-crop I3D (HKU OneDrive), full-length + ``(T, 10, 2048)``, RTFM-family scale L2~22 — COMPLETE 1610/290 but raw + pre-seg32 (needs T->32 to feed seg32 models). Do NOT mix with ``i3d`` + (DeepMIL scale ~2.5). See features/i3d_mgfn/PROVENANCE.md. + """ + return getattr(data_cfg, "feature_variant", None) or "i3d" + + +def _i3d_npy_dir(root: str, data_cfg, mode: str) -> str: + v = _i3d_variant(data_cfg) + new = os.path.join(_features_root(root, data_cfg), v, mode) + if os.path.isdir(new): + return new + return os.path.join(os.path.expanduser(root), v, mode) # legacy top-level + + def _load_clip( - root: str, mode: str, length: Optional[int], n_seg: Optional[int], single_crop: bool + root: str, + mode: str, + length: Optional[int], + n_seg: Optional[int], + single_crop: bool, + data_cfg=None, ) -> Dict[str, np.ndarray]: - d = os.path.join(root, "clip", mode) + d = _clip_dir(root, data_cfg, mode) out = {} for f in _list_npy(d): if single_crop and not _crop0(f): @@ -87,9 +128,16 @@ def _load_clip( return out -def _load_i3d(root: str, mode: str) -> Dict[str, np.ndarray]: - d = os.path.join(root, "i3d", mode) - out = {f: np.load(os.path.join(d, f)) for f in _list_npy(d)} +def _identity(x): # open_func for lazy npy-dir loading (FeatureDataset does np.load) + return x + + +def _load_i3d(root: str, mode: str, data_cfg=None) -> Dict[str, str]: + """Return ``{name: path}`` (LAZY). Eager-loading all npy into RAM OOM-kills WSL + for big variants (i3d_mgfn_seg200 train = 25 GB > 15 GB RAM -> session crash). + FeatureDataset(open_func=_identity) does ``np.load(path)`` per __getitem__.""" + d = _i3d_npy_dir(root, data_cfg, mode) + out = {f: os.path.join(d, f) for f in _list_npy(d)} if not out: raise FileNotFoundError(f"no I3D .npy under {d} (see docs/DATA_LOCAL.md)") return out @@ -104,8 +152,12 @@ def _dataset_root(root: str, data_cfg) -> str: def _i3d_zip_path(root: str, data_cfg, mode: str) -> Optional[str]: - p = os.path.join(_dataset_root(root, data_cfg), f"{mode}.zip") - return p if os.path.exists(p) else None + # new: /features//.zip ; legacy: /.zip + new = os.path.join(_features_root(root, data_cfg), _i3d_variant(data_cfg), f"{mode}.zip") + if os.path.exists(new): + return new + legacy = os.path.join(_dataset_root(root, data_cfg), f"{mode}.zip") + return legacy if os.path.exists(legacy) else None def _zip_feature_values( @@ -124,8 +176,14 @@ def _zip_feature_values( return names, {n: np.load(z.open(i)) for n, i in zip(names, infos)}, None -def has_local(root: str, backbone: str, mode: str) -> bool: - return bool(_list_npy(os.path.join(os.path.expanduser(root), backbone, mode))) +def has_local(root: str, backbone: str, mode: str, data_cfg=None) -> bool: + if backbone == "clip": + d = _clip_dir(root, data_cfg, mode) + elif backbone == "i3d": + d = _i3d_npy_dir(root, data_cfg, mode) + else: + d = os.path.join(os.path.expanduser(root), backbone, mode) + return bool(_list_npy(d)) def has_local_i3d_zip(root: str, data_cfg, mode: str = "train") -> bool: @@ -167,39 +225,48 @@ def build_datasets_local(data_cfg): """Build ``({normal, abnormal}, test)`` from ``~/data/wsad`` (local backbone).""" root = os.path.expanduser(getattr(data_cfg, "root", "~/data/wsad")) backbone = getattr(data_cfg, "backbone", "i3d") - with_mag = backbone == "i3d" + # magnitude defaults to the I3D convention; the comparison matrix overrides it + # so visual-magnitude heads (RTFM/MGFN/...) get a magnitude channel on CLIP too. + with_mag = bool(getattr(data_cfg, "with_magnitude", backbone == "i3d")) # I3D: prefer per-video npy dirs (i3d/{train,test}); else read the local # MGFN {train,test}.zip in place ("B": zip-direct, no extraction/duplication). - if backbone == "i3d" and not _list_npy(os.path.join(root, "i3d", "train")): + if backbone == "i3d" and not _list_npy(_i3d_npy_dir(root, data_cfg, "train")): if has_local_i3d_zip(root, data_cfg, "train"): return _build_i3d_from_zip(root, data_cfg) length = getattr(data_cfg, "clip_length", 256) n_seg = getattr(data_cfg, "segment", None) single_crop = getattr(data_cfg, "single_crop", True) + # VadCLIP trains on ALL 10 crops as separate samples (official ucf_CLIP_rgb.csv = + # 1610 vids x 10 crops = 16100 rows); test stays single-crop. 10x crop augmentation. + train_all_crops = bool(getattr(data_cfg, "train_all_crops", False)) def load(mode): if backbone == "clip": - return _load_clip(root, mode, length, n_seg, single_crop) - return _load_i3d(root, mode) + sc = single_crop and not (train_all_crops and mode == "train") + return _load_clip(root, mode, length, n_seg, sc, data_cfg) + return _load_i3d(root, mode, data_cfg) + # i3d npy dirs are loaded LAZILY (values = paths, np.load per __getitem__) to avoid + # OOM-killing WSL on big variants (seg200 = 25 GB); clip values are eager arrays. + i3d_open = _identity if backbone == "i3d" else None train_vals = load("train") names = list(train_vals) normal = [n for n in names if "Normal" in n] abnormal = [n for n in names if "Normal" not in n] train = { "normal": FeatureDataset( - normal, {f: train_vals[f] for f in normal}, with_magnitude=with_mag + normal, {f: train_vals[f] for f in normal}, open_func=i3d_open, with_magnitude=with_mag ), "abnormal": FeatureDataset( - abnormal, {f: train_vals[f] for f in abnormal}, with_magnitude=with_mag + abnormal, {f: train_vals[f] for f in abnormal}, open_func=i3d_open, with_magnitude=with_mag ), } test_vals = load("test") gt = _align_gt(list(test_vals), _load_ground_truth(data_cfg)) test = FeatureDataset( - list(test_vals), test_vals, labels=gt, with_magnitude=with_mag + list(test_vals), test_vals, labels=gt, open_func=i3d_open, with_magnitude=with_mag ) return train, test @@ -233,8 +300,14 @@ def _local_ground_truth_path(data_cfg) -> Optional[str]: if os.path.exists(gt): return gt root = getattr(data_cfg, "root", "~/data/wsad") - cand = os.path.join(_dataset_root(root, data_cfg), "ground_truth.json") - return cand if os.path.exists(cand) else None + ds = _dataset_root(root, data_cfg) + for cand in ( + os.path.join(ds, "annotations", "ground_truth.json"), # new layout + os.path.join(ds, "ground_truth.json"), # legacy dataset-root + ): + if os.path.exists(cand): + return cand + return None def _load_ground_truth(data_cfg): diff --git a/src/eval_matrix.py b/src/eval_matrix.py new file mode 100644 index 0000000..839dcec --- /dev/null +++ b/src/eval_matrix.py @@ -0,0 +1,157 @@ +"""Per-(backbone, head) eval shaping for the comparison matrix (additive). + +The core ``trainer.evaluate`` / ``inference.score_feature`` path is **I3D / +visual-head shaped** (test feature ``(T, ncrops, D)`` -> ``(1, ncrops, T, D)``, +no ``lengths``) and stays unchanged — design invariant: the original +training/offline-eval is never modified. But two real cases that path cannot +serve correctly: + + - **CLIP test layout is ``(ncrops=1, T, D)``** (transposed vs I3D's + ``(T, ncrops, D)``), so the I3D ``permute(1,0,2)`` collapses ``T`` to 1. + - **VadCLIP** consumes ``(B=windows, T<=256, D)`` + per-window ``lengths`` (the + official ``process_split`` windowed eval); the generic 4-D path crashes + (``DistanceAdj`` size mismatch) and is numerically wrong otherwise. + +This module supplies the per-head input shaping the matrix + CLIP serving need, +dispatching on ``(backbone, head)``. Visual heads slice their own magnitude +channel, so an appended-magnitude feature is fine. +""" + +from typing import Dict + +import numpy as np +import torch +from sklearn.metrics import auc, precision_recall_curve, roc_curve + +# heads whose forward consumes (B, ncrops, T, D) and slices [..., :feature_size] +_VISUAL_LAYOUT_HEADS = { + "mil", + "sultani", + "rtfm", + "mgfn", + "ur_dmu", + "bn_wvad", + "s3r", + "gs_moe", + "clip_tsa", +} +_VADCLIP_MAXLEN = 256 +# heads whose O(T^2) full-length test attention OOMs an 8 GB GPU with all 10 crops +# at once -> score each crop separately + average (the official per-crop protocol). +_PERCROP_HEADS = {"ur_dmu", "bn_wvad"} + + +def _to_crops_layout(feat: np.ndarray, backbone: str) -> np.ndarray: + """Shape a single video's cached feature to ``(1, ncrops, T, D)``. + + I3D test cache is ``(T, ncrops, D)``; CLIP cache is ``(ncrops=1, T, D)``. + """ + feat = np.asarray(feat, dtype=np.float32) + if feat.ndim == 2: # (T, D) -> single crop + feat = feat[None] # (1, T, D) == (ncrops, T, D) + return feat[None] # (1, 1, T, D) + if backbone == "i3d": # (T, ncrops, D) -> (ncrops, T, D) + feat = np.transpose(feat, (1, 0, 2)) + # CLIP: already (ncrops, T, D) + return feat[None] # (1, ncrops, T, D) + + +def _vadclip_lengths(length: int, maxlen: int = _VADCLIP_MAXLEN) -> torch.Tensor: + """Per-window valid lengths for an official ``process_split`` (UCF eval).""" + n = int(length / maxlen) + 1 + out = torch.zeros(n, dtype=torch.long) + rem = length + for j in range(n): + if rem > maxlen: + out[j] = maxlen + rem -= maxlen + else: + out[j] = rem + return out + + +def _vadclip_split(feat: np.ndarray, maxlen: int = _VADCLIP_MAXLEN): + """``(T, D)`` -> ``(windows, maxlen, D)`` (last window zero-padded).""" + t, d = feat.shape + if t < maxlen: + out = np.zeros((1, maxlen, d), dtype=np.float32) + out[0, :t] = feat + return out, t + n = int(t / maxlen) + 1 + out = np.zeros((n, maxlen, d), dtype=np.float32) + for i in range(n): + chunk = feat[i * maxlen : i * maxlen + maxlen] + out[i, : len(chunk)] = chunk + return out, t + + +@torch.no_grad() +def score_video( + model: torch.nn.Module, + feature: np.ndarray, + backbone: str, + head: str, + device: str = "cpu", + frames_per_clip: int = 16, +) -> np.ndarray: + """Per-frame anomaly scores for one video's cached feature. + + Returns a 1-D array of length ``T_snippets * frames_per_clip``. + """ + feat = np.asarray(feature, dtype=np.float32) + + if head == "vadclip": + if feat.ndim == 3: # (ncrops=1, T, D) -> (T, D) + feat = feat[0] + split, length = _vadclip_split(feat) + visual = torch.as_tensor(split, dtype=torch.float32, device=device) + lengths = _vadclip_lengths(length).to(device) + out = model(video=visual, lengths=lengths) + # (windows, maxlen, 1) -> (windows*maxlen,) -> first `length` snippets + prob = torch.sigmoid(out.binary_logits.reshape(-1))[:length] + scores = prob.cpu().numpy() + + else: # visual-layout heads + tpwng (all consume (B, ncrops, T, D); tpwng + # means over the crop dim inside _encode, so the 4-D layout is correct) + x = torch.as_tensor(_to_crops_layout(feat, backbone), device=device) + if head in _PERCROP_HEADS: + # UR-DMU/BN-WVAD: official eval scores each crop's FULL-LENGTH sequence + # separately then averages the 10 crops (ucf_infer.py: per-crop forward, + # mean over 10). Doing all crops in one forward is non-faithful AND ~10x + # the peak memory of the O(T^2) distance/attention -> OOMs 8 GB. Per-crop + # = faithful + fits. + per = [model(video=x[:, c : c + 1]).scores.squeeze(0).squeeze(-1) + for c in range(x.shape[1])] + scores = torch.stack(per).mean(0).float().cpu().numpy() + else: + out = model(video=x) + scores = out.scores.squeeze(0).squeeze(-1).float().cpu().numpy() + + return np.repeat(scores, frames_per_clip) + + +@torch.no_grad() +def evaluate( + model: torch.nn.Module, + test_dataset, + backbone: str, + head: str, + device: str = "cpu", + frames_per_clip: int = 16, +) -> Dict[str, float]: + """Frame-level ROC-AUC / PR-AUC over the UCF-Crime test set.""" + model.eval() + preds, labels = [], [] + for i in range(len(test_dataset)): + s = test_dataset[i] + p = score_video(model, s["feature"], backbone, head, device, frames_per_clip) + y = np.asarray(s["label"]) + n = min(len(p), len(y)) + preds.append(p[:n]) + labels.append(y[:n]) + preds = np.concatenate(preds) + labels = np.concatenate(labels) + + fpr, tpr, _ = roc_curve(labels, preds) + precision, recall, _ = precision_recall_curve(labels, preds) + return {"roc_auc": float(auc(fpr, tpr)), "pr_auc": float(auc(recall, precision))} diff --git a/src/features/__init__.py b/src/features/__init__.py index fd80e76..7a0a120 100644 --- a/src/features/__init__.py +++ b/src/features/__init__.py @@ -9,7 +9,7 @@ from src.features.base import FeatureExtractor # noqa from src.registry import FEATURE_EXTRACTORS # noqa -from . import clip, i3d, videomae, vggish # noqa (trigger registration) +from . import clip, i3d, videomae, vggish, xclip, internvideo # noqa (trigger registration) def build_extractor(name: str, **kwargs) -> FeatureExtractor: diff --git a/src/features/base.py b/src/features/base.py index 41747f3..98d4859 100644 --- a/src/features/base.py +++ b/src/features/base.py @@ -56,6 +56,48 @@ def extract(self, video: Union[str, List[Image.Image]]) -> np.ndarray: """ raise NotImplementedError + def iter_snippets(self, video_path: str, on_clips) -> np.ndarray: + """RAM-bounded streaming extraction over a video file. + + Reads only ``batch_size * snippet_len`` frames at a time from decord + (random-access ``get_batch``) instead of materializing every PIL frame — + a long surveillance clip can be 100k+ frames (decode-all = tens of GB RAM, + which has OOM-killed WSL before). ``on_clips(list_of_clips) -> (b, dim)`` + runs the backbone on one batch of clips (each clip = ``snippet_len`` PIL + frames); results are concatenated to ``(T, dim)``. + + Subclasses with the same snippet grid (videomae/xclip/internvideo) call this + with their per-batch encode hook; ``self.frequency``/``snippet_len``/ + ``batch_size`` must be set on the instance. + """ + import decord # lazy + + vr = decord.VideoReader(uri=video_path) + n = len(vr) + clip, freq, bs = self.snippet_len, self.frequency, self.batch_size + sample_to = getattr(self, "sample_to", None) + if sample_to: # extract only `sample_to` uniformly-spaced snippets (fast seg path: + # one snippet per segment instead of all-then-pool, ~10x fewer forwards on long + # surveillance clips). Output is already (sample_to, dim) — no segment pooling. + import numpy as _np + starts = sorted(set(_np.linspace(0, max(n - clip, 0), sample_to).astype(int).tolist())) + else: + starts = list(range(0, max(n - clip + 1, 1), freq)) + feats = [] + for i in range(0, len(starts), bs): + bstarts = starts[i : i + bs] + lo, hi = bstarts[0], min(bstarts[-1] + clip, n) + win = vr.get_batch(list(range(lo, hi))).asnumpy() # ONE contiguous read/batch + clips = [] + for s in bstarts: + a = win[s - lo : s - lo + clip] + frames = [Image.fromarray(f) for f in a] + if len(frames) < clip: # pad the tail snippet + frames += [frames[-1]] * (clip - len(frames)) + clips.append(frames) + feats.append(np.asarray(on_clips(clips))) + return np.concatenate(feats, axis=0) + def info(self) -> dict: return { "name": self.name, diff --git a/src/features/clip.py b/src/features/clip.py index 0f44916..4f8b3b5 100644 --- a/src/features/clip.py +++ b/src/features/clip.py @@ -11,6 +11,14 @@ - **dim = 512**; **preprocess** = the CLIP image transform (224, CLIP mean/std). - **text_aligned = True**. +**Pretrained = OpenAI ViT-B/16** (not laion). The local cached CLIP features +(``features/clip``) are the official VadCLIP ``UCFClipFeatures`` dump, extracted +with OpenAI CLIP ViT-B/16 — and every text-branch head is trained on them. Two +independently-trained CLIP models have *unaligned* 512-d bases (verified: +laion-vs-cache seg-pooled cosine ≈ 0; openai-vs-cache ≈ 0.92), so serving a raw +video through a head trained on the cache **requires** the OpenAI weights here. +Override ``pretrained="laion2b_s34b_b88k"`` only if re-extracting all caches. + Heavy deps (``open_clip``, ``decord``) are imported lazily so this module always imports; only ``extract()`` / ``__init__`` need them. Execution needs a GPU. """ @@ -37,7 +45,7 @@ class CLIPFeatureExtractor(FeatureExtractor): def __init__( self, model_name: str = "ViT-B-16", - pretrained: str = "laion2b_s34b_b88k", + pretrained: str = "openai", device: str = "cuda", batch_size: int = 256, fp16: bool = True, diff --git a/src/features/extract.py b/src/features/extract.py new file mode 100644 index 0000000..339c280 --- /dev/null +++ b/src/features/extract.py @@ -0,0 +1,56 @@ +"""Backbone-agnostic offline extraction core (issue #17). + +One generic loop that runs any registered ``FeatureExtractor`` over a set of +videos and caches ``_.npy``. The backbone-specific model/decord +work lives behind ``extractor.extract(path)``; this module stays dependency-light +(os, numpy) so it imports and unit-tests without decord / open_clip / a GPU. +""" + +import os +from typing import Iterable, List, Mapping + +import numpy as np + +try: + from tqdm.auto import tqdm +except Exception: # pragma: no cover - tqdm is optional here + + def tqdm(x, **_): + return x + + +def save_path_for(video_path: str, output_dir: str, backbone: str) -> str: + """``/_.npy`` (name-based cache, never a path).""" + stem = os.path.splitext(os.path.basename(video_path))[0] + return os.path.join(output_dir, f"{stem}_{backbone}.npy") + + +def run_extraction( + samples: Iterable[Mapping], + extractor, + output_dir: str, + backbone: str, + *, + skip_existing: bool = True, + path_key: str = "video_path", +) -> List[str]: + """Extract + cache features for each sample; return the saved paths. + + Args: + samples: iterable of mappings, each holding a video path under ``path_key``. + extractor: any object exposing ``extract(video_path) -> np.ndarray``. + output_dir: cache directory (created if missing). + backbone: name used for the ``_.npy`` suffix. + skip_existing: skip a video whose cache file already exists. + """ + os.makedirs(output_dir, exist_ok=True) + saved: List[str] = [] + for sample in tqdm(samples): + out = save_path_for(sample[path_key], output_dir, backbone) + if skip_existing and os.path.exists(out): + saved.append(out) + continue + feats = np.asarray(extractor.extract(sample[path_key])) + np.save(out, feats) + saved.append(out) + return saved diff --git a/src/features/i3d.py b/src/features/i3d.py index f027491..0475029 100644 --- a/src/features/i3d.py +++ b/src/features/i3d.py @@ -43,7 +43,7 @@ def __init__( def extract(self, video: Union[str, List[Image.Image]]) -> np.ndarray: from src.data import TenCropVideoFrameDataset - clips = TenCropVideoFrameDataset(video, frames_per_clip=self.snippet_len) + clips = TenCropVideoFrameDataset(video, clip_length=self.snippet_len) loader = DataLoader(clips, batch_size=self.batch_size, shuffle=False) outputs = [] diff --git a/src/features/i3d_gowtham.py b/src/features/i3d_gowtham.py new file mode 100644 index 0000000..cd9710c --- /dev/null +++ b/src/features/i3d_gowtham.py @@ -0,0 +1,131 @@ +"""Gowtham/RTFM-faithful I3D feature extraction (the standard WSVAD I3D features). + +A faithful port of GowthamGottimukkala/I3D_Feature_Extraction_resnet +(`extract_features.py` + `main.py`): ffmpeg -> per-frame JPG -> PIL resize +340x256 (ANTIALIAS) -> ``(x*2/255)-1`` -> exact 10-crop oversample -> I3Res50 -> +``(T, 10, 2048)`` per video. This is the lineage that produced RTFM's +``UCF_*_ten_crop_i3d`` (feature L2/snippet ~22), verified by +``scripts/verify_i3d_extract.py``. + +Distinct from :mod:`src.features.i3d` (the pytorchvideo ``i3d_8x8_r50`` path, +which uses Kinetics 0.45/0.225 normalization and torchvision TenCrop — a different +preprocessing, NOT scale-compatible with the MGFN/RTFM features). + +Weights: convert the facebookresearch Caffe2 blobs first:: + + python scripts/convert_i3d_caffe2.py pretrained/i3d/i3d_baseline_32x2_IN_pretrain_400k.pkl \\ + pretrained/i3d/i3d_baseline_r50_kinetics.pth --no-nl +""" + +import os +import subprocess +import tempfile + +import numpy as np +import torch +from PIL import Image + +from src.i3d import I3Res50 + +# Pillow >=10 removed Image.ANTIALIAS (it was an alias for LANCZOS). +_LANCZOS = getattr(getattr(Image, "Resampling", Image), "LANCZOS", getattr(Image, "ANTIALIAS", 1)) + +CHUNK_SIZE = 16 # frames per snippet (Gowtham chunk_size, fixed) + + +def build_model(pretrained_path: str, use_nl: bool, device: str = "cuda") -> I3Res50: + """``I3Res50(use_nl)`` with converted Caffe2 weights, eval mode (BN frozen).""" + model = I3Res50(use_nl=use_nl) + sd = torch.load(pretrained_path, map_location="cpu") + model.load_state_dict(sd, strict=False) # our model has no fc/drop head + return model.eval().to(device) + + +def load_frame(frame_file: str) -> np.ndarray: + """PIL open -> resize (W=340, H=256) ANTIALIAS -> ``(x*2/255)-1`` in [-1,1].""" + data = Image.open(frame_file).resize((340, 256), _LANCZOS) + data = np.array(data).astype(float) + data = (data * 2 / 255) - 1 + assert data.max() <= 1.0 and data.min() >= -1.0 + return data + + +def load_rgb_batch(frames_dir, rgb_files, frame_indices) -> np.ndarray: + batch = np.zeros(frame_indices.shape + (256, 340, 3)) + for i in range(frame_indices.shape[0]): + for j in range(frame_indices.shape[1]): + batch[i, j] = load_frame(os.path.join(frames_dir, rgb_files[frame_indices[i][j]])) + return batch + + +def oversample_data(data: np.ndarray): + """Exact Gowtham 10-crop: 5 spatial crops (of the 256x340 frame) + h-flips.""" + flip = np.array(data[:, :, :, ::-1, :]) + crops = lambda d: [ + np.array(d[:, :, :224, :224, :]), # top-left + np.array(d[:, :, :224, -224:, :]), # top-right + np.array(d[:, :, 16:240, 58:282, :]), # center + np.array(d[:, :, -224:, :224, :]), # bottom-left + np.array(d[:, :, -224:, -224:, :]), # bottom-right + ] + return crops(data) + crops(flip) + + +@torch.no_grad() +def extract_from_frames_dir( + model, frames_dir, frequency=16, batch_size=20, sample_mode="oversample", device="cuda" +) -> np.ndarray: + """Faithful port of Gowtham ``run()`` -> ``(num_chunks, 10 or 1, 2048)``.""" + assert sample_mode in ("oversample", "center_crop") + + def forward_batch(b_data): + b_data = b_data.transpose([0, 4, 1, 2, 3]) # b,t,h,w,c -> b,c,t,h,w + t = torch.from_numpy(b_data).to(device).float() + return model(t).cpu().numpy() + + rgb_files = sorted(os.listdir(frames_dir), key=lambda f: int(os.path.splitext(f)[0])) + frame_cnt = len(rgb_files) + assert frame_cnt > CHUNK_SIZE + clipped = ((frame_cnt - CHUNK_SIZE) // frequency) * frequency + frame_indices = np.array( + [[j for j in range(i * frequency, i * frequency + CHUNK_SIZE)] + for i in range(clipped // frequency + 1)] + ) + batch_num = int(np.ceil(frame_indices.shape[0] / batch_size)) + frame_indices = np.array_split(frame_indices, batch_num, axis=0) + + n_crop = 10 if sample_mode == "oversample" else 1 + full = [[] for _ in range(n_crop)] + for bi in range(batch_num): + batch_data = load_rgb_batch(frames_dir, rgb_files, frame_indices[bi]) + if sample_mode == "oversample": + for i, crop in enumerate(oversample_data(batch_data)): + assert crop.shape[-2] == 224 and crop.shape[-3] == 224 + full[i].append(forward_batch(crop)) + else: + crop = batch_data[:, :, 16:240, 58:282, :] + full[0].append(forward_batch(crop)) + + full = [np.concatenate(c, axis=0) for c in full] + full = np.concatenate([np.expand_dims(c, 0) for c in full], axis=0) + full = full[:, :, :, 0, 0, 0] # (n_crop, T, 2048) + return full.transpose([1, 0, 2]) # (T, n_crop, 2048) + + +def ffmpeg_extract_frames(video: str, out_dir: str) -> None: + """ffmpeg -> ``/%d.jpg`` (start 0, native fps) — exactly Gowtham/main.py.""" + os.makedirs(out_dir, exist_ok=True) + subprocess.run( + ["ffmpeg", "-loglevel", "quiet", "-i", video, "-start_number", "0", + os.path.join(out_dir, "%d.jpg")], + check=True, + ) + + +def extract_from_video( + model, video, frequency=16, batch_size=20, sample_mode="oversample", device="cuda" +) -> np.ndarray: + """ffmpeg-extract frames to a temp dir, then run the faithful pipeline.""" + with tempfile.TemporaryDirectory() as tmp: + ffmpeg_extract_frames(video, tmp) + return extract_from_frames_dir(model, tmp, frequency, batch_size, sample_mode, device) diff --git a/src/features/internvideo.py b/src/features/internvideo.py new file mode 100644 index 0000000..78e4cd0 --- /dev/null +++ b/src/features/internvideo.py @@ -0,0 +1,131 @@ +"""InternVideo2 backbone — current-SOTA video encoder (OpenGVLab), via HF +`AutoModel(trust_remote_code=True)`. + +⚠️ Size/feasibility: the public InternVideo2 checkpoints are large (Stage2 1B / 6B, +Chat 8B). The 1B vision tower in fp16 (~2 GB params + activations) is the only +realistic option on an 8 GB GPU and is still tight — extract with small `batch_size` +and `fp16`, or on a bigger GPU. This module imports cleanly (all heavy deps are lazy); +it only downloads/loads on first `extract()`. The exact `get_vid_fea` / preprocessing +interface differs per release, so **validate the output shape on a few videos (and the +forensic gate) before a full 1900-video run.** + +`model_name` e.g. `OpenGVLab/InternVideo2-Stage2_1B-224p-f4`. `text_aligned=True` +(InternVideo2 is contrastively trained with text), so in principle it can also feed the +VLM heads — but the head text encoders are CLIP-space, so treat as visual unless you +re-align the text tower. +""" + +from typing import List, Optional, Union + +import numpy as np +import torch +from PIL import Image + +from src.features.base import FeatureExtractor +from src.registry import FEATURE_EXTRACTORS + +# ImageNet/CLIP-style normalization (InternVideo2 uses OpenAI-CLIP mean/std) +_MEAN = (0.485, 0.456, 0.406) +_STD = (0.229, 0.224, 0.225) + + +@FEATURE_EXTRACTORS.register("internvideo") +class InternVideoFeatureExtractor(FeatureExtractor): + name = "internvideo" + dim = 0 # read from the model on load + unit = "snippet" + snippet_len = 16 + text_aligned = True + modality = "visual" + + def __init__( + self, + model_name: str = "OpenGVLab/InternVideo2-Stage2_1B-224p-f4", + device: str = "cuda", + batch_size: int = 2, + fp16: bool = True, + frequency: int = 16, + num_frames: int = 4, + size: int = 224, + segment_to: Optional[int] = None, + ): + from transformers import AutoModel # lazy (heavy) + + self.device = device + self.batch_size = batch_size + self.fp16 = fp16 and device != "cpu" + self.frequency = frequency + self.num_frames = num_frames + self.size = size + self.segment_to = segment_to + + self.model = AutoModel.from_pretrained( + model_name, trust_remote_code=True, + torch_dtype=torch.float16 if self.fp16 else torch.float32, + ).eval().to(device) + # vid-feature method name varies across releases + self._fea_fn = next( + (getattr(self.model, n) for n in ("get_vid_feat", "get_vid_fea", "encode_vision") + if hasattr(self.model, n)), None) + if self._fea_fn is None: + raise AttributeError( + f"{model_name} exposes no get_vid_feat/get_vid_fea/encode_vision; " + "inspect the model card and set the right call.") + + def _read_frames(self, video: Union[str, List[Image.Image]]) -> List[Image.Image]: + if isinstance(video, str): + import decord # lazy + + vr = decord.VideoReader(uri=video) + return [Image.fromarray(vr[i].asnumpy()) for i in range(len(vr))] + return video + + def _clip_tensor(self, frames: List[Image.Image]) -> torch.Tensor: + """`num_frames` uniformly sampled frames -> (T, 3, size, size) normalized.""" + import torchvision.transforms.functional as TF + + idx = np.linspace(0, len(frames) - 1, self.num_frames).round().astype(int) + out = [] + for i in idx: + im = frames[i].convert("RGB").resize((self.size, self.size)) + t = TF.to_tensor(im) + out.append(TF.normalize(t, _MEAN, _STD)) + return torch.stack(out) # (T, 3, H, W) + + @staticmethod + def _segment(feats: np.ndarray, n_seg: int) -> np.ndarray: + out = np.zeros((n_seg, feats.shape[1]), dtype=np.float32) + r = np.linspace(0, len(feats), n_seg + 1, dtype=int) + for i in range(n_seg): + lo, hi = r[i], r[i + 1] + out[i] = feats[lo:hi].mean(0) if hi > lo else feats[min(lo, len(feats) - 1)] + return out + + @torch.no_grad() + def _encode(self, clips: List[List[Image.Image]]) -> np.ndarray: + """One batch of 16-frame clips -> ``(b, dim)`` via the InternVideo2 vid-feature fn.""" + batch = torch.stack([self._clip_tensor(c) for c in clips]).to(self.device) # (b,T,3,H,W) + if self.fp16: + batch = batch.half() + emb = self._fea_fn(batch) + emb = emb[0] if isinstance(emb, (tuple, list)) else emb + out = emb.float().reshape(emb.shape[0], -1).cpu().numpy() + self.dim = out.shape[1] + return out + + @torch.no_grad() + def extract(self, video: Union[str, List[Image.Image]]) -> np.ndarray: + if isinstance(video, str): + feats = self.iter_snippets(video, self._encode) + else: + frames, clip = list(video), self.snippet_len + if len(frames) < clip: + frames += [frames[-1]] * (clip - len(frames)) + starts = list(range(0, len(frames) - clip + 1, self.frequency)) + clips = [frames[s : s + clip] for s in starts] + feats = np.concatenate( + [self._encode(clips[i : i + self.batch_size]) + for i in range(0, len(clips), self.batch_size)], axis=0) + if self.segment_to is not None: + feats = self._segment(feats, self.segment_to) + return feats diff --git a/src/features/videomae.py b/src/features/videomae.py index 2ac8eac..688943d 100644 --- a/src/features/videomae.py +++ b/src/features/videomae.py @@ -1,25 +1,28 @@ -"""VideoMAE / VideoMAEv2 ViT-B backbone — PLANNED (feature-ablation vs I3D). +"""VideoMAE / VideoMAEv2 ViT-B backbone — a strong modern *visual* backbone. -A strong modern *visual* backbone (not text-aligned). Use it for a clean -feature-vs-method ablation on visual-only heads (RTFM, MGFN, Sultani, UR-DMU): -swap I3D -> VideoMAE-B, keep everything else fixed. +Use it for a clean feature-vs-method ablation on visual-only heads (RTFM, MGFN, +Sultani, UR-DMU): swap I3D -> VideoMAE-B, keep everything else fixed. Differences vs I3D: - **unit = snippet** (16-frame clip) — same temporal unit as I3D, so the - 32-seg / full-T conventions and the loader carry over directly. - - **dim = 768** (ViT-B). ViT-L = 1024 (tight on 6 GB), ViT-g = 1408 (won't fit). - - **preprocess**: tubelet embedding, ImageNet-style normalize, 16x224x224 input. - Use `transformers.VideoMAEModel` + `VideoMAEImageProcessor`; take the mean of - patch tokens (or the pooled output) per clip. - - **text_aligned = False** -> visual-only heads only (no VadCLIP text branch - unless you add a learned projection / bridge into CLIP space). - -Status: not implemented (deferred). Run after CLIP unlocks the VLM methods. + 32-seg / full-T conventions and the loader carry over directly. We chunk the + video into non-overlapping 16-frame snippets (matching I3D's frequency=16). + - **dim = 768** (ViT-B). ViT-L = 1024 (tight on 8 GB), ViT-g = 1408 (won't fit). + - **preprocess**: ``VideoMAEImageProcessor`` (tubelet embedding, Kinetics-style + normalize, 16x224x224). Per clip we mean-pool the patch tokens of + ``VideoMAEModel.last_hidden_state`` -> one 768-d vector. + - **text_aligned = False** -> visual-only heads only. + +VideoMAEv2: pass ``model_name`` of a HF checkpoint that loads via +``transformers.VideoMAEModel`` (e.g. a v2 ViT-B export). Checkpoints published +only as OpenGVLab native weights need their own loader; ``hidden_size`` is read +from the model config so ``dim`` adapts automatically. """ -from typing import List, Union +from typing import List, Optional, Union import numpy as np +import torch from PIL import Image from src.features.base import FeatureExtractor @@ -39,12 +42,116 @@ def __init__( self, model_name: str = "MCG-NJU/videomae-base", device: str = "cuda", + batch_size: int = 16, + fp16: bool = True, + frequency: int = 16, + segment_to: Optional[int] = None, + sample_to: Optional[int] = None, ): - self.model_name = model_name + """Args: + frequency: stride between snippet starts (16 = non-overlapping, I3D-style). + segment_to: if set, uniform mean-pool the per-snippet features to this many + segments (32-seg convention); if ``None``, keep per-snippet (T, dim). + sample_to: FAST seg path — extract only this many uniformly-spaced snippets + (one per segment) instead of all-then-pool; ~10x fewer forwards on long + clips. Output is already ``(sample_to, dim)``; overrides ``segment_to``. + """ + from transformers import VideoMAEImageProcessor, VideoMAEModel # lazy (heavy) + self.device = device + self.batch_size = batch_size + self.fp16 = fp16 and device != "cpu" + self.frequency = frequency + self.segment_to = segment_to + self.sample_to = sample_to + + self.processor = VideoMAEImageProcessor.from_pretrained(model_name) + self.model = VideoMAEModel.from_pretrained(model_name).eval() + self._patch_qkv_bias(model_name) # before .half()/.to() — see method + self.model = self.model.to(device) + if self.fp16: + self.model.half() + self.dim = int(self.model.config.hidden_size) + + def _patch_qkv_bias(self, model_name: str) -> None: + """Restore the checkpoint's timm-style attention bias dropped by the HF loader. + + MCG-NJU VideoMAE checkpoints store attention bias as ``q_bias`` / ``v_bias`` + (with ``k_bias`` ≡ 0, the original qkv-fused convention). transformers' newer + VideoMAE port uses split ``query/key/value.bias`` Linear layers but does NOT + convert the old keys, so ``query.bias`` / ``value.bias`` load as ZERO — silently + discarding a large learned bias (||q_bias|| ≈ 17). That corrupts every attention + score and thus the features. Map ``q_bias -> query.bias``, ``v_bias -> value.bias`` + (key.bias stays 0). No-op on transformers versions that already load correctly. + """ + import torch + + enc = self.model.encoder.layer + a0 = enc[0].attention.attention + if a0.query.bias is None or a0.query.bias.abs().sum() > 0: + return # older transformers already mapped the bias (or arch has none) + + from huggingface_hub import hf_hub_download + try: + sd = torch.load(hf_hub_download(model_name, "pytorch_model.bin"), + map_location="cpu") + except Exception: + from safetensors.torch import load_file + sd = load_file(hf_hub_download(model_name, "model.safetensors")) + + patched = 0 + with torch.no_grad(): + for i, layer in enumerate(enc): + att = layer.attention.attention + for src, dst in (("q_bias", att.query), ("v_bias", att.value)): + key = f"videomae.encoder.layer.{i}.attention.attention.{src}" + if key in sd and dst.bias is not None: + dst.bias.copy_(sd[key].to(dst.bias.dtype)) + patched += 1 + if patched: + print(f"[videomae] restored {patched} timm q/v-bias tensors the HF loader dropped") + + def _read_frames(self, video: Union[str, List[Image.Image]]) -> List[Image.Image]: + if isinstance(video, str): + import decord # lazy + + vr = decord.VideoReader(uri=video) + return [Image.fromarray(vr[i].asnumpy()) for i in range(len(vr))] + return video + + @staticmethod + def _segment(feats: np.ndarray, n_seg: int) -> np.ndarray: + out = np.zeros((n_seg, feats.shape[1]), dtype=np.float32) + r = np.linspace(0, len(feats), n_seg + 1, dtype=int) + for i in range(n_seg): + lo, hi = r[i], r[i + 1] + out[i] = feats[lo:hi].mean(0) if hi > lo else feats[min(lo, len(feats) - 1)] + return out + + @torch.no_grad() + def _encode(self, clips: List[List[Image.Image]]) -> np.ndarray: + """One batch of clips (each = 16 PIL frames) -> ``(b, dim)`` (token mean-pool).""" + px = self.processor(clips, return_tensors="pt")["pixel_values"].to(self.device) + if self.fp16: + px = px.half() + hidden = self.model(pixel_values=px).last_hidden_state # (b, tokens, dim) + return hidden.float().mean(dim=1).cpu().numpy() + + @torch.no_grad() def extract(self, video: Union[str, List[Image.Image]]) -> np.ndarray: - raise NotImplementedError( - "VideoMAE extractor is planned (see module docstring / " - "docs/FEATURE_EXTRACTORS.md). Uses transformers.VideoMAEModel; deferred." - ) + if isinstance(video, str): # RAM-bounded streaming from the file + feats = self.iter_snippets(video, self._encode) + else: # in-memory frame list (tests / pre-decoded) + frames, clip = list(video), self.snippet_len + if len(frames) < clip: + frames += [frames[-1]] * (clip - len(frames)) + starts = list(range(0, len(frames) - clip + 1, self.frequency)) + clips = [frames[s : s + clip] for s in starts] + feats = np.concatenate( + [self._encode(clips[i : i + self.batch_size]) + for i in range(0, len(clips), self.batch_size)], axis=0) + + if self.segment_to is not None and not self.sample_to: + feats = self._segment(feats, self.segment_to) + return feats diff --git a/src/features/xclip.py b/src/features/xclip.py new file mode 100644 index 0000000..e62225a --- /dev/null +++ b/src/features/xclip.py @@ -0,0 +1,114 @@ +"""X-CLIP backbone — a **text-aligned VIDEO** CLIP (Microsoft, in HF Transformers). + +Unlike `clip.py` (per-frame image CLIP), X-CLIP encodes a short video clip into one +512-d embedding that lives in the CLIP **text** space, with a cross-frame attention + +video-specific prompting. So it is the only *temporal* text-aligned backbone here → +it unlocks the VLM/text-branch heads (VadCLIP, CLIP-TSA, TPWNG) on motion-aware video +features instead of frame-averaged CLIP. + + - **unit = snippet** (a window of `snippet_len` frames; X-CLIP base samples 8). + - **dim = 512**; **text_aligned = True**. + - per snippet: `XCLIPModel.get_video_features` over `num_frames` sampled frames. + +`model_name` examples: `microsoft/xclip-base-patch16` (8 frames), +`microsoft/xclip-base-patch32`, `microsoft/xclip-base-patch16-16-frames`. +`num_frames` is read from the model config so the frame sampling matches the checkpoint. +""" + +from typing import List, Optional, Union + +import numpy as np +import torch +from PIL import Image + +from src.features.base import FeatureExtractor +from src.registry import FEATURE_EXTRACTORS + + +@FEATURE_EXTRACTORS.register("xclip") +class XCLIPFeatureExtractor(FeatureExtractor): + name = "xclip" + dim = 512 + unit = "snippet" + snippet_len = 16 + text_aligned = True + modality = "visual" + + def __init__( + self, + model_name: str = "microsoft/xclip-base-patch16", + device: str = "cuda", + batch_size: int = 8, + fp16: bool = True, + frequency: int = 16, + segment_to: Optional[int] = None, + ): + from transformers import XCLIPModel, XCLIPProcessor # lazy (heavy) + + self.device = device + self.batch_size = batch_size + self.fp16 = fp16 and device != "cpu" + self.frequency = frequency + self.segment_to = segment_to + + self.processor = XCLIPProcessor.from_pretrained(model_name) + self.model = XCLIPModel.from_pretrained(model_name).eval().to(device) + if self.fp16: + self.model.half() + self.dim = int(self.model.config.projection_dim) + # frames the checkpoint expects per clip (8 or 16) + self.num_frames = int(getattr(self.model.config.vision_config, "num_frames", 8)) + + def _read_frames(self, video: Union[str, List[Image.Image]]) -> List[Image.Image]: + if isinstance(video, str): + import decord # lazy + + vr = decord.VideoReader(uri=video) + return [Image.fromarray(vr[i].asnumpy()) for i in range(len(vr))] + return video + + @staticmethod + def _sample(frames: List[Image.Image], k: int) -> List[Image.Image]: + idx = np.linspace(0, len(frames) - 1, k).round().astype(int) + return [frames[i] for i in idx] + + @staticmethod + def _segment(feats: np.ndarray, n_seg: int) -> np.ndarray: + out = np.zeros((n_seg, feats.shape[1]), dtype=np.float32) + r = np.linspace(0, len(feats), n_seg + 1, dtype=int) + for i in range(n_seg): + lo, hi = r[i], r[i + 1] + out[i] = feats[lo:hi].mean(0) if hi > lo else feats[min(lo, len(feats) - 1)] + return out + + @torch.no_grad() + def _encode(self, clips: List[List[Image.Image]]) -> np.ndarray: + """One batch of 16-frame clips -> ``(b, 512)`` (X-CLIP video embedding).""" + batch = [self._sample(c, self.num_frames) for c in clips] # 16 -> num_frames + px = self.processor(videos=batch, return_tensors="pt")["pixel_values"].to(self.device) + if self.fp16: + px = px.half() + emb = self.model.get_video_features(pixel_values=px) + if not torch.is_tensor(emb): # transformers>=5 returns an output object + emb = getattr(emb, "pooler_output", None) + if emb is None: + raise RuntimeError("XCLIP get_video_features returned no pooler_output") + return emb.float().cpu().numpy() + + @torch.no_grad() + def extract(self, video: Union[str, List[Image.Image]]) -> np.ndarray: + if isinstance(video, str): + feats = self.iter_snippets(video, self._encode) + else: + frames, clip = list(video), self.snippet_len + if len(frames) < clip: + frames += [frames[-1]] * (clip - len(frames)) + starts = list(range(0, len(frames) - clip + 1, self.frequency)) + clips = [frames[s : s + clip] for s in starts] + feats = np.concatenate( + [self._encode(clips[i : i + self.batch_size]) + for i in range(0, len(clips), self.batch_size)], axis=0) + + if self.segment_to is not None: + feats = self._segment(feats, self.segment_to) + return feats diff --git a/src/i3d.py b/src/i3d.py index 2bce8d6..561547c 100644 --- a/src/i3d.py +++ b/src/i3d.py @@ -3,7 +3,6 @@ import torch import torch.nn.functional as F from huggingface_hub import hf_hub_download -from pytorchvideo.models.resnet import create_resnet from torch import nn repo_id = "jinmang2/test_video_fe" @@ -334,6 +333,11 @@ def build_i3d_feature_extractor( if model_name == "tushar-n-baseline": model = I3Res50(use_nl=False) elif model_name == "i3d_8x8_r50": + # pytorchvideo is only needed for this SlowFast variant; import lazily so + # the self-contained tushar-n / nonlocal I3Res50 path works without it + # (pytorchvideo pins old torch and conflicts with the training stack). + from pytorchvideo.models.resnet import create_resnet + model = create_resnet( stem_conv_kernel_size=(5, 7, 7), stage1_pool=nn.MaxPool3d, diff --git a/src/models/gs_moe/modeling_gs_moe.py b/src/models/gs_moe/modeling_gs_moe.py index f6b1a9d..db65550 100644 --- a/src/models/gs_moe/modeling_gs_moe.py +++ b/src/models/gs_moe/modeling_gs_moe.py @@ -30,6 +30,7 @@ from typing import Optional import torch +from src.modules.amp import safe_bce from torch import nn from transformers import PreTrainedModel from transformers.utils import ModelOutput @@ -257,7 +258,7 @@ def _compute_loss(self, scores, expert_scores, class_labels): [torch.zeros(half, device=device), torch.ones(half, device=device)] ) vid = self._topk_mil(frame).clamp(1e-6, 1 - 1e-6) - loss = self.bce(vid, y) + loss = safe_bce(vid, y) # smoothness + sparsity (on abnormal scores) loss = loss + self.smooth(scores) + self.sparse(frame[half:].reshape(-1)) @@ -281,6 +282,6 @@ def _compute_loss(self, scores, expert_scores, class_labels): ).clamp( 1e-6, 1 - 1e-6 ) # (half, E) - loss = loss + self.bce(expert_vid, tgt) + loss = loss + safe_bce(expert_vid, tgt) return loss diff --git a/src/models/mgfn/modeling_mgfn.py b/src/models/mgfn/modeling_mgfn.py index 7d80934..9b99d4f 100644 --- a/src/models/mgfn/modeling_mgfn.py +++ b/src/models/mgfn/modeling_mgfn.py @@ -63,6 +63,9 @@ def __init__(self, config): super().__init__() init_dim = config.dims[0] self.mag_ratio = config.mag_ratio + # split point between feature channels and the appended magnitude channel; + # config-driven so non-I3D backbones (CLIP 512, VideoMAE 768) work too. + self.channels = config.channels self.to_tokens = nn.Conv1d( config.channels, init_dim, @@ -80,7 +83,7 @@ def forward(self, x): # c: feature dimension, `C` in paper. bs, ncrops, t, c = x.size() x = x.view(bs * ncrops, t, c).permute(0, 2, 1) - x_f, x_m = x[:, :2048, :], x[:, 2048:, :] + x_f, x_m = x[:, : self.channels, :], x[:, self.channels :, :] x_f = self.to_tokens(x_f) x_m = self.to_mag(x_m) # eq (1) x_f = x_f + self.mag_ratio * x_m # eq (2) diff --git a/src/models/s3r/modeling_s3r.py b/src/models/s3r/modeling_s3r.py index ee8ef3f..e7e7f89 100644 --- a/src/models/s3r/modeling_s3r.py +++ b/src/models/s3r/modeling_s3r.py @@ -32,6 +32,7 @@ import torch import torch.nn.functional as F +from src.modules.amp import safe_bce from einops import rearrange from torch import nn from transformers import PreTrainedModel @@ -302,7 +303,7 @@ def forward( nor_feamagnitude=n_sel, abn_feamagnitude=a_sel, ) labels = torch.cat([normal_labels, abnormal_labels], dim=0) - loss_macro = F.binary_cross_entropy( + loss_macro = safe_bce( macro_scores.squeeze(-1).clamp(1e-6, 1 - 1e-6), labels ) loss_smooth = TemporalSmoothnessLoss(self.config.lambda_smooth)(scores) diff --git a/src/models/tpwng/modeling_tpwng.py b/src/models/tpwng/modeling_tpwng.py index 71bd441..87089e8 100644 --- a/src/models/tpwng/modeling_tpwng.py +++ b/src/models/tpwng/modeling_tpwng.py @@ -30,6 +30,7 @@ import torch import torch.nn.functional as F +from src.modules.amp import safe_bce from torch import nn from transformers import PreTrainedModel from transformers.utils import ModelOutput @@ -129,6 +130,9 @@ def dummy_inputs(self): @MODELS.register("tpwng") class TPWNGForVideoAnomalyDetection(TPWNGPreTrainedModel): + # text-prompt-with-normality-guidance: uses a CLIP text encoder + requires_text_aligned = True + def __init__(self, config: TPWNGConfig): super().__init__(config) d = config.embed_dim @@ -227,7 +231,7 @@ def _compute_loss(self, hn, tf, sim, s_nn, s_an): psi = a * s_an + (1 - a) * (1 - s_nn) score_g = psi.clamp(0, 1) gamma = (psi.detach() > self.config.theta).float() - loss_cl = F.binary_cross_entropy(score_g.clamp(1e-6, 1 - 1e-6), gamma) + loss_cl = safe_bce(score_g.clamp(1e-6, 1 - 1e-6), gamma) # ranking: anomaly-sim higher in abnormal than normal; normal-sim high on normal an_max = s_an.max(dim=1).values # (B,) diff --git a/src/models/ur_dmu/modeling_ur_dmu.py b/src/models/ur_dmu/modeling_ur_dmu.py index de5ecdd..c27c01a 100644 --- a/src/models/ur_dmu/modeling_ur_dmu.py +++ b/src/models/ur_dmu/modeling_ur_dmu.py @@ -233,6 +233,8 @@ def _compute_loss(self, out, abnormal_labels, normal_labels): # triplet + KL + distance (weights from official ucf_main/config). import torch.nn.functional as F + from src.modules.amp import safe_bce # autocast-safe BCE (fp16 training) + frame = out["frame"] t = frame.size(1) k = t // 16 + 1 @@ -240,17 +242,17 @@ def _compute_loss(self, out, abnormal_labels, normal_labels): half = frame.size(0) // 2 y = torch.cat([torch.zeros(half, device=device), torch.ones(half, device=device)]) vid = torch.topk(frame, k, dim=1)[0].mean(1) - loss_mil = F.binary_cross_entropy(vid.clamp(1e-6, 1 - 1e-6), y) + loss_mil = safe_bce(vid.clamp(1e-6, 1 - 1e-6), y) a_score = torch.topk(out["A_att"], k, dim=1)[0].mean(1) n_score = torch.topk(out["N_att"], k, dim=1)[0].mean(1) an_score = torch.topk(out["A_Natt"], k, dim=1)[0].mean(1) na_score = torch.topk(out["N_Aatt"], k, dim=1)[0].mean(1) loss_mem = ( - F.binary_cross_entropy(a_score.clamp(1e-6, 1 - 1e-6), torch.ones_like(a_score)) - + F.binary_cross_entropy(n_score.clamp(1e-6, 1 - 1e-6), torch.ones_like(n_score)) - + F.binary_cross_entropy(an_score.clamp(1e-6, 1 - 1e-6), torch.zeros_like(an_score)) - + F.binary_cross_entropy(na_score.clamp(1e-6, 1 - 1e-6), torch.zeros_like(na_score)) + safe_bce(a_score.clamp(1e-6, 1 - 1e-6), torch.ones_like(a_score)) + + safe_bce(n_score.clamp(1e-6, 1 - 1e-6), torch.ones_like(n_score)) + + safe_bce(an_score.clamp(1e-6, 1 - 1e-6), torch.zeros_like(an_score)) + + safe_bce(na_score.clamp(1e-6, 1 - 1e-6), torch.zeros_like(na_score)) ) return ( loss_mil diff --git a/src/models/vadclip/modeling_vadclip.py b/src/models/vadclip/modeling_vadclip.py index 484ee3a..0dfec82 100644 --- a/src/models/vadclip/modeling_vadclip.py +++ b/src/models/vadclip/modeling_vadclip.py @@ -35,6 +35,7 @@ import numpy as np import torch import torch.nn.functional as F +from src.modules.amp import safe_bce from torch import nn from transformers import PreTrainedModel from transformers.utils import ModelOutput @@ -165,11 +166,13 @@ class VadCLIPPreTrainedModel(PreTrainedModel): base_model_prefix = "vadclip" def _init_weights(self, module): - if isinstance(module, nn.Linear): - nn.init.xavier_uniform_(module.weight) - if module.bias is not None: - nn.init.constant_(module.bias, 0) - elif isinstance(module, nn.Embedding): + # Match official VadCLIP init (model.py:119-122): ONLY the two task embeddings + # get a custom std=0.01; every Linear/MLP/GraphConv stays at PyTorch default + # (kaiming_uniform). xavier-overriding the Linears ~2.5x-inflates the binary + # head, saturating logits at init -> from-scratch training collapses to chance + # (verified 2026-06-22: xavier -> ROC 0.49; default -> learns). Inference is + # unaffected (load_state_dict overwrites init). + if isinstance(module, nn.Embedding): nn.init.normal_(module.weight, std=0.01) @property @@ -179,6 +182,9 @@ def dummy_inputs(self): @MODELS.register("vadclip") class VadCLIPForVideoAnomalyDetection(VadCLIPPreTrainedModel): + # uses a CLIP text encoder (prompts) -> needs a text-aligned backbone + requires_text_aligned = True + def __init__(self, config: VadCLIPConfig): super().__init__(config) vw = config.visual_width @@ -338,7 +344,7 @@ def forward( scores = torch.sigmoid(logits1) loss = None if abnormal_labels is not None and normal_labels is not None: - loss = self._compute_loss(logits1, logits2, text_features_ori, class_labels) + loss = self._compute_loss(logits1, logits2, text_features_ori, class_labels, lengths) return VadCLIPVideoAnomalyDetectionOutput( loss=loss, @@ -352,16 +358,25 @@ def forward( def _topk_k(self, t: int) -> int: return max(t // 16 + 1, 1) - def _compute_loss(self, binary_logits, alignment_logits, text_features_ori, class_labels): + def _compute_loss(self, binary_logits, alignment_logits, text_features_ori, + class_labels, lengths=None): bs, t, _ = binary_logits.size() half = bs // 2 device = binary_logits.device - k = self._topk_k(t) + # per-video valid length -> top-k over REAL frames only (official CLAS2/CLASM + # slice ``[0:lengths[i]]`` with k=lengths[i]//16+1). Padded positions are + # shared by normal+abnormal, so including them dilutes the MIL signal. + if lengths is None: + lens = [t] * bs + else: + lens = [int(max(min(L, t), 1)) for L in lengths] y = torch.cat([torch.zeros(half, device=device), torch.ones(half, device=device)]) probs = torch.sigmoid(binary_logits).squeeze(-1) - inst = torch.stack([torch.topk(probs[i], k)[0].mean() for i in range(bs)]) - loss1 = F.binary_cross_entropy(inst.clamp(1e-6, 1 - 1e-6), y) + inst = torch.stack([ + torch.topk(probs[i, : lens[i]], self._topk_k(lens[i]))[0].mean() for i in range(bs) + ]) + loss1 = safe_bce(inst.clamp(1e-6, 1 - 1e-6), y) tf = text_features_ori / (text_features_ori.norm(dim=-1, keepdim=True) + 1e-12) normal = tf[0] @@ -372,9 +387,10 @@ def _compute_loss(self, binary_logits, alignment_logits, text_features_ori, clas if class_labels is not None: labels = F.one_hot(class_labels.long().to(device), self.config.num_class).float() labels = labels / labels.sum(dim=1, keepdim=True).clamp_min(1e-6) - inst_logits = torch.stack( - [torch.topk(alignment_logits[i], k, dim=0)[0].mean(0) for i in range(bs)] - ) + inst_logits = torch.stack([ + torch.topk(alignment_logits[i, : lens[i]], self._topk_k(lens[i]), dim=0)[0].mean(0) + for i in range(bs) + ]) loss2 = -torch.mean(torch.sum(labels * F.log_softmax(inst_logits, dim=1), dim=1)) loss = loss + loss2 return loss @@ -393,3 +409,26 @@ def convert_official_vadclip(state_dict: dict) -> "OrderedDict": continue out[k] = v return out + + +def load_pretrained_clip_text(model, official_ckpt: str = "pretrained/vadclip/model_ucf.pth"): + """Initialise + FREEZE the CLIP text tower for *from-scratch* training. + + The official ``CLIPVAD`` loads pretrained CLIP ViT-B/16 and freezes it + (``model.py:111-113``: ``requires_grad = False``), so the ``clipmodel.*`` keys + in ``model_ucf.pth`` are the *unchanged* OpenAI CLIP weights — using them as the + from-scratch init is the frozen pretrained backbone, NOT trained head weights. + Everything else (temporal/gc/disAdj/linear/mlp/classifier/prompt embeddings) + stays at random init. Without this the text-alignment branch is random and the + model cannot learn (from-scratch ROC-AUC stays at chance ~0.49). + """ + sd = torch.load(official_ckpt, map_location="cpu", weights_only=False) + sd = sd.get("state_dict", sd) if isinstance(sd, dict) else sd + sd = convert_official_vadclip(sd) + clip_sd = {k: v for k, v in sd.items() if k.startswith("clipmodel.")} + missing, unexpected = model.load_state_dict(clip_sd, strict=False) + leaked = [k for k in unexpected if k.startswith("clipmodel.")] + assert not leaked, f"clipmodel keys failed to load: {leaked[:5]}" + for p in model.clipmodel.parameters(): # official freezes CLIP + p.requires_grad = False + return len(clip_sd) diff --git a/src/modules/amp.py b/src/modules/amp.py new file mode 100644 index 0000000..864e1b5 --- /dev/null +++ b/src/modules/amp.py @@ -0,0 +1,20 @@ +"""Autocast-safe loss helpers for mixed-precision (fp16/bf16) training. + +``F.binary_cross_entropy`` / ``nn.BCELoss`` operate on probabilities and are +explicitly **unsafe to autocast** (PyTorch raises). The WSVAD heads compute MIL/ +memory losses as BCE on sigmoid outputs (faithful to the official code), so AMP +training would crash. ``safe_bce`` computes the BCE in fp32 with autocast disabled. + +In fp32 (non-AMP) training/eval this is a *no-op*: autocast is already off and the +inputs are already fp32, so ``.float()`` returns the same tensor and the result is +numerically identical to ``F.binary_cross_entropy`` — the verified-checkpoint paths +are unchanged. It only fixes the fp16/bf16 path. +""" + +import torch +import torch.nn.functional as F + + +def safe_bce(inp: torch.Tensor, target: torch.Tensor, **kwargs) -> torch.Tensor: + with torch.autocast(device_type=inp.device.type, enabled=False): + return F.binary_cross_entropy(inp.float(), target.float(), **kwargs) diff --git a/src/modules/translayer.py b/src/modules/translayer.py index 50b61e1..c585b54 100644 --- a/src/modules/translayer.py +++ b/src/modules/translayer.py @@ -9,10 +9,22 @@ directly (``selfatt.layers.{i}.{0,1}.{norm,fn...}``). """ +import os + import torch +import torch.nn.functional as F from einops import rearrange from torch import nn +# Opt-in memory-efficient attention for the dual-branch temporal Transformer. Default +# "eager" keeps the bit-exact path the official-checkpoint verification relies on +# (UR-DMU max|Δ|=5.96e-08). "mem" routes branch-1 through scaled_dot_product_attention +# (flash/mem-efficient kernel, ~1e-7 deviation) and computes branch-2 without +# materializing the (b, h, n, n) decay matrix — so seg200 fits batch 64 on 8 GB GPUs +# instead of OOM-ing in the O(T²) ``q·kᵀ``. Set WSAD_ATTN=mem for training the memory +# heads at the official batch size; leave unset for verification/eval. +_ATTN_IMPL = os.environ.get("WSAD_ATTN", "eager").lower() + class PreNorm(nn.Module): def __init__(self, dim: int, fn: nn.Module): @@ -61,17 +73,25 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: qkvt = self.to_qkv(x).chunk(4, dim=-1) q, k, v, t = map(lambda u: rearrange(u, "b n (h d) -> b h n d", h=self.heads), qkvt) - dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale - attn1 = self.attend(dots) - - # fixed temporal distance-decay attention (official, device-safe) + # fixed temporal distance-decay attention (official, device-safe); (n, n) tmp_ones = torch.ones(n, device=x.device) tmp_n = torch.linspace(1, n, n, device=x.device) tg_tmp = torch.abs(tmp_n * tmp_ones - tmp_n.view(-1, 1)) attn2 = torch.exp(-tg_tmp / torch.exp(torch.tensor(1.0, device=x.device))) - attn2 = (attn2 / attn2.sum(-1)).unsqueeze(0).unsqueeze(1).repeat(b, self.heads, 1, 1) - - out = torch.cat([torch.matmul(attn1, v), torch.matmul(attn2, t)], dim=-1) + attn2 = attn2 / attn2.sum(-1) # row-normalized (n, n), identical across b/h + + if _ATTN_IMPL == "mem": + # branch-1: flash/mem-efficient kernel (no materialized (b,h,n,n) dots); + # branch-2: einsum reuses the shared (n,n) decay (no b*h repeat). Both avoid + # the O(T²) tensors that OOM at batch 64 / seg200. + out1 = F.scaled_dot_product_attention(q, k, v) + out2 = torch.einsum("nm,bhmd->bhnd", attn2, t) + out = torch.cat([out1, out2], dim=-1) + else: # eager: the bit-exact path the official-ckpt verification depends on + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + attn1 = self.attend(dots) + attn2b = attn2.unsqueeze(0).unsqueeze(1).repeat(b, self.heads, 1, 1) + out = torch.cat([torch.matmul(attn1, v), torch.matmul(attn2b, t)], dim=-1) out = rearrange(out, "b h n d -> b n (h d)") return self.to_out(out) @@ -90,9 +110,23 @@ def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.0): for _ in range(depth) ] ) + # activation checkpointing (off by default — eval/verify keep full activations). + # The O(T²) attention at seg200 is the activation bulk; recomputing it in the + # backward lets the full batch-64 forward fit 8 GB (BN sees the real batch). + self.gradient_checkpointing = False + + def gradient_checkpointing_enable(self): + self.gradient_checkpointing = True + + @staticmethod + def _block(attn, ff, x): + x = attn(x) + x + return ff(x) + x def forward(self, x): for attn, ff in self.layers: - x = attn(x) + x - x = ff(x) + x + if self.gradient_checkpointing and self.training: + x = torch.utils.checkpoint.checkpoint(self._block, attn, ff, x, use_reentrant=False) + else: + x = self._block(attn, ff, x) return x diff --git a/src/pipeline.py b/src/pipeline.py new file mode 100644 index 0000000..20facd7 --- /dev/null +++ b/src/pipeline.py @@ -0,0 +1,132 @@ +"""HF-style end-to-end anomaly-detection pipeline (issue #20). + +Bundles a feature backbone (the *processor*), a trained head, and post-process +into one serveable object: ``raw video -> per-frame anomaly scores``. No separate +offline extraction step at serve time — the extractor is absorbed, the way an HF +``pipeline`` absorbs its processor. + +Internals stay decoupled (the matrix harness swaps backbone/head freely); this +factory is the bundled surface. Spec 1 implements ``mode="offline"`` (bidirectional, +exact = training); ``window``/``causal`` extend the same object in Spec 2. + +Design notes: +- The backbone<->head text-alignment guard runs at construction (fail early). +- A fresh head config has its feature-dim field matched to the backbone dim, so + ``pipeline("clip", "sultani")`` builds a 512-d head without manual config. +- ``from_features`` is the extraction-free entry (cached features / tests); + ``__call__`` runs the full raw-video path. +""" + +from typing import Optional, Union + +import numpy as np +import torch + +from src.compat import assert_compatible +from src.registry import FEATURE_EXTRACTORS, MODELS + +# config field that carries the raw (pre-magnitude) feature dim, per head family +_DIM_FIELDS = ("feature_size", "channels", "visual_width") + + +def _set_feature_dim(config, dim: int) -> None: + for f in _DIM_FIELDS: + if hasattr(config, f): + setattr(config, f, dim) + + +class AnomalyDetectionPipeline: + """``raw video -> per-frame anomaly scores`` for a (backbone, head) pair.""" + + def __init__( + self, + backbone: str, + head: Union[str, "torch.nn.Module"], + *, + head_ckpt: Optional[str] = None, + model: Optional["torch.nn.Module"] = None, + device: str = "cpu", + with_magnitude: bool = True, + segment_to: Optional[int] = None, + extractor=None, + extractor_kwargs: Optional[dict] = None, + ): + self.backbone = backbone + self.device = device + self.with_magnitude = with_magnitude + self.segment_to = segment_to + self._extractor = extractor + self._extractor_kwargs = extractor_kwargs or {} + + # fail early on a text-branch head + visual-only backbone + if isinstance(head, str): + assert_compatible(backbone, head) + + if model is not None: + self.model = model + elif isinstance(head, str): + cls = MODELS.get(head) + if head_ckpt is not None: + self.model = cls.from_pretrained(head_ckpt) + else: + cfg = cls.config_class() + dim = getattr(FEATURE_EXTRACTORS.get(backbone), "dim", None) + if dim: + _set_feature_dim(cfg, dim) + self.model = cls(cfg) + else: # a ready nn.Module head instance + self.model = head + self.model.eval().to(device) + + @property + def extractor(self): + if self._extractor is None: + from src.features import build_extractor + + self._extractor = build_extractor( + self.backbone, device=self.device, **self._extractor_kwargs + ) + return self._extractor + + def _preprocess(self, feats: np.ndarray) -> torch.Tensor: + """``(T, D)`` or ``(T, ncrops, D)`` -> ``(1, ncrops, T', D'+mag)`` tensor.""" + from src.data.features import _add_magnitude + from src.data.local import segment + + feats = np.asarray(feats, dtype=np.float32) + if feats.ndim == 2: # single-crop (T, D) + feats = feats[:, None, :] + feats = np.transpose(feats, (1, 0, 2)) # (ncrops, T, D) + + crops = [] + for crop in feats: + if self.segment_to is not None: + crop = segment(crop, self.segment_to) + if self.with_magnitude: + crop = _add_magnitude(crop) + crops.append(crop.astype(np.float32)) + arr = np.stack(crops, axis=0) # (ncrops, T', D') + return torch.from_numpy(arr).unsqueeze(0).to(self.device) + + @torch.no_grad() + def from_features(self, feats: np.ndarray) -> np.ndarray: + """Run the head on already-extracted features -> ``(T',)`` scores.""" + x = self._preprocess(feats) + out = self.model(video=x) + return out.scores.squeeze(0).squeeze(-1).cpu().numpy() + + @torch.no_grad() + def __call__(self, video) -> np.ndarray: + """Full path: ``raw video -> (T',) per-index anomaly scores``.""" + return self.from_features(self.extractor.extract(video)) + + def visualize(self, video_or_scores, gt=None, **kwargs): + """Run (if needed) and plot the score curve with GT shading.""" + from src.viz import plot_anomaly_scores + + scores = ( + video_or_scores + if isinstance(video_or_scores, np.ndarray) + else self(video_or_scores) + ) + return plot_anomaly_scores(scores, gt=gt, **kwargs) diff --git a/src/trainer.py b/src/trainer.py index 94fab31..66aefd6 100644 --- a/src/trainer.py +++ b/src/trainer.py @@ -50,7 +50,7 @@ def build_datasets(data_cfg) -> tuple: local_present = ( backbone == "clip" - or has_local(root, backbone, "train") + or has_local(root, backbone, "train", data_cfg) or (backbone == "i3d" and has_local_i3d_zip(root, data_cfg, "train")) ) use_local = source == "local" or (source == "auto" and local_present) @@ -81,26 +81,81 @@ def __init__( batch_size: int = 32, num_workers: int = 2, frames_per_clip: int = 16, - mixed_precision: str = "no", # "no" | "fp16" | "bf16" + mixed_precision: str = "no", # "no" | "fp16" | "bf16" (AMP halves activation mem) + grad_clip_norm: Optional[float] = None, # e.g. 1.0 for BN-WVAD (official train.py:16) + optimizer_name: str = "adam", # "adam" | "adamw" (VadCLIP uses AdamW) + scheduler_milestones: Optional[list] = None, # MultiStepLR epochs, e.g. [4, 8] (VadCLIP) + scheduler_gamma: float = 0.1, + gradient_checkpointing: bool = False, # recompute activations in backward -> fit + # the full batch-64 forward on 8 GB (BN-WVAD/UR-DMU need the real batch for BN). + gradient_accumulation_steps: int = 1, # NB: does NOT enlarge BN's batch (per-micro + # batch stats) — use real batch + checkpointing for BatchNorm-based heads. ): - self.accelerator = Accelerator(mixed_precision=mixed_precision) + self.accelerator = Accelerator( + mixed_precision=mixed_precision, + gradient_accumulation_steps=gradient_accumulation_steps, + ) + self.grad_accum = gradient_accumulation_steps + if gradient_checkpointing: + n = 0 + for m in model.modules(): + if m is not model and hasattr(m, "gradient_checkpointing_enable"): + m.gradient_checkpointing_enable() + n += 1 + print(f"[trainer] gradient checkpointing enabled on {n} submodule(s)", flush=True) self.model = model - self.optimizer = torch.optim.Adam( + opt_cls = {"adam": torch.optim.Adam, "adamw": torch.optim.AdamW}[optimizer_name.lower()] + self.optimizer = opt_cls( model.parameters(), lr=learning_rate, weight_decay=weight_decay ) + # optional per-epoch LR schedule (faithful recipes: VadCLIP MultiStepLR[4,8] x0.1). + # Stepped once per epoch in fit(); ignored by the step-based fit_steps path. + self.scheduler = ( + torch.optim.lr_scheduler.MultiStepLR( + self.optimizer, milestones=list(scheduler_milestones), gamma=scheduler_gamma + ) + if scheduler_milestones + else None + ) self.batch_size = batch_size self.num_workers = num_workers self.frames_per_clip = frames_per_clip - self._accepts_class = ( - "class_labels" in inspect.signature(model.forward).parameters - ) + self.grad_clip_norm = grad_clip_norm + sig = inspect.signature(model.forward).parameters + self._accepts_class = "class_labels" in sig + # heads that mask padded frames by per-video length (VadCLIP windowed-256). + self._accepts_lengths = "lengths" in sig + + @staticmethod + def _valid_lengths(feature: torch.Tensor) -> torch.Tensor: + """Per-video valid (non-padded) length from a windowed feature. + + Clip windows are zero-padded to ``visual_length``; the real length is the + count of non-zero frames (official ``process_feat`` returns this length). + ``feature`` is ``(B, ncrops, T, D)`` or ``(B, T, D)``. + """ + x = feature[:, 0] if feature.dim() == 4 else feature # (B, T, D) + return (x.abs().sum(-1) > 0).sum(-1).clamp(min=1) # (B,) def fit( self, train_datasets: Dict[str, Dataset], test_dataset: Optional[Dataset] = None, epochs: int = 1, + eval_fn=None, + select_metric: str = "roc_auc", + eval_every: Optional[int] = None, ): + """Epoch-based training (faithful recipes that count epochs + use an LR + schedule, e.g. VadCLIP: AdamW + MultiStepLR[4,8], 10 epochs). + + ``eval_fn(model) -> {roc_auc, pr_auc}`` (e.g. a per-head + ``src.eval_matrix.evaluate`` closure) overrides the default I3D-shaped + ``self.evaluate``; the BEST checkpoint by ``select_metric`` is returned. + ``eval_every`` (optimizer steps) adds intra-epoch evals + best-checkpoint + selection — VadCLIP's official eval at ``step % 1280 == 0`` (~12x/epoch on + the 16100-sample 10-crop train), which catches a higher peak than 1/epoch. + """ loaders = { k: DataLoader( train_datasets[k], @@ -116,6 +171,18 @@ def fit( self.model, self.optimizer, loaders["normal"], loaders["abnormal"] ) + best = {"roc_auc": 0.0, "pr_auc": 0.0, "best_epoch": -1, "last": 0.0, "_sel": -1.0} + + def _record(epoch): # eval + keep best by select_metric + m = eval_fn(self.accelerator.unwrap_model(model)) + best["last"] = m["roc_auc"] + sel = m.get(select_metric, m["roc_auc"]) + if sel > best["_sel"]: + best.update(roc_auc=m["roc_auc"], pr_auc=m["pr_auc"], best_epoch=epoch, _sel=sel) + model.train() + return m + + gstep = 0 for epoch in range(epochs): model.train() total = 0.0 @@ -128,6 +195,8 @@ def fit( kwargs["class_labels"] = torch.cat( [nb["class_id"], ab["class_id"]], dim=0 ) + if self._accepts_lengths: + kwargs["lengths"] = self._valid_lengths(feature) out = model( video=feature, abnormal_labels=ab["anomaly"], @@ -135,15 +204,118 @@ def fit( **kwargs, ) self.accelerator.backward(out.loss) + if self.grad_clip_norm is not None: + self.accelerator.clip_grad_norm_(model.parameters(), self.grad_clip_norm) optimizer.step() optimizer.zero_grad() total += out.loss.item() + gstep += 1 + if eval_fn is not None and eval_every and gstep % eval_every == 0: + _record(epoch) + if self.scheduler is not None: + self.scheduler.step() log = {"epoch": epoch, "train_loss": total / max(len(nl), 1)} - if test_dataset is not None: + if eval_fn is not None: + m = _record(epoch) + log.update(roc_auc=round(m["roc_auc"], 4), pr_auc=round(m["pr_auc"], 4), + best=round(best["roc_auc"], 4)) + elif test_dataset is not None: log.update(self.evaluate(test_dataset)) self.accelerator.print(log) - return log + best.pop("_sel", None) + return best if eval_fn is not None else log + + def fit_steps( + self, + train_datasets: Dict[str, Dataset], + max_steps: int, + eval_fn, + eval_interval: int, + eval_start: int = 0, + select_metric: str = "roc_auc", + lr_decay: Optional[str] = None, + ) -> Dict[str, float]: + """Iteration-based training (the WSVAD convention: RTFM/S3R/UR-DMU/... count + gradient *steps*, not epochs). Cycles the normal/abnormal loaders, takes + ``max_steps`` balanced mini-batches, calls ``eval_fn(model) -> {roc_auc, + pr_auc}`` every ``eval_interval`` steps (after ``eval_start``), and keeps the + BEST checkpoint by ``select_metric`` (AUC, or 'pr_auc' for BN-WVAD's AP). + + ``eval_fn`` is injected so the trainer stays decoupled from any specific eval + (the matrix passes a per-head ``src.eval_matrix.evaluate`` closure). + + ``lr_decay="cosine"`` adds a per-step CosineAnnealingLR over ``max_steps`` — + the official constant-lr RTFM recipe destabilizes (train loss climbs, AUC + oscillates around an early lucky peak); decaying the lr lets it CONVERGE so + the eval AUC settles high instead of bouncing. Returns ``auc_mean``/``auc_std`` + over all eval points (the honest stability metric, alongside the optimistic + best-test-AUC the field reports). + """ + loaders = { + k: DataLoader(train_datasets[k], batch_size=self.batch_size, shuffle=True, + drop_last=True, num_workers=self.num_workers, + collate_fn=_collate_train) + for k in ("normal", "abnormal") + } + model, optimizer, nl, al = self.accelerator.prepare( + self.model, self.optimizer, loaders["normal"], loaders["abnormal"] + ) + scheduler = ( + torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max_steps) + if lr_decay == "cosine" + else None + ) + + def _cycle(dl): + while True: + for x in dl: + yield x + + nit, ait = _cycle(nl), _cycle(al) + best = {"roc_auc": 0.0, "pr_auc": 0.0, "best_step": -1, "last": 0.0, "_sel": -1.0} + eval_aucs = [] + run_loss = 0.0 + for step in range(1, max_steps + 1): + model.train() + nb, ab = next(nit), next(ait) + feature = torch.cat([nb["feature"], ab["feature"]], dim=0) + kwargs = {} + if self._accepts_class: + kwargs["class_labels"] = torch.cat([nb["class_id"], ab["class_id"]], dim=0) + if self._accepts_lengths: + kwargs["lengths"] = self._valid_lengths(feature) + with self.accelerator.accumulate(model): + out = model(video=feature, abnormal_labels=ab["anomaly"], + normal_labels=nb["anomaly"], **kwargs) + self.accelerator.backward(out.loss) + if self.accelerator.sync_gradients and self.grad_clip_norm is not None: + self.accelerator.clip_grad_norm_(model.parameters(), self.grad_clip_norm) + optimizer.step() + optimizer.zero_grad() + if scheduler is not None and self.accelerator.sync_gradients: + scheduler.step() + run_loss += out.loss.item() + if step >= eval_start and (step % eval_interval == 0 or step == max_steps): + m = eval_fn(self.accelerator.unwrap_model(model)) + best["last"] = m["roc_auc"] + eval_aucs.append(m["roc_auc"]) + sel = m.get(select_metric, m["roc_auc"]) + if sel > best["_sel"]: + best.update(roc_auc=m["roc_auc"], pr_auc=m["pr_auc"], best_step=step, _sel=sel) + self.accelerator.print( + {"step": step, "lr": round(optimizer.param_groups[0]["lr"], 6), + "train_loss": round(run_loss / eval_interval, 4), + "roc_auc": round(m["roc_auc"], 4), "best": round(best["roc_auc"], 4)}) + run_loss = 0.0 + best.pop("_sel", None) + # honest stability metric: mean/std over the LATE half of eval points + late = eval_aucs[len(eval_aucs) // 2:] or eval_aucs + if late: + import statistics + best["auc_mean"] = round(statistics.mean(late), 4) + best["auc_std"] = round(statistics.pstdev(late), 4) if len(late) > 1 else 0.0 + return best @torch.no_grad() def evaluate(self, test_dataset: Dataset) -> Dict[str, float]: diff --git a/src/viz.py b/src/viz.py new file mode 100644 index 0000000..4615be5 --- /dev/null +++ b/src/viz.py @@ -0,0 +1,105 @@ +"""Qualitative figure for video anomaly detection (issue #21). + +The figure every WSVAD paper plots: a per-frame/snippet anomaly-score curve with +the ground-truth anomaly region(s) shaded. This is the liveness proof for the +serving pipeline; the full qualitative suite (top-k thumbnails, PR/ROC, cross- +matrix panels) is Spec 3. + +``matplotlib`` is imported lazily so importing this module is cheap and never +fails when plotting isn't needed. +""" + +from __future__ import annotations + +from typing import List, Optional, Sequence, Tuple, Union + +import numpy as np + +Interval = Tuple[int, int] + + +def _mask_to_intervals(mask: Sequence[int]) -> List[Interval]: + """Contiguous True runs of a binary mask -> list of half-open [start, end).""" + m = np.asarray(mask).astype(np.int8) + if m.size == 0: + return [] + padded = np.concatenate(([0], m, [0])) + diff = np.diff(padded) + starts = np.where(diff == 1)[0] + ends = np.where(diff == -1)[0] + return list(zip(starts.tolist(), ends.tolist())) + + +def _coerce_gt(gt, n: int) -> List[Interval]: + """Accept a binary mask (len == n) or a list of (start, end) intervals.""" + if gt is None: + return [] + arr = np.asarray(gt) + if arr.ndim == 1 and arr.size == n and set(np.unique(arr).tolist()).issubset({0, 1}): + return _mask_to_intervals(arr) + return [(int(s), int(e)) for s, e in gt] + + +def plot_anomaly_scores( + scores: Sequence[float], + gt: Optional[Union[Sequence[int], Sequence[Interval]]] = None, + *, + threshold: Optional[float] = None, + title: Optional[str] = None, + xlabel: str = "snippet / frame index", + ylabel: str = "anomaly score", + legend: bool = True, + save_path: Optional[str] = None, + show: bool = False, +): + """Plot a per-index anomaly-score curve with GT anomaly regions shaded. + + Args: + scores: 1-D per-index anomaly scores. + gt: a binary mask (``len == len(scores)``) or a list of ``(start, end)`` + half-open index intervals marking ground-truth anomalies. + threshold: optional horizontal decision line. + title: optional figure title. + save_path: if set, save the figure (forces the headless Agg backend). + show: if True, call ``plt.show()``. + Returns: + ``(fig, ax)``. + """ + import matplotlib + + if save_path is not None and not show: + matplotlib.use("Agg", force=True) + import matplotlib.pyplot as plt + + scores = np.asarray(scores, dtype=float).ravel() + x = np.arange(len(scores)) + + fig, ax = plt.subplots(figsize=(10, 3)) + ax.plot(x, scores, color="#1f77b4", lw=1.5, label="anomaly score") + + for i, (s, e) in enumerate(_coerce_gt(gt, len(scores))): + # light-orange GT region with dashed red boundaries (paper convention) + ax.axvspan(s, e, color="#ff9e4a", alpha=0.35, + label="ground truth" if i == 0 else None) + for boundary in (s, e): + ax.axvline(boundary, color="#d62728", ls="--", lw=1.0) + + if threshold is not None: + ax.axhline(threshold, color="gray", ls="--", lw=1, label="threshold") + + top = 1.05 if (scores.size and scores.max() <= 1) else (float(scores.max()) * 1.05 if scores.size else 1.0) + ax.set_xlim(0, max(len(scores) - 1, 1)) + ax.set_ylim(0, top) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + if title: + ax.set_title(title) + if legend: + ax.legend(loc="upper right", fontsize=8) + fig.tight_layout() + + if save_path is not None: + fig.savefig(save_path, dpi=120) + if show: + plt.show() + return fig, ax diff --git a/tests/test_compat.py b/tests/test_compat.py new file mode 100644 index 0000000..591d0df --- /dev/null +++ b/tests/test_compat.py @@ -0,0 +1,83 @@ +"""Dim-agnostic heads + backbone<->head text-alignment guard (issue #18). + +Covers: (1) MGFN no longer hardcodes 2048 (works on 512/768 backbones) while the +2048 split is byte-for-byte unchanged; (2) the compatibility guard rejects a +text-branch head on a visual-only backbone. +""" + +import pytest +import torch + +import src.features # noqa: F401 (registration) +import src.models # noqa: F401 (registration) +from src.compat import ( + assert_compatible, + backbone_is_text_aligned, + head_requires_text_aligned, + is_compatible, +) + +T = 32 +NCROPS = 10 + + +def _mgfn(channels): + from src.models.mgfn import MGFNConfig, MGFNForVideoAnomalyDetection + + return MGFNForVideoAnomalyDetection(MGFNConfig(channels=channels)) + + +# --- MGFN dim-agnosticism --- + + +@pytest.mark.parametrize("channels", [2048, 768, 512]) +def test_mgfn_is_dim_agnostic(channels): + model = _mgfn(channels).eval() + with torch.no_grad(): + # +1 for the appended magnitude channel + out = model(video=torch.randn(1, NCROPS, T, channels + 1)) + assert out.scores.shape == (1, T, 1) + assert torch.all(torch.isfinite(out.scores)) + + +def test_mgfn_2048_split_unchanged(): + """Regression: at channels=2048 the amplifier splits exactly as the old + hardcoded literal did (first 2048 = features, last 1 = magnitude).""" + from src.models.mgfn import MGFNConfig + from src.models.mgfn.modeling_mgfn import MGFNFeatureAmplifier + + cfg = MGFNConfig(channels=2048) + amp = MGFNFeatureAmplifier(cfg).eval() + x = torch.randn(1, NCROPS, T, 2049) + with torch.no_grad(): + out = amp(x) + assert out.shape == (NCROPS, cfg.dims[0], T) + + +# --- compatibility guard --- + + +def test_text_branch_heads_flagged(): + assert head_requires_text_aligned("vadclip") is True + assert head_requires_text_aligned("tpwng") is True + assert head_requires_text_aligned("mgfn") is False + assert head_requires_text_aligned("clip_tsa") is False # CLIP image feats, no text + + +def test_backbone_text_alignment(): + assert backbone_is_text_aligned("clip") is True + assert backbone_is_text_aligned("i3d") is False + assert backbone_is_text_aligned("videomae") is False + + +def test_assert_compatible_allows_valid_pairs(): + assert_compatible("clip", "vadclip") # text head + text backbone + assert_compatible("i3d", "mgfn") # visual head + visual backbone + assert_compatible("clip", "mgfn") # visual head on a text backbone is fine + assert is_compatible("i3d", "mgfn") + + +def test_assert_compatible_rejects_text_head_on_visual_backbone(): + assert is_compatible("i3d", "vadclip") is False + with pytest.raises(ValueError, match="text-aligned"): + assert_compatible("i3d", "vadclip") diff --git a/tests/test_extract.py b/tests/test_extract.py new file mode 100644 index 0000000..c3db7df --- /dev/null +++ b/tests/test_extract.py @@ -0,0 +1,45 @@ +"""Backbone-agnostic extraction core (issue #17). + +Tests the generic loop with a fake extractor — no decord/GPU/models needed. +""" + +import os + +import numpy as np + +from src.features.extract import run_extraction, save_path_for + + +class _FakeExtractor: + def __init__(self): + self.calls = 0 + + def extract(self, path): + self.calls += 1 + return np.zeros((4, 8), dtype=np.float32) + + +def test_save_path_for(): + assert ( + save_path_for("/v/Robbery001_x264.mp4", "/out", "clip") + == "/out/Robbery001_x264_clip.npy" + ) + + +def test_run_extraction_saves(tmp_path): + ex = _FakeExtractor() + samples = [{"video_path": "/v/A.mp4"}, {"video_path": "/v/B.mp4"}] + saved = run_extraction(samples, ex, str(tmp_path), "videomae") + assert len(saved) == 2 + assert all(os.path.exists(p) for p in saved) + assert saved[0].endswith("A_videomae.npy") + assert np.load(saved[0]).shape == (4, 8) + assert ex.calls == 2 + + +def test_run_extraction_skips_existing(tmp_path): + ex = _FakeExtractor() + samples = [{"video_path": "/v/A.mp4"}] + run_extraction(samples, ex, str(tmp_path), "i3d") + run_extraction(samples, ex, str(tmp_path), "i3d") # second call must skip + assert ex.calls == 1 diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..d8065c6 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,53 @@ +"""HF-style AnomalyDetectionPipeline (issue #20). + +Covers the extraction-free path (cached/dummy features), the compatibility guard +at construction, backbone-dim auto-matching, and the visualize hook. The raw-video +path (``__call__``) needs a backbone + decord and is exercised manually, not here. +""" + +import numpy as np +import pytest + +import src.features # noqa: F401 (registration) +import src.models # noqa: F401 (registration) +from src.pipeline import AnomalyDetectionPipeline + +T = 32 +NCROPS = 10 + + +def test_guard_rejects_text_head_on_visual_backbone(): + with pytest.raises(ValueError, match="text-aligned"): + AnomalyDetectionPipeline("i3d", "vadclip") + + +def test_from_features_mgfn_i3d(): + pipe = AnomalyDetectionPipeline("i3d", "mgfn") + feats = np.random.randn(T, NCROPS, 2048).astype(np.float32) # (T, ncrops, D) + scores = pipe.from_features(feats) + assert scores.shape == (T,) + assert np.all(np.isfinite(scores)) + + +def test_backbone_dim_auto_matched_for_clip(): + pipe = AnomalyDetectionPipeline("clip", "mgfn") + assert pipe.model.config.channels == 512 # CLIP dim, not the 2048 default + feats = np.random.randn(T, 512).astype(np.float32) # single-crop CLIP + scores = pipe.from_features(feats) + assert scores.shape == (T,) + + +def test_from_features_sultani_clip_single_crop(): + pipe = AnomalyDetectionPipeline("clip", "sultani") + assert pipe.model.config.feature_size == 512 + feats = np.random.randn(T, 512).astype(np.float32) + scores = pipe.from_features(feats) + assert scores.shape == (T,) + + +def test_visualize_from_scores(tmp_path): + pipe = AnomalyDetectionPipeline("i3d", "mgfn") + scores = np.random.rand(T) + out = tmp_path / "p.png" + fig, ax = pipe.visualize(scores, gt=[(5, 12)], save_path=str(out)) + assert out.exists() diff --git a/tests/test_viz.py b/tests/test_viz.py new file mode 100644 index 0000000..4d99d0e --- /dev/null +++ b/tests/test_viz.py @@ -0,0 +1,39 @@ +"""Qualitative figure util (issue #21).""" + +import numpy as np + +from src.viz import _mask_to_intervals, plot_anomaly_scores + + +def test_mask_to_intervals(): + m = np.array([0, 1, 1, 0, 0, 1, 0]) + assert _mask_to_intervals(m) == [(1, 3), (5, 6)] + + +def test_mask_to_intervals_edges(): + assert _mask_to_intervals(np.array([1, 1, 0, 1])) == [(0, 2), (3, 4)] + assert _mask_to_intervals(np.array([0, 0])) == [] + + +def test_plot_returns_curve_of_right_length(): + scores = np.linspace(0, 1, 32) + fig, ax = plot_anomaly_scores(scores) + assert len(ax.get_lines()[0].get_xdata()) == 32 + + +def test_plot_shades_interval_gt_and_saves(tmp_path): + scores = np.random.rand(50) + out = tmp_path / "fig.png" + fig, ax = plot_anomaly_scores( + scores, gt=[(10, 20)], threshold=0.5, save_path=str(out) + ) + assert out.exists() + assert len(ax.patches) >= 1 # axvspan polygon + + +def test_plot_accepts_binary_mask_gt(): + scores = np.random.rand(20) + mask = np.zeros(20) + mask[5:10] = 1 + fig, ax = plot_anomaly_scores(scores, gt=mask) + assert len(ax.patches) >= 1