perf(ingest): streaming beamtime ingest + maintainability refactor

Cargo.toml

@@ -18,8 +18,13 @@ name = "browser"
 path = "src/bin/browser.rs"
 required-features = ["tui"]
 
+[[bin]]
+name = "migrate-zarr-3d"
+path = "src/bin/migrate_zarr_3d.rs"
+required-features = ["catalog"]
+
 [features]
-default = ["bindings", "catalog", "parallel_ingest"]
+default = ["bindings", "catalog", "parallel_ingest", "watch"]
 bindings = ["dep:pyo3", "dep:pyo3-polars", "dep:numpy"]
 extension-module = ["bindings", "pyo3/extension-module", "pyo3/generate-import-lib"]
 catalog = [
@@ -30,6 +35,7 @@ catalog = [
     "dep:walkdir",
     "dep:zarrs",
     "dep:crossbeam-channel",
+    "dep:zstd",
 ]
 parallel_ingest = ["catalog"]
 zarr = ["catalog"]
@@ -61,7 +67,8 @@ directories = {version = "6", optional = true}
 notify-debouncer-mini = {version = "0.4", optional = true}
 log = {version = "0.4", optional = true}
 eframe = {version = "0.28", optional = true, default-features = true}
-zarrs = {version = "0.23", optional = true, default-features = false, features = ["filesystem", "ndarray"]}
+zarrs = {version = "0.23", optional = true, default-features = false, features = ["filesystem", "ndarray", "zstd", "sharding", "crc32c"]}
+zstd = {version = "0.13", optional = true, default-features = false}
 diesel = { version = "2.2", optional = true, features = ["sqlite", "returning_clauses_for_sqlite_3_35", "32-column-tables"] }
 diesel_migrations = {version = "2.2", optional = true, features = ["sqlite"]}
 libsqlite3-sys = {version = "0.28", optional = true, features = ["bundled"]}

docs/README.md

@@ -0,0 +1,37 @@
+# Documentation Index
+
+## Prototyping and Development
+
+Current prototyping and development guidance lives under:
+
+- `docs/prototyping/README.md`
+- `docs/prototyping/comprehensive_implementation_plan.md`
+- `docs/prototyping/development_documentation_guidelines.md`
+
+## Workflow and API Specifications
+
+The active workflow/API specs are maintained under `docs/prototyping/`:
+
+### API Design
+
+- `docs/prototyping/api-design/api_flow_design.md`
+- `docs/prototyping/api-design/catalog_api_reference.md`
+- `docs/prototyping/api-design/header_normalization_spec.md`
+
+### References
+
+- `docs/prototyping/references/header_summary_rules_schema.md`
+
+### Workflows
+
+- `docs/prototyping/workflows/cell_01_catalog_and_selection.md`
+- `docs/prototyping/workflows/cell_02_profile_split.md`
+- `docs/prototyping/workflows/cell_03_classify_and_i0_linking.md`
+- `docs/prototyping/workflows/cell_04_mask_and_beamspot.md`
+- `docs/prototyping/workflows/cell_05_persist_mask_and_spot_overrides.md`
+- `docs/prototyping/workflows/cell_06_reduction_and_uncertainty.md`
+- `docs/prototyping/workflows/cell_07_role_diagnostics.md`
+- `docs/prototyping/workflows/cell_08_stitch_compute.md`
+- `docs/prototyping/workflows/cell_09_profile_comparison_and_export.md`
+- `docs/prototyping/workflows/just_process_flow.md`
+- `docs/prototyping/workflows/lazy_refresh_with_watcher.md`

docs/prototyping/README.md

@@ -0,0 +1,53 @@
+# Prototyping and Development Documentation
+
+## Purpose
+
+This section is the canonical planning surface for the catalog-first PRSoXR API and notebook-driven prototyping pipeline.
+
+All agents and contributors should start here before proposing implementation changes.
+
+## Primary Documents
+
+- `docs/prototyping/comprehensive_implementation_plan.md`
+- `docs/prototyping/development_documentation_guidelines.md`
+
+## Workflow Specifications
+
+### API Design
+
+- `docs/prototyping/api-design/api_flow_design.md`
+- `docs/prototyping/api-design/catalog_api_reference.md`
+- `docs/prototyping/api-design/header_normalization_spec.md`
+
+### References
+
+- `docs/prototyping/references/header_summary_rules_schema.md`
+
+### Workflows
+
+- `docs/prototyping/workflows/cell_01_catalog_and_selection.md`
+- `docs/prototyping/workflows/cell_02_profile_split.md`
+- `docs/prototyping/workflows/cell_03_classify_and_i0_linking.md`
+- `docs/prototyping/workflows/cell_04_mask_and_beamspot.md`
+- `docs/prototyping/workflows/cell_05_persist_mask_and_spot_overrides.md`
+- `docs/prototyping/workflows/cell_06_reduction_and_uncertainty.md`
+- `docs/prototyping/workflows/cell_07_role_diagnostics.md`
+- `docs/prototyping/workflows/cell_08_stitch_compute.md`
+- `docs/prototyping/workflows/cell_09_profile_comparison_and_export.md`
+- `docs/prototyping/workflows/just_process_flow.md`
+- `docs/prototyping/workflows/lazy_refresh_with_watcher.md`
+
+## Notebook Prototyping Targets
+
+The `notebooks` directory should contain one prototype notebook per workflow stage:
+
+- `notebooks/00_catalog_backend_and_migration.ipynb`
+- `notebooks/01_beamtime_scan_sample_views.ipynb`
+- `notebooks/02_profile_split_and_header_normalization.ipynb`
+- `notebooks/03_classification_and_i0_linking.ipynb`
+- `notebooks/04_mask_and_beamspot_interactive.ipynb`
+- `notebooks/05_reduction_and_uncertainty_budget.ipynb`
+- `notebooks/06_role_qc_and_bad_point_flagging.ipynb`
+- `notebooks/07_stitch_factors_and_propagation.ipynb`
+- `notebooks/08_profile_comparison_and_export.ipynb`
+- `notebooks/09_just_process_end_to_end.ipynb`