diff --git a/docs/developer_notes/NDR_PORTING_INSTRUCTIONS.md b/docs/developer_notes/NDR_PORTING_INSTRUCTIONS.md new file mode 100644 index 0000000..035ff52 --- /dev/null +++ b/docs/developer_notes/NDR_PORTING_INSTRUCTIONS.md @@ -0,0 +1,810 @@ +# Instructions: Port NDR-matlab to NDR-python + +**For:** Claude Code agent working in the new `NDR-python` repository +**Reference implementation:** [NDR-matlab](https://github.com/VH-Lab/NDR-matlab) +**Pattern to follow:** [NDI-python](https://github.com/Waltham-Data-Science/NDI-python) (this repo) + +--- + +## 1. Overview + +NDR (Neuroscience Data Reader) is a lower-level data-reading library used by NDI. NDR-matlab lives at `https://github.com/VH-Lab/NDR-matlab`. Your job is to create `NDR-python` as a faithful Python mirror, following the exact same lead-follow architecture, developer notes, bridge YAML protocol, and symmetry test framework used in NDI-python. + +NDR-matlab provides: +- An abstract base reader class (`ndr.reader.base`) +- A high-level reader wrapper (`ndr.reader`) +- 10 format-specific reader subclasses (Intan RHD, Axon ABF, CED SMR, etc.) +- Format handler packages with low-level file I/O +- Time, string, data, and file utilities + +--- + +## 2. Repository Setup + +### 2.1 Create the repo structure + +``` +NDR-python/ +├── src/ +│ └── ndr/ +│ ├── __init__.py +│ ├── globals.py # from +ndr/globals.m +│ ├── known_readers.py # from +ndr/known_readers.m +│ ├── reader_wrapper.py # from +ndr/reader.m (the high-level wrapper) +│ ├── reader/ +│ │ ├── __init__.py +│ │ ├── base.py # from +ndr/+reader/base.m +│ │ ├── intan_rhd.py # from +ndr/+reader/intan_rhd.m +│ │ ├── axon_abf.py # from +ndr/+reader/axon_abf.m +│ │ ├── ced_smr.py # from +ndr/+reader/ced_smr.m +│ │ ├── bjg.py # from +ndr/+reader/bjg.m +│ │ ├── dabrowska.py # from +ndr/+reader/dabrowska.m +│ │ ├── neo.py # from +ndr/+reader/neo.m +│ │ ├── spikegadgets_rec.py # from +ndr/+reader/spikegadgets_rec.m +│ │ ├── tdt_sev.py # from +ndr/+reader/tdt_sev.m +│ │ └── whitematter.py # from +ndr/+reader/whitematter.m +│ ├── format/ +│ │ ├── __init__.py +│ │ ├── intan/ +│ │ │ ├── __init__.py +│ │ │ └── ... # from +ndr/+format/+intan/ +│ │ ├── axon/ +│ │ ├── ced/ +│ │ ├── bjg/ +│ │ ├── dabrowska/ +│ │ ├── spikegadgets/ +│ │ ├── tdt/ +│ │ ├── textSignal/ +│ │ └── whitematter/ +│ ├── data/ +│ │ └── ... # from +ndr/+data/ +│ ├── file/ +│ │ └── ... # from +ndr/+file/ +│ ├── fun/ +│ │ └── ... # from +ndr/+fun/ +│ ├── string/ +│ │ └── ... # from +ndr/+string/ +│ └── time/ +│ └── ... # from +ndr/+time/ +├── tests/ +│ ├── __init__.py +│ ├── test_reader_base.py +│ ├── test_readers.py +│ └── symmetry/ # Cross-language symmetry tests +│ ├── conftest.py +│ ├── make_artifacts/ +│ │ └── reader/ +│ │ └── test_read_data.py +│ └── read_artifacts/ +│ └── reader/ +│ └── test_read_data.py +├── docs/ +│ └── developer_notes/ +│ ├── PYTHON_PORTING_GUIDE.md +│ ├── ndr_xlang_principles.md +│ ├── ndr_matlab_python_bridge.yaml +│ └── symmetry_tests.md +├── example_data/ # Copy from NDR-matlab for testing +├── pyproject.toml +├── README.md +├── LICENSE +└── AGENTS.md +``` + +### 2.2 Directory Parity Rule + +MATLAB `+namespace` paths map directly to Python package directories: + +| MATLAB | Python | +|--------|--------| +| `+ndr/globals.m` | `src/ndr/globals.py` | +| `+ndr/reader.m` | `src/ndr/reader_wrapper.py` | +| `+ndr/+reader/base.m` | `src/ndr/reader/base.py` | +| `+ndr/+reader/intan_rhd.m` | `src/ndr/reader/intan_rhd.py` | +| `+ndr/+format/+intan/` | `src/ndr/format/intan/` | +| `+ndr/+time/` | `src/ndr/time/` | + +**Exception:** `+ndr/reader.m` is the high-level wrapper class. Since `ndr/reader/` is the reader subpackage, put the wrapper in `ndr/reader_wrapper.py` and re-export from `ndr/__init__.py`. + +### 2.3 pyproject.toml + +Model after NDI-python's pyproject.toml: + +```toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "ndr" +version = "0.1.0" +description = "Neuroscience Data Reader - Python implementation" +readme = "README.md" +license = {text = "CC-BY-NC-SA-4.0"} +authors = [{name = "VH-Lab", email = "vhlab@brandeis.edu"}] +maintainers = [{name = "Waltham Data Science"}] +requires-python = ">=3.10" +dependencies = [ + "numpy>=1.20.0", + "pydantic>=2.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "ruff>=0.1.0", + "mypy>=1.0.0", +] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["src/ndr"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +addopts = "-v --tb=short --ignore=tests/symmetry" +markers = [ + "slow: marks tests as slow", + "symmetry: marks cross-language symmetry tests", +] + +[tool.black] +line-length = 100 +target-version = ["py310", "py311", "py312"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "B", "C4", "UP"] +ignore = ["E501", "B905", "E402", "B017", "B028"] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401", "UP035"] +``` + +Add additional dependencies as needed when porting format readers that require third-party libraries (e.g., `neo` for the Neo reader, `scipy` for certain file formats). + +--- + +## 3. Developer Notes (Create These Files) + +You must create four developer notes files in `docs/developer_notes/`. These are adapted from NDI-python's equivalents but scoped to NDR. + +### 3.1 `PYTHON_PORTING_GUIDE.md` + +Create this file with the following content (adapt from NDI-python's version): + +```markdown +# NDR MATLAB to Python Porting Guide + +## 1. The Core Philosophy: Lead-Follow Architecture + +The MATLAB codebase is the **Source of Truth**. The Python version is a "faithful mirror." +When a conflict arises between "Pythonic" style and MATLAB symmetry, **symmetry wins**. + +- **Lead-Follow:** MATLAB defines the logic, hierarchy, and naming. +- **The Contract:** Every package contains an `ndr_matlab_python_bridge.yaml`. + This file is the binding contract for function names, arguments, and return types. + +## 2. Naming & Discovery (The Mirror Rule) + +Function and class names must match MATLAB exactly. + +- **Naming Source:** Refer to the local `ndr_matlab_python_bridge.yaml`. +- **Missing Entries:** If a function is not in the bridge file, refer to the MATLAB + source, add the entry to the bridge file, and notify the user. +- **Case Preservation:** Use `readchannels_epochsamples`, not `read_channels_epoch_samples`. +- **Directory Parity:** Python file paths must mirror MATLAB `+namespace` paths + (e.g., `+ndr/+reader` -> `src/ndr/reader/`). + +## 3. The Porting Workflow (The Bridge Protocol) + +1. **Check the Bridge:** Open the `ndr_matlab_python_bridge.yaml` in the target package. +2. **Sync the Interface:** If the function is missing or outdated, update the YAML first. +3. **Record the Sync Hash:** Store the short git hash of the MATLAB `.m` file: + `git log -1 --format="%h" -- ` +4. **Implement:** Write Python code to satisfy the interface defined in the YAML. +5. **Log & Notify:** Record the sync date in the YAML's `decision_log`. + +## 4. Input Validation: Pydantic is Mandatory + +Use `@pydantic.validate_call` on all public-facing API functions. + +- MATLAB `double`/`numeric` -> Python `float | int` +- MATLAB `char`/`string` -> Python `str` +- MATLAB `{member1, member2}` -> Python `Literal["member1", "member2"]` + +## 5. Multiple Returns (Outputs) + +Return multiple values as a **tuple** in the exact order defined in the YAML. + +## 6. Code Style & Linting + +- **Black:** The sole code formatter. Line length 100. +- **Ruff:** The primary linter. Run `ruff check --fix` before committing. + +## 7. Error Handling + +If MATLAB throws an `error`, Python MUST raise a corresponding Exception. +``` + +### 3.2 `ndr_xlang_principles.md` + +Create this file, adapted from NDI-python's `ndi_xlang_principles.md`: + +```markdown +# NDR Cross-Language (MATLAB/Python) Principles + +- **Status:** Active +- **Scope:** Universal (Applies to all NDR implementations) + +## 1. Indexing & Counting (The Semantic Parity Rule) + +- Python uses 0-indexing internally. +- User-facing concepts (Epochs, Channels) use 1-based numbering in both languages. +- Python code accepts `channel_number=1` from user, maps to `data[0]` internally. + +## 2. Data Containers + +- Prefer NumPy over lists for numerical data. +- MATLAB `double` array -> `numpy.ndarray` in Python. + +## 3. Multiple Returns + +- Python returns multiple values as a tuple in MATLAB signature order. + +## 4. Booleans + +- MATLAB `1`/`0` (logical) -> Python `True`/`False`. + +## 5. Strings + +- MATLAB `char` and `string` -> Python `str`. +- MATLAB cell array of strings -> Python `list[str]`. + +## 6. Error Philosophy + +- No silent failures. If MATLAB errors, Python raises an exception. +``` + +### 3.3 `ndr_matlab_python_bridge.yaml` + +Create the bridge YAML spec file. This defines the contract format: + +```yaml +# The NDR Bridge Protocol: YAML Specification +# +# Name: ndr_matlab_python_bridge.yaml +# Location: One file per sub-package directory +# (e.g., src/ndr/reader/ndr_matlab_python_bridge.yaml). +# Role: Primary Contract. Defines how MATLAB names and types map to Python. + +project_metadata: + bridge_version: "1.1" + naming_policy: "Strict MATLAB Mirror" + indexing_policy: "Semantic Parity (1-based for user concepts, 0-based for internal data)" + +# When porting a function: +# 1. Check: Does the function/class exist in the YAML? +# 2. Add/Update: If missing or changed, update the YAML first. +# 3. Record Hash: git log -1 --format="%h" -- +# 4. Notify: Tell the user what was added/changed. + +# --- Example: Class --- +# - name: base +# type: class +# matlab_path: "+ndr/+reader/base.m" +# python_path: "ndr/reader/base.py" +# matlab_last_sync_hash: "a4c9e07" +# methods: +# - name: readchannels_epochsamples +# input_arguments: +# - name: channeltype +# type_python: "str" +# - name: channel +# type_python: "int | list[int]" +# - name: epoch +# type_python: "str | int" +# - name: s0 +# type_python: "int" +# - name: s1 +# type_python: "int" +# output_arguments: +# - name: data +# type_python: "numpy.ndarray" +``` + +Then create **one bridge YAML per sub-package** as you port it: +- `src/ndr/reader/ndr_matlab_python_bridge.yaml` +- `src/ndr/format/intan/ndr_matlab_python_bridge.yaml` +- `src/ndr/time/ndr_matlab_python_bridge.yaml` +- etc. + +### 3.4 `symmetry_tests.md` + +Create this file, adapted from NDI-python's version: + +```markdown +# Cross-Language Symmetry Test Framework + +**Status:** Active +**Scope:** NDR-python <-> NDR-matlab parity + +## Purpose + +Symmetry tests verify that data read by one language implementation matches +the other. This ensures the Python and MATLAB NDR stacks remain interoperable. + +## Architecture + +| Phase | Python location | MATLAB location | +|-------|----------------|-----------------| +| **makeArtifacts** | `tests/symmetry/make_artifacts/` | `tests/+ndr/+symmetry/+makeArtifacts/` | +| **readArtifacts** | `tests/symmetry/read_artifacts/` | `tests/+ndr/+symmetry/+readArtifacts/` | + +### Artifact Directory Layout + +``` +/NDR/symmetryTest/ +├── pythonArtifacts/ +│ └── /// +│ ├── readData.json # Channel data, timestamps, etc. +│ └── metadata.json # Channel list, sample rates, epoch info +└── matlabArtifacts/ + └── /// + └── ... (same structure) +``` + +### Workflow + +1. **makeArtifacts** (Python or MATLAB) reads example data files and writes + JSON artifacts containing: channel lists, sample rates, epoch clocks, + t0/t1 boundaries, and actual data samples. +2. **readArtifacts** (the other language) loads the same example data files, + reads the same channels/epochs, and compares against the stored artifacts. + +Each `readArtifacts` test is parameterized over `{matlabArtifacts, pythonArtifacts}`. + +## Running + +```bash +# Generate artifacts +pytest tests/symmetry/make_artifacts/ -v + +# Verify artifacts +pytest tests/symmetry/read_artifacts/ -v +``` + +## Writing a New Symmetry Test + +See NDI-python's `docs/developer_notes/symmetry_tests.md` for the full template. +Adapt the pattern for NDR's reader-centric API. +``` + +--- + +## 4. AGENTS.md + +Create an `AGENTS.md` at the repo root: + +```markdown +# Instructions for AI Agents + +## Overview + +NDR-python is a faithful Python port of NDR-matlab (Neuroscience Data Reader). + +## Architecture + +- **Lead-Follow:** MATLAB is the source of truth. Python mirrors it exactly. +- **Bridge Contract:** Each sub-package has an `ndr_matlab_python_bridge.yaml` + defining the function names, arguments, and return types. +- **Naming:** Preserve MATLAB names exactly. Use `readchannels_epochsamples`, + not `read_channels_epoch_samples`. + +## Key Classes + +- `ndr.reader.base` — Abstract base class. All readers inherit from this. +- `ndr.reader` (wrapper) — High-level interface that delegates to a base reader. +- `ndr.reader.intan_rhd`, `ndr.reader.axon_abf`, etc. — Format-specific readers. + +## Workflow + +1. Check the bridge YAML in the target package. +2. If the function is missing, add it based on the MATLAB source. +3. Record the MATLAB git hash in `matlab_last_sync_hash`. +4. Implement the Python code. +5. Run `black` and `ruff check --fix` before committing. +6. Run `pytest` to verify. + +## Testing + +- Unit tests: `pytest tests/` +- Symmetry tests: `pytest tests/symmetry/` (excluded from default run) + +## Environment + +- Python 3.10+ +- NumPy for all numerical data +- Pydantic for input validation (`@validate_call`) +``` + +--- + +## 5. Classes to Port + +### 5.1 Priority Order + +Port in this order (each builds on the previous): + +1. **`ndr.reader.base`** — Abstract base class with all method signatures +2. **`ndr.reader` (wrapper)** — High-level reader interface +3. **`ndr.format.intan`** — Intan RHD format handler (most commonly used) +4. **`ndr.reader.intan_rhd`** — Intan reader subclass +5. **Utility packages** — `ndr.time`, `ndr.data`, `ndr.string`, `ndr.file`, `ndr.fun` +6. **Remaining readers** — `axon_abf`, `ced_smr`, `bjg`, `dabrowska`, `spikegadgets_rec`, `tdt_sev`, `whitematter`, `neo` +7. **`ndr.globals`** and **`ndr.known_readers`** + +### 5.2 The Abstract Base: `ndr.reader.base` + +This is the most important class. It defines the interface all readers must implement. + +**MATLAB source:** `+ndr/+reader/base.m` + +**Properties:** +- `MightHaveTimeGaps` (bool) + +**Abstract methods (subclasses MUST implement):** +- `readchannels_epochsamples(channeltype, channel, epoch, s0, s1)` -> `numpy.ndarray` +- `readevents_epochsamples_native(channeltype, channel, epoch, s0, s1)` -> tuple + +**Concrete methods (base provides default implementations):** +- `canbereadtogether(channels)` -> `bool` +- `daqchannels2internalchannels(channeltype, channel, epoch)` -> internal channel struct +- `epochclock(epoch)` -> list of clock types +- `getchannelsepoch(epoch)` -> list of channel descriptors +- `underlying_datatype(channeltype, channel, epoch)` -> str +- `samplerate(epoch, channeltype, channel)` -> float +- `t0_t1(epoch)` -> tuple[float, float] +- `samples2times(epoch, samples, channeltype, channel)` -> numpy.ndarray +- `times2samples(epoch, times, channeltype, channel)` -> numpy.ndarray + +**Static methods:** +- `mfdaq_channeltypes()` -> list[str] +- `mfdaq_prefix(channeltype)` -> str +- `mfdaq_type(channeltype)` -> str + +### 5.3 The Reader Wrapper: `ndr.reader` + +**MATLAB source:** `+ndr/reader.m` + +This wraps a `base` subclass and provides the high-level `read()` method plus delegation to all base methods. The constructor takes a format name string and instantiates the appropriate reader subclass. + +### 5.4 Format-Specific Readers + +Each reader in `+ndr/+reader/` subclasses `base` and overrides the abstract methods. Each has a corresponding format handler package in `+ndr/+format/` that does the low-level binary file I/O. + +| Reader class | Format package | File format | +|-------------|---------------|-------------| +| `intan_rhd` | `+ndr/+format/+intan/` | Intan RHD2000 (.rhd) | +| `axon_abf` | `+ndr/+format/+axon/` | Axon ABF (.abf) | +| `ced_smr` | `+ndr/+format/+ced/` | CED Spike2 (.smr) | +| `bjg` | `+ndr/+format/+bjg/` | BJG format | +| `dabrowska` | `+ndr/+format/+dabrowska/` | Dabrowska format | +| `spikegadgets_rec` | `+ndr/+format/+spikegadgets/` | SpikeGadgets (.rec) | +| `tdt_sev` | `+ndr/+format/+tdt/` | TDT (.sev) | +| `whitematter` | `+ndr/+format/+whitematter/` | White Matter format | +| `neo` | `+ndr/+format/+neo/` | Neo Python bridge | + +**Important:** The `intan_rhd` reader should use `vhlab-toolbox-python` for the low-level RHD file reading if available, as NDI-python does. Add `vhlab-toolbox-python` as an optional dependency: + +```toml +dependencies = [ + "numpy>=1.20.0", + "pydantic>=2.0", + "vhlab-toolbox-python @ git+https://github.com/VH-Lab/vhlab-toolbox-python.git@main", +] +``` + +### 5.5 Skip the Template + +`+ndr/+reader/somecompany_someformat.m` is a template for creating new readers. Do NOT port it. It's documentation, not functional code. + +--- + +## 6. Symmetry Tests + +### 6.1 conftest.py + +Create `tests/symmetry/conftest.py`: + +```python +"""Shared fixtures and configuration for NDR symmetry tests.""" + +import tempfile +from pathlib import Path + +# Base directory where all symmetry artifacts live: +# /NDR/symmetryTest///// +SYMMETRY_BASE = Path(tempfile.gettempdir()) / "NDR" / "symmetryTest" +PYTHON_ARTIFACTS = SYMMETRY_BASE / "pythonArtifacts" +MATLAB_ARTIFACTS = SYMMETRY_BASE / "matlabArtifacts" + +SOURCE_TYPES = ["matlabArtifacts", "pythonArtifacts"] +``` + +Note: `NDR` not `NDI` in the path. + +### 6.2 makeArtifacts Example + +Create `tests/symmetry/make_artifacts/reader/test_read_data.py`: + +```python +"""Generate symmetry artifacts for NDR reader tests. + +Reads example data files using NDR readers and exports: +- Channel metadata (names, types, sample rates) +- Epoch clock types and t0/t1 boundaries +- Actual data samples for comparison +""" + +import json +import shutil + +import numpy as np +import pytest + +from ndr.reader.intan_rhd import IntanRHDReader +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "reader" / "readData" / "testReadDataArtifacts" +EXAMPLE_DATA = Path(__file__).parents[4] / "example_data" + + +class TestReadData: + @pytest.fixture(autouse=True) + def _setup(self): + rhd_file = EXAMPLE_DATA / "Intan_160317_125049_short.rhd" + if not rhd_file.exists(): + pytest.skip("Example RHD file not available") + self.reader = IntanRHDReader() + self.epochfiles = [str(rhd_file)] + + def test_read_data_artifacts(self): + if ARTIFACT_DIR.exists(): + shutil.rmtree(ARTIFACT_DIR) + ARTIFACT_DIR.mkdir(parents=True) + + # Export channel metadata + channels = self.reader.getchannelsepoch(self.epochfiles) + metadata = { + "channels": channels, + "samplerate": self.reader.samplerate(self.epochfiles, "ai", 1), + "t0_t1": list(self.reader.t0_t1(self.epochfiles)), + "epochclock": self.reader.epochclock(self.epochfiles), + } + (ARTIFACT_DIR / "metadata.json").write_text( + json.dumps(metadata, indent=2, default=str), encoding="utf-8" + ) + + # Export a small data sample + data = self.reader.readchannels_epochsamples("ai", [1], self.epochfiles, 0, 100) + (ARTIFACT_DIR / "readData.json").write_text( + json.dumps({"ai_channel_1_samples_0_100": data.tolist()}, indent=2), + encoding="utf-8", + ) +``` + +### 6.3 readArtifacts Example + +Create `tests/symmetry/read_artifacts/reader/test_read_data.py`: + +```python +"""Read and verify symmetry artifacts for NDR reader tests.""" + +import json + +import numpy as np +import pytest + +from ndr.reader.intan_rhd import IntanRHDReader +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE + +EXAMPLE_DATA = Path(__file__).parents[4] / "example_data" + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + + +class TestReadData: + def _artifact_dir(self, source_type): + return SYMMETRY_BASE / source_type / "reader" / "readData" / "testReadDataArtifacts" + + def test_read_data_metadata(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip(f"No artifacts from {source_type}") + + metadata = json.loads((artifact_dir / "metadata.json").read_text()) + + rhd_file = EXAMPLE_DATA / "Intan_160317_125049_short.rhd" + if not rhd_file.exists(): + pytest.skip("Example RHD file not available") + + reader = IntanRHDReader() + epochfiles = [str(rhd_file)] + + actual_sr = reader.samplerate(epochfiles, "ai", 1) + assert actual_sr == metadata["samplerate"], ( + f"Sample rate mismatch: {actual_sr} vs {metadata['samplerate']}" + ) + + actual_t0_t1 = reader.t0_t1(epochfiles) + assert np.allclose(actual_t0_t1, metadata["t0_t1"], atol=1e-6) + + def test_read_data_samples(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip(f"No artifacts from {source_type}") + + read_data = json.loads((artifact_dir / "readData.json").read_text()) + expected = np.array(read_data["ai_channel_1_samples_0_100"]) + + rhd_file = EXAMPLE_DATA / "Intan_160317_125049_short.rhd" + if not rhd_file.exists(): + pytest.skip("Example RHD file not available") + + reader = IntanRHDReader() + actual = reader.readchannels_epochsamples("ai", [1], [str(rhd_file)], 0, 100) + + assert np.allclose(actual, expected, atol=1e-9), ( + f"Data mismatch for ai channel 1, samples 0-100 ({source_type})" + ) +``` + +### 6.4 Symmetry Test Guidelines + +- Artifact paths use **camelCase** so both Python and MATLAB write to the same directories. +- `readArtifacts` tests must `pytest.skip()` if the artifact directory doesn't exist (the other language may not have been run yet). +- Symmetry tests are excluded from the default pytest run via `--ignore=tests/symmetry` in `pyproject.toml`. +- For NDR, the primary thing to test for symmetry is **data values** — both languages reading the same binary file should produce the same numerical results (within floating-point tolerance). + +--- + +## 7. Porting Rules + +### 7.1 Naming + +- **Preserve MATLAB names exactly.** `readchannels_epochsamples`, not `read_channels_epoch_samples`. +- **Class names:** `base`, `intan_rhd`, `axon_abf` — match the MATLAB filenames. +- **Method names:** Match MATLAB exactly: `getchannelsepoch`, `t0_t1`, `epochclock`. + +### 7.2 Method Signatures + +Fetch each MATLAB `.m` file from GitHub and port the exact method signature. The MATLAB source is at: + +``` +https://raw.githubusercontent.com/VH-Lab/NDR-matlab/main/+ndr/+reader/.m +``` + +For each method, create a bridge YAML entry BEFORE writing the implementation. + +### 7.3 Epoch Files Convention + +In NDR-matlab, `epoch` arguments to reader methods are typically file paths (a cell array of filenames). In Python, use `list[str]` — a list of file path strings. + +### 7.4 Channel Types + +NDR uses standard channel type strings. The static method `mfdaq_channeltypes()` returns: +`['ai', 'ao', 'di', 'do', 'time', 'auxiliary', 'mark', 'event', 'text']` + +With prefixes from `mfdaq_prefix()`: +`{'ai': 'ai', 'ao': 'ao', 'di': 'di', 'do': 'do', ...}` + +### 7.5 Return Types + +- Numerical data: `numpy.ndarray` +- Channel lists: `list[dict]` where each dict has keys like `name`, `type`, `samplerate` +- Time boundaries: `tuple[float, float]` +- Clock types: `list[dict]` with `type` key + +### 7.6 Error Handling + +- If MATLAB throws an error, Python raises `ValueError` or `TypeError`. +- Never silently return empty/None when MATLAB would error. + +--- + +## 8. Format Handler Packages + +Each format handler in `+ndr/+format/` contains low-level file I/O functions. These are the workhorses that actually read binary data from disk. + +For the Intan format (`+ndr/+format/+intan/`), consider using `vhlab-toolbox-python` which already has Intan RHD reading capability, rather than porting the MATLAB binary reader from scratch. + +For other formats, check if Python libraries already exist: +- **Axon ABF:** `pyabf` package +- **CED SMR:** `neo` or `sonpy` packages +- **TDT:** `tdt` package +- **SpikeGadgets:** May need native port +- **Neo:** Already wraps Python's `neo` package + +If a well-maintained Python library exists for a format, use it as the backend and wrap it to match the NDR interface. Document this in the bridge YAML's `decision_log`. + +--- + +## 9. Testing + +### 9.1 Unit Tests + +Write unit tests for each reader using the example data files from `example_data/`: + +```python +def test_intan_rhd_getchannelsepoch(): + reader = IntanRHDReader() + channels = reader.getchannelsepoch(["example_data/Intan_160317_125049_short.rhd"]) + assert len(channels) > 0 + assert any(ch["type"] == "ai" for ch in channels) + +def test_intan_rhd_readchannels(): + reader = IntanRHDReader() + data = reader.readchannels_epochsamples( + "ai", [1], ["example_data/Intan_160317_125049_short.rhd"], 0, 100 + ) + assert data.shape == (100, 1) +``` + +### 9.2 Run Before Committing + +```bash +black src/ tests/ +ruff check --fix src/ tests/ +pytest tests/ -v +``` + +--- + +## 10. Relationship to NDI-python + +NDR-python will be a dependency of NDI-python. Once NDR-python is ready: + +1. Add it to NDI-python's `pyproject.toml`: + ```toml + "ndr @ git+https://github.com/VH-Lab/NDR-python.git@main", + ``` + +2. NDI-python's `ndi.daq.reader.mfdaq.intan` will delegate to `ndr.reader.intan_rhd` instead of implementing its own reading logic. + +3. The NDI `ndi.file.navigator` will use NDR readers to access epoch data. + +This mirrors the MATLAB architecture where NDI-matlab depends on NDR-matlab. + +--- + +## 11. Quick-Start Checklist + +- [ ] Create repo structure per Section 2 +- [ ] Create `pyproject.toml` per Section 2.3 +- [ ] Create all four developer notes files per Section 3 +- [ ] Create `AGENTS.md` per Section 4 +- [ ] Create `tests/symmetry/conftest.py` per Section 6.1 +- [ ] Port `ndr.reader.base` (abstract base class) +- [ ] Create bridge YAML for `src/ndr/reader/` +- [ ] Port `ndr.reader` wrapper +- [ ] Port `ndr.format.intan` (or wire to vhlab-toolbox-python) +- [ ] Port `ndr.reader.intan_rhd` +- [ ] Write unit tests using example data +- [ ] Write symmetry makeArtifacts test +- [ ] Write symmetry readArtifacts test +- [ ] Port utility packages (`time`, `data`, `string`, `file`, `fun`) +- [ ] Port remaining readers +- [ ] Run `black`, `ruff`, `pytest` +- [ ] Commit and push diff --git a/examples/integration_demo.py b/examples/integration_demo.py index 0d07fd9..ca5153a 100644 --- a/examples/integration_demo.py +++ b/examples/integration_demo.py @@ -12,12 +12,12 @@ This is the workflow the cofounder is building toward! """ -import numpy as np import os import tempfile -# Our NDI core (Phase 2 implementation) -from ndi import Document, Query, Ido +import numpy as np + +from ndi import Document, Ido, Query from ndi.common import timestamp # Cofounder's compression library @@ -52,7 +52,7 @@ def demo_full_workflow(): sample_rate = 30000 # Simulate neural data: noise + spikes - t = np.linspace(0, 1, num_samples) + _t = np.linspace(0, 1, num_samples) data = np.random.randn(num_samples, num_channels) * 50 # noise # Add some "spikes" @@ -138,7 +138,7 @@ def demo_full_workflow(): q3 = Query('base.name').contains('ephys') # Combined query - q_combined = q1 & q2 & q3 + _q_combined = q1 & q2 & q3 print(f" Query 1: {q1.to_searchstructure()}") print(f" Query 2: {q2.to_searchstructure()}") @@ -225,7 +225,7 @@ def demo_document_features(): doc = doc.setproperties(**{ 'base.name': 'neural_recording', }) - print(f"3. Set name via setproperties") + print("3. Set name via setproperties") # Document equality (by ID) doc2 = Document(doc.document_properties) diff --git a/pyproject.toml b/pyproject.toml index 1ef270d..6e90fa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "did @ git+https://github.com/VH-Lab/DID-python.git@main", + "ndr @ git+https://github.com/VH-lab/NDR-python.git@main", + "vhlab-toolbox-python @ git+https://github.com/VH-Lab/vhlab-toolbox-python.git@main", "numpy>=1.20.0", "networkx>=2.6", "jsonschema>=4.0.0", diff --git a/src/ndi/calculator.py b/src/ndi/calculator.py index 8fb7bea..97c813e 100644 --- a/src/ndi/calculator.py +++ b/src/ndi/calculator.py @@ -16,6 +16,7 @@ from .app import ndi_app from .app.appdoc import DocExistsAction, ndi_app_appdoc +from .util.classname import ndi_matlab_classname if TYPE_CHECKING: from .document import ndi_document @@ -59,7 +60,7 @@ def __init__( path_to_doc_type: Path to the document type schema """ # Initialize ndi_app (session + name from class) - name = type(self).__name__ + name = ndi_matlab_classname(self) ndi_app.__init__(self, session=session, name=name) # Initialize ndi_app_appdoc @@ -353,6 +354,8 @@ def search_for_calculator_docs( # Add dependency constraints depends_on = parameters.get("depends_on", []) for dep in depends_on: + if not isinstance(dep, dict): + continue dep_name = dep.get("name", "") dep_value = dep.get("value", "") if dep_value: diff --git a/src/ndi/daq/metadatareader/__init__.py b/src/ndi/daq/metadatareader/__init__.py index 86d9fe8..ab270f0 100644 --- a/src/ndi/daq/metadatareader/__init__.py +++ b/src/ndi/daq/metadatareader/__init__.py @@ -14,6 +14,7 @@ from typing import Any from ...ido import ndi_ido +from ...util.classname import ndi_matlab_classname class ndi_daq_metadatareader(ndi_ido): @@ -270,7 +271,7 @@ def newdocument(self) -> Any: doc = ndi_document( "daq/daqmetadatareader", **{ - "daqmetadatareader.ndi_daqmetadatareader_class": self.__class__.__name__, + "daqmetadatareader.ndi_daqmetadatareader_class": ndi_matlab_classname(self), "daqmetadatareader.tab_separated_file_parameter": self._tab_separated_file_parameter, "base.id": self.id, }, diff --git a/src/ndi/daq/reader/mfdaq/intan.py b/src/ndi/daq/reader/mfdaq/intan.py index 52c2c79..5ada9dc 100644 --- a/src/ndi/daq/reader/mfdaq/intan.py +++ b/src/ndi/daq/reader/mfdaq/intan.py @@ -1,8 +1,8 @@ """ ndi.daq.reader.mfdaq.intan - Intan RHD/RHS reader. -Thin wrapper around ndi_daq_reader_SpikeInterfaceReader for Intan data files. -Falls back gracefully if spikeinterface is not installed. +Native reader for Intan Technologies RHD2000 data files. +Uses ndr.format.intan for header parsing and data file reading. MATLAB equivalent: src/ndi/+ndi/+daq/+reader/+mfdaq/intan.m """ @@ -13,21 +13,24 @@ from typing import Any import numpy as np +from ndr.format.intan import ( + Intan_RHD2000_blockinfo, + read_Intan_RHD2000_datafile, + read_Intan_RHD2000_header, +) -logger = logging.getLogger(__name__) +from ...mfdaq import ChannelInfo, ndi_daq_reader_mfdaq, standardize_channel_type -from ...mfdaq import ChannelInfo, ndi_daq_reader_mfdaq +logger = logging.getLogger(__name__) class ndi_daq_reader_mfdaq_intan(ndi_daq_reader_mfdaq): """ - Reader for Intan RHD/RHS data files. + Reader for Intan RHD2000 data files. - Supports Intan Technologies recording systems including - RHD2000 and RHS2000 series. Uses spikeinterface for data access - when available. + Uses ndr.format.intan for header parsing and data file reading. - File extensions: .rhd, .rhs + File extensions: .rhd Example: >>> reader = ndi_daq_reader_mfdaq_intan() @@ -45,27 +48,112 @@ def __init__( ): super().__init__(identifier=identifier, session=session, document=document) self._ndi_daqreader_class = self.NDI_DAQREADER_CLASS + self._header_cache: dict[str, dict] = {} - def _get_si_reader(self): - """Get ndi_daq_reader_SpikeInterfaceReader lazily.""" - try: - from ..spikeinterface_adapter import ndi_daq_reader_SpikeInterfaceReader + def _get_header(self, epochfiles: list[str]) -> dict | None: + """Read and cache the Intan RHD header for the first .rhd file.""" + for filepath in epochfiles: + if filepath.lower().endswith(".rhd"): + if filepath not in self._header_cache: + try: + self._header_cache[filepath] = read_Intan_RHD2000_header(filepath) + except Exception as exc: + logger.warning("Failed to read Intan header from %s: %s", filepath, exc) + return None + return self._header_cache[filepath] + return None - return ndi_daq_reader_SpikeInterfaceReader - except ImportError: - return None + @staticmethod + def _rhd_file(epochfiles: list[str]) -> str | None: + """Return the first .rhd file path from epochfiles, or None.""" + for f in epochfiles: + if f.lower().endswith(".rhd"): + return f + return None def getchannelsepoch(self, epochfiles: list[str]) -> list[ChannelInfo]: - SI = self._get_si_reader() - if SI is None: - return [] - try: - reader = SI() - return reader.getchannelsepoch(epochfiles) - except Exception as exc: - logger.warning("ndi_daq_reader_mfdaq_intan.getchannelsepoch failed: %s", exc) + header = self._get_header(epochfiles) + if header is None: return [] + channels: list[ChannelInfo] = [] + sr = header["frequency_parameters"]["amplifier_sample_rate"] + + # Amplifier channels → analog_in + for i, _ch in enumerate(header.get("amplifier_channels", [])): + channels.append( + ChannelInfo( + name=f"ai{i + 1}", + type="analog_in", + time_channel=1, + number=i + 1, + sample_rate=sr, + ) + ) + + # Auxiliary input channels → auxiliary_in + aux_sr = header["frequency_parameters"].get("aux_input_sample_rate", sr / 4) + for i, _ch in enumerate(header.get("aux_input_channels", [])): + channels.append( + ChannelInfo( + name=f"aux{i + 1}", + type="auxiliary_in", + time_channel=1, + number=i + 1, + sample_rate=aux_sr, + ) + ) + + # Board ADC channels → analog_in (numbered after amplifier channels) + n_amp = len(header.get("amplifier_channels", [])) + for i, _ch in enumerate(header.get("board_adc_channels", [])): + channels.append( + ChannelInfo( + name=f"ai{n_amp + i + 1}", + type="analog_in", + time_channel=1, + number=n_amp + i + 1, + sample_rate=sr, + ) + ) + + # Digital input channels + for i, _ch in enumerate(header.get("board_dig_in_channels", [])): + channels.append( + ChannelInfo( + name=f"di{i + 1}", + type="digital_in", + time_channel=1, + number=i + 1, + sample_rate=sr, + ) + ) + + # Digital output channels + for i, _ch in enumerate(header.get("board_dig_out_channels", [])): + channels.append( + ChannelInfo( + name=f"do{i + 1}", + type="digital_out", + time_channel=1, + number=i + 1, + sample_rate=sr, + ) + ) + + # Time channel + channels.append( + ChannelInfo( + name="t1", + type="time", + time_channel=None, + number=1, + sample_rate=sr, + ) + ) + + return channels + def readchannels_epochsamples( self, channeltype, @@ -74,18 +162,178 @@ def readchannels_epochsamples( s0, s1, ) -> np.ndarray: - SI = self._get_si_reader() - if SI is None: - raise ImportError("spikeinterface required for reading Intan data") - reader = SI() - return reader.readchannels_epochsamples(channeltype, channel, epochfiles, s0, s1) + header = self._get_header(epochfiles) + if header is None: + raise FileNotFoundError("No valid .rhd file found in epochfiles") + + filepath = self._rhd_file(epochfiles) + + if isinstance(channel, int): + channel = [channel] + if isinstance(channeltype, str): + channeltype = [channeltype] * len(channel) + + channeltype = [standardize_channel_type(ct) for ct in channeltype] + + sr = header["frequency_parameters"]["amplifier_sample_rate"] + n_amp = len(header.get("amplifier_channels", [])) + + # Convert 1-indexed samples to time (seconds) for NDR + t0_sec = (s0 - 1) / sr + t1_sec = (s1 - 1) / sr + + n_samples = s1 - s0 + 1 + result = np.zeros((n_samples, len(channel))) + + # Cache NDR reads by channel type to avoid re-reading the file + ndr_cache: dict[str, np.ndarray] = {} + + for col, (ct, ch_num) in enumerate(zip(channeltype, channel)): + if ct == "time": + if "time" not in ndr_cache: + ndr_cache["time"] = read_Intan_RHD2000_datafile( + filepath, channeltype="time", channel=1, t0=t0_sec, t1=t1_sec + ) + data = ndr_cache["time"] + n = min(n_samples, data.shape[0]) + result[:n, col] = data[:n, 0] + + elif ct == "analog_in": + if ch_num <= n_amp: + # Amplifier channel + if "amp" not in ndr_cache: + ndr_cache["amp"] = read_Intan_RHD2000_datafile( + filepath, + channeltype="amp", + channel=list(range(1, n_amp + 1)), + t0=t0_sec, + t1=t1_sec, + ) + data = ndr_cache["amp"] + n = min(n_samples, data.shape[0]) + result[:n, col] = data[:n, ch_num - 1] + else: + # Board ADC channel + n_adc = len(header.get("board_adc_channels", [])) + if "adc" not in ndr_cache and n_adc > 0: + raw_adc = read_Intan_RHD2000_datafile( + filepath, + channeltype="adc", + channel=list(range(1, n_adc + 1)), + t0=t0_sec, + t1=t1_sec, + ) + # NDR returns raw ADC values; apply voltage conversion + eval_board_mode = header.get("eval_board_mode", 0) + if eval_board_mode == 1: + ndr_cache["adc"] = (raw_adc - 32768) * 312.5e-6 + else: + ndr_cache["adc"] = (raw_adc - 32768) * 0.0003125 + if "adc" in ndr_cache: + adc_idx = ch_num - n_amp - 1 + data = ndr_cache["adc"] + n = min(n_samples, data.shape[0]) + result[:n, col] = data[:n, adc_idx] + + elif ct == "auxiliary_in": + if "aux" not in ndr_cache: + n_aux = len(header.get("aux_input_channels", [])) + if n_aux > 0: + # Read all aux data to interpolate correctly + ndr_cache["aux"] = read_Intan_RHD2000_datafile( + filepath, + channeltype="aux", + channel=list(range(1, n_aux + 1)), + t0=0.0, + t1=float("inf"), + ) + if "aux" in ndr_cache: + ch_idx = ch_num - 1 + aux_data = ndr_cache["aux"] + # Aux is sampled at 1/4 rate; interpolate to main rate + i0 = s0 - 1 + i1 = s1 # exclusive end + aux_i0 = i0 // 4 + aux_i1 = i1 // 4 + if aux_i1 > aux_i0 and aux_i1 <= aux_data.shape[0]: + result[:, col] = np.interp( + np.arange(i0, i1), + np.arange(aux_i0 * 4, aux_i1 * 4, 4), + aux_data[aux_i0:aux_i1, ch_idx], + ) + + elif ct == "digital_in": + if "din" not in ndr_cache: + ndr_cache["din"] = read_Intan_RHD2000_datafile( + filepath, + channeltype="din", + channel=list(range(1, 17)), + t0=t0_sec, + t1=t1_sec, + ) + data = ndr_cache["din"] + n = min(n_samples, data.shape[0]) + result[:n, col] = data[:n, ch_num - 1] + + elif ct == "digital_out": + if "dout" not in ndr_cache: + ndr_cache["dout"] = read_Intan_RHD2000_datafile( + filepath, + channeltype="dout", + channel=list(range(1, 17)), + t0=t0_sec, + t1=t1_sec, + ) + data = ndr_cache["dout"] + n = min(n_samples, data.shape[0]) + result[:n, col] = data[:n, ch_num - 1] + + return result + + def t0_t1(self, epochfiles: list[str]) -> list[tuple[float, float]]: + header = self._get_header(epochfiles) + filepath = self._rhd_file(epochfiles) + if header is None or filepath is None: + return [(np.nan, np.nan)] + sr = header["frequency_parameters"]["amplifier_sample_rate"] + if sr == 0: + return [(np.nan, np.nan)] + blockinfo, _, _, num_data_blocks = Intan_RHD2000_blockinfo(filepath, header) + total_samples = blockinfo["samples_per_block"] * num_data_blocks + if total_samples == 0: + return [(np.nan, np.nan)] + t0 = 0.0 + t1 = (total_samples - 1) / sr + return [(t0, t1)] def samplerate(self, epochfiles, channeltype, channel) -> np.ndarray: - SI = self._get_si_reader() - if SI is None: - raise ImportError("spikeinterface required for reading Intan data") - reader = SI() - return reader.samplerate(epochfiles, channeltype, channel) + header = self._get_header(epochfiles) + if header is None: + raise FileNotFoundError("No valid .rhd file found in epochfiles") + + if isinstance(channel, int): + channel = [channel] + if isinstance(channeltype, str): + channeltype = [channeltype] * len(channel) + + channeltype = [standardize_channel_type(ct) for ct in channeltype] + sr = header["frequency_parameters"]["amplifier_sample_rate"] + freq = header["frequency_parameters"] + + rates = [] + for ct in channeltype: + if ct == "auxiliary_in": + rates.append(freq.get("aux_input_sample_rate", sr / 4)) + elif ct in ("supply_voltage",): + rates.append( + freq.get( + "supply_voltage_sample_rate", sr / header["num_samples_per_data_block"] + ) + ) + else: + rates.append(sr) + + return np.array(rates) def __repr__(self) -> str: return f"ndi_daq_reader_mfdaq_intan(id={self.id[:8]}...)" diff --git a/src/ndi/daq/reader_base.py b/src/ndi/daq/reader_base.py index e1f3564..e047226 100644 --- a/src/ndi/daq/reader_base.py +++ b/src/ndi/daq/reader_base.py @@ -14,6 +14,7 @@ from ..ido import ndi_ido from ..time import NO_TIME, ndi_time_clocktype +from ..util.classname import ndi_matlab_classname class ndi_daq_reader(ndi_ido, ABC): @@ -314,7 +315,7 @@ def newdocument(self) -> Any: "daq/daqreader", **{ "daqreader.ndi_daqreader_class": getattr( - self, "NDI_DAQREADER_CLASS", self.__class__.__name__ + self, "NDI_DAQREADER_CLASS", ndi_matlab_classname(self) ), "base.id": self.id, }, diff --git a/src/ndi/daq/system.py b/src/ndi/daq/system.py index 3eabc46..22a37b8 100644 --- a/src/ndi/daq/system.py +++ b/src/ndi/daq/system.py @@ -19,6 +19,63 @@ logger = logging.getLogger(__name__) +def _serialize_clocktype(ct: Any) -> dict[str, str]: + """Convert a ClockType enum (or dict) to a MATLAB-compatible dict.""" + if isinstance(ct, dict): + return ct + # ClockType enum — value is the type string + return {"type": str(ct.value) if hasattr(ct, "value") else str(ct)} + + +def _serialize_t0_t1(t0_t1: Any) -> list: + """Convert t0_t1 to MATLAB-compatible format: [t0, t1] with null for NaN.""" + import math + + if isinstance(t0_t1, (list, tuple)) and len(t0_t1) == 1: + # Single-element list of tuples: [(t0, t1)] -> [t0, t1] + t0_t1 = t0_t1[0] + if isinstance(t0_t1, (list, tuple)) and len(t0_t1) == 2: + t0, t1 = t0_t1 + t0 = None if (isinstance(t0, float) and math.isnan(t0)) else t0 + t1 = None if (isinstance(t1, float) and math.isnan(t1)) else t1 + return [t0, t1] + return t0_t1 + + +def _serialize_epochnode(node: dict[str, Any]) -> None: + """Normalize epoch node dict in-place to MATLAB-compatible JSON format.""" + # epoch_clock: list of ClockType -> single dict (MATLAB unwraps single) + ec = node.get("epoch_clock") + if isinstance(ec, list): + node["epoch_clock"] = ( + _serialize_clocktype(ec[0]) if len(ec) == 1 else [_serialize_clocktype(c) for c in ec] + ) + elif ec is not None: + node["epoch_clock"] = _serialize_clocktype(ec) + + # t0_t1 + t = node.get("t0_t1") + if t is not None: + node["t0_t1"] = _serialize_t0_t1(t) + + # underlying_epochs: recursively normalize + ue = node.get("underlying_epochs") + if isinstance(ue, dict): + ue_ec = ue.get("epoch_clock") + if isinstance(ue_ec, list): + ue["epoch_clock"] = [_serialize_clocktype(c) for c in ue_ec] + ue_t = ue.get("t0_t1") + if isinstance(ue_t, list): + ue["t0_t1"] = [_serialize_t0_t1(t) for t in ue_t] + + # epochprobemap: if it's a list, handle; if it has a to_dict method, use it + epm = node.get("epochprobemap") + if epm is not None and hasattr(epm, "to_dict"): + node["epochprobemap"] = epm.to_dict() + elif epm is not None and hasattr(epm, "__dict__") and not isinstance(epm, dict): + node["epochprobemap"] = {k: v for k, v in epm.__dict__.items() if not k.startswith("_")} + + class ndi_daq_system(ndi_ido): """ Complete data acquisition system. @@ -50,7 +107,7 @@ class ndi_daq_system(ndi_ido): >>> et = sys.epochtable() """ - NDI_DAQSYSTEM_CLASS = "ndi.daq.system.mfdaq" + NDI_DAQSYSTEM_CLASS = "ndi.daq.system" def __init__( self, @@ -199,6 +256,14 @@ def session(self) -> Any: return self._filenavigator.session return self._session + def _get_session_id(self) -> str | None: + """Return the session id, handling both method and property access.""" + s = self.session + if s is None: + return None + sid = s.id + return sid() if callable(sid) else sid + def set_daqmetadatareaders( self, readers: list[Any], @@ -340,7 +405,7 @@ def epochtable(self) -> list[dict[str, Any]]: { "epoch_number": epoch_number, "epoch_id": epoch_id, - "epoch_session_id": self.session.id if self.session else None, + "epoch_session_id": self._get_session_id(), "epochprobemap": epochprobemap, "epoch_clock": epoch_clock, "t0_t1": t0_t1, @@ -350,6 +415,24 @@ def epochtable(self) -> list[dict[str, Any]]: return et + def epochnodes(self) -> list[dict[str, Any]]: + """Return epoch node structs for this DAQ system. + + Each node mirrors the MATLAB ``epochnodes`` output: the same fields + as ``epochtable`` (minus ``epoch_number``) plus ``objectname`` and + ``objectclass``. Values are JSON-serializable and match MATLAB's + format for cross-language comparison. + """ + et = self.epochtable() + nodes = [] + for entry in et: + node = {k: v for k, v in entry.items() if k != "epoch_number"} + node["objectname"] = self._name + node["objectclass"] = self.NDI_DAQSYSTEM_CLASS + _serialize_epochnode(node) + nodes.append(node) + return nodes + def getprobes(self) -> list[dict[str, Any]]: """ Return all probes associated with this DAQ system. diff --git a/src/ndi/daq/system_mfdaq.py b/src/ndi/daq/system_mfdaq.py index 841c466..9d9b25f 100644 --- a/src/ndi/daq/system_mfdaq.py +++ b/src/ndi/daq/system_mfdaq.py @@ -35,6 +35,8 @@ class ndi_daq_system_mfdaq(ndi_daq_system): >>> data = sys.readchannels_epochsamples('ai', [1, 2], 1, 0, 1000) """ + NDI_DAQSYSTEM_CLASS = "ndi.daq.system.mfdaq" + CHANNEL_TYPES = { "analog_in": "ai", "analog_out": "ao", @@ -73,7 +75,8 @@ def t0_t1(self, epoch_number: int) -> list[tuple[float, float]]: List of (t0, t1) tuples per clock type """ if self._daqreader is not None and self._filenavigator is not None: - epochfiles = self._filenavigator.getepochfiles(epoch_number) + result = self._filenavigator.getepochfiles(epoch_number) + epochfiles = result[0] if isinstance(result, tuple) else result return self._daqreader.t0_t1(epochfiles) return [(np.nan, np.nan)] diff --git a/src/ndi/dataset/__init__.py b/src/ndi/dataset/__init__.py index f3108c6..e02814e 100644 --- a/src/ndi/dataset/__init__.py +++ b/src/ndi/dataset/__init__.py @@ -21,7 +21,11 @@ # dataset, mirroring the MATLAB constructor ``ndi.dataset.dir``. dir = ndi_dataset_dir +# Pythonic alias +Dataset = ndi_dataset_dir + __all__ = [ + "Dataset", "ndi_dataset", "ndi_dataset_dir", "dir", diff --git a/src/ndi/dataset/_dataset.py b/src/ndi/dataset/_dataset.py index ec7a0a6..e1dfb0c 100644 --- a/src/ndi/dataset/_dataset.py +++ b/src/ndi/dataset/_dataset.py @@ -18,6 +18,7 @@ from ..document import ndi_document from ..query import ndi_query +from ..util.classname import ndi_matlab_classname, ndi_python_classname logger = logging.getLogger(__name__) @@ -638,7 +639,7 @@ def _make_session_info(session: Any, is_linked: bool) -> dict[str, Any]: "session_id": session.id(), "session_reference": session.reference, "is_linked": is_linked, - "session_creator": type(session).__name__, + "session_creator": ndi_matlab_classname(session), } for i in range(1, 7): key = f"session_creator_input{i}" @@ -652,9 +653,9 @@ def _recreate_session( session_id: str, ) -> Any | None: """Recreate a session from stored creator args.""" - creator = info.get("session_creator", "") + creator = ndi_python_classname(info.get("session_creator", "")) - if creator == "ndi_session_dir" or creator == "ndi.session.dir": + if creator == "ndi_session_dir": from ..session.dir import ndi_session_dir ref = info.get("session_creator_input1", "") diff --git a/src/ndi/document.py b/src/ndi/document.py index dbc5200..24615f0 100644 --- a/src/ndi/document.py +++ b/src/ndi/document.py @@ -774,3 +774,7 @@ def read_blank_definition(document_type: str) -> dict: def __repr__(self) -> str: return f"ndi_document('{self.doc_class()}', id='{self.id}')" + + +# Pythonic alias +Document = ndi_document diff --git a/src/ndi/file/navigator/__init__.py b/src/ndi/file/navigator/__init__.py index 674fa6d..35e6c85 100644 --- a/src/ndi/file/navigator/__init__.py +++ b/src/ndi/file/navigator/__init__.py @@ -147,6 +147,8 @@ class ndi_file_navigator(ndi_ido): Attributes: session: The NDI session fileparameters: Parameters for finding epoch files + Class Attributes: + NDI_FILENAVIGATOR_CLASS: MATLAB-compatible class name string. epochprobemap_fileparameters: Parameters for finding probe map files epochprobemap_class: Class to use for epoch probe maps @@ -213,6 +215,7 @@ def _prop(obj: Any, key: str, default: Any = None) -> Any: base_id = _prop(base, "id") if base_id is not None: self.identifier = base_id + self._name = _prop(base, "name", "") or "unknown" filenavigator = _prop(doc_props, "filenavigator") if filenavigator: @@ -230,6 +233,26 @@ def _prop(obj: Any, key: str, default: Any = None) -> Any: self._fileparameters = {"filematch": []} self._epochprobemap_fileparameters = {"filematch": []} + @staticmethod + def _parse_fileparameters(raw: str) -> Any: + """Parse a fileparameters string, handling MATLAB cell array syntax. + + MATLAB stores cell arrays as ``"{ '#.rhd', '#.dat' }"`` which + ``eval()`` turns into a Python ``set`` (losing order). We detect + this pattern and parse it as an ordered list instead. + """ + stripped = raw.strip() + if stripped.startswith("{") and stripped.endswith("}"): + # MATLAB cell array — extract quoted strings in order + items = re.findall(r"'([^']*)'", stripped) + if items: + return items + # Fall back to eval for Python-native formats (list, dict) + try: + return eval(stripped) # noqa: S307 + except Exception: + return stripped + @staticmethod def _normalize_fileparameters( params: str | list[str] | dict[str, Any] | None, @@ -247,14 +270,26 @@ def _normalize_fileparameters( fm = params.get("filematch", []) if isinstance(fm, str): fm = [fm] - return {"filematch": fm} - return {"filematch": []} + return {"filematch": list(fm)} + # list, set, tuple, or any iterable (MATLAB cell arrays eval to + # Python sets when they use { 'a', 'b' } syntax) + try: + return {"filematch": list(params)} + except TypeError: + return {"filematch": []} @property def session(self) -> Any: """Get the session.""" return self._session + def _get_session_id(self) -> str | None: + """Return the session id, handling both method and property access.""" + if self._session is None: + return None + sid = self._session.id + return sid() if callable(sid) else sid + @property def fileparameters(self) -> dict[str, list[str]]: """Get the file parameters.""" @@ -465,7 +500,8 @@ def epochtable(self) -> list[dict[str, Any]]: underlying = { "underlying": files, "epoch_id": epoch_id, - "epoch_session_id": self._session.id if self._session else None, + "epoch_session_id": self._get_session_id(), + "epochprobemap": [], "epoch_clock": [NO_TIME], "t0_t1": [(float("nan"), float("nan"))], } @@ -473,7 +509,7 @@ def epochtable(self) -> list[dict[str, Any]]: entry = { "epoch_number": epoch_number, "epoch_id": epoch_id, - "epoch_session_id": self._session.id if self._session else None, + "epoch_session_id": self._get_session_id(), "epochprobemap": epochprobemaps[i] or self.getepochprobemap(epoch_number, files), "epoch_clock": [NO_TIME], "t0_t1": [(float("nan"), float("nan"))], @@ -483,6 +519,26 @@ def epochtable(self) -> list[dict[str, Any]]: return table + def epochnodes(self) -> list[dict[str, Any]]: + """Return epoch node structs for this file navigator. + + Same as ``epochtable`` (minus ``epoch_number``) with + ``objectname`` and ``objectclass`` appended, matching MATLAB's + ``epochnodes`` output. Values are serialized for cross-language + comparison. + """ + from ...daq.system import _serialize_epochnode + + et = self.epochtable() + nodes = [] + for entry in et: + node = {k: v for k, v in entry.items() if k != "epoch_number"} + node["objectname"] = self._name if hasattr(self, "_name") else "unknown" + node["objectclass"] = self.NDI_FILENAVIGATOR_CLASS + _serialize_epochnode(node) + nodes.append(node) + return nodes + def epochid( self, epoch_number: int, @@ -549,10 +605,22 @@ def epochidfilename( return None fmstr = self.filematch_hashstring() - parent, filename = os.path.split(epochfiles[0]) - stem, _ = os.path.splitext(filename) + parent = os.path.dirname(epochfiles[0]) + stem = self._epoch_stem(epochfiles) return os.path.join(parent, f".{stem}.{fmstr}.epochid.ndi") + @staticmethod + def _epoch_stem(epochfiles: list[str]) -> str: + """Return the common filename stem for a group of epoch files. + + MATLAB uses the common prefix of basenames (stripping trailing + dot) so that ``['foo.rhd', 'foo.epochprobemap.ndi']`` yields + ``'foo'``. + """ + basenames = [os.path.basename(f) for f in epochfiles] + stem = os.path.commonprefix(basenames).rstrip(".") + return stem if stem else basenames[0] + def epochprobemapfilename( self, epoch_number: int, @@ -603,8 +671,9 @@ def defaultepochprobemapfilename( return None fmstr = self.filematch_hashstring() - parent, filename = os.path.split(epochfiles[0]) - return os.path.join(parent, f".{filename}.{fmstr}.epochprobemap.ndi") + parent = os.path.dirname(epochfiles[0]) + stem = self._epoch_stem(epochfiles) + return os.path.join(parent, f".{stem}.{fmstr}.epochprobemap.ndi") def getepochprobemap( self, @@ -631,15 +700,61 @@ def getepochprobemap( props = doc.document_properties return getattr(props.epochfiles_ingested, "epochprobemap", None) - # Try to load from file + # Try to find a probe map file within the epoch files + epm_patterns = self._epochprobemap_fileparameters.get("filematch", []) + for f in epochfiles: + fname = os.path.basename(f) + for pat in epm_patterns: + glob_pat = pat.replace("#", "*") + if fnmatch.fnmatch(fname, glob_pat): + return self._load_epochprobemap_file(f) + try: + if re.search(pat, fname): + return self._load_epochprobemap_file(f) + except re.error: + pass + + # Fall back to generated probe map file filename = self.epochprobemapfilename(epoch_number) if filename and Path(filename).is_file(): - # Load probe map from file - # This would need to be implemented based on the probe map format - pass + return self._load_epochprobemap_file(filename) return None + @staticmethod + def _load_epochprobemap_file(filepath: str) -> Any | None: + """Load an epoch probe map from a TSV file. + + The file format is tab-separated with a header row: + ``name reference type devicestring subjectstring`` + """ + from ...epoch.epochprobemap import ndi_epoch_epochprobemap as EpochProbeMap + + try: + with open(filepath, encoding="utf-8") as f: + lines = f.read().strip().splitlines() + if len(lines) < 2: + return None + # Skip header, parse data lines + maps = [] + for line in lines[1:]: + parts = line.split("\t") + if len(parts) >= 3: + maps.append( + EpochProbeMap( + name=parts[0].strip(), + reference=int(parts[1].strip()), + type=parts[2].strip(), + devicestring=parts[3].strip() if len(parts) > 3 else "", + subjectstring=parts[4].strip() if len(parts) > 4 else "", + ) + ) + if len(maps) == 1: + return maps[0] + return maps if maps else None + except Exception: + return None + def getepochingesteddoc( self, epochfiles: list[str], @@ -763,11 +878,9 @@ def filematch_hashstring(self) -> str: # Prefer the raw document string so epoch-ID filenames match MATLAB if self._raw_fileparameters_str: return hashlib.md5(self._raw_fileparameters_str.encode()).hexdigest() - patterns = self._fileparameters.get("filematch", []) if not patterns: return "" - concat = "".join(patterns) return hashlib.md5(concat.encode()).hexdigest() diff --git a/src/ndi/query.py b/src/ndi/query.py index 5bb39b6..2689eea 100644 --- a/src/ndi/query.py +++ b/src/ndi/query.py @@ -581,3 +581,7 @@ def __repr__(self) -> str: if self._resolved: return f"ndi_query('{self.field}' {self.operator} {self.value!r})" return f"ndi_query('{self.field}')" + + +# Pythonic alias +Query = ndi_query diff --git a/src/ndi/session/dir.py b/src/ndi/session/dir.py index 60b278a..b50dc1f 100644 --- a/src/ndi/session/dir.py +++ b/src/ndi/session/dir.py @@ -285,3 +285,7 @@ def __eq__(self, other: Any) -> bool: def __repr__(self) -> str: """String representation.""" return f"ndi_session_dir(reference='{self._reference}', path='{self._path}')" + + +# Pythonic alias +DirSession = ndi_session_dir diff --git a/src/ndi/session/session_base.py b/src/ndi/session/session_base.py index c3af8a4..3315534 100644 --- a/src/ndi/session/session_base.py +++ b/src/ndi/session/session_base.py @@ -19,6 +19,7 @@ from ..query import ndi_query from ..time.syncgraph import ndi_time_syncgraph from ..time.syncrule_base import ndi_time_syncrule +from ..util.classname import ndi_matlab_classname logger = logging.getLogger(__name__) @@ -981,7 +982,7 @@ def findexpobj(self, obj_name: str, obj_classname: str) -> Any | None: if not isinstance(obj, list): obj = [obj] for o in obj: - if type(o).__name__ == obj_classname: + if ndi_matlab_classname(o) == ndi_matlab_classname(obj_classname): return o return None @@ -990,7 +991,10 @@ def findexpobj(self, obj_name: str, obj_classname: str) -> Any | None: probes = self.getprobes() for p in probes: p_name = p.name if hasattr(p, "name") else "" - if type(p).__name__ == obj_classname and p_name == obj_name: + if ( + ndi_matlab_classname(p) == ndi_matlab_classname(obj_classname) + and p_name == obj_name + ): return p return None @@ -998,7 +1002,10 @@ def findexpobj(self, obj_name: str, obj_classname: str) -> Any | None: probes = self.getprobes() for p in probes: p_name = p.name if hasattr(p, "name") else "" - if type(p).__name__ == obj_classname and p_name == obj_name: + if ( + ndi_matlab_classname(p) == ndi_matlab_classname(obj_classname) + and p_name == obj_name + ): return p obj = self.daqsystem_load(name=obj_name) @@ -1006,7 +1013,7 @@ def findexpobj(self, obj_name: str, obj_classname: str) -> Any | None: if not isinstance(obj, list): obj = [obj] for o in obj: - if type(o).__name__ == obj_classname: + if ndi_matlab_classname(o) == ndi_matlab_classname(obj_classname): return o return None @@ -1098,6 +1105,16 @@ def _document_to_object(self, document: ndi_document) -> Any: # Check document type if document.doc_isa("daqsystem"): + props = document.document_properties + daq_class_name = "" + if isinstance(props, dict): + daq_class_name = props.get("daqsystem", {}).get("ndi_daqsystem_class", "") + + if "mfdaq" in daq_class_name: + from ..daq.system_mfdaq import ndi_daq_system_mfdaq + + return ndi_daq_system_mfdaq(session=self, document=document) + from ..daq.system import ndi_daq_system return ndi_daq_system(session=self, document=document) diff --git a/src/ndi/time/syncgraph.py b/src/ndi/time/syncgraph.py index 1af4f11..1ce0a31 100644 --- a/src/ndi/time/syncgraph.py +++ b/src/ndi/time/syncgraph.py @@ -20,6 +20,7 @@ HAS_NETWORKX = False from ..ido import ndi_ido +from ..util.classname import ndi_matlab_classname from .clocktype import ndi_time_clocktype from .syncrule_base import ndi_time_syncrule from .timemapping import ndi_time_timemapping @@ -570,7 +571,7 @@ def new_document(self) -> list[ndi_document]: sg_doc = ndi_document( document_type="daq/syncgraph", **{ - "syncgraph.ndi_syncgraph_class": type(self).__name__, + "syncgraph.ndi_syncgraph_class": ndi_matlab_classname(self), "base.id": self.id, "base.session_id": self._session.id() if self._session else "", }, diff --git a/src/ndi/time/syncrule/common_triggers_overlapping_epochs.py b/src/ndi/time/syncrule/common_triggers_overlapping_epochs.py index 8cbd697..6d05f27 100644 --- a/src/ndi/time/syncrule/common_triggers_overlapping_epochs.py +++ b/src/ndi/time/syncrule/common_triggers_overlapping_epochs.py @@ -168,7 +168,7 @@ def is_valid_parameters(self, parameters: dict[str, Any]) -> tuple[bool, str]: def eligible_epochsets(self) -> list[str]: """Return eligible epochset class names.""" - return ["ndi.daq.system", "ndi_daq_system"] + return ["ndi.daq.system"] def ineligible_epochsets(self) -> list[str]: """Return ineligible epochset class names.""" diff --git a/src/ndi/time/syncrule/filefind.py b/src/ndi/time/syncrule/filefind.py index 624d284..23d5190 100644 --- a/src/ndi/time/syncrule/filefind.py +++ b/src/ndi/time/syncrule/filefind.py @@ -101,7 +101,7 @@ def is_valid_parameters(self, parameters: dict[str, Any]) -> tuple[bool, str]: def eligible_epochsets(self) -> list[str]: """Return eligible epochset class names.""" - return ["ndi.daq.system", "ndi_daq_system"] + return ["ndi.daq.system"] def ineligible_epochsets(self) -> list[str]: """Return ineligible epochset class names.""" diff --git a/src/ndi/time/syncrule/filematch.py b/src/ndi/time/syncrule/filematch.py index a35a700..fa43ddf 100644 --- a/src/ndi/time/syncrule/filematch.py +++ b/src/ndi/time/syncrule/filematch.py @@ -9,6 +9,7 @@ from typing import Any +from ...util.classname import ndi_matlab_classname from ..syncrule_base import ndi_time_syncrule from ..timemapping import ndi_time_timemapping @@ -75,7 +76,7 @@ def is_valid_parameters(self, parameters: dict[str, Any]) -> tuple[bool, str]: def eligible_epochsets(self) -> list[str]: """Return eligible epochset class names.""" - return ["ndi.daq.system", "ndi_daq_system"] + return ["ndi.daq.system"] def ineligible_epochsets(self) -> list[str]: """Return ineligible epochset class names.""" @@ -140,12 +141,7 @@ def apply( @staticmethod def _is_daq_system(classname: str) -> bool: """Check if a classname represents a DAQ system.""" - daq_classes = [ - "ndi.daq.system", - "ndi_daq_system", - "daq.system", - ] - return any(c in classname for c in daq_classes) + return "ndi.daq.system" in ndi_matlab_classname(classname) @staticmethod def _get_underlying_files(underlying_epochs: Any) -> list[str]: diff --git a/src/ndi/time/syncrule/random_pulses.py b/src/ndi/time/syncrule/random_pulses.py index a573eac..fb2e430 100644 --- a/src/ndi/time/syncrule/random_pulses.py +++ b/src/ndi/time/syncrule/random_pulses.py @@ -160,7 +160,7 @@ def is_valid_parameters(self, parameters: dict[str, Any]) -> tuple[bool, str]: def eligible_epochsets(self) -> list[str]: """Return eligible epochset class names.""" - return ["ndi.daq.system.mfdaq", "ndi_daq_system"] + return ["ndi.daq.system.mfdaq"] def ineligible_epochsets(self) -> list[str]: """Return ineligible epochset class names.""" diff --git a/src/ndi/time/syncrule_base.py b/src/ndi/time/syncrule_base.py index aefc27d..be9ed16 100644 --- a/src/ndi/time/syncrule_base.py +++ b/src/ndi/time/syncrule_base.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any from ..ido import ndi_ido +from ..util.classname import ndi_matlab_classname from .clocktype import ndi_time_clocktype from .timemapping import ndi_time_timemapping @@ -184,7 +185,7 @@ def new_document(self) -> ndi_document: doc = ndi_document( document_type="daq/syncrule", **{ - "syncrule.ndi_syncrule_class": type(self).__name__, + "syncrule.ndi_syncrule_class": ndi_matlab_classname(self), "syncrule.parameters": self._parameters, "base.id": self.id, }, @@ -211,7 +212,7 @@ def to_dict(self) -> dict[str, Any]: """ return { "id": self.id, - "class": type(self).__name__, + "class": ndi_matlab_classname(self), "parameters": self._parameters, } diff --git a/src/ndi/util/__init__.py b/src/ndi/util/__init__.py index 9616283..96fca72 100644 --- a/src/ndi/util/__init__.py +++ b/src/ndi/util/__init__.py @@ -13,6 +13,7 @@ ``ndi.openminds_convert``. """ +from .classname import ndi_matlab_classname, ndi_python_classname from .compare_session_summary import compareSessionSummary from .datestamp2datetime import datestamp2datetime from .downsampleTimeseries import downsampleTimeseries @@ -25,6 +26,8 @@ from .unwrapTableCellContent import unwrapTableCellContent __all__ = [ + "ndi_matlab_classname", + "ndi_python_classname", "compareSessionSummary", "datestamp2datetime", "downsampleTimeseries", diff --git a/src/ndi/util/classname.py b/src/ndi/util/classname.py new file mode 100644 index 0000000..8270cfb --- /dev/null +++ b/src/ndi/util/classname.py @@ -0,0 +1,78 @@ +""" +ndi.util.classname - Convert between Python and MATLAB class names. + +The NDI naming convention maps mechanically between the two languages: + +* MATLAB uses dots as package separators: ``ndi.session.dir`` +* Python uses single underscores: ``ndi_session_dir`` +* A literal underscore in MATLAB (``ndi.calc.tuning_fit``) becomes a + double underscore in Python (``ndi_calc_tuning__fit``). + +The two helper functions in this module perform the conversion in each +direction so that class names stored in documents or artifacts are +always in the canonical MATLAB form and can be read by either language. +""" + +from __future__ import annotations + + +def ndi_matlab_classname(obj_or_name: object | str) -> str: + """Return the MATLAB-style dotted class name. + + Parameters + ---------- + obj_or_name + Either a Python object whose ``type().__name__`` will be used, + or a string that is already a Python-style underscore name + (e.g. ``"ndi_session_dir"``). If the string is already in + MATLAB dot-notation it is returned unchanged. + + Returns + ------- + str + MATLAB class name, e.g. ``"ndi.session.dir"``. + + Examples + -------- + >>> ndi_matlab_classname("ndi_session_dir") + 'ndi.session.dir' + >>> ndi_matlab_classname("ndi_calc_tuning__fit") + 'ndi.calc.tuning_fit' + >>> ndi_matlab_classname("ndi.session.dir") + 'ndi.session.dir' + """ + name = obj_or_name if isinstance(obj_or_name, str) else type(obj_or_name).__name__ + if "." in name: + return name # already MATLAB-style + # "__" → literal underscore, then "_" → dot + return name.replace("__", "\x00").replace("_", ".").replace("\x00", "_") + + +def ndi_python_classname(name: str) -> str: + """Return the Python-style underscore class name. + + Parameters + ---------- + name + A MATLAB-style dotted name (e.g. ``"ndi.session.dir"``). + If the string is already in Python underscore notation it is + returned unchanged. + + Returns + ------- + str + Python class name, e.g. ``"ndi_session_dir"``. + + Examples + -------- + >>> ndi_python_classname("ndi.session.dir") + 'ndi_session_dir' + >>> ndi_python_classname("ndi.calc.tuning_fit") + 'ndi_calc_tuning__fit' + >>> ndi_python_classname("ndi_session_dir") + 'ndi_session_dir' + """ + if "." not in name: + return name # already Python-style + # literal "_" → "__", then "." → "_" + return name.replace("_", "__").replace(".", "_") diff --git a/src/ndi/util/session_summary.py b/src/ndi/util/session_summary.py index 4ad2d5d..ec0c457 100644 --- a/src/ndi/util/session_summary.py +++ b/src/ndi/util/session_summary.py @@ -12,6 +12,8 @@ import os from typing import Any +from .classname import ndi_matlab_classname + def sessionSummary(session_obj: Any) -> dict[str, Any]: """Create a summary structure of an ndi.session object. @@ -62,7 +64,7 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: details: dict[str, Any] = {} - # Get filenavigator class + # Get filenavigator class (use MATLAB-compatible name for symmetry) fn = getattr(sys, "filenavigator", None) if fn is not None: details["filenavigator_class"] = getattr( @@ -76,10 +78,12 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: details["filenavigator_class"] = "" details["epochNodes_filenavigator"] = [] - # Get daqreader class + # Get daqreader class (use MATLAB-compatible name for symmetry) dr = getattr(sys, "daqreader", None) if dr is not None: - details["daqreader_class"] = getattr(dr, "NDI_DAQREADER_CLASS", type(dr).__qualname__) + details["daqreader_class"] = getattr( + dr, "NDI_DAQREADER_CLASS", ndi_matlab_classname(dr) + ) else: details["daqreader_class"] = "" diff --git a/tests/matlab_tests/test_daq.py b/tests/matlab_tests/test_daq.py index 9cfcde5..690e2be 100644 --- a/tests/matlab_tests/test_daq.py +++ b/tests/matlab_tests/test_daq.py @@ -4,18 +4,17 @@ MATLAB source files: mfdaqIntanTest.m -> TestIntanReader mfdaqNDRAxonTest.m -> skipped (ABF reader not ported; uses NDR) - mfdaqNDRIntanTest.m -> skipped (NDR Intan reader not ported; uses SpikeInterface) + mfdaqNDRIntanTest.m -> TestIntanReader (now uses ndr.format.intan) Python replacement modules: - ndi.daq.reader.mfdaq.intan.ndi_daq_reader_mfdaq_intan (wraps SpikeInterface) - ndi.daq.reader.spikeinterface_adapter.ndi_daq_reader_SpikeInterfaceReader + ndi.daq.reader.mfdaq.intan.ndi_daq_reader_mfdaq_intan (uses ndr.format.intan) ndi.daq.mfdaq.ndi_daq_reader_mfdaq (base with epochsamples2times / epochtimes2samples) ndi.fun.utils.channelname2prefixnumber """ import os from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch import numpy as np import pytest @@ -73,7 +72,7 @@ class TestIntanReader: The MATLAB test creates an ndi_daq_reader_mfdaq_intan, points it at real .rhd files, and checks channel discovery, epochsamples2times, epochtimes2samples. - The Python reader delegates to SpikeInterface under the hood. + The Python reader uses ndr.format.intan for header parsing and data reading. """ def test_intan_reader_instantiation(self): @@ -125,67 +124,74 @@ def test_intan_reader_channel_types(self): assert len(types) == len(abbrevs) def test_intan_reader_mocked_getchannels(self): - """ndi_daq_reader_mfdaq_intan.getchannelsepoch works via mocked SpikeInterface. + """ndi_daq_reader_mfdaq_intan.getchannelsepoch works via mocked ndr header. - The ndi_daq_reader_mfdaq_intan delegates to ndi_daq_reader_SpikeInterfaceReader internally. - We mock that to simulate a successful channel discovery without - needing real data files. + We mock ndr.format.intan.read_Intan_RHD2000_header to simulate a + successful channel discovery without needing real data files. """ reader = ndi_daq_reader_mfdaq_intan() - mock_channels = [ - ChannelInfo( - name="ai1", type="analog_in", time_channel=1, number=1, sample_rate=30000.0 - ), - ChannelInfo( - name="ai2", type="analog_in", time_channel=1, number=2, sample_rate=30000.0 - ), - ChannelInfo(name="t1", type="time", time_channel=None, number=1, sample_rate=30000.0), - ] - - # Mock the _get_si_reader method to return a mock reader class - mock_si_class = MagicMock() - mock_si_instance = MagicMock() - mock_si_instance.getchannelsepoch.return_value = mock_channels - mock_si_class.return_value = mock_si_instance - - with patch.object(reader, "_get_si_reader", return_value=mock_si_class): + mock_header = { + "sample_rate": 30000.0, + "num_amplifier_channels": 2, + "num_aux_input_channels": 0, + "num_supply_voltage_channels": 0, + "num_board_adc_channels": 0, + "num_board_dig_in_channels": 0, + "num_board_dig_out_channels": 0, + "num_temp_sensor_channels": 0, + "num_samples_per_data_block": 128, + "num_samples": 128000, + "header_size": 512, + "amplifier_channels": [ + {"native_channel_name": "A-000", "signal_type": 0}, + {"native_channel_name": "A-001", "signal_type": 0}, + ], + "aux_input_channels": [], + "board_adc_channels": [], + "board_dig_in_channels": [], + "board_dig_out_channels": [], + "supply_voltage_channels": [], + "frequency_parameters": { + "amplifier_sample_rate": 30000.0, + "aux_input_sample_rate": 7500.0, + }, + } + + with patch( + "ndi.daq.reader.mfdaq.intan.read_Intan_RHD2000_header", + return_value=mock_header, + ): + # Clear any cached header + reader._header_cache.clear() channels = reader.getchannelsepoch(["fake_data.rhd"]) - assert len(channels) == 3 + assert len(channels) == 3 # 2 amplifier + 1 time assert channels[0].name == "ai1" assert channels[0].type == "analog_in" assert channels[0].sample_rate == 30000.0 + assert channels[1].name == "ai2" assert channels[2].type == "time" - def test_intan_reader_no_spikeinterface(self): - """ndi_daq_reader_mfdaq_intan.getchannelsepoch returns [] when SI is unavailable. - - MATLAB tests always had NDR; Python gracefully degrades. - """ + def test_intan_reader_no_rhd_file(self): + """ndi_daq_reader_mfdaq_intan.getchannelsepoch returns [] with no .rhd file.""" reader = ndi_daq_reader_mfdaq_intan() - - # Mock _get_si_reader to return None (no spikeinterface) - with patch.object(reader, "_get_si_reader", return_value=None): - channels = reader.getchannelsepoch(["nonexistent.rhd"]) - + channels = reader.getchannelsepoch(["nonexistent.txt"]) assert channels == [] - def test_intan_reader_readchannels_no_si_raises(self): - """ndi_daq_reader_mfdaq_intan.readchannels_epochsamples raises ImportError without SI.""" + def test_intan_reader_readchannels_no_file_raises(self): + """ndi_daq_reader_mfdaq_intan.readchannels_epochsamples raises with no .rhd file.""" reader = ndi_daq_reader_mfdaq_intan() - with patch.object(reader, "_get_si_reader", return_value=None): - with pytest.raises(ImportError, match="spikeinterface"): - reader.readchannels_epochsamples("ai", [1], ["fake.rhd"], 1, 100) + with pytest.raises(FileNotFoundError): + reader.readchannels_epochsamples("ai", [1], ["fake.txt"], 1, 100) - def test_intan_reader_samplerate_no_si_raises(self): - """ndi_daq_reader_mfdaq_intan.samplerate raises ImportError without SI.""" + def test_intan_reader_samplerate_no_file_raises(self): + """ndi_daq_reader_mfdaq_intan.samplerate raises with no .rhd file.""" reader = ndi_daq_reader_mfdaq_intan() - with patch.object(reader, "_get_si_reader", return_value=None): - with pytest.raises(ImportError, match="spikeinterface"): - reader.samplerate(["fake.rhd"], "ai", [1]) + with pytest.raises(FileNotFoundError): + reader.samplerate(["fake.txt"], "ai", [1]) def test_intan_reader_repr(self): """ndi_daq_reader_mfdaq_intan has a useful repr.""" @@ -217,6 +223,92 @@ def test_intan_reader_live(self): assert ch.sample_rate is not None assert ch.sample_rate > 0 + def test_intan_reader_samplerate_mocked(self): + """ndi_daq_reader_mfdaq_intan.samplerate returns correct rates via mocked header.""" + reader = ndi_daq_reader_mfdaq_intan() + + mock_header = { + "sample_rate": 30000.0, + "num_amplifier_channels": 2, + "num_aux_input_channels": 1, + "num_supply_voltage_channels": 0, + "num_board_adc_channels": 0, + "num_board_dig_in_channels": 0, + "num_board_dig_out_channels": 0, + "num_temp_sensor_channels": 0, + "num_samples_per_data_block": 128, + "num_samples": 128000, + "header_size": 512, + "amplifier_channels": [ + {"native_channel_name": "A-000", "signal_type": 0}, + {"native_channel_name": "A-001", "signal_type": 0}, + ], + "aux_input_channels": [{"native_channel_name": "AUX1", "signal_type": 1}], + "board_adc_channels": [], + "board_dig_in_channels": [], + "board_dig_out_channels": [], + "supply_voltage_channels": [], + "frequency_parameters": { + "amplifier_sample_rate": 30000.0, + "aux_input_sample_rate": 7500.0, + }, + } + + with patch( + "ndi.daq.reader.mfdaq.intan.read_Intan_RHD2000_header", + return_value=mock_header, + ): + reader._header_cache.clear() + rates = reader.samplerate(["fake.rhd"], "ai", [1]) + assert rates[0] == 30000.0 + + reader._header_cache.clear() + aux_rates = reader.samplerate(["fake.rhd"], "auxiliary_in", [1]) + assert aux_rates[0] == 7500.0 + + def test_intan_reader_t0_t1_mocked(self): + """ndi_daq_reader_mfdaq_intan.t0_t1 returns correct time range.""" + reader = ndi_daq_reader_mfdaq_intan() + + mock_header = { + "num_amplifier_channels": 1, + "num_aux_input_channels": 0, + "num_supply_voltage_channels": 0, + "num_board_adc_channels": 0, + "num_board_dig_in_channels": 0, + "num_board_dig_out_channels": 0, + "num_temp_sensor_channels": 0, + "num_samples_per_data_block": 60, + "amplifier_channels": [{"native_channel_name": "A-000", "signal_type": 0}], + "aux_input_channels": [], + "board_adc_channels": [], + "board_dig_in_channels": [], + "board_dig_out_channels": [], + "supply_voltage_channels": [], + "frequency_parameters": {"amplifier_sample_rate": 30000.0}, + } + + # Intan_RHD2000_blockinfo returns (blockinfo, bytes_per_block, bytes_present, num_data_blocks) + mock_blockinfo = ({"samples_per_block": 60}, 0, 0, 5000) + # total_samples = 60 * 5000 = 300000 + + with ( + patch( + "ndi.daq.reader.mfdaq.intan.read_Intan_RHD2000_header", + return_value=mock_header, + ), + patch( + "ndi.daq.reader.mfdaq.intan.Intan_RHD2000_blockinfo", + return_value=mock_blockinfo, + ), + ): + reader._header_cache.clear() + t0_t1 = reader.t0_t1(["fake.rhd"]) + + assert len(t0_t1) == 1 + assert t0_t1[0][0] == 0.0 + np.testing.assert_allclose(t0_t1[0][1], (300000 - 1) / 30000.0) + # =========================================================================== # TestEpochSampleTimeConversion diff --git a/tests/matlab_tests/test_dataset.py b/tests/matlab_tests/test_dataset.py index 10f3520..9351e6e 100644 --- a/tests/matlab_tests/test_dataset.py +++ b/tests/matlab_tests/test_dataset.py @@ -198,10 +198,10 @@ def test_session_list_outputs(self, build_dataset): # Check session_reference matches assert props.get("session_reference") == "exp_demo", "session_reference should match" - # Check session_creator + # Check session_creator (MATLAB-compatible name) assert ( - props.get("session_creator") == "ndi_session_dir" - ), "session_creator should be ndi_session_dir" + props.get("session_creator") == "ndi.session.dir" + ), "session_creator should be ndi.session.dir" # =========================================================================== diff --git a/tests/symmetry/make_artifacts/dataset/__init__.py b/tests/symmetry/make_artifacts/dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py new file mode 100644 index 0000000..e903029 --- /dev/null +++ b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py @@ -0,0 +1,135 @@ +"""Generate symmetry artifacts for an NDI dataset with an ingested session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+dataset/buildDataset.m + +This test creates an NDI Dataset containing an ingested DirSession with +5 demoNDI documents (each with a file attachment), then persists the +dataset database, a ``datasetSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/dataset/buildDataset/ + testBuildDatasetArtifacts/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +from ndi.dataset import Dataset +from ndi.document import Document +from ndi.query import Query +from ndi.session.dir import DirSession +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "dataset" / "buildDataset" / "testBuildDatasetArtifacts" + + +def _add_doc_with_file(session: DirSession, doc_number: int) -> None: + """Add a demoNDI document with a file attachment to the session.""" + docname = f"doc_{doc_number}" + filepath = session.path / docname + filepath.write_text(docname) + + doc = Document("demoNDI") + props = doc.document_properties + props["base"]["name"] = docname + props["demoNDI"]["value"] = doc_number + props["base"]["session_id"] = session.id() + doc = Document(props) + doc = doc.add_file("filename1.ext", str(filepath)) + session.database_add(doc) + + +def _dataset_summary(dataset: Dataset) -> dict: + """Create a summary structure for a dataset. + + Mirrors MATLAB's ``ndi.symmetry.makeArtifacts.dataset.buildDataset`` + which writes: numSessions, references, sessionIds, sessionSummaries. + """ + refs, session_ids, *_ = dataset.session_list() + num_sessions = len(refs) + + # Build a session summary for each session in the dataset + session_summaries = [] + for sid in session_ids: + sess = dataset.open_session(sid) + session_summaries.append(sessionSummary(sess)) + + return { + "numSessions": num_sessions, + "references": refs, + "sessionIds": session_ids, + "sessionSummaries": session_summaries, + } + + +class TestBuildDataset: + """Mirror of ndi.symmetry.makeArtifacts.dataset.buildDataset.""" + + # -- setup ---------------------------------------------------------------- + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Build a dataset with an ingested session in a temporary directory. + + Mirrors the MATLAB ``buildDatasetSetup`` method. + """ + # Create a session with 5 demoNDI documents + file attachments + session_dir = tmp_path / "session_src" + session_dir.mkdir() + session = DirSession("exp_demo", session_dir) + + for i in range(1, 6): + _add_doc_with_file(session, i) + + # Create the dataset and ingest the session + dataset_dir = tmp_path / "ds_demo" + dataset_dir.mkdir() + dataset = Dataset(dataset_dir, "ds_demo") + dataset.add_ingested_session(session) + + self.dataset = dataset + self.session = session + + # -- tests ---------------------------------------------------------------- + + def test_build_dataset_artifacts(self): + """Export the dataset to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + # Clear previous artifacts + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Copy the entire dataset directory to the persistent artifact dir. + shutil.copytree(str(self.dataset.getpath()), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.dataset.database_search(Query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write datasetSummary.json – open from artifact_dir so the session + # path lists files that are actually present (including jsonDocuments). + artifact_dataset = Dataset(artifact_dir) + summary = _dataset_summary(artifact_dataset) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + summary_path = artifact_dir / "datasetSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + # Verify artifacts were created + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() + assert len(list(json_docs_dir.glob("*.json"))) > 0 diff --git a/tests/symmetry/read_artifacts/dataset/__init__.py b/tests/symmetry/read_artifacts/dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py new file mode 100644 index 0000000..79fb452 --- /dev/null +++ b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py @@ -0,0 +1,161 @@ +"""Read and verify symmetry artifacts for an NDI dataset with ingested session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+dataset/buildDataset.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can: + +1. Open the copied dataset database. +2. Verify the session list matches ``datasetSummary.json``. +3. Compare session summaries for each session in the dataset. +4. Load every document whose JSON was exported to ``jsonDocuments/``. + +The MATLAB ``datasetSummary.json`` contains: + numSessions, references, sessionIds, sessionSummaries + +The test is parameterized over ``source_type`` so that a single test class +covers both ``matlabArtifacts`` and ``pythonArtifacts``. +""" + +import json + +import pytest + +from ndi.dataset import Dataset +from ndi.query import Query +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBuildDataset: + """Mirror of ndi.symmetry.readArtifacts.dataset.buildDataset.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE / source_type / "dataset" / "buildDataset" / "testBuildDatasetArtifacts" + ) + + def _open_dataset(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, Dataset(artifact_dir) + + # -- tests ---------------------------------------------------------------- + + def test_build_dataset_summary(self, source_type): + """Verify session counts, references, and IDs match datasetSummary.json.""" + artifact_dir, dataset = self._open_dataset(source_type) + + summary_path = artifact_dir / "datasetSummary.json" + if not summary_path.exists(): + pytest.skip(f"datasetSummary.json not found in {source_type} artifact directory.") + + expected = json.loads(summary_path.read_text(encoding="utf-8")) + + # Verify session list + refs, session_ids, *_ = dataset.session_list() + num_sessions = len(refs) + + expected_num = expected.get("numSessions", 0) + expected_ids = expected.get("sessionIds", []) + expected_refs = expected.get("references", []) + + assert num_sessions == expected_num, ( + f"Session count mismatch in {source_type}: " + f"got {num_sessions}, expected {expected_num}" + ) + + for exp_id in expected_ids: + assert exp_id in session_ids, ( + f"Expected session ID {exp_id!r} not found in dataset " f"from {source_type}" + ) + + for exp_ref in expected_refs: + assert exp_ref in refs, ( + f"Expected session reference {exp_ref!r} not found in dataset " + f"from {source_type}" + ) + + def test_build_dataset_session_summaries(self, source_type): + """Compare per-session summaries against those stored in datasetSummary.json.""" + artifact_dir, dataset = self._open_dataset(source_type) + + summary_path = artifact_dir / "datasetSummary.json" + if not summary_path.exists(): + pytest.skip(f"datasetSummary.json not found in {source_type} artifact directory.") + + expected = json.loads(summary_path.read_text(encoding="utf-8")) + expected_summaries = expected.get("sessionSummaries", []) + expected_ids = expected.get("sessionIds", []) + + if not expected_summaries: + pytest.skip(f"No sessionSummaries in {source_type} datasetSummary.json.") + + for i, sid in enumerate(expected_ids): + if i >= len(expected_summaries): + break + + sess = dataset.open_session(sid) + if sess is None: + pytest.fail(f"Could not open session {sid} from {source_type} dataset.") + + actual_summary = sessionSummary(sess) + expected_summary = expected_summaries[i] + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["datasetSummary.json", "jsonDocuments"], + ) + + assert len(report) == 0, ( + f"Session summary mismatch for session {sid} " + f"in {source_type} dataset:\n" + "\n".join(report) + ) + + def test_build_dataset_documents(self, source_type): + """Verify that every exported JSON document can be loaded from the dataset DB.""" + artifact_dir, dataset = self._open_dataset(source_type) + + json_docs_dir = artifact_dir / "jsonDocuments" + if not json_docs_dir.exists(): + pytest.skip(f"jsonDocuments directory not found in {source_type}.") + + json_files = list(json_docs_dir.glob("*.json")) + + actual_docs = dataset.database_search(Query("base.id").match("(.*)")) + + assert len(actual_docs) == len(json_files), ( + f"Number of documents in dataset ({len(actual_docs)}) does not match " + f"{source_type} JSON artifacts ({len(json_files)})." + ) + + for jf in json_files: + expected_doc = json.loads(jf.read_text(encoding="utf-8")) + expected_id = expected_doc.get("base", {}).get("id", "") + + found = False + for actual in actual_docs: + if actual.id == expected_id: + found = True + actual_props = actual.document_properties + assert actual_props.get("document_class", {}).get( + "class_name" + ) == expected_doc.get("document_class", {}).get("class_name"), ( + f"Document class mismatch for id: {expected_id} " f"in {source_type}" + ) + break + assert found, ( + f"Document from {source_type} artifact not found in dataset: " f"{expected_id}" + ) diff --git a/tests/test_phase9.py b/tests/test_phase9.py index 2cb5b9f..e133987 100644 --- a/tests/test_phase9.py +++ b/tests/test_phase9.py @@ -311,7 +311,7 @@ def test_construction_with_session(self, session): def test_name_is_class_name(self): calc = ndi_calculator() - assert calc.name == "ndi_calculator" + assert calc.name == "ndi.calculator" def test_repr(self): calc = ndi_calculator(document_type="my_calc") @@ -504,7 +504,7 @@ def test_repr(self): def test_name_is_class_name(self): sc = ndi_calc_example_simple() - assert sc.name == "ndi_calc_example_simple" + assert sc.name == "ndi.calc.example.simple" class TestSimpleCalcDefaultParameters: