From 0a1207428603fd10ede0c67a8e5c64abd68ffa9c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 13:06:40 +0000 Subject: [PATCH 01/15] Add blank session symmetry tests for kjnielsenlab, marderlab, and vhlab Mirror the new NDI-matlab symmetry tests that create blank sessions with lab-specific DAQ system configurations. Each lab has a makeArtifacts test that creates a session with the lab's DAQ systems (from ndi_common/daq_systems/ JSON configs) and exports artifacts, and a readArtifacts test that verifies the session summary round-trips correctly. New files: - _lab_setup.py: shared helper that reads DAQ config JSONs and creates filenavigator/daqreader/daqsystem documents (Python equivalent of ndi.setup.lab) - _summary_helpers.py: normalizes DAQ system ordering for comparison - 3 make_artifacts tests + 3 read_artifacts tests https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- .../make_artifacts/session/_lab_setup.py | 133 ++++++++++++++++++ .../test_blank_session_kjnielsenlab.py | 86 +++++++++++ .../session/test_blank_session_marderlab.py | 86 +++++++++++ .../session/test_blank_session_vhlab.py | 83 +++++++++++ .../session/_summary_helpers.py | 21 +++ .../test_blank_session_kjnielsenlab.py | 75 ++++++++++ .../session/test_blank_session_marderlab.py | 73 ++++++++++ .../session/test_blank_session_vhlab.py | 73 ++++++++++ 8 files changed, 630 insertions(+) create mode 100644 tests/symmetry/make_artifacts/session/_lab_setup.py create mode 100644 tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py create mode 100644 tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py create mode 100644 tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py create mode 100644 tests/symmetry/read_artifacts/session/_summary_helpers.py create mode 100644 tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py create mode 100644 tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py create mode 100644 tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py diff --git a/tests/symmetry/make_artifacts/session/_lab_setup.py b/tests/symmetry/make_artifacts/session/_lab_setup.py new file mode 100644 index 0000000..1d193d7 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/_lab_setup.py @@ -0,0 +1,133 @@ +"""Shared helper for creating blank lab sessions in symmetry tests. + +Reads DAQ system configuration JSON files from +``ndi_common/daq_systems//`` and creates the corresponding +filenavigator, daqreader, (optionally daqmetadatareader), and daqsystem +documents in the given session. + +This mirrors the MATLAB ``ndi.setup.lab()`` function. +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def _to_matlab_cell_str(items: list[str]) -> str: + """Convert a list of strings to MATLAB cell-array syntax. + + Example: ``['#.rhd']`` becomes ``"{ '#.rhd' }"``. + """ + if not items: + return "" + quoted = ", ".join(f"'{s}'" for s in items) + return f"{{ {quoted} }}" + + +def _find_daq_configs(lab_name: str) -> list[dict]: + """Read all DAQ system JSON configs for a given lab.""" + import ndi.ndi_common + + common_dir = Path(ndi.ndi_common.__path__[0]) + lab_dir = common_dir / "daq_systems" / lab_name + + if not lab_dir.exists(): + raise FileNotFoundError(f"No DAQ system configs found for lab: {lab_name}") + + configs = [] + for json_file in sorted(lab_dir.glob("*.json")): + with open(json_file, encoding="utf-8") as f: + configs.append(json.load(f)) + return configs + + +def setup_lab_daq_systems(session, lab_name: str) -> None: + """Add DAQ system documents to a session based on lab JSON configs. + + For each JSON config in ``ndi_common/daq_systems//``, creates: + - A ``daq/filenavigator`` document + - A ``daq/daqreader`` document + - Optionally a ``daq/daqmetadatareader`` document + - A ``daq/daqsystem`` document linking them together + + This is the Python equivalent of MATLAB's ``ndi.setup.lab(lab_name, ...)``. + """ + configs = _find_daq_configs(lab_name) + + for config in configs: + name = config["Name"] + file_params = config.get("FileParameters", []) + epm_class = config.get("EpochProbeMapClass", "ndi.epoch.epochprobemap_daqsystem") + epm_file_params = config.get("EpochProbeMapFileParameters", "") + reader_class = config.get("DaqReaderClass", "ndi.daq.reader.mfdaq") + system_class = config.get("DaqSystemClass", "ndi.daq.system.mfdaq") + metadata_reader_class = config.get("MetadataReaderClass", []) + + # Convert file parameters to MATLAB cell string format + fp_str = _to_matlab_cell_str(file_params) + + # EpochProbeMapFileParameters may be a string or list + if isinstance(epm_file_params, list): + epm_fp_str = _to_matlab_cell_str(epm_file_params) + else: + epm_fp_str = f"{{ '{epm_file_params}' }}" if epm_file_params else "" + + # Create filenavigator document + fn_doc = session.newdocument( + "daq/filenavigator", + **{ + "base.name": name, + "filenavigator.ndi_filenavigator_class": "ndi.file.navigator", + "filenavigator.fileparameters": fp_str, + "filenavigator.epochprobemap_class": epm_class, + "filenavigator.epochprobemap_fileparameters": epm_fp_str, + }, + ) + session.database_add(fn_doc) + + # Create daqreader document + dr_doc = session.newdocument( + "daq/daqreader", + **{ + "base.name": name, + "daqreader.ndi_daqreader_class": reader_class, + }, + ) + session.database_add(dr_doc) + + # Create daqsystem document + daq_doc = session.newdocument( + "daq/daqsystem", + **{ + "base.name": name, + "daqsystem.ndi_daqsystem_class": system_class, + }, + ) + daq_doc = daq_doc.set_dependency_value( + "filenavigator_id", fn_doc.id, error_if_not_found=False + ) + daq_doc = daq_doc.set_dependency_value( + "daqreader_id", dr_doc.id, error_if_not_found=False + ) + + # Create daqmetadatareader document if configured + if metadata_reader_class and metadata_reader_class != []: + mr_class = ( + metadata_reader_class + if isinstance(metadata_reader_class, str) + else metadata_reader_class[0] + ) + mr_doc = session.newdocument( + "daq/daqmetadatareader", + **{ + "base.name": name, + "daqmetadatareader.ndi_daqmetadatareader_class": mr_class, + }, + ) + session.database_add(mr_doc) + daq_doc = daq_doc.add_dependency_value_n( + "daqmetadatareader_id", mr_doc.id + ) + + session.database_add(daq_doc) diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py new file mode 100644 index 0000000..a102d74 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py @@ -0,0 +1,86 @@ +"""Generate symmetry artifacts for a blank kjnielsenlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionKjnielsenlab.m + +This test creates a blank NDI session configured with the kjnielsenlab DAQ +systems (from ``ndi_common/daq_systems/kjnielsenlab/``), then persists the +session database, a ``sessionSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionKjnielsenlab/ + testBlankSessionKjnielsenlab/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS +from tests.symmetry.make_artifacts.session._lab_setup import setup_lab_daq_systems + +ARTIFACT_DIR = ( + PYTHON_ARTIFACTS + / "session" + / "blankSessionKjnielsenlab" + / "testBlankSessionKjnielsenlab" +) + + +class TestBlankSessionKjnielsenlab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionKjnielsenlab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with kjnielsenlab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + setup_lab_daq_systems(session, "kjnielsenlab") + + self.session = session + + def test_blank_session_kjnielsenlab(self): + """Export the blank kjnielsenlab session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py new file mode 100644 index 0000000..2c6d2af --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py @@ -0,0 +1,86 @@ +"""Generate symmetry artifacts for a blank marderlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionMarderlab.m + +This test creates a blank NDI session configured with the marderlab DAQ +systems (from ``ndi_common/daq_systems/marderlab/``), then persists the +session database, a ``sessionSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionMarderlab/ + testBlankSessionMarderlab/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS +from tests.symmetry.make_artifacts.session._lab_setup import setup_lab_daq_systems + +ARTIFACT_DIR = ( + PYTHON_ARTIFACTS + / "session" + / "blankSessionMarderlab" + / "testBlankSessionMarderlab" +) + + +class TestBlankSessionMarderlab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionMarderlab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with marderlab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + setup_lab_daq_systems(session, "marderlab") + + self.session = session + + def test_blank_session_marderlab(self): + """Export the blank marderlab session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py new file mode 100644 index 0000000..e7d5909 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py @@ -0,0 +1,83 @@ +"""Generate symmetry artifacts for a blank vhlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionVhlab.m + +This test creates a blank NDI session configured with the vhlab DAQ +systems (from ``ndi_common/daq_systems/vhlab/``), then persists the +session database, a ``sessionSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionVhlab/ + testBlankSessionVhlab/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS +from tests.symmetry.make_artifacts.session._lab_setup import setup_lab_daq_systems + +ARTIFACT_DIR = ( + PYTHON_ARTIFACTS / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" +) + + +class TestBlankSessionVhlab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionVhlab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with vhlab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + setup_lab_daq_systems(session, "vhlab") + + self.session = session + + def test_blank_session_vhlab(self): + """Export the blank vhlab session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/read_artifacts/session/_summary_helpers.py b/tests/symmetry/read_artifacts/session/_summary_helpers.py new file mode 100644 index 0000000..6a94777 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/_summary_helpers.py @@ -0,0 +1,21 @@ +"""Shared helpers for read_artifacts session symmetry tests.""" + +from __future__ import annotations + +from typing import Any + + +def sort_daq_systems_by_name(summary: dict[str, Any]) -> None: + """Sort daqSystemNames and daqSystemDetails in a summary by name. + + The database may return DAQ systems in different order on re-open. + Sorting both summaries by name before comparison avoids spurious + ordering mismatches. Modifies *summary* in place. + """ + names = summary.get("daqSystemNames", []) + details = summary.get("daqSystemDetails", []) + + if len(names) == len(details) and len(names) > 1: + paired = sorted(zip(names, details), key=lambda x: x[0]) + summary["daqSystemNames"] = [p[0] for p in paired] + summary["daqSystemDetails"] = [p[1] for p in paired] diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py new file mode 100644 index 0000000..89bcb46 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py @@ -0,0 +1,75 @@ +"""Read and verify symmetry artifacts for a blank kjnielsenlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionKjnielsenlab.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can open the +session and produce a matching session summary. +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBlankSessionKjnielsenlab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionKjnielsenlab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE + / source_type + / "session" + / "blankSessionKjnielsenlab" + / "testBlankSessionKjnielsenlab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_kjnielsenlab_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + # Normalize DAQ system ordering — the database may return them in + # different order on re-open. + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert len(report) == 0, ( + f"Session summary mismatch against {source_type} generated artifacts:\n" + + "\n".join(report) + ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py new file mode 100644 index 0000000..785f945 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py @@ -0,0 +1,73 @@ +"""Read and verify symmetry artifacts for a blank marderlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionMarderlab.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can open the +session and produce a matching session summary. +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBlankSessionMarderlab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionMarderlab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE + / source_type + / "session" + / "blankSessionMarderlab" + / "testBlankSessionMarderlab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_marderlab_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert len(report) == 0, ( + f"Session summary mismatch against {source_type} generated artifacts:\n" + + "\n".join(report) + ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py new file mode 100644 index 0000000..8bf27e6 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py @@ -0,0 +1,73 @@ +"""Read and verify symmetry artifacts for a blank vhlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionVhlab.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can open the +session and produce a matching session summary. +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBlankSessionVhlab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionVhlab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE + / source_type + / "session" + / "blankSessionVhlab" + / "testBlankSessionVhlab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_vhlab_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert len(report) == 0, ( + f"Session summary mismatch against {source_type} generated artifacts:\n" + + "\n".join(report) + ) From e8930eee575062090355901e2b2180a350698348 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 14:30:57 +0000 Subject: [PATCH 02/15] Add ndi.setup package with lab() function for DAQ system configuration Move lab setup logic from test helper to public ndi.setup.lab() API, matching the MATLAB ndi.setup.lab() convention. Users can now call ndi.setup.lab(session, "vhlab") to configure a session with lab-specific DAQ systems from the JSON configs in ndi_common/daq_systems/. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/__init__.py | 3 +- src/ndi/ndi_matlab_python_bridge.yaml | 12 +- src/ndi/setup/__init__.py | 13 ++ src/ndi/setup/lab.py | 146 ++++++++++++++++++ .../make_artifacts/session/_lab_setup.py | 134 +--------------- .../test_blank_session_kjnielsenlab.py | 4 +- .../session/test_blank_session_marderlab.py | 4 +- .../session/test_blank_session_vhlab.py | 4 +- 8 files changed, 179 insertions(+), 141 deletions(-) create mode 100644 src/ndi/setup/__init__.py create mode 100644 src/ndi/setup/lab.py diff --git a/src/ndi/__init__.py b/src/ndi/__init__.py index 3128ebb..fb69954 100644 --- a/src/ndi/__init__.py +++ b/src/ndi/__init__.py @@ -32,7 +32,7 @@ # Import epoch module (Phase 6) # Import Phase 10: Cloud API client # Import Phase 11: Schema validation -from . import calc, cloud, common, daq, epoch, file, session, time, util, validate, validators +from . import calc, cloud, common, daq, epoch, file, session, setup, time, util, validate, validators # Import Phase 9: ndi_app framework and calculators from .app import ndi_app @@ -124,6 +124,7 @@ def version() -> tuple: "ndi_calculator", "calc", "cloud", + "setup", "util", "validate", "validators", diff --git a/src/ndi/ndi_matlab_python_bridge.yaml b/src/ndi/ndi_matlab_python_bridge.yaml index b03f0e8..bd58701 100644 --- a/src/ndi/ndi_matlab_python_bridge.yaml +++ b/src/ndi/ndi_matlab_python_bridge.yaml @@ -985,13 +985,13 @@ not_applicable: # ========================================================================= - name: setup matlab_path: "+ndi/+setup/" - status: not_applicable + python_path: "ndi/setup/" + status: implemented decision_log: > - Entire MATLAB package containing lab-specific DAQ configurations - (angeluccilab, dbkatzlab, marderlab, vhlab, yangyangwang) and - sub-packages (+NDIMaker, +conv, +daq, +epoch, +stimulus, - @DaqSystemConfiguration). These are MATLAB-specific lab setup - wizards. Python labs configure via JSON/YAML files instead. + Python equivalent of MATLAB lab setup wizards. The ndi.setup.lab() + function reads DAQ system JSON configs from ndi_common/daq_systems/ + and creates the corresponding documents in a session. Usage: + ndi.setup.lab(session, "vhlab"). # ========================================================================= # ndi.docs namespace — MATLAB documentation generation diff --git a/src/ndi/setup/__init__.py b/src/ndi/setup/__init__.py new file mode 100644 index 0000000..cbb382b --- /dev/null +++ b/src/ndi/setup/__init__.py @@ -0,0 +1,13 @@ +"""ndi.setup — Lab configuration and session setup utilities. + +Python equivalent of MATLAB's ``+ndi/+setup/`` package. + +Usage:: + + import ndi + ndi.setup.lab(session, "vhlab") +""" + +from .lab import lab + +__all__ = ["lab"] diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py new file mode 100644 index 0000000..83b5117 --- /dev/null +++ b/src/ndi/setup/lab.py @@ -0,0 +1,146 @@ +"""Set up a session with DAQ systems from lab configuration files. + +Reads DAQ system configuration JSON files from +``ndi_common/daq_systems//`` and creates the corresponding +filenavigator, daqreader, (optionally daqmetadatareader), and daqsystem +documents in the given session. + +This is the Python equivalent of MATLAB's ``ndi.setup.lab()``. + +Example:: + + import ndi + session = ndi.ndi_session_dir("exp1", "/path/to/experiment") + ndi.setup.lab(session, "vhlab") +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def _to_matlab_cell_str(items: list[str]) -> str: + """Convert a list of strings to MATLAB cell-array syntax. + + Example: ``['#.rhd']`` becomes ``"{ '#.rhd' }"``. + """ + if not items: + return "" + quoted = ", ".join(f"'{s}'" for s in items) + return f"{{ {quoted} }}" + + +def _find_daq_configs(lab_name: str) -> list[dict]: + """Read all DAQ system JSON configs for a given lab.""" + import ndi.ndi_common + + common_dir = Path(ndi.ndi_common.__path__[0]) + lab_dir = common_dir / "daq_systems" / lab_name + + if not lab_dir.exists(): + raise FileNotFoundError(f"No DAQ system configs found for lab: {lab_name}") + + configs = [] + for json_file in sorted(lab_dir.glob("*.json")): + with open(json_file, encoding="utf-8") as f: + configs.append(json.load(f)) + return configs + + +def lab(session, lab_name: str) -> None: + """Add DAQ system documents to a session based on lab JSON configs. + + For each JSON config in ``ndi_common/daq_systems//``, creates: + + - A ``daq/filenavigator`` document + - A ``daq/daqreader`` document + - Optionally a ``daq/daqmetadatareader`` document + - A ``daq/daqsystem`` document linking them together + + Parameters + ---------- + session : ndi.session.session_base + The NDI session to add DAQ systems to. + lab_name : str + Name of the lab directory under ``ndi_common/daq_systems/`` + (e.g. ``"vhlab"``, ``"marderlab"``, ``"kjnielsenlab"``). + """ + configs = _find_daq_configs(lab_name) + + for config in configs: + name = config["Name"] + file_params = config.get("FileParameters", []) + epm_class = config.get("EpochProbeMapClass", "ndi.epoch.epochprobemap_daqsystem") + epm_file_params = config.get("EpochProbeMapFileParameters", "") + reader_class = config.get("DaqReaderClass", "ndi.daq.reader.mfdaq") + system_class = config.get("DaqSystemClass", "ndi.daq.system.mfdaq") + metadata_reader_class = config.get("MetadataReaderClass", []) + + # Convert file parameters to MATLAB cell string format + fp_str = _to_matlab_cell_str(file_params) + + # EpochProbeMapFileParameters may be a string or list + if isinstance(epm_file_params, list): + epm_fp_str = _to_matlab_cell_str(epm_file_params) + else: + epm_fp_str = f"{{ '{epm_file_params}' }}" if epm_file_params else "" + + # Create filenavigator document + fn_doc = session.newdocument( + "daq/filenavigator", + **{ + "base.name": name, + "filenavigator.ndi_filenavigator_class": "ndi.file.navigator", + "filenavigator.fileparameters": fp_str, + "filenavigator.epochprobemap_class": epm_class, + "filenavigator.epochprobemap_fileparameters": epm_fp_str, + }, + ) + session.database_add(fn_doc) + + # Create daqreader document + dr_doc = session.newdocument( + "daq/daqreader", + **{ + "base.name": name, + "daqreader.ndi_daqreader_class": reader_class, + }, + ) + session.database_add(dr_doc) + + # Create daqsystem document + daq_doc = session.newdocument( + "daq/daqsystem", + **{ + "base.name": name, + "daqsystem.ndi_daqsystem_class": system_class, + }, + ) + daq_doc = daq_doc.set_dependency_value( + "filenavigator_id", fn_doc.id, error_if_not_found=False + ) + daq_doc = daq_doc.set_dependency_value( + "daqreader_id", dr_doc.id, error_if_not_found=False + ) + + # Create daqmetadatareader document if configured + if metadata_reader_class and metadata_reader_class != []: + mr_class = ( + metadata_reader_class + if isinstance(metadata_reader_class, str) + else metadata_reader_class[0] + ) + mr_doc = session.newdocument( + "daq/daqmetadatareader", + **{ + "base.name": name, + "daqmetadatareader.ndi_daqmetadatareader_class": mr_class, + }, + ) + session.database_add(mr_doc) + daq_doc = daq_doc.add_dependency_value_n( + "daqmetadatareader_id", mr_doc.id + ) + + session.database_add(daq_doc) diff --git a/tests/symmetry/make_artifacts/session/_lab_setup.py b/tests/symmetry/make_artifacts/session/_lab_setup.py index 1d193d7..55a3ff4 100644 --- a/tests/symmetry/make_artifacts/session/_lab_setup.py +++ b/tests/symmetry/make_artifacts/session/_lab_setup.py @@ -1,133 +1,11 @@ -"""Shared helper for creating blank lab sessions in symmetry tests. +"""Thin wrapper that re-exports ``ndi.setup.lab`` for backwards compatibility. -Reads DAQ system configuration JSON files from -``ndi_common/daq_systems//`` and creates the corresponding -filenavigator, daqreader, (optionally daqmetadatareader), and daqsystem -documents in the given session. +Tests should prefer importing from ``ndi.setup`` directly:: -This mirrors the MATLAB ``ndi.setup.lab()`` function. + import ndi + ndi.setup.lab(session, "vhlab") """ -from __future__ import annotations +from ndi.setup.lab import lab as setup_lab_daq_systems -import json -from pathlib import Path - - -def _to_matlab_cell_str(items: list[str]) -> str: - """Convert a list of strings to MATLAB cell-array syntax. - - Example: ``['#.rhd']`` becomes ``"{ '#.rhd' }"``. - """ - if not items: - return "" - quoted = ", ".join(f"'{s}'" for s in items) - return f"{{ {quoted} }}" - - -def _find_daq_configs(lab_name: str) -> list[dict]: - """Read all DAQ system JSON configs for a given lab.""" - import ndi.ndi_common - - common_dir = Path(ndi.ndi_common.__path__[0]) - lab_dir = common_dir / "daq_systems" / lab_name - - if not lab_dir.exists(): - raise FileNotFoundError(f"No DAQ system configs found for lab: {lab_name}") - - configs = [] - for json_file in sorted(lab_dir.glob("*.json")): - with open(json_file, encoding="utf-8") as f: - configs.append(json.load(f)) - return configs - - -def setup_lab_daq_systems(session, lab_name: str) -> None: - """Add DAQ system documents to a session based on lab JSON configs. - - For each JSON config in ``ndi_common/daq_systems//``, creates: - - A ``daq/filenavigator`` document - - A ``daq/daqreader`` document - - Optionally a ``daq/daqmetadatareader`` document - - A ``daq/daqsystem`` document linking them together - - This is the Python equivalent of MATLAB's ``ndi.setup.lab(lab_name, ...)``. - """ - configs = _find_daq_configs(lab_name) - - for config in configs: - name = config["Name"] - file_params = config.get("FileParameters", []) - epm_class = config.get("EpochProbeMapClass", "ndi.epoch.epochprobemap_daqsystem") - epm_file_params = config.get("EpochProbeMapFileParameters", "") - reader_class = config.get("DaqReaderClass", "ndi.daq.reader.mfdaq") - system_class = config.get("DaqSystemClass", "ndi.daq.system.mfdaq") - metadata_reader_class = config.get("MetadataReaderClass", []) - - # Convert file parameters to MATLAB cell string format - fp_str = _to_matlab_cell_str(file_params) - - # EpochProbeMapFileParameters may be a string or list - if isinstance(epm_file_params, list): - epm_fp_str = _to_matlab_cell_str(epm_file_params) - else: - epm_fp_str = f"{{ '{epm_file_params}' }}" if epm_file_params else "" - - # Create filenavigator document - fn_doc = session.newdocument( - "daq/filenavigator", - **{ - "base.name": name, - "filenavigator.ndi_filenavigator_class": "ndi.file.navigator", - "filenavigator.fileparameters": fp_str, - "filenavigator.epochprobemap_class": epm_class, - "filenavigator.epochprobemap_fileparameters": epm_fp_str, - }, - ) - session.database_add(fn_doc) - - # Create daqreader document - dr_doc = session.newdocument( - "daq/daqreader", - **{ - "base.name": name, - "daqreader.ndi_daqreader_class": reader_class, - }, - ) - session.database_add(dr_doc) - - # Create daqsystem document - daq_doc = session.newdocument( - "daq/daqsystem", - **{ - "base.name": name, - "daqsystem.ndi_daqsystem_class": system_class, - }, - ) - daq_doc = daq_doc.set_dependency_value( - "filenavigator_id", fn_doc.id, error_if_not_found=False - ) - daq_doc = daq_doc.set_dependency_value( - "daqreader_id", dr_doc.id, error_if_not_found=False - ) - - # Create daqmetadatareader document if configured - if metadata_reader_class and metadata_reader_class != []: - mr_class = ( - metadata_reader_class - if isinstance(metadata_reader_class, str) - else metadata_reader_class[0] - ) - mr_doc = session.newdocument( - "daq/daqmetadatareader", - **{ - "base.name": name, - "daqmetadatareader.ndi_daqmetadatareader_class": mr_class, - }, - ) - session.database_add(mr_doc) - daq_doc = daq_doc.add_dependency_value_n( - "daqmetadatareader_id", mr_doc.id - ) - - session.database_add(daq_doc) +__all__ = ["setup_lab_daq_systems"] diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py index a102d74..d8c161d 100644 --- a/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py +++ b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py @@ -24,7 +24,7 @@ from ndi.session.dir import ndi_session_dir from ndi.util import sessionSummary from tests.symmetry.conftest import PYTHON_ARTIFACTS -from tests.symmetry.make_artifacts.session._lab_setup import setup_lab_daq_systems +import ndi.setup ARTIFACT_DIR = ( PYTHON_ARTIFACTS @@ -47,7 +47,7 @@ def _setup(self, tmp_path): session.database_clear("yes") session.cache.clear() - setup_lab_daq_systems(session, "kjnielsenlab") + ndi.setup.lab(session, "kjnielsenlab") self.session = session diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py index 2c6d2af..62db073 100644 --- a/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py +++ b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py @@ -24,7 +24,7 @@ from ndi.session.dir import ndi_session_dir from ndi.util import sessionSummary from tests.symmetry.conftest import PYTHON_ARTIFACTS -from tests.symmetry.make_artifacts.session._lab_setup import setup_lab_daq_systems +import ndi.setup ARTIFACT_DIR = ( PYTHON_ARTIFACTS @@ -47,7 +47,7 @@ def _setup(self, tmp_path): session.database_clear("yes") session.cache.clear() - setup_lab_daq_systems(session, "marderlab") + ndi.setup.lab(session, "marderlab") self.session = session diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py index e7d5909..21be0f8 100644 --- a/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py +++ b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py @@ -24,7 +24,7 @@ from ndi.session.dir import ndi_session_dir from ndi.util import sessionSummary from tests.symmetry.conftest import PYTHON_ARTIFACTS -from tests.symmetry.make_artifacts.session._lab_setup import setup_lab_daq_systems +import ndi.setup ARTIFACT_DIR = ( PYTHON_ARTIFACTS / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" @@ -44,7 +44,7 @@ def _setup(self, tmp_path): session.database_clear("yes") session.cache.clear() - setup_lab_daq_systems(session, "vhlab") + ndi.setup.lab(session, "vhlab") self.session = session From 5ec4e4a3c4e6a370b9929a0dc4ddff6b8dac24db Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 17:46:29 +0000 Subject: [PATCH 03/15] Fix filenavigator_class and daqreader_class for MATLAB symmetry - Add NDI_FILENAVIGATOR_CLASS = "ndi.file.navigator.epochdir" to the epochdir navigator class - Register ndi_file_navigator_epochdir in the class registry - Use HasEpochDirectories from JSON config to pick the correct filenavigator class in ndi.setup.lab() - Use class registry to reconstruct the correct navigator subclass when loading DAQ systems from documents - Fall back to a generic daqreader with the stored class name when the specific reader class is not in the registry https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/class_registry.py | 4 +++- src/ndi/daq/system.py | 29 +++++++++++++++++++++++++++-- src/ndi/file/navigator/epochdir.py | 2 ++ src/ndi/setup/lab.py | 8 +++++++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index bde135c..060e70c 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -34,6 +34,7 @@ def _build_registry() -> dict[str, type]: from .daq.system import ndi_daq_system from .element import ndi_element from .file.navigator import ndi_file_navigator + from .file.navigator.epochdir import ndi_file_navigator_epochdir from .probe import ndi_probe from .probe.timeseries import ndi_probe_timeseries from .probe.timeseries_mfdaq import ndi_probe_timeseries_mfdaq @@ -63,8 +64,9 @@ def _build_registry() -> dict[str, type]: # DAQ system registry[ndi_daq_system.NDI_DAQSYSTEM_CLASS] = ndi_daq_system - # File navigator + # File navigators registry[ndi_file_navigator.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator + registry[ndi_file_navigator_epochdir.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_epochdir return registry diff --git a/src/ndi/daq/system.py b/src/ndi/daq/system.py index 22a37b8..66b3e8b 100644 --- a/src/ndi/daq/system.py +++ b/src/ndi/daq/system.py @@ -217,15 +217,40 @@ def _load_from_document(self, session: Any, document: Any) -> None: "Could not reconstruct DAQ reader %s: %s", reader_class_name, exc ) else: + # Unknown reader class — create a generic reader that preserves + # the MATLAB-compatible class name for symmetry testing. logger.debug("Unknown DAQ reader class: %s", reader_class_name) + if reader_class_name: + from .reader_base import ndi_daq_reader + + try: + reader = ndi_daq_reader(session=session, document=reader_doc) + reader.NDI_DAQREADER_CLASS = reader_class_name + self._daqreader = reader + except Exception: + pass # Reconstruct file navigator from its document self._filenavigator = None if len(nav_docs) == 1: - from ..file.navigator import ndi_file_navigator + nav_doc = nav_docs[0] + nav_class_name = "" + nav_props = nav_doc.document_properties + if isinstance(nav_props, dict): + nav_class_name = nav_props.get("filenavigator", {}).get( + "ndi_filenavigator_class", "" + ) + + from ..class_registry import get_class as get_nav_class + + NavCls = get_nav_class(nav_class_name) + if NavCls is None: + from ..file.navigator import ndi_file_navigator + + NavCls = ndi_file_navigator try: - self._filenavigator = ndi_file_navigator(session=session, document=nav_docs[0]) + self._filenavigator = NavCls(session=session, document=nav_doc) except Exception as exc: logger.warning("Could not reconstruct file navigator: %s", exc) diff --git a/src/ndi/file/navigator/epochdir.py b/src/ndi/file/navigator/epochdir.py index 503cc6c..28ae344 100644 --- a/src/ndi/file/navigator/epochdir.py +++ b/src/ndi/file/navigator/epochdir.py @@ -32,6 +32,8 @@ class ndi_file_navigator_epochdir(ndi_file_navigator): >>> # trial_002/file.rhd -> epoch 2 """ + NDI_FILENAVIGATOR_CLASS = "ndi.file.navigator.epochdir" + def epochid( self, epoch_number: int, diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py index 83b5117..468f366 100644 --- a/src/ndi/setup/lab.py +++ b/src/ndi/setup/lab.py @@ -76,6 +76,12 @@ def lab(session, lab_name: str) -> None: reader_class = config.get("DaqReaderClass", "ndi.daq.reader.mfdaq") system_class = config.get("DaqSystemClass", "ndi.daq.system.mfdaq") metadata_reader_class = config.get("MetadataReaderClass", []) + has_epoch_dirs = config.get("HasEpochDirectories", False) + + # Choose the correct filenavigator class + filenavigator_class = ( + "ndi.file.navigator.epochdir" if has_epoch_dirs else "ndi.file.navigator" + ) # Convert file parameters to MATLAB cell string format fp_str = _to_matlab_cell_str(file_params) @@ -91,7 +97,7 @@ def lab(session, lab_name: str) -> None: "daq/filenavigator", **{ "base.name": name, - "filenavigator.ndi_filenavigator_class": "ndi.file.navigator", + "filenavigator.ndi_filenavigator_class": filenavigator_class, "filenavigator.fileparameters": fp_str, "filenavigator.epochprobemap_class": epm_class, "filenavigator.epochprobemap_fileparameters": epm_fp_str, From 6b0dca9af4b9599551c29a8936436c7ac45e7057 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 19:39:54 +0000 Subject: [PATCH 04/15] Port ndi.setup.daq.reader.mfdaq.stimulus readers (vhlabvisspike2, nielsenvisintan) Add lab-specific stimulus DAQ readers: - vhlabvisspike2: extends CED Spike2 reader for VH Lab visual stimulus data - nielsenvisintan: extends Intan reader for Nielsen Lab visual stimulus data Both are registered in the class registry so DAQ systems can reconstruct them from documents, fixing the daqreader_class symmetry mismatches. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/class_registry.py | 8 ++++ src/ndi/setup/daq/__init__.py | 1 + src/ndi/setup/daq/reader/__init__.py | 1 + src/ndi/setup/daq/reader/mfdaq/__init__.py | 1 + .../daq/reader/mfdaq/stimulus/__init__.py | 15 +++++++ .../reader/mfdaq/stimulus/nielsenvisintan.py | 41 ++++++++++++++++++ .../reader/mfdaq/stimulus/vhlabvisspike2.py | 42 +++++++++++++++++++ 7 files changed, 109 insertions(+) create mode 100644 src/ndi/setup/daq/__init__.py create mode 100644 src/ndi/setup/daq/reader/__init__.py create mode 100644 src/ndi/setup/daq/reader/mfdaq/__init__.py create mode 100644 src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py create mode 100644 src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py create mode 100644 src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index 060e70c..9d88171 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -32,6 +32,12 @@ def _build_registry() -> dict[str, type]: from .daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan from .daq.reader.mfdaq.spikegadgets import ndi_daq_reader_mfdaq_spikegadgets from .daq.system import ndi_daq_system + from .setup.daq.reader.mfdaq.stimulus.nielsenvisintan import ( + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, + ) + from .setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 import ( + ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, + ) from .element import ndi_element from .file.navigator import ndi_file_navigator from .file.navigator.epochdir import ndi_file_navigator_epochdir @@ -58,6 +64,8 @@ def _build_registry() -> dict[str, type]: ndi_daq_reader_mfdaq_blackrock, ndi_daq_reader_mfdaq_cedspike2, ndi_daq_reader_mfdaq_spikegadgets, + ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, ): registry[cls.NDI_DAQREADER_CLASS] = cls diff --git a/src/ndi/setup/daq/__init__.py b/src/ndi/setup/daq/__init__.py new file mode 100644 index 0000000..a001625 --- /dev/null +++ b/src/ndi/setup/daq/__init__.py @@ -0,0 +1 @@ +"""ndi.setup.daq — Lab-specific DAQ extensions.""" diff --git a/src/ndi/setup/daq/reader/__init__.py b/src/ndi/setup/daq/reader/__init__.py new file mode 100644 index 0000000..5d6d7c9 --- /dev/null +++ b/src/ndi/setup/daq/reader/__init__.py @@ -0,0 +1 @@ +"""ndi.setup.daq.reader — Lab-specific DAQ reader extensions.""" diff --git a/src/ndi/setup/daq/reader/mfdaq/__init__.py b/src/ndi/setup/daq/reader/mfdaq/__init__.py new file mode 100644 index 0000000..f60f048 --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/__init__.py @@ -0,0 +1 @@ +"""ndi.setup.daq.reader.mfdaq — Lab-specific MFDAQ reader extensions.""" diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py new file mode 100644 index 0000000..e389f0e --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py @@ -0,0 +1,15 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus — Stimulus-specific DAQ readers. + +Lab-specific DAQ reader variants that handle stimulus presentation +data in addition to standard electrophysiology recordings. + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/`` +""" + +from .nielsenvisintan import ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan +from .vhlabvisspike2 import ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2 + +__all__ = [ + "ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2", + "ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan", +] diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py new file mode 100644 index 0000000..9c5ce94 --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py @@ -0,0 +1,41 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisintan — Nielsen Lab visual stimulus Intan reader. + +Extends the Intan RHD reader with support for Nielsen Lab visual +stimulus files (.analyzer). + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/nielsenvisintan.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``#_info.rhd`` — Intan recording info header +- ``#_digitalin.dat`` — digital input data +- ``#.analyzer`` — Nielsen Lab Analyzer stimulus structure +- ``epochprobemap.txt`` — epoch-to-probe mapping + +Stimulus metadata is read by the companion +:class:`~ndi.daq.metadatareader.nielsenlab_stims.ndi_daq_metadatareader_NielsenLabStims`. +""" + +from __future__ import annotations + +import logging + +from ndi.daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan(ndi_daq_reader_mfdaq_intan): + """Nielsen Lab visual stimulus reader built on Intan RHD. + + Inherits all channel reading and sample-rate logic from the Intan + reader. The stimulus-specific behaviour (extracting Analyzer + parameters) is handled by the metadata reader + ``ndi.daq.metadatareader.NielsenLabStims`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisintan" + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan(id={self.id[:8]}...)" diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py new file mode 100644 index 0000000..6b1227f --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py @@ -0,0 +1,42 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 — VH Lab visual stimulus Spike2 reader. + +Extends the CED Spike2 reader with support for VH Lab visual stimulus +files (stims.mat, stimtimes.txt, verticalblanking.txt). + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/vhlabvisspike2.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``reference.txt`` — epoch reference file +- ``stimtimes.txt`` — stimulus timing information +- ``verticalblanking.txt`` — vertical blanking timestamps +- ``stims.mat`` — NewStim stimulus script parameters +- ``spike2data.smr`` — CED Spike2 electrophysiology data + +Stimulus metadata is read by the companion +:class:`~ndi.daq.metadatareader.newstim_stims.ndi_daq_metadatareader_NewStimStims`. +""" + +from __future__ import annotations + +import logging + +from ndi.daq.reader.mfdaq.cedspike2 import ndi_daq_reader_mfdaq_cedspike2 + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2(ndi_daq_reader_mfdaq_cedspike2): + """VH Lab visual stimulus reader built on CED Spike2. + + Inherits all channel reading and sample-rate logic from the CED Spike2 + reader. The stimulus-specific behaviour (extracting stim parameters, + timing alignment) is handled by the metadata reader + ``ndi.daq.metadatareader.NewStimStims`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.vhlabvisspike2" + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2(id={self.id[:8]}...)" From 747c05a6986bbc9c93def3e4664bf7bca76fc44e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 19:41:44 +0000 Subject: [PATCH 05/15] Raise errors instead of falling back to generic classes for unknown DAQ readers/navigators When a DAQ reader or file navigator class is not found in the registry, raise ValueError instead of silently falling back to a base class. This ensures missing class implementations are caught immediately. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/daq/system.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/ndi/daq/system.py b/src/ndi/daq/system.py index 66b3e8b..b88d785 100644 --- a/src/ndi/daq/system.py +++ b/src/ndi/daq/system.py @@ -209,26 +209,17 @@ def _load_from_document(self, session: Any, document: Any) -> None: from ..class_registry import get_class ReaderCls = get_class(reader_class_name) - if ReaderCls is not None: - try: - self._daqreader = ReaderCls(session=session, document=reader_doc) - except Exception as exc: - logger.warning( - "Could not reconstruct DAQ reader %s: %s", reader_class_name, exc - ) - else: - # Unknown reader class — create a generic reader that preserves - # the MATLAB-compatible class name for symmetry testing. - logger.debug("Unknown DAQ reader class: %s", reader_class_name) - if reader_class_name: - from .reader_base import ndi_daq_reader - - try: - reader = ndi_daq_reader(session=session, document=reader_doc) - reader.NDI_DAQREADER_CLASS = reader_class_name - self._daqreader = reader - except Exception: - pass + if ReaderCls is None: + raise ValueError( + f"Unknown DAQ reader class: {reader_class_name!r}. " + f"Register it in ndi.class_registry." + ) + try: + self._daqreader = ReaderCls(session=session, document=reader_doc) + except Exception as exc: + raise RuntimeError( + f"Could not reconstruct DAQ reader {reader_class_name!r}: {exc}" + ) from exc # Reconstruct file navigator from its document self._filenavigator = None @@ -245,14 +236,16 @@ def _load_from_document(self, session: Any, document: Any) -> None: NavCls = get_nav_class(nav_class_name) if NavCls is None: - from ..file.navigator import ndi_file_navigator - - NavCls = ndi_file_navigator - + raise ValueError( + f"Unknown file navigator class: {nav_class_name!r}. " + f"Register it in ndi.class_registry." + ) try: self._filenavigator = NavCls(session=session, document=nav_doc) except Exception as exc: - logger.warning("Could not reconstruct file navigator: %s", exc) + raise RuntimeError( + f"Could not reconstruct file navigator {nav_class_name!r}: {exc}" + ) from exc @property def name(self) -> str: From 92b083d525d86fd56788e7e3b5ee825f595d0ba7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 19:46:39 +0000 Subject: [PATCH 06/15] Fix lint and remove element/probe fallback to base classes - Fix ruff import sorting in class_registry.py - Fix black formatting in setup/lab.py and session_base.py - Remove element/probe fallback in _document_to_object: raise ValueError for unknown element classes instead of silently falling back to ndi_element or ndi_probe https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/class_registry.py | 12 ++++++------ src/ndi/session/session_base.py | 16 +++++----------- src/ndi/setup/lab.py | 8 ++------ 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index 9d88171..e03194b 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -32,12 +32,6 @@ def _build_registry() -> dict[str, type]: from .daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan from .daq.reader.mfdaq.spikegadgets import ndi_daq_reader_mfdaq_spikegadgets from .daq.system import ndi_daq_system - from .setup.daq.reader.mfdaq.stimulus.nielsenvisintan import ( - ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, - ) - from .setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 import ( - ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, - ) from .element import ndi_element from .file.navigator import ndi_file_navigator from .file.navigator.epochdir import ndi_file_navigator_epochdir @@ -45,6 +39,12 @@ def _build_registry() -> dict[str, type]: from .probe.timeseries import ndi_probe_timeseries from .probe.timeseries_mfdaq import ndi_probe_timeseries_mfdaq from .probe.timeseries_stimulator import ndi_probe_timeseries_stimulator + from .setup.daq.reader.mfdaq.stimulus.nielsenvisintan import ( + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, + ) + from .setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 import ( + ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, + ) registry: dict[str, type] = {} diff --git a/src/ndi/session/session_base.py b/src/ndi/session/session_base.py index 3315534..de53380 100644 --- a/src/ndi/session/session_base.py +++ b/src/ndi/session/session_base.py @@ -1123,17 +1123,11 @@ def _document_to_object(self, document: ndi_document) -> Any: props = document.document_properties ndi_class = props.get("element", {}).get("ndi_element_class", "") cls = get_class(ndi_class) - if cls is not None: - return cls(session=self, document=document) - # Fallback: if ndi_element_class contains "probe" but - # isn't a known key (e.g. "ndi.probe.timage"), use ndi_probe. - from ..probe import ndi_probe - - if "probe" in ndi_class: - return ndi_probe(session=self, document=document) - from ..element import ndi_element - - return ndi_element(session=self, document=document) + if cls is None: + raise ValueError( + f"Unknown element class: {ndi_class!r}. " f"Register it in ndi.class_registry." + ) + return cls(session=self, document=document) if document.doc_isa("syncgraph"): return ndi_time_syncgraph(session=self, document=document) diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py index 468f366..6c36919 100644 --- a/src/ndi/setup/lab.py +++ b/src/ndi/setup/lab.py @@ -126,9 +126,7 @@ def lab(session, lab_name: str) -> None: daq_doc = daq_doc.set_dependency_value( "filenavigator_id", fn_doc.id, error_if_not_found=False ) - daq_doc = daq_doc.set_dependency_value( - "daqreader_id", dr_doc.id, error_if_not_found=False - ) + daq_doc = daq_doc.set_dependency_value("daqreader_id", dr_doc.id, error_if_not_found=False) # Create daqmetadatareader document if configured if metadata_reader_class and metadata_reader_class != []: @@ -145,8 +143,6 @@ def lab(session, lab_name: str) -> None: }, ) session.database_add(mr_doc) - daq_doc = daq_doc.add_dependency_value_n( - "daqmetadatareader_id", mr_doc.id - ) + daq_doc = daq_doc.add_dependency_value_n("daqmetadatareader_id", mr_doc.id) session.database_add(daq_doc) From 724c8d592ea4bae2b9c08d0b029dc5f6169fa66e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 19:48:32 +0000 Subject: [PATCH 07/15] Fix black and ruff formatting across all modified files https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/__init__.py | 15 ++++++++++++++- .../session/test_blank_session_kjnielsenlab.py | 7 ++----- .../session/test_blank_session_marderlab.py | 9 ++------- .../session/test_blank_session_vhlab.py | 6 ++---- .../session/test_blank_session_kjnielsenlab.py | 7 ++++--- .../session/test_blank_session_marderlab.py | 7 ++++--- .../session/test_blank_session_vhlab.py | 13 +++++-------- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/ndi/__init__.py b/src/ndi/__init__.py index fb69954..889785b 100644 --- a/src/ndi/__init__.py +++ b/src/ndi/__init__.py @@ -32,7 +32,20 @@ # Import epoch module (Phase 6) # Import Phase 10: Cloud API client # Import Phase 11: Schema validation -from . import calc, cloud, common, daq, epoch, file, session, setup, time, util, validate, validators +from . import ( + calc, + cloud, + common, + daq, + epoch, + file, + session, + setup, + time, + util, + validate, + validators, +) # Import Phase 9: ndi_app framework and calculators from .app import ndi_app diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py index d8c161d..2203944 100644 --- a/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py +++ b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py @@ -20,17 +20,14 @@ import pytest +import ndi.setup from ndi.query import ndi_query from ndi.session.dir import ndi_session_dir from ndi.util import sessionSummary from tests.symmetry.conftest import PYTHON_ARTIFACTS -import ndi.setup ARTIFACT_DIR = ( - PYTHON_ARTIFACTS - / "session" - / "blankSessionKjnielsenlab" - / "testBlankSessionKjnielsenlab" + PYTHON_ARTIFACTS / "session" / "blankSessionKjnielsenlab" / "testBlankSessionKjnielsenlab" ) diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py index 62db073..52ac1f7 100644 --- a/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py +++ b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py @@ -20,18 +20,13 @@ import pytest +import ndi.setup from ndi.query import ndi_query from ndi.session.dir import ndi_session_dir from ndi.util import sessionSummary from tests.symmetry.conftest import PYTHON_ARTIFACTS -import ndi.setup -ARTIFACT_DIR = ( - PYTHON_ARTIFACTS - / "session" - / "blankSessionMarderlab" - / "testBlankSessionMarderlab" -) +ARTIFACT_DIR = PYTHON_ARTIFACTS / "session" / "blankSessionMarderlab" / "testBlankSessionMarderlab" class TestBlankSessionMarderlab: diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py index 21be0f8..3b00191 100644 --- a/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py +++ b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py @@ -20,15 +20,13 @@ import pytest +import ndi.setup from ndi.query import ndi_query from ndi.session.dir import ndi_session_dir from ndi.util import sessionSummary from tests.symmetry.conftest import PYTHON_ARTIFACTS -import ndi.setup -ARTIFACT_DIR = ( - PYTHON_ARTIFACTS / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" -) +ARTIFACT_DIR = PYTHON_ARTIFACTS / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" class TestBlankSessionVhlab: diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py index 89bcb46..2c96c35 100644 --- a/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py +++ b/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py @@ -69,7 +69,8 @@ def test_blank_session_kjnielsenlab_summary(self, source_type): excludeFiles=["sessionSummary.json", "jsonDocuments"], ) - assert len(report) == 0, ( - f"Session summary mismatch against {source_type} generated artifacts:\n" - + "\n".join(report) + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py index 785f945..b38a0cb 100644 --- a/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py +++ b/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py @@ -67,7 +67,8 @@ def test_blank_session_marderlab_summary(self, source_type): excludeFiles=["sessionSummary.json", "jsonDocuments"], ) - assert len(report) == 0, ( - f"Session summary mismatch against {source_type} generated artifacts:\n" - + "\n".join(report) + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py index 8bf27e6..938c925 100644 --- a/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py +++ b/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py @@ -31,11 +31,7 @@ class TestBlankSessionVhlab: def _artifact_dir(self, source_type: str): return ( - SYMMETRY_BASE - / source_type - / "session" - / "blankSessionVhlab" - / "testBlankSessionVhlab" + SYMMETRY_BASE / source_type / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" ) def _open_session(self, source_type): @@ -67,7 +63,8 @@ def test_blank_session_vhlab_summary(self, source_type): excludeFiles=["sessionSummary.json", "jsonDocuments"], ) - assert len(report) == 0, ( - f"Session summary mismatch against {source_type} generated artifacts:\n" - + "\n".join(report) + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report ) From 919f644615469c3ac07bc50e664a6fdcb19468ed Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 20:02:50 +0000 Subject: [PATCH 08/15] Fix sessionSummary dropping DAQ systems with unimplemented readers sessionSummary previously relied on daqsystem_load() which silently drops DAQ systems whose reader classes are not implemented in Python (e.g. ndi.daq.reader.mfdaq.ndr). This caused length mismatches when comparing against MATLAB artifacts that include those systems. Now queries all daqsystem documents directly from the database, tries full object reconstruction first, and falls back to extracting class info from raw document properties for unimplemented readers. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/util/session_summary.py | 127 ++++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 37 deletions(-) diff --git a/src/ndi/util/session_summary.py b/src/ndi/util/session_summary.py index ec0c457..f9373e1 100644 --- a/src/ndi/util/session_summary.py +++ b/src/ndi/util/session_summary.py @@ -15,6 +15,74 @@ from .classname import ndi_matlab_classname +def _daq_detail_from_object(sys: Any) -> dict[str, Any]: + """Extract DAQ detail dict from a fully-loaded DAQ system object.""" + details: dict[str, Any] = {} + + fn = getattr(sys, "filenavigator", None) + if fn is not None: + details["filenavigator_class"] = getattr( + fn, "NDI_FILENAVIGATOR_CLASS", type(fn).__qualname__ + ) + try: + details["epochNodes_filenavigator"] = fn.epochnodes() + except Exception: + details["epochNodes_filenavigator"] = [] + else: + details["filenavigator_class"] = "" + details["epochNodes_filenavigator"] = [] + + dr = getattr(sys, "daqreader", None) + if dr is not None: + details["daqreader_class"] = getattr(dr, "NDI_DAQREADER_CLASS", ndi_matlab_classname(dr)) + else: + details["daqreader_class"] = "" + + try: + details["epochNodes_daqsystem"] = sys.epochnodes() + except Exception: + details["epochNodes_daqsystem"] = [] + + return details + + +def _daq_detail_from_document(session_obj: Any, doc: Any) -> dict[str, Any]: + """Extract DAQ detail dict from a raw database document. + + Used as a fallback when the DAQ system object cannot be fully + reconstructed (e.g. reader class not implemented in Python). + """ + from ..query import ndi_query + + details: dict[str, Any] = {} + + # Look up the reader document via dependency + reader_class = "" + reader_id = doc.dependency_value("daqreader_id", error_if_not_found=False) + if reader_id: + reader_docs = session_obj.database_search(ndi_query("base.id") == reader_id) + if reader_docs: + props = reader_docs[0].document_properties + if isinstance(props, dict): + reader_class = props.get("daqreader", {}).get("ndi_daqreader_class", "") + details["daqreader_class"] = reader_class + + # Look up the navigator document via dependency + nav_class = "" + nav_id = doc.dependency_value("filenavigator_id", error_if_not_found=False) + if nav_id: + nav_docs = session_obj.database_search(ndi_query("base.id") == nav_id) + if nav_docs: + props = nav_docs[0].document_properties + if isinstance(props, dict): + nav_class = props.get("filenavigator", {}).get("ndi_filenavigator_class", "") + details["filenavigator_class"] = nav_class + details["epochNodes_filenavigator"] = [] + details["epochNodes_daqsystem"] = [] + + return details + + def sessionSummary(session_obj: Any) -> dict[str, Any]: """Create a summary structure of an ndi.session object. @@ -27,6 +95,8 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: Dict with keys: reference, sessionId, files, filesInDotNDI, daqSystemNames, daqSystemDetails, probes. """ + from ..query import ndi_query + summary: dict[str, Any] = {} # 1. ndi_session basics @@ -49,51 +119,34 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: else: summary["filesInDotNDI"] = [] - # 4. DAQ Systems - daqs = session_obj.daqsystem_load(name="(.*)") - if daqs is None: - daqs = [] - elif not isinstance(daqs, list): - daqs = [daqs] + # 4. DAQ Systems — query all daqsystem documents from the database, + # then try full object reconstruction. Fall back to reading raw + # document properties for systems whose reader/navigator classes + # are not yet implemented in Python. + q = ndi_query("").isa("daqsystem") & (ndi_query("base.session_id") == session_obj.id()) + all_docs = session_obj.database_search(q) daq_names: list[str] = [] daq_details: list[dict[str, Any]] = [] - for sys in daqs: - daq_names.append(getattr(sys, "name", "")) - - details: dict[str, Any] = {} - - # Get filenavigator class (use MATLAB-compatible name for symmetry) - fn = getattr(sys, "filenavigator", None) - if fn is not None: - details["filenavigator_class"] = getattr( - fn, "NDI_FILENAVIGATOR_CLASS", type(fn).__qualname__ - ) - try: - details["epochNodes_filenavigator"] = fn.epochnodes() - except Exception: - details["epochNodes_filenavigator"] = [] - else: - details["filenavigator_class"] = "" - details["epochNodes_filenavigator"] = [] - - # Get daqreader class (use MATLAB-compatible name for symmetry) - dr = getattr(sys, "daqreader", None) - if dr is not None: - details["daqreader_class"] = getattr( - dr, "NDI_DAQREADER_CLASS", ndi_matlab_classname(dr) - ) - else: - details["daqreader_class"] = "" + for doc in all_docs: + props = doc.document_properties + name = "" + if isinstance(props, dict): + name = props.get("base", {}).get("name", "") + daq_names.append(name) - # Get epoch nodes of daq system + # Try full reconstruction first try: - details["epochNodes_daqsystem"] = sys.epochnodes() + obj = session_obj._document_to_object(doc) + if obj is not None: + daq_details.append(_daq_detail_from_object(obj)) + continue except Exception: - details["epochNodes_daqsystem"] = [] + pass - daq_details.append(details) + # Fallback: extract what we can from the raw documents + daq_details.append(_daq_detail_from_document(session_obj, doc)) summary["daqSystemNames"] = daq_names summary["daqSystemDetails"] = daq_details From 725a3b403ca9e623aeaaa651dc5d9e2cf9fb6f7d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 20:14:35 +0000 Subject: [PATCH 09/15] Add missing DAQ reader classes and vhlab configs for MATLAB symmetry - Implement ndi.daq.reader.mfdaq.ndr reader (SpikeInterface wrapper for ABF files), fixing marderlab's marder_abf DAQ system reconstruction - Add vhtaste_bpod.json and vhtaste_sync.json configs to match MATLAB's 5 vhlab DAQ systems - Implement VHAudreyBPod stimulus reader and metadata reader for BPod behavioral task data - Register all new classes in class_registry.py - Revert sessionSummary fallback: DAQ systems that fail to reconstruct should surface as errors, not be silently papered over https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/class_registry.py | 6 + src/ndi/daq/metadatareader/__init__.py | 2 + .../daq/metadatareader/vhaudreybpod_stims.py | 107 +++++++++++++++ src/ndi/daq/reader/__init__.py | 2 + src/ndi/daq/reader/mfdaq/__init__.py | 2 + src/ndi/daq/reader/mfdaq/ndr.py | 66 +++++++++ .../daq_systems/vhlab/vhtaste_bpod.json | 15 +++ .../daq_systems/vhlab/vhtaste_sync.json | 17 +++ .../daq/reader/mfdaq/stimulus/__init__.py | 2 + .../daq/reader/mfdaq/stimulus/vhaudreybpod.py | 48 +++++++ src/ndi/util/session_summary.py | 127 +++++------------- 11 files changed, 304 insertions(+), 90 deletions(-) create mode 100644 src/ndi/daq/metadatareader/vhaudreybpod_stims.py create mode 100644 src/ndi/daq/reader/mfdaq/ndr.py create mode 100644 src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json create mode 100644 src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json create mode 100644 src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index e03194b..5c5e13a 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -30,6 +30,7 @@ def _build_registry() -> dict[str, type]: from .daq.reader.mfdaq.blackrock import ndi_daq_reader_mfdaq_blackrock from .daq.reader.mfdaq.cedspike2 import ndi_daq_reader_mfdaq_cedspike2 from .daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan + from .daq.reader.mfdaq.ndr import ndi_daq_reader_mfdaq_ndr from .daq.reader.mfdaq.spikegadgets import ndi_daq_reader_mfdaq_spikegadgets from .daq.system import ndi_daq_system from .element import ndi_element @@ -42,6 +43,9 @@ def _build_registry() -> dict[str, type]: from .setup.daq.reader.mfdaq.stimulus.nielsenvisintan import ( ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, ) + from .setup.daq.reader.mfdaq.stimulus.vhaudreybpod import ( + ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, + ) from .setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 import ( ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, ) @@ -63,9 +67,11 @@ def _build_registry() -> dict[str, type]: ndi_daq_reader_mfdaq_intan, ndi_daq_reader_mfdaq_blackrock, ndi_daq_reader_mfdaq_cedspike2, + ndi_daq_reader_mfdaq_ndr, ndi_daq_reader_mfdaq_spikegadgets, ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, + ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, ): registry[cls.NDI_DAQREADER_CLASS] = cls diff --git a/src/ndi/daq/metadatareader/__init__.py b/src/ndi/daq/metadatareader/__init__.py index ab270f0..c8c6a79 100644 --- a/src/ndi/daq/metadatareader/__init__.py +++ b/src/ndi/daq/metadatareader/__init__.py @@ -306,9 +306,11 @@ def __hash__(self) -> int: # Import subclass readers from .newstim_stims import ndi_daq_metadatareader_NewStimStims from .nielsenlab_stims import ndi_daq_metadatareader_NielsenLabStims +from .vhaudreybpod_stims import ndi_daq_metadatareader_VHAudreyBPod __all__ = [ "ndi_daq_metadatareader", "ndi_daq_metadatareader_NewStimStims", "ndi_daq_metadatareader_NielsenLabStims", + "ndi_daq_metadatareader_VHAudreyBPod", ] diff --git a/src/ndi/daq/metadatareader/vhaudreybpod_stims.py b/src/ndi/daq/metadatareader/vhaudreybpod_stims.py new file mode 100644 index 0000000..e7d61b0 --- /dev/null +++ b/src/ndi/daq/metadatareader/vhaudreybpod_stims.py @@ -0,0 +1,107 @@ +""" +ndi.daq.metadatareader.VHAudreyBPod - VH Lab Audrey BPod stimulus metadata reader. + +Reads stimulus parameters from BPod behavioral task summary log JSON files +used in the VH Lab taste experiments. + +MATLAB equivalent: src/ndi/+ndi/+daq/+metadatareader/VHAudreyBPod.m +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from ..metadatareader import ndi_daq_metadatareader + + +class ndi_daq_metadatareader_VHAudreyBPod(ndi_daq_metadatareader): + """ + Metadata reader for VH Lab Audrey BPod stimulus systems. + + Reads stimulus parameters from JSON summary log files produced by + BPod behavioral task systems. + + Example: + >>> reader = ndi_daq_metadatareader_VHAudreyBPod() + >>> params = reader.readmetadata(['triggers.tsv', 'summary_log.json']) + """ + + SUMMARY_FILE_PATTERN = r"_summary_log\.json$" + + def __init__( + self, + tsv_pattern: str = "", + identifier: str | None = None, + session: Any | None = None, + document: Any | None = None, + ): + super().__init__( + tsv_pattern=tsv_pattern, + identifier=identifier, + session=session, + document=document, + ) + + def readmetadata( + self, + epochfiles: list[str], + ) -> list[dict[str, Any]]: + """ + Read stimulus metadata from BPod summary log files. + + Args: + epochfiles: List of file paths for the epoch + + Returns: + List of parameter dictionaries + """ + # First try TSV-based reading from base class + if self._tab_separated_file_parameter: + try: + return super().readmetadata(epochfiles) + except (ValueError, FileNotFoundError): + pass + + # Look for summary_log.json + summary_file = self._find_summary_file(epochfiles) + if summary_file is None: + return [] + + return self._read_summary_json(summary_file) + + def _find_summary_file(self, epochfiles: list[str]) -> str | None: + """Find the BPod summary log JSON file in epoch files.""" + pattern = re.compile(self.SUMMARY_FILE_PATTERN, re.IGNORECASE) + for f in epochfiles: + if pattern.search(f): + return f + return None + + @staticmethod + def _read_summary_json(filepath: str) -> list[dict[str, Any]]: + """ + Read stimulus parameters from a BPod summary log JSON file. + + Args: + filepath: Path to the summary_log.json file + + Returns: + List of parameter dicts + """ + if not Path(filepath).is_file(): + return [] + + with open(filepath, encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, list): + return data + if isinstance(data, dict): + return [data] + return [] + + def __repr__(self) -> str: + return f"ndi_daq_metadatareader_VHAudreyBPod(id='{self.id[:8]}...')" diff --git a/src/ndi/daq/reader/__init__.py b/src/ndi/daq/reader/__init__.py index 62efa91..4942ae5 100644 --- a/src/ndi/daq/reader/__init__.py +++ b/src/ndi/daq/reader/__init__.py @@ -18,6 +18,7 @@ ndi_daq_reader_mfdaq_blackrock, ndi_daq_reader_mfdaq_cedspike2, ndi_daq_reader_mfdaq_intan, + ndi_daq_reader_mfdaq_ndr, ndi_daq_reader_mfdaq_spikegadgets, ) from .spikeinterface_adapter import ndi_daq_reader_SpikeInterfaceReader @@ -27,5 +28,6 @@ "ndi_daq_reader_mfdaq_intan", "ndi_daq_reader_mfdaq_blackrock", "ndi_daq_reader_mfdaq_cedspike2", + "ndi_daq_reader_mfdaq_ndr", "ndi_daq_reader_mfdaq_spikegadgets", ] diff --git a/src/ndi/daq/reader/mfdaq/__init__.py b/src/ndi/daq/reader/mfdaq/__init__.py index f20ddf5..af949d6 100644 --- a/src/ndi/daq/reader/mfdaq/__init__.py +++ b/src/ndi/daq/reader/mfdaq/__init__.py @@ -12,11 +12,13 @@ from .blackrock import ndi_daq_reader_mfdaq_blackrock from .cedspike2 import ndi_daq_reader_mfdaq_cedspike2 from .intan import ndi_daq_reader_mfdaq_intan +from .ndr import ndi_daq_reader_mfdaq_ndr from .spikegadgets import ndi_daq_reader_mfdaq_spikegadgets __all__ = [ "ndi_daq_reader_mfdaq_intan", "ndi_daq_reader_mfdaq_blackrock", "ndi_daq_reader_mfdaq_cedspike2", + "ndi_daq_reader_mfdaq_ndr", "ndi_daq_reader_mfdaq_spikegadgets", ] diff --git a/src/ndi/daq/reader/mfdaq/ndr.py b/src/ndi/daq/reader/mfdaq/ndr.py new file mode 100644 index 0000000..5f9c6da --- /dev/null +++ b/src/ndi/daq/reader/mfdaq/ndr.py @@ -0,0 +1,66 @@ +""" +ndi.daq.reader.mfdaq.ndr - NDR (Neuroscience Data Reader) wrapper. + +Thin wrapper around ndi_daq_reader_SpikeInterfaceReader for file formats +supported by the NDR library (e.g. Axon ABF files). + +MATLAB equivalent: src/ndi/+ndi/+daq/+reader/+mfdaq/ndr.m +""" + +from __future__ import annotations + +import logging + +from ...mfdaq import ChannelInfo, ndi_daq_reader_mfdaq + +logger = logging.getLogger(__name__) + + +class ndi_daq_reader_mfdaq_ndr(ndi_daq_reader_mfdaq): + """ + Reader for data files handled by the NDR library. + + Currently supports Axon ABF files via spikeinterface/neo. + + File extensions: .abf + """ + + NDI_DAQREADER_CLASS = "ndi.daq.reader.mfdaq.ndr" + FILE_EXTENSIONS = [".abf"] + + def __init__(self, identifier=None, session=None, document=None): + super().__init__(identifier=identifier, session=session, document=document) + self._ndi_daqreader_class = self.NDI_DAQREADER_CLASS + + def _get_si_reader(self): + try: + from ..spikeinterface_adapter import ndi_daq_reader_SpikeInterfaceReader + + return ndi_daq_reader_SpikeInterfaceReader + except ImportError: + return None + + def getchannelsepoch(self, epochfiles: list[str]) -> list[ChannelInfo]: + SI = self._get_si_reader() + if SI is None: + return [] + try: + return SI().getchannelsepoch(epochfiles) + except Exception as exc: + logger.warning("ndi_daq_reader_mfdaq_ndr.getchannelsepoch failed: %s", exc) + return [] + + def readchannels_epochsamples(self, channeltype, channel, epochfiles, s0, s1): + SI = self._get_si_reader() + if SI is None: + raise ImportError("spikeinterface required for reading NDR data") + return SI().readchannels_epochsamples(channeltype, channel, epochfiles, s0, s1) + + def samplerate(self, epochfiles, channeltype, channel): + SI = self._get_si_reader() + if SI is None: + raise ImportError("spikeinterface required for reading NDR data") + return SI().samplerate(epochfiles, channeltype, channel) + + def __repr__(self): + return f"ndi_daq_reader_mfdaq_ndr(id={self.id[:8]}...)" diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json new file mode 100644 index 0000000..1eaf263 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json @@ -0,0 +1,15 @@ +{ + "Name": "vhtaste_bpod", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod", + "MetadataReaderClass": "ndi.daq.metadatareader.VHAudreyBPod", + "EpochProbeMapClass": "ndi.setup.epoch.epochprobemap_daqsystem_vhlab", + "FileParameters": [ + "#_stimulus_triggers_log.tsv", + "#.*_summary_log.json" + ], + "DaqReaderFileParameters": [], + "MetadataReaderFileParameters": "(.*)\\.json", + "EpochProbeMapFileParameters": "#_stimulus_triggers_log.tsv", + "HasEpochDirectories": false +} diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json new file mode 100644 index 0000000..249aa45 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json @@ -0,0 +1,17 @@ +{ + "Name": "vhtaste_sync", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.daq.reader.mfdaq.intan", + "MetadataReaderClass": [], + "EpochProbeMapClass": "ndi.setup.epoch.epochprobemap_daqsystem_vhlab", + "FileParameters": [ + "reference.txt", + ".*\\.rhd\\>", + "vhintan_channelgrouping.txt", + "vhtaste_sync.txt" + ], + "DaqReaderFileParameters": [], + "MetadataReaderFileParameters": [], + "EpochProbeMapFileParameters": "vhtaste_sync.txt", + "HasEpochDirectories": true +} diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py index e389f0e..a3e5cdf 100644 --- a/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py @@ -7,9 +7,11 @@ """ from .nielsenvisintan import ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan +from .vhaudreybpod import ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod from .vhlabvisspike2 import ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2 __all__ = [ "ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2", "ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan", + "ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod", ] diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py new file mode 100644 index 0000000..52eab5e --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py @@ -0,0 +1,48 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod — VH Lab Audrey BPod stimulus reader. + +Reads stimulus trigger data from BPod behavioral task systems used in +the VH Lab taste experiments. + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/VHAudreyBPod.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``#_stimulus_triggers_log.tsv`` — stimulus trigger timing log +- ``#.*_summary_log.json`` — experiment summary metadata +""" + +from __future__ import annotations + +import logging + +from ndi.daq.mfdaq import ChannelInfo, ndi_daq_reader_mfdaq + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod(ndi_daq_reader_mfdaq): + """VH Lab Audrey BPod stimulus reader. + + Reads stimulus timing and trigger information from BPod behavioral + task systems. Stimulus metadata is read by the companion + ``ndi.daq.metadatareader.VHAudreyBPod`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod" + + def __init__(self, identifier=None, session=None, document=None): + super().__init__(identifier=identifier, session=session, document=document) + self._ndi_daqreader_class = self.NDI_DAQREADER_CLASS + + def getchannelsepoch(self, epochfiles: list[str]) -> list[ChannelInfo]: + return [] + + def readchannels_epochsamples(self, channeltype, channel, epochfiles, s0, s1): + raise NotImplementedError("VHAudreyBPod stimulus reader does not support sample reading") + + def samplerate(self, epochfiles, channeltype, channel): + raise NotImplementedError("VHAudreyBPod stimulus reader does not support sample rates") + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod(id={self.id[:8]}...)" diff --git a/src/ndi/util/session_summary.py b/src/ndi/util/session_summary.py index f9373e1..ec0c457 100644 --- a/src/ndi/util/session_summary.py +++ b/src/ndi/util/session_summary.py @@ -15,74 +15,6 @@ from .classname import ndi_matlab_classname -def _daq_detail_from_object(sys: Any) -> dict[str, Any]: - """Extract DAQ detail dict from a fully-loaded DAQ system object.""" - details: dict[str, Any] = {} - - fn = getattr(sys, "filenavigator", None) - if fn is not None: - details["filenavigator_class"] = getattr( - fn, "NDI_FILENAVIGATOR_CLASS", type(fn).__qualname__ - ) - try: - details["epochNodes_filenavigator"] = fn.epochnodes() - except Exception: - details["epochNodes_filenavigator"] = [] - else: - details["filenavigator_class"] = "" - details["epochNodes_filenavigator"] = [] - - dr = getattr(sys, "daqreader", None) - if dr is not None: - details["daqreader_class"] = getattr(dr, "NDI_DAQREADER_CLASS", ndi_matlab_classname(dr)) - else: - details["daqreader_class"] = "" - - try: - details["epochNodes_daqsystem"] = sys.epochnodes() - except Exception: - details["epochNodes_daqsystem"] = [] - - return details - - -def _daq_detail_from_document(session_obj: Any, doc: Any) -> dict[str, Any]: - """Extract DAQ detail dict from a raw database document. - - Used as a fallback when the DAQ system object cannot be fully - reconstructed (e.g. reader class not implemented in Python). - """ - from ..query import ndi_query - - details: dict[str, Any] = {} - - # Look up the reader document via dependency - reader_class = "" - reader_id = doc.dependency_value("daqreader_id", error_if_not_found=False) - if reader_id: - reader_docs = session_obj.database_search(ndi_query("base.id") == reader_id) - if reader_docs: - props = reader_docs[0].document_properties - if isinstance(props, dict): - reader_class = props.get("daqreader", {}).get("ndi_daqreader_class", "") - details["daqreader_class"] = reader_class - - # Look up the navigator document via dependency - nav_class = "" - nav_id = doc.dependency_value("filenavigator_id", error_if_not_found=False) - if nav_id: - nav_docs = session_obj.database_search(ndi_query("base.id") == nav_id) - if nav_docs: - props = nav_docs[0].document_properties - if isinstance(props, dict): - nav_class = props.get("filenavigator", {}).get("ndi_filenavigator_class", "") - details["filenavigator_class"] = nav_class - details["epochNodes_filenavigator"] = [] - details["epochNodes_daqsystem"] = [] - - return details - - def sessionSummary(session_obj: Any) -> dict[str, Any]: """Create a summary structure of an ndi.session object. @@ -95,8 +27,6 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: Dict with keys: reference, sessionId, files, filesInDotNDI, daqSystemNames, daqSystemDetails, probes. """ - from ..query import ndi_query - summary: dict[str, Any] = {} # 1. ndi_session basics @@ -119,34 +49,51 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: else: summary["filesInDotNDI"] = [] - # 4. DAQ Systems — query all daqsystem documents from the database, - # then try full object reconstruction. Fall back to reading raw - # document properties for systems whose reader/navigator classes - # are not yet implemented in Python. - q = ndi_query("").isa("daqsystem") & (ndi_query("base.session_id") == session_obj.id()) - all_docs = session_obj.database_search(q) + # 4. DAQ Systems + daqs = session_obj.daqsystem_load(name="(.*)") + if daqs is None: + daqs = [] + elif not isinstance(daqs, list): + daqs = [daqs] daq_names: list[str] = [] daq_details: list[dict[str, Any]] = [] - for doc in all_docs: - props = doc.document_properties - name = "" - if isinstance(props, dict): - name = props.get("base", {}).get("name", "") - daq_names.append(name) + for sys in daqs: + daq_names.append(getattr(sys, "name", "")) + + details: dict[str, Any] = {} + + # Get filenavigator class (use MATLAB-compatible name for symmetry) + fn = getattr(sys, "filenavigator", None) + if fn is not None: + details["filenavigator_class"] = getattr( + fn, "NDI_FILENAVIGATOR_CLASS", type(fn).__qualname__ + ) + try: + details["epochNodes_filenavigator"] = fn.epochnodes() + except Exception: + details["epochNodes_filenavigator"] = [] + else: + details["filenavigator_class"] = "" + details["epochNodes_filenavigator"] = [] + + # Get daqreader class (use MATLAB-compatible name for symmetry) + dr = getattr(sys, "daqreader", None) + if dr is not None: + details["daqreader_class"] = getattr( + dr, "NDI_DAQREADER_CLASS", ndi_matlab_classname(dr) + ) + else: + details["daqreader_class"] = "" - # Try full reconstruction first + # Get epoch nodes of daq system try: - obj = session_obj._document_to_object(doc) - if obj is not None: - daq_details.append(_daq_detail_from_object(obj)) - continue + details["epochNodes_daqsystem"] = sys.epochnodes() except Exception: - pass + details["epochNodes_daqsystem"] = [] - # Fallback: extract what we can from the raw documents - daq_details.append(_daq_detail_from_document(session_obj, doc)) + daq_details.append(details) summary["daqSystemNames"] = daq_names summary["daqSystemDetails"] = daq_details From 73ec09b07085af7e30cd0c100eb6c20373a71c3a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 20:28:13 +0000 Subject: [PATCH 10/15] Use daq/daqreader_ndr document type for ndr readers in lab setup MATLAB's ndi.daq.reader.mfdaq.ndr constructor expects the document to have a daqreader_ndr section with ndr_reader_string. The lab setup was creating a plain daq/daqreader document, causing MATLAB to error with "Unrecognized field name daqreader_ndr" when reading Python artifacts. Now creates daq/daqreader_ndr documents with the ndr_reader_string populated from the config's DaqReaderFileParameters (e.g. "abf"). https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/setup/lab.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py index 6c36919..518ee5b 100644 --- a/src/ndi/setup/lab.py +++ b/src/ndi/setup/lab.py @@ -105,14 +105,31 @@ def lab(session, lab_name: str) -> None: ) session.database_add(fn_doc) - # Create daqreader document - dr_doc = session.newdocument( - "daq/daqreader", - **{ - "base.name": name, - "daqreader.ndi_daqreader_class": reader_class, - }, - ) + # Create daqreader document — use the specialised document type + # for readers that require extra properties (e.g. ndr needs + # daqreader_ndr.ndr_reader_string so MATLAB can reconstruct it). + reader_file_params = config.get("DaqReaderFileParameters", "") + if isinstance(reader_file_params, list): + reader_file_params = reader_file_params[0] if reader_file_params else "" + + if reader_class == "ndi.daq.reader.mfdaq.ndr": + dr_doc = session.newdocument( + "daq/daqreader_ndr", + **{ + "base.name": name, + "daqreader.ndi_daqreader_class": reader_class, + "daqreader_ndr.ndr_reader_string": reader_file_params, + "daqreader_ndr.ndi_daqreader_ndr_class": reader_class, + }, + ) + else: + dr_doc = session.newdocument( + "daq/daqreader", + **{ + "base.name": name, + "daqreader.ndi_daqreader_class": reader_class, + }, + ) session.database_add(dr_doc) # Create daqsystem document From 027e806e83a9b5006e06292f5972a4e58882ca95 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Mar 2026 22:52:32 +0000 Subject: [PATCH 11/15] Sort DAQ systems by name for order-independent symmetry comparison Python and MATLAB may load DAQ system configs in different filesystem order. Sort by name in sessionSummary output and compare by name in compareSessionSummary so the positional comparison doesn't produce false mismatches. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- src/ndi/util/compare_session_summary.py | 15 +++++++++++++++ src/ndi/util/session_summary.py | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/ndi/util/compare_session_summary.py b/src/ndi/util/compare_session_summary.py index 0cfa213..d4dae0c 100644 --- a/src/ndi/util/compare_session_summary.py +++ b/src/ndi/util/compare_session_summary.py @@ -87,6 +87,21 @@ def compareSessionSummary( ) continue + # For DAQ system lists, sort by name so order doesn't matter. + # daqSystemNames and daqSystemDetails are parallel arrays keyed + # by daqSystemNames; sort both sides consistently. + if field == "daqSystemNames": + val1 = sorted(val1) + val2 = sorted(val2) + elif field == "daqSystemDetails": + # Sort details by their paired name from the parent summary + names1 = summary1.get("daqSystemNames", []) + names2 = summary2.get("daqSystemNames", []) + if len(names1) == len(val1): + val1 = [d for _, d in sorted(zip(names1, val1))] + if len(names2) == len(val2): + val2 = [d for _, d in sorted(zip(names2, val2))] + for j, (item1, item2) in enumerate(zip(val1, val2)): if isinstance(item1, str) and isinstance(item2, str): s1 = re.sub(r"[\r\n]+", "", item1) diff --git a/src/ndi/util/session_summary.py b/src/ndi/util/session_summary.py index ec0c457..dbf9d00 100644 --- a/src/ndi/util/session_summary.py +++ b/src/ndi/util/session_summary.py @@ -95,8 +95,11 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: daq_details.append(details) - summary["daqSystemNames"] = daq_names - summary["daqSystemDetails"] = daq_details + # Sort by name so the order is deterministic across languages + # (Python and MATLAB may load configs in different filesystem order). + paired = sorted(zip(daq_names, daq_details), key=lambda x: x[0]) + summary["daqSystemNames"] = [p[0] for p in paired] + summary["daqSystemDetails"] = [p[1] for p in paired] # 5. Probes probes = session_obj.getprobes() From 55668c9a8adf91eebdb52431e59b9aa45c8fd455 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Mar 2026 10:59:56 +0000 Subject: [PATCH 12/15] Group dataset jsonDocuments by session ID to match MATLAB structure MATLAB's buildDataset exports JSON documents into per-session subdirectories (jsonDocuments//.json) rather than flat files. Update both makeArtifacts (write all dataset docs grouped by base.session_id) and readArtifacts (glob **/*.json) to match this structure. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- .../dataset/test_build_dataset.py | 22 +++++++++++++------ .../dataset/test_build_dataset.py | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py index e903029..7d39c68 100644 --- a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py @@ -110,19 +110,27 @@ def test_build_dataset_artifacts(self): # Copy the entire dataset directory to the persistent artifact dir. shutil.copytree(str(self.dataset.getpath()), str(artifact_dir)) - # Write individual JSON documents. + # Write individual JSON documents, grouped by session ID. + # Mirrors MATLAB which creates jsonDocuments//.json json_docs_dir = artifact_dir / "jsonDocuments" json_docs_dir.mkdir(exist_ok=True) - docs = self.dataset.database_search(Query("base.id").match("(.*)")) - for doc in docs: + artifact_dataset = Dataset(artifact_dir) + + # Query all documents from the dataset and group by session_id. + all_docs = artifact_dataset.database_search(Query("base.id").match("(.*)")) + for doc in all_docs: props = doc.document_properties - doc_path = json_docs_dir / f"{doc.id}.json" - doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + sid = props.get("base", {}).get("session_id", "unknown") + session_json_dir = json_docs_dir / sid + session_json_dir.mkdir(exist_ok=True) + doc_path = session_json_dir / f"{doc.id}.json" + doc_path.write_text( + json.dumps(props, indent=2, allow_nan=True), encoding="utf-8" + ) # Write datasetSummary.json – open from artifact_dir so the session # path lists files that are actually present (including jsonDocuments). - artifact_dataset = Dataset(artifact_dir) summary = _dataset_summary(artifact_dataset) summary_json = json.dumps(summary, indent=2, allow_nan=True) summary_path = artifact_dir / "datasetSummary.json" @@ -132,4 +140,4 @@ def test_build_dataset_artifacts(self): assert artifact_dir.exists() assert summary_path.exists() assert json_docs_dir.exists() - assert len(list(json_docs_dir.glob("*.json"))) > 0 + assert len(list(json_docs_dir.glob("**/*.json"))) > 0 diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py index 79fb452..dff7028 100644 --- a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py @@ -132,7 +132,7 @@ def test_build_dataset_documents(self, source_type): if not json_docs_dir.exists(): pytest.skip(f"jsonDocuments directory not found in {source_type}.") - json_files = list(json_docs_dir.glob("*.json")) + json_files = list(json_docs_dir.glob("**/*.json")) actual_docs = dataset.database_search(Query("base.id").match("(.*)")) From d1f2ee11bee1e1f4aace9d655c08babe2863385e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Mar 2026 11:15:00 +0000 Subject: [PATCH 13/15] Fix dataset tests to query session DBs, not dataset DB MATLAB's buildDataset iterates session_list() and queries each session's database individually. The dataset's database contains extra documents (session, session_in_a_dataset) that aren't in the session DBs. Updated both make_artifacts and read_artifacts to match MATLAB's approach. https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- .../dataset/test_build_dataset.py | 23 +++++++++++-------- .../dataset/test_build_dataset.py | 15 +++++++++--- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py index 7d39c68..3da7882 100644 --- a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py @@ -111,23 +111,26 @@ def test_build_dataset_artifacts(self): shutil.copytree(str(self.dataset.getpath()), str(artifact_dir)) # Write individual JSON documents, grouped by session ID. - # Mirrors MATLAB which creates jsonDocuments//.json + # Mirrors MATLAB which iterates session_list() and queries each + # session's database (NOT the dataset's database). json_docs_dir = artifact_dir / "jsonDocuments" json_docs_dir.mkdir(exist_ok=True) artifact_dataset = Dataset(artifact_dir) - # Query all documents from the dataset and group by session_id. - all_docs = artifact_dataset.database_search(Query("base.id").match("(.*)")) - for doc in all_docs: - props = doc.document_properties - sid = props.get("base", {}).get("session_id", "unknown") + # Export JSON documents for each session in the dataset + refs, session_ids, *_ = artifact_dataset.session_list() + for sid in session_ids: + sess = artifact_dataset.open_session(sid) session_json_dir = json_docs_dir / sid session_json_dir.mkdir(exist_ok=True) - doc_path = session_json_dir / f"{doc.id}.json" - doc_path.write_text( - json.dumps(props, indent=2, allow_nan=True), encoding="utf-8" - ) + docs = sess.database_search(Query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = session_json_dir / f"{doc.id}.json" + doc_path.write_text( + json.dumps(props, indent=2, allow_nan=True), encoding="utf-8" + ) # Write datasetSummary.json – open from artifact_dir so the session # path lists files that are actually present (including jsonDocuments). diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py index dff7028..4d47553 100644 --- a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py @@ -125,7 +125,11 @@ def test_build_dataset_session_summaries(self, source_type): ) def test_build_dataset_documents(self, source_type): - """Verify that every exported JSON document can be loaded from the dataset DB.""" + """Verify that every exported JSON document can be loaded from session DBs. + + Mirrors MATLAB which queries each session's database individually + (not the dataset's database). + """ artifact_dir, dataset = self._open_dataset(source_type) json_docs_dir = artifact_dir / "jsonDocuments" @@ -134,10 +138,15 @@ def test_build_dataset_documents(self, source_type): json_files = list(json_docs_dir.glob("**/*.json")) - actual_docs = dataset.database_search(Query("base.id").match("(.*)")) + # Collect docs from each session's database, matching MATLAB's approach + actual_docs = [] + _refs, session_ids, *_ = dataset.session_list() + for sid in session_ids: + sess = dataset.open_session(sid) + actual_docs.extend(sess.database_search(Query("base.id").match("(.*)"))) assert len(actual_docs) == len(json_files), ( - f"Number of documents in dataset ({len(actual_docs)}) does not match " + f"Number of documents across sessions ({len(actual_docs)}) does not match " f"{source_type} JSON artifacts ({len(json_files)})." ) From ed277af536f0bd76c18220962ac4ee253b004d0d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Mar 2026 11:31:44 +0000 Subject: [PATCH 14/15] Fix formatting for CI lint https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- .../make_artifacts/dataset/test_build_dataset.py | 4 +--- .../read_artifacts/dataset/test_build_dataset.py | 12 +++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py index 3da7882..327a76f 100644 --- a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py @@ -128,9 +128,7 @@ def test_build_dataset_artifacts(self): for doc in docs: props = doc.document_properties doc_path = session_json_dir / f"{doc.id}.json" - doc_path.write_text( - json.dumps(props, indent=2, allow_nan=True), encoding="utf-8" - ) + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") # Write datasetSummary.json – open from artifact_dir so the session # path lists files that are actually present (including jsonDocuments). diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py index 4d47553..beae0c5 100644 --- a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py @@ -72,19 +72,17 @@ def test_build_dataset_summary(self, source_type): expected_refs = expected.get("references", []) assert num_sessions == expected_num, ( - f"Session count mismatch in {source_type}: " - f"got {num_sessions}, expected {expected_num}" + f"Session count mismatch in {source_type}: got {num_sessions}, expected {expected_num}" ) for exp_id in expected_ids: assert exp_id in session_ids, ( - f"Expected session ID {exp_id!r} not found in dataset " f"from {source_type}" + f"Expected session ID {exp_id!r} not found in dataset from {source_type}" ) for exp_ref in expected_refs: assert exp_ref in refs, ( - f"Expected session reference {exp_ref!r} not found in dataset " - f"from {source_type}" + f"Expected session reference {exp_ref!r} not found in dataset from {source_type}" ) def test_build_dataset_session_summaries(self, source_type): @@ -162,9 +160,9 @@ def test_build_dataset_documents(self, source_type): assert actual_props.get("document_class", {}).get( "class_name" ) == expected_doc.get("document_class", {}).get("class_name"), ( - f"Document class mismatch for id: {expected_id} " f"in {source_type}" + f"Document class mismatch for id: {expected_id} in {source_type}" ) break assert found, ( - f"Document from {source_type} artifact not found in dataset: " f"{expected_id}" + f"Document from {source_type} artifact not found in dataset: {expected_id}" ) From 4bba819d2c3a5cf07dbed021a5dae03b32a7a57d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Mar 2026 11:34:57 +0000 Subject: [PATCH 15/15] Fix black formatting for CI lint https://claude.ai/code/session_01XKnremisjfbzcM4aK7YMKq --- .../dataset/test_build_dataset.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py index beae0c5..d1212e4 100644 --- a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py @@ -71,19 +71,19 @@ def test_build_dataset_summary(self, source_type): expected_ids = expected.get("sessionIds", []) expected_refs = expected.get("references", []) - assert num_sessions == expected_num, ( - f"Session count mismatch in {source_type}: got {num_sessions}, expected {expected_num}" - ) + assert ( + num_sessions == expected_num + ), f"Session count mismatch in {source_type}: got {num_sessions}, expected {expected_num}" for exp_id in expected_ids: - assert exp_id in session_ids, ( - f"Expected session ID {exp_id!r} not found in dataset from {source_type}" - ) + assert ( + exp_id in session_ids + ), f"Expected session ID {exp_id!r} not found in dataset from {source_type}" for exp_ref in expected_refs: - assert exp_ref in refs, ( - f"Expected session reference {exp_ref!r} not found in dataset from {source_type}" - ) + assert ( + exp_ref in refs + ), f"Expected session reference {exp_ref!r} not found in dataset from {source_type}" def test_build_dataset_session_summaries(self, source_type): """Compare per-session summaries against those stored in datasetSummary.json.""" @@ -159,10 +159,10 @@ def test_build_dataset_documents(self, source_type): actual_props = actual.document_properties assert actual_props.get("document_class", {}).get( "class_name" - ) == expected_doc.get("document_class", {}).get("class_name"), ( - f"Document class mismatch for id: {expected_id} in {source_type}" - ) + ) == expected_doc.get("document_class", {}).get( + "class_name" + ), f"Document class mismatch for id: {expected_id} in {source_type}" break - assert found, ( - f"Document from {source_type} artifact not found in dataset: {expected_id}" - ) + assert ( + found + ), f"Document from {source_type} artifact not found in dataset: {expected_id}"