diff --git a/src/ndi/__init__.py b/src/ndi/__init__.py index 3128ebb..889785b 100644 --- a/src/ndi/__init__.py +++ b/src/ndi/__init__.py @@ -32,7 +32,20 @@ # Import epoch module (Phase 6) # Import Phase 10: Cloud API client # Import Phase 11: Schema validation -from . import calc, cloud, common, daq, epoch, file, session, time, util, validate, validators +from . import ( + calc, + cloud, + common, + daq, + epoch, + file, + session, + setup, + time, + util, + validate, + validators, +) # Import Phase 9: ndi_app framework and calculators from .app import ndi_app @@ -124,6 +137,7 @@ def version() -> tuple: "ndi_calculator", "calc", "cloud", + "setup", "util", "validate", "validators", diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index bde135c..5c5e13a 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -30,14 +30,25 @@ def _build_registry() -> dict[str, type]: from .daq.reader.mfdaq.blackrock import ndi_daq_reader_mfdaq_blackrock from .daq.reader.mfdaq.cedspike2 import ndi_daq_reader_mfdaq_cedspike2 from .daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan + from .daq.reader.mfdaq.ndr import ndi_daq_reader_mfdaq_ndr from .daq.reader.mfdaq.spikegadgets import ndi_daq_reader_mfdaq_spikegadgets from .daq.system import ndi_daq_system from .element import ndi_element from .file.navigator import ndi_file_navigator + from .file.navigator.epochdir import ndi_file_navigator_epochdir from .probe import ndi_probe from .probe.timeseries import ndi_probe_timeseries from .probe.timeseries_mfdaq import ndi_probe_timeseries_mfdaq from .probe.timeseries_stimulator import ndi_probe_timeseries_stimulator + from .setup.daq.reader.mfdaq.stimulus.nielsenvisintan import ( + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, + ) + from .setup.daq.reader.mfdaq.stimulus.vhaudreybpod import ( + ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, + ) + from .setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 import ( + ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, + ) registry: dict[str, type] = {} @@ -56,15 +67,20 @@ def _build_registry() -> dict[str, type]: ndi_daq_reader_mfdaq_intan, ndi_daq_reader_mfdaq_blackrock, ndi_daq_reader_mfdaq_cedspike2, + ndi_daq_reader_mfdaq_ndr, ndi_daq_reader_mfdaq_spikegadgets, + ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, + ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, ): registry[cls.NDI_DAQREADER_CLASS] = cls # DAQ system registry[ndi_daq_system.NDI_DAQSYSTEM_CLASS] = ndi_daq_system - # File navigator + # File navigators registry[ndi_file_navigator.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator + registry[ndi_file_navigator_epochdir.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_epochdir return registry diff --git a/src/ndi/daq/metadatareader/__init__.py b/src/ndi/daq/metadatareader/__init__.py index ab270f0..c8c6a79 100644 --- a/src/ndi/daq/metadatareader/__init__.py +++ b/src/ndi/daq/metadatareader/__init__.py @@ -306,9 +306,11 @@ def __hash__(self) -> int: # Import subclass readers from .newstim_stims import ndi_daq_metadatareader_NewStimStims from .nielsenlab_stims import ndi_daq_metadatareader_NielsenLabStims +from .vhaudreybpod_stims import ndi_daq_metadatareader_VHAudreyBPod __all__ = [ "ndi_daq_metadatareader", "ndi_daq_metadatareader_NewStimStims", "ndi_daq_metadatareader_NielsenLabStims", + "ndi_daq_metadatareader_VHAudreyBPod", ] diff --git a/src/ndi/daq/metadatareader/vhaudreybpod_stims.py b/src/ndi/daq/metadatareader/vhaudreybpod_stims.py new file mode 100644 index 0000000..e7d61b0 --- /dev/null +++ b/src/ndi/daq/metadatareader/vhaudreybpod_stims.py @@ -0,0 +1,107 @@ +""" +ndi.daq.metadatareader.VHAudreyBPod - VH Lab Audrey BPod stimulus metadata reader. + +Reads stimulus parameters from BPod behavioral task summary log JSON files +used in the VH Lab taste experiments. + +MATLAB equivalent: src/ndi/+ndi/+daq/+metadatareader/VHAudreyBPod.m +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from ..metadatareader import ndi_daq_metadatareader + + +class ndi_daq_metadatareader_VHAudreyBPod(ndi_daq_metadatareader): + """ + Metadata reader for VH Lab Audrey BPod stimulus systems. + + Reads stimulus parameters from JSON summary log files produced by + BPod behavioral task systems. + + Example: + >>> reader = ndi_daq_metadatareader_VHAudreyBPod() + >>> params = reader.readmetadata(['triggers.tsv', 'summary_log.json']) + """ + + SUMMARY_FILE_PATTERN = r"_summary_log\.json$" + + def __init__( + self, + tsv_pattern: str = "", + identifier: str | None = None, + session: Any | None = None, + document: Any | None = None, + ): + super().__init__( + tsv_pattern=tsv_pattern, + identifier=identifier, + session=session, + document=document, + ) + + def readmetadata( + self, + epochfiles: list[str], + ) -> list[dict[str, Any]]: + """ + Read stimulus metadata from BPod summary log files. + + Args: + epochfiles: List of file paths for the epoch + + Returns: + List of parameter dictionaries + """ + # First try TSV-based reading from base class + if self._tab_separated_file_parameter: + try: + return super().readmetadata(epochfiles) + except (ValueError, FileNotFoundError): + pass + + # Look for summary_log.json + summary_file = self._find_summary_file(epochfiles) + if summary_file is None: + return [] + + return self._read_summary_json(summary_file) + + def _find_summary_file(self, epochfiles: list[str]) -> str | None: + """Find the BPod summary log JSON file in epoch files.""" + pattern = re.compile(self.SUMMARY_FILE_PATTERN, re.IGNORECASE) + for f in epochfiles: + if pattern.search(f): + return f + return None + + @staticmethod + def _read_summary_json(filepath: str) -> list[dict[str, Any]]: + """ + Read stimulus parameters from a BPod summary log JSON file. + + Args: + filepath: Path to the summary_log.json file + + Returns: + List of parameter dicts + """ + if not Path(filepath).is_file(): + return [] + + with open(filepath, encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, list): + return data + if isinstance(data, dict): + return [data] + return [] + + def __repr__(self) -> str: + return f"ndi_daq_metadatareader_VHAudreyBPod(id='{self.id[:8]}...')" diff --git a/src/ndi/daq/reader/__init__.py b/src/ndi/daq/reader/__init__.py index 62efa91..4942ae5 100644 --- a/src/ndi/daq/reader/__init__.py +++ b/src/ndi/daq/reader/__init__.py @@ -18,6 +18,7 @@ ndi_daq_reader_mfdaq_blackrock, ndi_daq_reader_mfdaq_cedspike2, ndi_daq_reader_mfdaq_intan, + ndi_daq_reader_mfdaq_ndr, ndi_daq_reader_mfdaq_spikegadgets, ) from .spikeinterface_adapter import ndi_daq_reader_SpikeInterfaceReader @@ -27,5 +28,6 @@ "ndi_daq_reader_mfdaq_intan", "ndi_daq_reader_mfdaq_blackrock", "ndi_daq_reader_mfdaq_cedspike2", + "ndi_daq_reader_mfdaq_ndr", "ndi_daq_reader_mfdaq_spikegadgets", ] diff --git a/src/ndi/daq/reader/mfdaq/__init__.py b/src/ndi/daq/reader/mfdaq/__init__.py index f20ddf5..af949d6 100644 --- a/src/ndi/daq/reader/mfdaq/__init__.py +++ b/src/ndi/daq/reader/mfdaq/__init__.py @@ -12,11 +12,13 @@ from .blackrock import ndi_daq_reader_mfdaq_blackrock from .cedspike2 import ndi_daq_reader_mfdaq_cedspike2 from .intan import ndi_daq_reader_mfdaq_intan +from .ndr import ndi_daq_reader_mfdaq_ndr from .spikegadgets import ndi_daq_reader_mfdaq_spikegadgets __all__ = [ "ndi_daq_reader_mfdaq_intan", "ndi_daq_reader_mfdaq_blackrock", "ndi_daq_reader_mfdaq_cedspike2", + "ndi_daq_reader_mfdaq_ndr", "ndi_daq_reader_mfdaq_spikegadgets", ] diff --git a/src/ndi/daq/reader/mfdaq/ndr.py b/src/ndi/daq/reader/mfdaq/ndr.py new file mode 100644 index 0000000..5f9c6da --- /dev/null +++ b/src/ndi/daq/reader/mfdaq/ndr.py @@ -0,0 +1,66 @@ +""" +ndi.daq.reader.mfdaq.ndr - NDR (Neuroscience Data Reader) wrapper. + +Thin wrapper around ndi_daq_reader_SpikeInterfaceReader for file formats +supported by the NDR library (e.g. Axon ABF files). + +MATLAB equivalent: src/ndi/+ndi/+daq/+reader/+mfdaq/ndr.m +""" + +from __future__ import annotations + +import logging + +from ...mfdaq import ChannelInfo, ndi_daq_reader_mfdaq + +logger = logging.getLogger(__name__) + + +class ndi_daq_reader_mfdaq_ndr(ndi_daq_reader_mfdaq): + """ + Reader for data files handled by the NDR library. + + Currently supports Axon ABF files via spikeinterface/neo. + + File extensions: .abf + """ + + NDI_DAQREADER_CLASS = "ndi.daq.reader.mfdaq.ndr" + FILE_EXTENSIONS = [".abf"] + + def __init__(self, identifier=None, session=None, document=None): + super().__init__(identifier=identifier, session=session, document=document) + self._ndi_daqreader_class = self.NDI_DAQREADER_CLASS + + def _get_si_reader(self): + try: + from ..spikeinterface_adapter import ndi_daq_reader_SpikeInterfaceReader + + return ndi_daq_reader_SpikeInterfaceReader + except ImportError: + return None + + def getchannelsepoch(self, epochfiles: list[str]) -> list[ChannelInfo]: + SI = self._get_si_reader() + if SI is None: + return [] + try: + return SI().getchannelsepoch(epochfiles) + except Exception as exc: + logger.warning("ndi_daq_reader_mfdaq_ndr.getchannelsepoch failed: %s", exc) + return [] + + def readchannels_epochsamples(self, channeltype, channel, epochfiles, s0, s1): + SI = self._get_si_reader() + if SI is None: + raise ImportError("spikeinterface required for reading NDR data") + return SI().readchannels_epochsamples(channeltype, channel, epochfiles, s0, s1) + + def samplerate(self, epochfiles, channeltype, channel): + SI = self._get_si_reader() + if SI is None: + raise ImportError("spikeinterface required for reading NDR data") + return SI().samplerate(epochfiles, channeltype, channel) + + def __repr__(self): + return f"ndi_daq_reader_mfdaq_ndr(id={self.id[:8]}...)" diff --git a/src/ndi/daq/system.py b/src/ndi/daq/system.py index 22a37b8..b88d785 100644 --- a/src/ndi/daq/system.py +++ b/src/ndi/daq/system.py @@ -209,25 +209,43 @@ def _load_from_document(self, session: Any, document: Any) -> None: from ..class_registry import get_class ReaderCls = get_class(reader_class_name) - if ReaderCls is not None: - try: - self._daqreader = ReaderCls(session=session, document=reader_doc) - except Exception as exc: - logger.warning( - "Could not reconstruct DAQ reader %s: %s", reader_class_name, exc - ) - else: - logger.debug("Unknown DAQ reader class: %s", reader_class_name) + if ReaderCls is None: + raise ValueError( + f"Unknown DAQ reader class: {reader_class_name!r}. " + f"Register it in ndi.class_registry." + ) + try: + self._daqreader = ReaderCls(session=session, document=reader_doc) + except Exception as exc: + raise RuntimeError( + f"Could not reconstruct DAQ reader {reader_class_name!r}: {exc}" + ) from exc # Reconstruct file navigator from its document self._filenavigator = None if len(nav_docs) == 1: - from ..file.navigator import ndi_file_navigator + nav_doc = nav_docs[0] + nav_class_name = "" + nav_props = nav_doc.document_properties + if isinstance(nav_props, dict): + nav_class_name = nav_props.get("filenavigator", {}).get( + "ndi_filenavigator_class", "" + ) + from ..class_registry import get_class as get_nav_class + + NavCls = get_nav_class(nav_class_name) + if NavCls is None: + raise ValueError( + f"Unknown file navigator class: {nav_class_name!r}. " + f"Register it in ndi.class_registry." + ) try: - self._filenavigator = ndi_file_navigator(session=session, document=nav_docs[0]) + self._filenavigator = NavCls(session=session, document=nav_doc) except Exception as exc: - logger.warning("Could not reconstruct file navigator: %s", exc) + raise RuntimeError( + f"Could not reconstruct file navigator {nav_class_name!r}: {exc}" + ) from exc @property def name(self) -> str: diff --git a/src/ndi/file/navigator/epochdir.py b/src/ndi/file/navigator/epochdir.py index 503cc6c..28ae344 100644 --- a/src/ndi/file/navigator/epochdir.py +++ b/src/ndi/file/navigator/epochdir.py @@ -32,6 +32,8 @@ class ndi_file_navigator_epochdir(ndi_file_navigator): >>> # trial_002/file.rhd -> epoch 2 """ + NDI_FILENAVIGATOR_CLASS = "ndi.file.navigator.epochdir" + def epochid( self, epoch_number: int, diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json new file mode 100644 index 0000000..1eaf263 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_bpod.json @@ -0,0 +1,15 @@ +{ + "Name": "vhtaste_bpod", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod", + "MetadataReaderClass": "ndi.daq.metadatareader.VHAudreyBPod", + "EpochProbeMapClass": "ndi.setup.epoch.epochprobemap_daqsystem_vhlab", + "FileParameters": [ + "#_stimulus_triggers_log.tsv", + "#.*_summary_log.json" + ], + "DaqReaderFileParameters": [], + "MetadataReaderFileParameters": "(.*)\\.json", + "EpochProbeMapFileParameters": "#_stimulus_triggers_log.tsv", + "HasEpochDirectories": false +} diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json new file mode 100644 index 0000000..249aa45 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhtaste_sync.json @@ -0,0 +1,17 @@ +{ + "Name": "vhtaste_sync", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.daq.reader.mfdaq.intan", + "MetadataReaderClass": [], + "EpochProbeMapClass": "ndi.setup.epoch.epochprobemap_daqsystem_vhlab", + "FileParameters": [ + "reference.txt", + ".*\\.rhd\\>", + "vhintan_channelgrouping.txt", + "vhtaste_sync.txt" + ], + "DaqReaderFileParameters": [], + "MetadataReaderFileParameters": [], + "EpochProbeMapFileParameters": "vhtaste_sync.txt", + "HasEpochDirectories": true +} diff --git a/src/ndi/ndi_matlab_python_bridge.yaml b/src/ndi/ndi_matlab_python_bridge.yaml index b03f0e8..bd58701 100644 --- a/src/ndi/ndi_matlab_python_bridge.yaml +++ b/src/ndi/ndi_matlab_python_bridge.yaml @@ -985,13 +985,13 @@ not_applicable: # ========================================================================= - name: setup matlab_path: "+ndi/+setup/" - status: not_applicable + python_path: "ndi/setup/" + status: implemented decision_log: > - Entire MATLAB package containing lab-specific DAQ configurations - (angeluccilab, dbkatzlab, marderlab, vhlab, yangyangwang) and - sub-packages (+NDIMaker, +conv, +daq, +epoch, +stimulus, - @DaqSystemConfiguration). These are MATLAB-specific lab setup - wizards. Python labs configure via JSON/YAML files instead. + Python equivalent of MATLAB lab setup wizards. The ndi.setup.lab() + function reads DAQ system JSON configs from ndi_common/daq_systems/ + and creates the corresponding documents in a session. Usage: + ndi.setup.lab(session, "vhlab"). # ========================================================================= # ndi.docs namespace — MATLAB documentation generation diff --git a/src/ndi/session/session_base.py b/src/ndi/session/session_base.py index 3315534..de53380 100644 --- a/src/ndi/session/session_base.py +++ b/src/ndi/session/session_base.py @@ -1123,17 +1123,11 @@ def _document_to_object(self, document: ndi_document) -> Any: props = document.document_properties ndi_class = props.get("element", {}).get("ndi_element_class", "") cls = get_class(ndi_class) - if cls is not None: - return cls(session=self, document=document) - # Fallback: if ndi_element_class contains "probe" but - # isn't a known key (e.g. "ndi.probe.timage"), use ndi_probe. - from ..probe import ndi_probe - - if "probe" in ndi_class: - return ndi_probe(session=self, document=document) - from ..element import ndi_element - - return ndi_element(session=self, document=document) + if cls is None: + raise ValueError( + f"Unknown element class: {ndi_class!r}. " f"Register it in ndi.class_registry." + ) + return cls(session=self, document=document) if document.doc_isa("syncgraph"): return ndi_time_syncgraph(session=self, document=document) diff --git a/src/ndi/setup/__init__.py b/src/ndi/setup/__init__.py new file mode 100644 index 0000000..cbb382b --- /dev/null +++ b/src/ndi/setup/__init__.py @@ -0,0 +1,13 @@ +"""ndi.setup — Lab configuration and session setup utilities. + +Python equivalent of MATLAB's ``+ndi/+setup/`` package. + +Usage:: + + import ndi + ndi.setup.lab(session, "vhlab") +""" + +from .lab import lab + +__all__ = ["lab"] diff --git a/src/ndi/setup/daq/__init__.py b/src/ndi/setup/daq/__init__.py new file mode 100644 index 0000000..a001625 --- /dev/null +++ b/src/ndi/setup/daq/__init__.py @@ -0,0 +1 @@ +"""ndi.setup.daq — Lab-specific DAQ extensions.""" diff --git a/src/ndi/setup/daq/reader/__init__.py b/src/ndi/setup/daq/reader/__init__.py new file mode 100644 index 0000000..5d6d7c9 --- /dev/null +++ b/src/ndi/setup/daq/reader/__init__.py @@ -0,0 +1 @@ +"""ndi.setup.daq.reader — Lab-specific DAQ reader extensions.""" diff --git a/src/ndi/setup/daq/reader/mfdaq/__init__.py b/src/ndi/setup/daq/reader/mfdaq/__init__.py new file mode 100644 index 0000000..f60f048 --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/__init__.py @@ -0,0 +1 @@ +"""ndi.setup.daq.reader.mfdaq — Lab-specific MFDAQ reader extensions.""" diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py new file mode 100644 index 0000000..a3e5cdf --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py @@ -0,0 +1,17 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus — Stimulus-specific DAQ readers. + +Lab-specific DAQ reader variants that handle stimulus presentation +data in addition to standard electrophysiology recordings. + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/`` +""" + +from .nielsenvisintan import ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan +from .vhaudreybpod import ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod +from .vhlabvisspike2 import ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2 + +__all__ = [ + "ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2", + "ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan", + "ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod", +] diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py new file mode 100644 index 0000000..9c5ce94 --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisintan.py @@ -0,0 +1,41 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisintan — Nielsen Lab visual stimulus Intan reader. + +Extends the Intan RHD reader with support for Nielsen Lab visual +stimulus files (.analyzer). + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/nielsenvisintan.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``#_info.rhd`` — Intan recording info header +- ``#_digitalin.dat`` — digital input data +- ``#.analyzer`` — Nielsen Lab Analyzer stimulus structure +- ``epochprobemap.txt`` — epoch-to-probe mapping + +Stimulus metadata is read by the companion +:class:`~ndi.daq.metadatareader.nielsenlab_stims.ndi_daq_metadatareader_NielsenLabStims`. +""" + +from __future__ import annotations + +import logging + +from ndi.daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan(ndi_daq_reader_mfdaq_intan): + """Nielsen Lab visual stimulus reader built on Intan RHD. + + Inherits all channel reading and sample-rate logic from the Intan + reader. The stimulus-specific behaviour (extracting Analyzer + parameters) is handled by the metadata reader + ``ndi.daq.metadatareader.NielsenLabStims`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisintan" + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan(id={self.id[:8]}...)" diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py new file mode 100644 index 0000000..52eab5e --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhaudreybpod.py @@ -0,0 +1,48 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod — VH Lab Audrey BPod stimulus reader. + +Reads stimulus trigger data from BPod behavioral task systems used in +the VH Lab taste experiments. + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/VHAudreyBPod.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``#_stimulus_triggers_log.tsv`` — stimulus trigger timing log +- ``#.*_summary_log.json`` — experiment summary metadata +""" + +from __future__ import annotations + +import logging + +from ndi.daq.mfdaq import ChannelInfo, ndi_daq_reader_mfdaq + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod(ndi_daq_reader_mfdaq): + """VH Lab Audrey BPod stimulus reader. + + Reads stimulus timing and trigger information from BPod behavioral + task systems. Stimulus metadata is read by the companion + ``ndi.daq.metadatareader.VHAudreyBPod`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod" + + def __init__(self, identifier=None, session=None, document=None): + super().__init__(identifier=identifier, session=session, document=document) + self._ndi_daqreader_class = self.NDI_DAQREADER_CLASS + + def getchannelsepoch(self, epochfiles: list[str]) -> list[ChannelInfo]: + return [] + + def readchannels_epochsamples(self, channeltype, channel, epochfiles, s0, s1): + raise NotImplementedError("VHAudreyBPod stimulus reader does not support sample reading") + + def samplerate(self, epochfiles, channeltype, channel): + raise NotImplementedError("VHAudreyBPod stimulus reader does not support sample rates") + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod(id={self.id[:8]}...)" diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py new file mode 100644 index 0000000..6b1227f --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/vhlabvisspike2.py @@ -0,0 +1,42 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.vhlabvisspike2 — VH Lab visual stimulus Spike2 reader. + +Extends the CED Spike2 reader with support for VH Lab visual stimulus +files (stims.mat, stimtimes.txt, verticalblanking.txt). + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/vhlabvisspike2.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``reference.txt`` — epoch reference file +- ``stimtimes.txt`` — stimulus timing information +- ``verticalblanking.txt`` — vertical blanking timestamps +- ``stims.mat`` — NewStim stimulus script parameters +- ``spike2data.smr`` — CED Spike2 electrophysiology data + +Stimulus metadata is read by the companion +:class:`~ndi.daq.metadatareader.newstim_stims.ndi_daq_metadatareader_NewStimStims`. +""" + +from __future__ import annotations + +import logging + +from ndi.daq.reader.mfdaq.cedspike2 import ndi_daq_reader_mfdaq_cedspike2 + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2(ndi_daq_reader_mfdaq_cedspike2): + """VH Lab visual stimulus reader built on CED Spike2. + + Inherits all channel reading and sample-rate logic from the CED Spike2 + reader. The stimulus-specific behaviour (extracting stim parameters, + timing alignment) is handled by the metadata reader + ``ndi.daq.metadatareader.NewStimStims`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.vhlabvisspike2" + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2(id={self.id[:8]}...)" diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py new file mode 100644 index 0000000..518ee5b --- /dev/null +++ b/src/ndi/setup/lab.py @@ -0,0 +1,165 @@ +"""Set up a session with DAQ systems from lab configuration files. + +Reads DAQ system configuration JSON files from +``ndi_common/daq_systems//`` and creates the corresponding +filenavigator, daqreader, (optionally daqmetadatareader), and daqsystem +documents in the given session. + +This is the Python equivalent of MATLAB's ``ndi.setup.lab()``. + +Example:: + + import ndi + session = ndi.ndi_session_dir("exp1", "/path/to/experiment") + ndi.setup.lab(session, "vhlab") +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def _to_matlab_cell_str(items: list[str]) -> str: + """Convert a list of strings to MATLAB cell-array syntax. + + Example: ``['#.rhd']`` becomes ``"{ '#.rhd' }"``. + """ + if not items: + return "" + quoted = ", ".join(f"'{s}'" for s in items) + return f"{{ {quoted} }}" + + +def _find_daq_configs(lab_name: str) -> list[dict]: + """Read all DAQ system JSON configs for a given lab.""" + import ndi.ndi_common + + common_dir = Path(ndi.ndi_common.__path__[0]) + lab_dir = common_dir / "daq_systems" / lab_name + + if not lab_dir.exists(): + raise FileNotFoundError(f"No DAQ system configs found for lab: {lab_name}") + + configs = [] + for json_file in sorted(lab_dir.glob("*.json")): + with open(json_file, encoding="utf-8") as f: + configs.append(json.load(f)) + return configs + + +def lab(session, lab_name: str) -> None: + """Add DAQ system documents to a session based on lab JSON configs. + + For each JSON config in ``ndi_common/daq_systems//``, creates: + + - A ``daq/filenavigator`` document + - A ``daq/daqreader`` document + - Optionally a ``daq/daqmetadatareader`` document + - A ``daq/daqsystem`` document linking them together + + Parameters + ---------- + session : ndi.session.session_base + The NDI session to add DAQ systems to. + lab_name : str + Name of the lab directory under ``ndi_common/daq_systems/`` + (e.g. ``"vhlab"``, ``"marderlab"``, ``"kjnielsenlab"``). + """ + configs = _find_daq_configs(lab_name) + + for config in configs: + name = config["Name"] + file_params = config.get("FileParameters", []) + epm_class = config.get("EpochProbeMapClass", "ndi.epoch.epochprobemap_daqsystem") + epm_file_params = config.get("EpochProbeMapFileParameters", "") + reader_class = config.get("DaqReaderClass", "ndi.daq.reader.mfdaq") + system_class = config.get("DaqSystemClass", "ndi.daq.system.mfdaq") + metadata_reader_class = config.get("MetadataReaderClass", []) + has_epoch_dirs = config.get("HasEpochDirectories", False) + + # Choose the correct filenavigator class + filenavigator_class = ( + "ndi.file.navigator.epochdir" if has_epoch_dirs else "ndi.file.navigator" + ) + + # Convert file parameters to MATLAB cell string format + fp_str = _to_matlab_cell_str(file_params) + + # EpochProbeMapFileParameters may be a string or list + if isinstance(epm_file_params, list): + epm_fp_str = _to_matlab_cell_str(epm_file_params) + else: + epm_fp_str = f"{{ '{epm_file_params}' }}" if epm_file_params else "" + + # Create filenavigator document + fn_doc = session.newdocument( + "daq/filenavigator", + **{ + "base.name": name, + "filenavigator.ndi_filenavigator_class": filenavigator_class, + "filenavigator.fileparameters": fp_str, + "filenavigator.epochprobemap_class": epm_class, + "filenavigator.epochprobemap_fileparameters": epm_fp_str, + }, + ) + session.database_add(fn_doc) + + # Create daqreader document — use the specialised document type + # for readers that require extra properties (e.g. ndr needs + # daqreader_ndr.ndr_reader_string so MATLAB can reconstruct it). + reader_file_params = config.get("DaqReaderFileParameters", "") + if isinstance(reader_file_params, list): + reader_file_params = reader_file_params[0] if reader_file_params else "" + + if reader_class == "ndi.daq.reader.mfdaq.ndr": + dr_doc = session.newdocument( + "daq/daqreader_ndr", + **{ + "base.name": name, + "daqreader.ndi_daqreader_class": reader_class, + "daqreader_ndr.ndr_reader_string": reader_file_params, + "daqreader_ndr.ndi_daqreader_ndr_class": reader_class, + }, + ) + else: + dr_doc = session.newdocument( + "daq/daqreader", + **{ + "base.name": name, + "daqreader.ndi_daqreader_class": reader_class, + }, + ) + session.database_add(dr_doc) + + # Create daqsystem document + daq_doc = session.newdocument( + "daq/daqsystem", + **{ + "base.name": name, + "daqsystem.ndi_daqsystem_class": system_class, + }, + ) + daq_doc = daq_doc.set_dependency_value( + "filenavigator_id", fn_doc.id, error_if_not_found=False + ) + daq_doc = daq_doc.set_dependency_value("daqreader_id", dr_doc.id, error_if_not_found=False) + + # Create daqmetadatareader document if configured + if metadata_reader_class and metadata_reader_class != []: + mr_class = ( + metadata_reader_class + if isinstance(metadata_reader_class, str) + else metadata_reader_class[0] + ) + mr_doc = session.newdocument( + "daq/daqmetadatareader", + **{ + "base.name": name, + "daqmetadatareader.ndi_daqmetadatareader_class": mr_class, + }, + ) + session.database_add(mr_doc) + daq_doc = daq_doc.add_dependency_value_n("daqmetadatareader_id", mr_doc.id) + + session.database_add(daq_doc) diff --git a/src/ndi/util/compare_session_summary.py b/src/ndi/util/compare_session_summary.py index 0cfa213..d4dae0c 100644 --- a/src/ndi/util/compare_session_summary.py +++ b/src/ndi/util/compare_session_summary.py @@ -87,6 +87,21 @@ def compareSessionSummary( ) continue + # For DAQ system lists, sort by name so order doesn't matter. + # daqSystemNames and daqSystemDetails are parallel arrays keyed + # by daqSystemNames; sort both sides consistently. + if field == "daqSystemNames": + val1 = sorted(val1) + val2 = sorted(val2) + elif field == "daqSystemDetails": + # Sort details by their paired name from the parent summary + names1 = summary1.get("daqSystemNames", []) + names2 = summary2.get("daqSystemNames", []) + if len(names1) == len(val1): + val1 = [d for _, d in sorted(zip(names1, val1))] + if len(names2) == len(val2): + val2 = [d for _, d in sorted(zip(names2, val2))] + for j, (item1, item2) in enumerate(zip(val1, val2)): if isinstance(item1, str) and isinstance(item2, str): s1 = re.sub(r"[\r\n]+", "", item1) diff --git a/src/ndi/util/session_summary.py b/src/ndi/util/session_summary.py index ec0c457..dbf9d00 100644 --- a/src/ndi/util/session_summary.py +++ b/src/ndi/util/session_summary.py @@ -95,8 +95,11 @@ def sessionSummary(session_obj: Any) -> dict[str, Any]: daq_details.append(details) - summary["daqSystemNames"] = daq_names - summary["daqSystemDetails"] = daq_details + # Sort by name so the order is deterministic across languages + # (Python and MATLAB may load configs in different filesystem order). + paired = sorted(zip(daq_names, daq_details), key=lambda x: x[0]) + summary["daqSystemNames"] = [p[0] for p in paired] + summary["daqSystemDetails"] = [p[1] for p in paired] # 5. Probes probes = session_obj.getprobes() diff --git a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py index e903029..327a76f 100644 --- a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py @@ -110,19 +110,28 @@ def test_build_dataset_artifacts(self): # Copy the entire dataset directory to the persistent artifact dir. shutil.copytree(str(self.dataset.getpath()), str(artifact_dir)) - # Write individual JSON documents. + # Write individual JSON documents, grouped by session ID. + # Mirrors MATLAB which iterates session_list() and queries each + # session's database (NOT the dataset's database). json_docs_dir = artifact_dir / "jsonDocuments" json_docs_dir.mkdir(exist_ok=True) - docs = self.dataset.database_search(Query("base.id").match("(.*)")) - for doc in docs: - props = doc.document_properties - doc_path = json_docs_dir / f"{doc.id}.json" - doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + artifact_dataset = Dataset(artifact_dir) + + # Export JSON documents for each session in the dataset + refs, session_ids, *_ = artifact_dataset.session_list() + for sid in session_ids: + sess = artifact_dataset.open_session(sid) + session_json_dir = json_docs_dir / sid + session_json_dir.mkdir(exist_ok=True) + docs = sess.database_search(Query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = session_json_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") # Write datasetSummary.json – open from artifact_dir so the session # path lists files that are actually present (including jsonDocuments). - artifact_dataset = Dataset(artifact_dir) summary = _dataset_summary(artifact_dataset) summary_json = json.dumps(summary, indent=2, allow_nan=True) summary_path = artifact_dir / "datasetSummary.json" @@ -132,4 +141,4 @@ def test_build_dataset_artifacts(self): assert artifact_dir.exists() assert summary_path.exists() assert json_docs_dir.exists() - assert len(list(json_docs_dir.glob("*.json"))) > 0 + assert len(list(json_docs_dir.glob("**/*.json"))) > 0 diff --git a/tests/symmetry/make_artifacts/session/_lab_setup.py b/tests/symmetry/make_artifacts/session/_lab_setup.py new file mode 100644 index 0000000..55a3ff4 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/_lab_setup.py @@ -0,0 +1,11 @@ +"""Thin wrapper that re-exports ``ndi.setup.lab`` for backwards compatibility. + +Tests should prefer importing from ``ndi.setup`` directly:: + + import ndi + ndi.setup.lab(session, "vhlab") +""" + +from ndi.setup.lab import lab as setup_lab_daq_systems + +__all__ = ["setup_lab_daq_systems"] diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py new file mode 100644 index 0000000..2203944 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_kjnielsenlab.py @@ -0,0 +1,83 @@ +"""Generate symmetry artifacts for a blank kjnielsenlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionKjnielsenlab.m + +This test creates a blank NDI session configured with the kjnielsenlab DAQ +systems (from ``ndi_common/daq_systems/kjnielsenlab/``), then persists the +session database, a ``sessionSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionKjnielsenlab/ + testBlankSessionKjnielsenlab/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +import ndi.setup +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = ( + PYTHON_ARTIFACTS / "session" / "blankSessionKjnielsenlab" / "testBlankSessionKjnielsenlab" +) + + +class TestBlankSessionKjnielsenlab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionKjnielsenlab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with kjnielsenlab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + ndi.setup.lab(session, "kjnielsenlab") + + self.session = session + + def test_blank_session_kjnielsenlab(self): + """Export the blank kjnielsenlab session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py new file mode 100644 index 0000000..52ac1f7 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_marderlab.py @@ -0,0 +1,81 @@ +"""Generate symmetry artifacts for a blank marderlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionMarderlab.m + +This test creates a blank NDI session configured with the marderlab DAQ +systems (from ``ndi_common/daq_systems/marderlab/``), then persists the +session database, a ``sessionSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionMarderlab/ + testBlankSessionMarderlab/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +import ndi.setup +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "session" / "blankSessionMarderlab" / "testBlankSessionMarderlab" + + +class TestBlankSessionMarderlab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionMarderlab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with marderlab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + ndi.setup.lab(session, "marderlab") + + self.session = session + + def test_blank_session_marderlab(self): + """Export the blank marderlab session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py new file mode 100644 index 0000000..3b00191 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_vhlab.py @@ -0,0 +1,81 @@ +"""Generate symmetry artifacts for a blank vhlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionVhlab.m + +This test creates a blank NDI session configured with the vhlab DAQ +systems (from ``ndi_common/daq_systems/vhlab/``), then persists the +session database, a ``sessionSummary.json`` manifest, and individual JSON +representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionVhlab/ + testBlankSessionVhlab/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +import ndi.setup +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" + + +class TestBlankSessionVhlab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionVhlab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with vhlab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + ndi.setup.lab(session, "vhlab") + + self.session = session + + def test_blank_session_vhlab(self): + """Export the blank vhlab session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py index 79fb452..d1212e4 100644 --- a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py +++ b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py @@ -71,21 +71,19 @@ def test_build_dataset_summary(self, source_type): expected_ids = expected.get("sessionIds", []) expected_refs = expected.get("references", []) - assert num_sessions == expected_num, ( - f"Session count mismatch in {source_type}: " - f"got {num_sessions}, expected {expected_num}" - ) + assert ( + num_sessions == expected_num + ), f"Session count mismatch in {source_type}: got {num_sessions}, expected {expected_num}" for exp_id in expected_ids: - assert exp_id in session_ids, ( - f"Expected session ID {exp_id!r} not found in dataset " f"from {source_type}" - ) + assert ( + exp_id in session_ids + ), f"Expected session ID {exp_id!r} not found in dataset from {source_type}" for exp_ref in expected_refs: - assert exp_ref in refs, ( - f"Expected session reference {exp_ref!r} not found in dataset " - f"from {source_type}" - ) + assert ( + exp_ref in refs + ), f"Expected session reference {exp_ref!r} not found in dataset from {source_type}" def test_build_dataset_session_summaries(self, source_type): """Compare per-session summaries against those stored in datasetSummary.json.""" @@ -125,19 +123,28 @@ def test_build_dataset_session_summaries(self, source_type): ) def test_build_dataset_documents(self, source_type): - """Verify that every exported JSON document can be loaded from the dataset DB.""" + """Verify that every exported JSON document can be loaded from session DBs. + + Mirrors MATLAB which queries each session's database individually + (not the dataset's database). + """ artifact_dir, dataset = self._open_dataset(source_type) json_docs_dir = artifact_dir / "jsonDocuments" if not json_docs_dir.exists(): pytest.skip(f"jsonDocuments directory not found in {source_type}.") - json_files = list(json_docs_dir.glob("*.json")) + json_files = list(json_docs_dir.glob("**/*.json")) - actual_docs = dataset.database_search(Query("base.id").match("(.*)")) + # Collect docs from each session's database, matching MATLAB's approach + actual_docs = [] + _refs, session_ids, *_ = dataset.session_list() + for sid in session_ids: + sess = dataset.open_session(sid) + actual_docs.extend(sess.database_search(Query("base.id").match("(.*)"))) assert len(actual_docs) == len(json_files), ( - f"Number of documents in dataset ({len(actual_docs)}) does not match " + f"Number of documents across sessions ({len(actual_docs)}) does not match " f"{source_type} JSON artifacts ({len(json_files)})." ) @@ -152,10 +159,10 @@ def test_build_dataset_documents(self, source_type): actual_props = actual.document_properties assert actual_props.get("document_class", {}).get( "class_name" - ) == expected_doc.get("document_class", {}).get("class_name"), ( - f"Document class mismatch for id: {expected_id} " f"in {source_type}" - ) + ) == expected_doc.get("document_class", {}).get( + "class_name" + ), f"Document class mismatch for id: {expected_id} in {source_type}" break - assert found, ( - f"Document from {source_type} artifact not found in dataset: " f"{expected_id}" - ) + assert ( + found + ), f"Document from {source_type} artifact not found in dataset: {expected_id}" diff --git a/tests/symmetry/read_artifacts/session/_summary_helpers.py b/tests/symmetry/read_artifacts/session/_summary_helpers.py new file mode 100644 index 0000000..6a94777 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/_summary_helpers.py @@ -0,0 +1,21 @@ +"""Shared helpers for read_artifacts session symmetry tests.""" + +from __future__ import annotations + +from typing import Any + + +def sort_daq_systems_by_name(summary: dict[str, Any]) -> None: + """Sort daqSystemNames and daqSystemDetails in a summary by name. + + The database may return DAQ systems in different order on re-open. + Sorting both summaries by name before comparison avoids spurious + ordering mismatches. Modifies *summary* in place. + """ + names = summary.get("daqSystemNames", []) + details = summary.get("daqSystemDetails", []) + + if len(names) == len(details) and len(names) > 1: + paired = sorted(zip(names, details), key=lambda x: x[0]) + summary["daqSystemNames"] = [p[0] for p in paired] + summary["daqSystemDetails"] = [p[1] for p in paired] diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py new file mode 100644 index 0000000..2c96c35 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_kjnielsenlab.py @@ -0,0 +1,76 @@ +"""Read and verify symmetry artifacts for a blank kjnielsenlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionKjnielsenlab.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can open the +session and produce a matching session summary. +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBlankSessionKjnielsenlab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionKjnielsenlab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE + / source_type + / "session" + / "blankSessionKjnielsenlab" + / "testBlankSessionKjnielsenlab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_kjnielsenlab_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + # Normalize DAQ system ordering — the database may return them in + # different order on re-open. + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report + ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py new file mode 100644 index 0000000..b38a0cb --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_marderlab.py @@ -0,0 +1,74 @@ +"""Read and verify symmetry artifacts for a blank marderlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionMarderlab.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can open the +session and produce a matching session summary. +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBlankSessionMarderlab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionMarderlab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE + / source_type + / "session" + / "blankSessionMarderlab" + / "testBlankSessionMarderlab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_marderlab_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report + ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py b/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py new file mode 100644 index 0000000..938c925 --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_vhlab.py @@ -0,0 +1,70 @@ +"""Read and verify symmetry artifacts for a blank vhlab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionVhlab.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can open the +session and produce a matching session summary. +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBlankSessionVhlab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionVhlab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE / source_type / "session" / "blankSessionVhlab" / "testBlankSessionVhlab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_vhlab_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report + )