diff --git a/DID_PYTHON_UPDATE_INSTRUCTIONS.md b/DID_PYTHON_UPDATE_INSTRUCTIONS.md new file mode 100644 index 0000000..51f8e3e --- /dev/null +++ b/DID_PYTHON_UPDATE_INSTRUCTIONS.md @@ -0,0 +1,285 @@ +# Instructions: Update DID-python to Match DID-matlab SQLite Behavior + +## Background + +DID-python (`pip show did`, installed at `/usr/local/lib/python3.11/dist-packages/did/`) is a Python port of [VH-Lab/DID-matlab](https://github.com/VH-Lab/DID-matlab). The SQLite implementation has two critical gaps versus the MATLAB original: + +1. **`_do_add_doc` does not populate `fields`/`doc_data` tables** — it only stores JSON blobs in `docs`. The MATLAB version (`did.implementations.sqlitedb.do_add_doc`) uses `did.implementations.doc2sql` to flatten documents and populate both tables. + +2. **`search` uses brute-force JSON deserialization** instead of SQL queries against `doc_data`. The MATLAB version builds SQL queries from query structs and runs them directly. + +This means databases created by Python cannot be searched by MATLAB, and vice versa (MATLAB's search relies on `doc_data`). + +## Reference Files + +**DID-matlab** (authoritative, on `main` branch of https://github.com/VH-Lab/DID-matlab): +- `src/did/+did/+implementations/sqlitedb.m` — SQLite implementation (uses mksqlite) +- `src/did/+did/+implementations/doc2sql.m` — Document-to-SQL flattener +- `src/did/+did/database.m` — Base class with `search_doc_ids`, `get_sql_query_str`, `query_struct_to_sql_str` +- `src/did/+did/query.m` — Query class with `to_searchstructure` resolving `isa`/`depends_on` +- `src/did/+did/+datastructures/fieldsearch.m` — In-memory field search (used by non-SQL backends) + +**DID-python** (to be updated): +- `/usr/local/lib/python3.11/dist-packages/did/implementations/sqlitedb.py` +- `/usr/local/lib/python3.11/dist-packages/did/implementations/doc2sql.py` +- `/usr/local/lib/python3.11/dist-packages/did/database.py` +- `/usr/local/lib/python3.11/dist-packages/did/query.py` +- `/usr/local/lib/python3.11/dist-packages/did/datastructures.py` + +**NDI-python** (consumer — has workarounds to remove after DID is fixed): +- `/home/user/NDI-python/src/ndi/database.py` — `SQLiteDriver` class with `_populate_doc_data` workaround + +## Task 1: Populate `fields`/`doc_data` in `_do_add_doc` + +### What MATLAB Does + +In `sqlitedb.m`, `do_add_doc` (line ~256-404): + +1. Calls `did.implementations.doc2sql(document_obj)` to produce a struct array of "meta-tables" +2. The first meta-table is always `"meta"` with columns: `doc_id`, `class`, `superclass`, `datestamp`, `creation`, `deletion`, `depends_on` +3. Subsequent meta-tables are named after top-level document fields (e.g., `"base"`, `"element"`, `"daqsystem"`) with sub-field columns +4. For each column (skipping `doc_id`), it calls `get_field_idx(group_name, field_name)` to look up or create a `fields` row +5. Inserts all `(doc_idx, field_idx, value)` triples into `doc_data` in bulk + +### How `doc2sql` Flattens Documents + +Given a document with properties: +```json +{ + "document_class": {"class_name": "element", "superclasses": [{"definition": "$NDIDOCUMENTPATH/base.json"}]}, + "base": {"id": "abc", "name": "elec1", "session_id": "sess1", "datestamp": "2024-01-01"}, + "element": {"type": "probe", "reference": 1}, + "depends_on": [{"name": "subject_id", "value": "xyz"}] +} +``` + +`doc2sql` produces: + +**Meta table** (`name="meta"`): +| column name | value | +|---|---| +| doc_id | `"abc"` | +| class | `"element"` | +| superclass | `"base"` (extracted from definitions, stripped path and `.json`) | +| datestamp | `"2024-01-01"` | +| creation | `""` | +| deletion | `""` | +| depends_on | `"subject_id,xyz;"` (formatted as `name,value;name,value;...`) | + +**Base table** (`name="base"`): +| column name | value | +|---|---| +| doc_id | `"abc"` | +| id | `"abc"` | +| name | `"elec1"` | +| session_id | `"sess1"` | +| datestamp | `"2024-01-01"` | + +**Element table** (`name="element"`): +| column name | value | +|---|---| +| doc_id | `"abc"` | +| type | `"probe"` | +| reference | `1` | + +### How `get_field_idx` Works + +The field name stored in the `fields` table uses the format `{group_name}.{field_name}` — e.g., `"meta.class"`, `"base.name"`, `"element.type"`. + +Triple-underscores in column names from `doc2sql` are converted back to dots: `"___"` → `"."`. + +The `fields` table columns are: +- `class`: the group/table name (e.g., `"meta"`, `"base"`, `"element"`) +- `field_name`: the dot-separated path (e.g., `"meta.class"`, `"base.name"`) +- `json_name`: dots replaced with `___` (e.g., `"meta___class"`) +- `field_idx`: auto-increment primary key + +A `fields_cache` (in-memory dict) avoids repeated DB lookups. + +### What to Change in DID-python + +In `sqlitedb.py`, method `_do_add_doc` (currently lines 145-180): + +After the `docs` INSERT (line 168) and before the `branch_docs` INSERT (line 173), add logic to: + +1. Call a Python equivalent of `doc2sql` to flatten the document +2. For each (field_name, value) pair, look up or create a `field_idx` in the `fields` table +3. Bulk-insert all `(doc_idx, field_idx, value)` triples into `doc_data` + +You can either: +- **Option A**: Update the existing `doc2sql.py` to match MATLAB's `doc2sql.m` and call it from `_do_add_doc` +- **Option B**: Implement the flattening inline in `_do_add_doc` (simpler) + +**Critical**: The field names in the `fields` table MUST match what MATLAB's `query_struct_to_sql_str` expects. Specifically, the search queries use these field names: +- `"meta.class"` — the document class name (from `document_class.class_name`) +- `"meta.superclass"` — comma-separated superclass names (extracted from `document_class.superclasses[].definition`, with path and `.json` stripped) +- `"meta.depends_on"` — semicolon-separated `name,value;` pairs +- `"meta.datestamp"` — from `base.datestamp` +- `"{group}.{field}"` — for all other top-level groups and their sub-fields (e.g., `"base.name"`, `"element.type"`) + +### Superclass Extraction + +MATLAB extracts superclass names from the `definition` field of each superclass entry: +```matlab +superclass = getField(doc_props, 'document_class.superclasses'); +if isstruct(superclass) + superclass = regexprep({superclass.definition},{'.+/','\..+$'},''); + superclass = strjoin(unique(superclass), ', '); +end +``` + +This takes `"$NDIDOCUMENTPATH/daq/daqsystem.json"` → strips path → strips extension → `"daqsystem"`. + +Multiple superclasses are joined as `"base, daqsystem"` (comma-space separated, alphabetically sorted/unique). + +### Depends_on Serialization + +MATLAB serializes `depends_on` as: +```matlab +allData = [{dependsOn.name}; {dependsOn.value}]; +dependsOn = sprintf('%s,%s;',allData{:}); +``` + +This produces: `"filenavigator_id,abc123;daqreader_id,def456;"`. + +The search query for `depends_on` uses `LIKE "%name,value;%"`. + +### Nested Struct Fields (via `getMetaTableFrom`) + +For each top-level field group (excluding `depends_on`, `document_class`, `files`), MATLAB creates a "meta-table" with: +- A `doc_id` column +- One column per sub-field, with nested structs flattened using `___` separator + +Example: if `element` has `{"type": "probe", "details": {"count": 3}}`, the columns would be: +- `type` → `"probe"` (stored as field `element.type`) +- `details___count` → `3` (stored as field `element.details.count` after `___` → `.` conversion) + +## Task 2: Update `query.to_search_structure` to Resolve `isa` and `depends_on` + +### What MATLAB Does + +In `query.m`, the `to_searchstructure` method (line ~173-227) resolves `isa` and `depends_on` into lower-level operations BEFORE passing to search: + +**`isa` resolution** — converts to `OR(hasanysubfield_contains_string, contains_string)`: +```matlab +if strcmpi('isa', operation) + findinsubfield = struct('field','document_class.superclasses',... + 'operation','hasanysubfield_contains_string',... + 'param1','definition', 'param2', classname); + findinmainfield = struct('field','document_class.definition', ... + 'operation','contains_string', 'param1', classname, 'param2', ''); + ss.field = ''; + ss.operation = 'or'; + ss.param1 = findinsubfield; + ss.param2 = findinmainfield; +end +``` + +**`depends_on` resolution** — converts to `hasanysubfield_exact_string`: +```matlab +if strcmpi('depends_on', operation) + param1 = {'name','value'}; + param2 = {name_param, value_param}; + if strcmp(param2{1},'*') % wildcard: ignore name + param1 = param1(2); + param2 = param2(2); + end + ss = struct('field','depends_on','operation','hasanysubfield_exact_string'); + ss.param1 = param1; + ss.param2 = param2; +end +``` + +### What DID-python Currently Does + +In `query.py` line 52-55: +```python +def to_search_structure(self): + # A full implementation would recursively resolve 'isa', 'depends_on', etc. + # This is a simplified version for now. + return self.search_structure +``` + +It passes `isa` and `depends_on` through unchanged, relying on `field_search` in `datastructures.py` to handle them directly. This works for brute-force search but NOT for SQL-based search. + +### What to Change + +Update `query.py`'s `to_search_structure` to resolve `isa` and `depends_on` into lower-level operations, matching MATLAB's logic. This is needed so the SQL query builder can translate them. + +However, note that `datastructures.py`'s `field_search` also handles `isa` and `depends_on` directly (for non-SQL backends), so you should NOT break that path. The cleanest approach: resolve in `to_search_structure` and let `field_search` handle the resolved operations. + +## Task 3: Add SQL-Based Search (Optional but Recommended) + +### What MATLAB Does + +MATLAB's `did.database` base class (on `main` branch) has a SQL-based `do_search` that: + +1. Calls `search_doc_ids(query_struct, branch_id)` which recursively: + - For AND queries (struct arrays): intersects results from each sub-query + - For OR queries: unions results from each sub-query + - For leaf queries: calls `get_sql_query_str` → runs SQL → returns doc_ids + +2. `get_sql_query_str` builds: +```sql +SELECT DISTINCT docs.doc_id +FROM docs, branch_docs, doc_data, fields +WHERE docs.doc_idx = doc_data.doc_idx + AND docs.doc_idx = branch_docs.doc_idx + AND branch_docs.branch_id = "a" + AND fields.field_idx = doc_data.field_idx + AND {per-operation clause} +``` + +3. `query_struct_to_sql_str` maps operations to SQL: + - `exact_string` → `fields.field_name="X" AND doc_data.value = "Y"` + - `contains_string` → `fields.field_name="X" AND doc_data.value LIKE "%Y%"` + - `regexp` → `fields.field_name="X" AND regex(doc_data.value, "Y") NOT NULL` + - Note: MATLAB uses mksqlite's built-in `regex()` function. Python's sqlite3 does NOT have regex by default — you'd need `connection.create_function("regexp", 2, ...)`. + - `isa` → `(fields.field_name="meta.class" AND doc_data.value = "X") OR (fields.field_name="meta.superclass" AND regex(doc_data.value, "(^|, )X(,|$)") NOT NULL)` + - `depends_on` → `fields.field_name="meta.depends_on" AND doc_data.value LIKE "%name,value;%"` + - `hasfield` → `fields.field_name="X" OR fields.field_name LIKE "X.%"` + - Numeric comparisons: `doc_data.value < Y`, etc. + - `NOT` prefix: adds `NOT` before the value check + +### What to Change + +You can either: +- **Option A (Recommended for parity)**: Override `search` in `SQLiteDB` to build SQL queries against `doc_data`, matching MATLAB's `query_struct_to_sql_str` logic +- **Option B (Minimum viable)**: Keep the brute-force `field_search` approach in `database.py` but ensure `doc_data` is populated (Task 1) so MATLAB can search Python-created DBs + +Option A gives full symmetry. Option B gives cross-language write compatibility but not search performance parity. + +## Task 4: Remove NDI-python Workarounds + +After DID-python is updated, remove the workaround code from NDI-python's `database.py`: + +1. Remove `SQLiteDriver._flatten_document()` (static method) +2. Remove `SQLiteDriver._get_or_create_field_idx()` +3. Remove `SQLiteDriver._populate_doc_data()` +4. Remove `SQLiteDriver._populate_doc_data_with_cursor()` +5. Remove the `_populate_doc_data` calls in `add()`, `bulk_add()`, and `update()` +6. Remove the `doc_data` cleanup in `update()` + +The `bulk_add` method bypasses DID-python's `_do_add_doc` for performance — after DID is fixed, either: +- Route `bulk_add` through DID's `add_docs` (simpler, slightly slower), or +- Keep the direct SQL but call DID's flattening logic instead of NDI's + +## Testing + +1. Create a fresh database with Python, add documents, verify `fields` and `doc_data` tables are populated correctly +2. Search the Python-created database with MATLAB — verify `isa`, `depends_on`, `exact_string` queries all work +3. Create a database with MATLAB, search it with Python — verify all query types work +4. Run the existing symmetry tests in `/home/user/NDI-python/tests/symmetry/` + +## Key Gotcha: Field Name Format + +The most critical detail for cross-language compatibility is that **field names in the `fields` table must match between MATLAB and Python**. MATLAB uses `doc2sql` which produces: + +- `meta.class` (not `document_class.class_name`) +- `meta.superclass` (not `document_class.superclasses`) +- `meta.depends_on` (not `depends_on`) +- `meta.datestamp` (not `base.datestamp`) +- `base.id`, `base.name`, etc. +- `element.type`, `element.reference`, etc. + +Python's current workaround in NDI uses raw dotted paths like `document_class.class_name` and `depends_on(0).name` — these do NOT match what MATLAB expects. The Python `doc2sql` must match MATLAB's `doc2sql` field naming exactly. diff --git a/README.md b/README.md index 673e942..9dcb6ac 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Dataset (multi-session container) ### Running Tests ```bash -# Run all tests +# Run all tests (excludes symmetry tests by default) pytest tests/ -v # Run specific test module @@ -249,6 +249,30 @@ pytest tests/test_document.py -v pytest tests/ --cov=src/ndi --cov-report=term-missing ``` +### Cross-Language Symmetry Tests + +Symmetry tests verify that NDI-python and NDI-matlab produce identical artifacts. +They live in `tests/symmetry/` and are **excluded from the default test run** +because they require artifacts that may have been generated by a prior MATLAB session. + +```bash +# Step 1 — Generate Python artifacts +pytest tests/symmetry/make_artifacts/ -v + +# Step 2 — (Optional) Run MATLAB makeArtifacts to generate MATLAB artifacts +# In MATLAB: results = runtests('ndi.symmetry.makeArtifacts'); + +# Step 3 — Verify artifacts from both languages +pytest tests/symmetry/read_artifacts/ -v + +# Run all symmetry tests at once (make + read) +pytest tests/symmetry/ -v +``` + +Tests that cannot find their expected artifact directory are **skipped** (not +failed), so the suite runs safely on machines with only one language installed. +See `docs/developer_notes/symmetry_tests.md` for the full framework description. + ### Code Quality CI enforces formatting and lint on every push/PR: diff --git a/docs/developer_notes/symmetry_tests.md b/docs/developer_notes/symmetry_tests.md new file mode 100644 index 0000000..eb5600d --- /dev/null +++ b/docs/developer_notes/symmetry_tests.md @@ -0,0 +1,180 @@ +# Cross-Language Symmetry Test Framework + +**Status:** Active +**Scope:** NDI-python ↔ NDI-matlab parity + +## Purpose + +Symmetry tests verify that NDI sessions, documents, and probes created by one +language implementation can be correctly read and interpreted by the other. +This is the primary mechanism for ensuring that the Python and MATLAB NDI stacks +remain interoperable as both codebases evolve. + +## Architecture + +The framework has two halves, each existing in both languages: + +| Phase | Python location | MATLAB location | +|-------|----------------|-----------------| +| **makeArtifacts** | `tests/symmetry/make_artifacts/` | `tests/+ndi/+symmetry/+makeArtifacts/` | +| **readArtifacts** | `tests/symmetry/read_artifacts/` | `tests/+ndi/+symmetry/+readArtifacts/` | + +### Artifact Directory Layout + +All artifacts are written to the OS temporary directory under a fixed path: + +``` +/NDI/symmetryTest/ +├── pythonArtifacts/ +│ └── /// +│ ├── .ndi/ # NDI session database +│ ├── jsonDocuments/ # One JSON file per document +│ │ ├── .json +│ │ └── .json +│ └── probes.json # Array of probe descriptors +└── matlabArtifacts/ + └── /// + └── ... (same structure) +``` + +- **``** — the NDI domain being tested (e.g., `session`). +- **``** — the test class name, in camelCase (e.g., `buildSession`). +- **``** — the test method name, in camelCase (e.g., `testBuildSessionArtifacts`). + +### Workflow + +``` +┌──────────────────────────┐ ┌──────────────────────────┐ +│ Python makeArtifacts │ │ MATLAB makeArtifacts │ +│ pytest tests/symmetry/ │ │ runtests('ndi.symmetry. │ +│ make_artifacts/ -v │ │ makeArtifacts') │ +└──────────┬───────────────┘ └──────────┬───────────────┘ + │ writes │ writes + ▼ ▼ + pythonArtifacts/ matlabArtifacts/ + │ │ + └────────────┬────────────────────┘ + │ reads + ┌────────────┴────────────────────┐ + │ │ + ▼ ▼ +┌──────────────────────────┐ ┌──────────────────────────┐ +│ Python readArtifacts │ │ MATLAB readArtifacts │ +│ pytest tests/symmetry/ │ │ runtests('ndi.symmetry. │ +│ read_artifacts/ -v │ │ readArtifacts') │ +└──────────────────────────┘ └──────────────────────────┘ +``` + +Each `readArtifacts` test is parameterized over `{matlabArtifacts, pythonArtifacts}` +so a single test class validates both directions of compatibility. + +## Running the Tests + +### From Python + +```bash +# Generate artifacts +pytest tests/symmetry/make_artifacts/ -v + +# Verify artifacts (skips missing sources) +pytest tests/symmetry/read_artifacts/ -v + +# Both phases at once +pytest tests/symmetry/ -v +``` + +### From MATLAB + +```matlab +% Generate artifacts +results = runtests('ndi.symmetry.makeArtifacts'); + +% Verify artifacts +results = runtests('ndi.symmetry.readArtifacts'); +``` + +### Why Separate from Regular Tests? + +Symmetry tests are **excluded from the default `pytest` run** (via +`--ignore=tests/symmetry` in `pyproject.toml`) because: + +1. **readArtifacts** tests will mostly just be skipped unless the user has + previously run MATLAB's `makeArtifacts` suite on the same machine. +2. **makeArtifacts** tests write to the system temp directory, which is a + side-effect that doesn't belong in routine CI. +3. The full cross-language cycle requires both runtimes and is better suited + to integration / nightly CI pipelines. + +## Writing a New Symmetry Test + +### 1. Choose a namespace + +Pick the NDI domain being tested (e.g., `session`, `document`, `probe`). + +### 2. Create the makeArtifacts test + +**Python:** `tests/symmetry/make_artifacts//test_.py` + +```python +import json, shutil +from pathlib import Path +from tests.symmetry.conftest import PYTHON_ARTIFACTS +from ndi.session.dir import DirSession + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "" / "" / "" + +class TestMyFeature: + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + # Build session in tmp_path ... + self.session = DirSession("exp1", tmp_path / "exp1") + + def test_my_feature_artifacts(self): + if ARTIFACT_DIR.exists(): + shutil.rmtree(ARTIFACT_DIR) + shutil.copytree(str(self.session.path), str(ARTIFACT_DIR)) + # Write jsonDocuments/, probes.json, etc. +``` + +**MATLAB:** `tests/+ndi/+symmetry/+makeArtifacts/+/.m` + +Follow the INSTRUCTIONS.md in the MATLAB `+makeArtifacts` folder. + +### 3. Create the readArtifacts test + +**Python:** `tests/symmetry/read_artifacts//test_.py` + +```python +import json, pytest +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from ndi.session.dir import DirSession + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + +class TestMyFeature: + def test_my_feature_artifacts(self, source_type): + artifact_dir = SYMMETRY_BASE / source_type / "" / ... + if not artifact_dir.exists(): + pytest.skip(f"No artifacts from {source_type}") + session = DirSession("exp1", artifact_dir) + # Assertions ... +``` + +**MATLAB:** `tests/+ndi/+symmetry/+readArtifacts/+/.m` + +Follow the INSTRUCTIONS.md in the MATLAB `+readArtifacts` folder. + +### 4. Naming Conventions + +| Concept | Python | MATLAB | +|---------|--------|--------| +| Test directory | `tests/symmetry/make_artifacts/session/` | `tests/+ndi/+symmetry/+makeArtifacts/+session/` | +| Test file | `test_build_session.py` | `buildSession.m` | +| Test class | `TestBuildSession` | `buildSession` (classdef) | +| Artifact className | `buildSession` (camelCase) | `buildSession` | +| Artifact testName | `testBuildSessionArtifacts` (camelCase) | `testBuildSessionArtifacts` | + +Use **camelCase** for the artifact directory components (`className`, `testName`) +so that both languages write to and read from the exact same paths. diff --git a/pyproject.toml b/pyproject.toml index b5c1ff9..1ef270d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,9 +99,10 @@ include = ["/src", "/tests", "/examples", "LICENSE", "README.md", "CHANGELOG.md" [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py", "*_test.py"] -addopts = "-v --tb=short" +addopts = "-v --tb=short --ignore=tests/symmetry" markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "symmetry: marks cross-language symmetry tests (run with 'pytest tests/symmetry/')", ] [tool.black] diff --git a/src/ndi/daq/system.py b/src/ndi/daq/system.py index 9e23e94..c3b0ede 100644 --- a/src/ndi/daq/system.py +++ b/src/ndi/daq/system.py @@ -94,20 +94,27 @@ def _load_from_document(self, session: Any, document: Any) -> None: """Load DAQSystem from a document.""" doc_props = getattr(document, "document_properties", document) - # Get IDs from document - daqreader_id = document.dependency_value("daqreader_id") - filenavigator_id = document.dependency_value("filenavigator_id") + # Extract basic properties using dict access + base = doc_props.get("base", {}) if isinstance(doc_props, dict) else {} + self._name = base.get("name", "") + if "id" in base: + self._id = base["id"] + self._session = session + + # Get dependency IDs from document + daqreader_id = document.dependency_value("daqreader_id", error_if_not_found=False) + filenavigator_id = document.dependency_value("filenavigator_id", error_if_not_found=False) # Load reader and navigator from database from ..query import Query - reader_docs = session.database_search(Query("base.id") == daqreader_id) - if len(reader_docs) != 1: - raise ValueError(f"Could not find daqreader with id {daqreader_id}") + reader_docs = [] + if daqreader_id: + reader_docs = session.database_search(Query("base.id") == daqreader_id) - nav_docs = session.database_search(Query("base.id") == filenavigator_id) - if len(nav_docs) != 1: - raise ValueError(f"Could not find filenavigator with id {filenavigator_id}") + nav_docs = [] + if filenavigator_id: + nav_docs = session.database_search(Query("base.id") == filenavigator_id) # Load metadata readers metadata_ids = ( @@ -117,49 +124,65 @@ def _load_from_document(self, session: Any, document: Any) -> None: for mid in metadata_ids: m_docs = session.database_search(Query("base.id") == mid) if len(m_docs) == 1: - # Create reader from document from .metadatareader import MetadataReader - metadata_readers.append(MetadataReader(session=session, document=m_docs[0])) - - self._name = doc_props.base.name if hasattr(doc_props, "base") else "" - self.identifier = doc_props.base.id if hasattr(doc_props, "base") else self.identifier - self._session = session + try: + metadata_readers.append(MetadataReader(session=session, document=m_docs[0])) + except Exception: + pass self._daqmetadatareaders = metadata_readers # Reconstruct reader from its document - reader_doc = reader_docs[0] - reader_class_name = reader_doc._get_nested_property("daqreader.ndi_daqreader_class", "") - _READER_CLASSES = { - "IntanReader": "ndi.daq.reader.mfdaq.intan.IntanReader", - "BlackrockReader": "ndi.daq.reader.mfdaq.blackrock.BlackrockReader", - "CEDSpike2Reader": "ndi.daq.reader.mfdaq.cedspike2.CEDSpike2Reader", - "SpikeGadgetsReader": "ndi.daq.reader.mfdaq.spikegadgets.SpikeGadgetsReader", - } - reader_path = _READER_CLASSES.get(reader_class_name) - if reader_path: - try: - module_path, cls_name = reader_path.rsplit(".", 1) - import importlib + self._daqreader = None + if len(reader_docs) == 1: + reader_doc = reader_docs[0] + reader_class_name = "" + reader_props = reader_doc.document_properties + if isinstance(reader_props, dict): + reader_class_name = reader_props.get("daqreader", {}).get("ndi_daqreader_class", "") + else: + reader_class_name = reader_doc._get_nested_property( + "daqreader.ndi_daqreader_class", "" + ) - mod = importlib.import_module(module_path) - ReaderCls = getattr(mod, cls_name) - self._daqreader = ReaderCls(session=session, document=reader_doc) - except Exception as exc: - logger.warning("Could not reconstruct DAQ reader %s: %s", reader_class_name, exc) - self._daqreader = None - else: - logger.debug("Unknown DAQ reader class: %s", reader_class_name) - self._daqreader = None + # Map both Python class names and MATLAB class names + _READER_CLASSES = { + # Python class names + "IntanReader": "ndi.daq.reader.mfdaq.intan.IntanReader", + "BlackrockReader": "ndi.daq.reader.mfdaq.blackrock.BlackrockReader", + "CEDSpike2Reader": "ndi.daq.reader.mfdaq.cedspike2.CEDSpike2Reader", + "SpikeGadgetsReader": "ndi.daq.reader.mfdaq.spikegadgets.SpikeGadgetsReader", + # MATLAB class names + "ndi.daq.reader.mfdaq.intan": "ndi.daq.reader.mfdaq.intan.IntanReader", + "ndi.daq.reader.mfdaq.blackrock": "ndi.daq.reader.mfdaq.blackrock.BlackrockReader", + "ndi.daq.reader.mfdaq.cedspike2": "ndi.daq.reader.mfdaq.cedspike2.CEDSpike2Reader", + "ndi.daq.reader.mfdaq.spikegadgets": "ndi.daq.reader.mfdaq.spikegadgets.SpikeGadgetsReader", + } + reader_path = _READER_CLASSES.get(reader_class_name) + if reader_path: + try: + module_path, cls_name = reader_path.rsplit(".", 1) + import importlib + + mod = importlib.import_module(module_path) + ReaderCls = getattr(mod, cls_name) + self._daqreader = ReaderCls(session=session, document=reader_doc) + except Exception as exc: + logger.warning( + "Could not reconstruct DAQ reader %s: %s", reader_class_name, exc + ) + else: + logger.debug("Unknown DAQ reader class: %s", reader_class_name) # Reconstruct file navigator from its document - from ..file.navigator import FileNavigator + self._filenavigator = None + if len(nav_docs) == 1: + from ..file.navigator import FileNavigator - try: - self._filenavigator = FileNavigator(session=session, document=nav_docs[0]) - except Exception as exc: - logger.warning("Could not reconstruct file navigator: %s", exc) - self._filenavigator = None + try: + self._filenavigator = FileNavigator(session=session, document=nav_docs[0]) + except Exception as exc: + logger.warning("Could not reconstruct file navigator: %s", exc) @property def name(self) -> str: diff --git a/src/ndi/database.py b/src/ndi/database.py index c529d0b..4481323 100644 --- a/src/ndi/database.py +++ b/src/ndi/database.py @@ -33,12 +33,14 @@ class SQLiteDriver: field_search() for query evaluation. """ - def __init__(self, db_path: Path, branch_id: str = "main"): + def __init__(self, db_path: Path, branch_id: str = "a"): """Initialize the SQLite driver. Args: db_path: Path to the SQLite database file. - branch_id: Default branch ID to use. + branch_id: Default branch ID to use. DID-matlab and + NDI-matlab have always used ``"a"`` as the default + branch, so we match that for cross-language compatibility. """ from did.datastructures import field_search from did.document import Document as DIDDocument @@ -52,7 +54,7 @@ def __init__(self, db_path: Path, branch_id: str = "main"): # Initialize SQLiteDB self._db = SQLiteDB(str(db_path)) - # Create main branch if it doesn't exist + # Create branch if it doesn't exist existing_branches = self._db.all_branch_ids() if branch_id not in existing_branches: self._db.add_branch(branch_id, "") # Empty string for root branch @@ -72,6 +74,11 @@ def add(self, document: dict) -> None: did_doc = self._DIDDocument(document) self._db.add_docs([did_doc], self._branch_id) + # Populate doc_data indexed fields for cross-language compatibility. + # DID-python's _do_add_doc does not populate the fields/doc_data + # tables, but DID-matlab's search relies on them. + self._populate_doc_data(doc_id, document) + def bulk_add(self, documents: list[dict]) -> tuple[int, int]: """Add many documents at once, bypassing per-doc duplicate checks. @@ -109,6 +116,9 @@ def bulk_add(self, documents: list[dict]) -> tuple[int, int]: ) except Exception: pass + # Populate doc_data for cross-language compatibility + self._populate_doc_data_with_cursor(cursor, doc_idx, doc) + existing_ids.add(doc_id) added += 1 self._db.dbid.commit() @@ -118,6 +128,69 @@ def bulk_add(self, documents: list[dict]) -> tuple[int, int]: return added, skipped + # ----------------------------------------------------------------- + # doc_data population for MATLAB DID compatibility + # ----------------------------------------------------------------- + + @staticmethod + def _flatten_document(doc: dict, prefix: str = "") -> list[tuple[str, str]]: + """Flatten a document dict into (field_path, value) pairs. + + Only includes leaf scalar values (str, int, float, bool). + Skips lists and nested dicts (those are traversed recursively). + """ + pairs: list[tuple[str, str]] = [] + for key, val in doc.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(val, dict): + pairs.extend(SQLiteDriver._flatten_document(val, path)) + elif isinstance(val, list): + # Index list elements for depends_on etc. + for i, item in enumerate(val): + if isinstance(item, dict): + pairs.extend(SQLiteDriver._flatten_document(item, f"{path}({i})")) + elif item is not None: + pairs.append((f"{path}({i})", str(item))) + elif val is not None: + pairs.append((path, str(val))) + return pairs + + def _get_or_create_field_idx(self, cursor, field_name: str, doc_class: str = "") -> int: + """Get field_idx from the fields table, creating a new entry if needed.""" + cursor.execute("SELECT field_idx FROM fields WHERE field_name = ?", (field_name,)) + row = cursor.fetchone() + if row: + return row[0] if isinstance(row, (tuple, list)) else row["field_idx"] + cursor.execute( + "INSERT INTO fields (class, field_name, json_name, field_idx) " + "VALUES (?, ?, ?, NULL)", + (doc_class, field_name, field_name), + ) + return cursor.lastrowid + + def _populate_doc_data(self, doc_id: str, document: dict) -> None: + """Populate the doc_data table for a document after insertion.""" + cursor = self._db.dbid.cursor() + # Look up doc_idx + cursor.execute("SELECT doc_idx FROM docs WHERE doc_id = ?", (doc_id,)) + row = cursor.fetchone() + if not row: + return + doc_idx = row[0] if isinstance(row, (tuple, list)) else row["doc_idx"] + self._populate_doc_data_with_cursor(cursor, doc_idx, document) + self._db.dbid.commit() + + def _populate_doc_data_with_cursor(self, cursor, doc_idx: int, document: dict) -> None: + """Populate doc_data for a document using an existing cursor.""" + doc_class = document.get("document_class", {}).get("class_name", "") + pairs = self._flatten_document(document) + for field_name, value in pairs: + field_idx = self._get_or_create_field_idx(cursor, field_name, doc_class) + cursor.execute( + "INSERT INTO doc_data (doc_idx, field_idx, value) VALUES (?, ?, ?)", + (doc_idx, field_idx, value), + ) + def update(self, document: dict) -> None: """Update an existing document.""" doc_id = document.get("base", {}).get("id", "") @@ -127,11 +200,23 @@ def update(self, document: dict) -> None: if doc_id not in existing_ids: raise FileNotFoundError(f"Document {doc_id} not found") + # Remove old doc_data entries + cursor = self._db.dbid.cursor() + cursor.execute("SELECT doc_idx FROM docs WHERE doc_id = ?", (doc_id,)) + row = cursor.fetchone() + if row: + old_idx = row[0] if isinstance(row, (tuple, list)) else row["doc_idx"] + cursor.execute("DELETE FROM doc_data WHERE doc_idx = ?", (old_idx,)) + self._db.dbid.commit() + # Remove old and add new (SQLiteDB doesn't have direct update) self._db.remove_docs([doc_id], self._branch_id) did_doc = self._DIDDocument(document) self._db.add_docs([did_doc], self._branch_id) + # Repopulate doc_data + self._populate_doc_data(doc_id, document) + def delete_by_id(self, doc_id: str) -> bool: """Delete a document by ID.""" existing_ids = self._db.get_doc_ids(self._branch_id) @@ -203,7 +288,7 @@ def __init__(self, session_path: str | Path, db_name: str = ".ndi", **backend_kw db_name: Name of the database directory within session. Default is '.ndi'. **backend_kwargs: Additional arguments passed to SQLiteDriver - (e.g., branch_id='main'). + (e.g., branch_id='a'). """ self.session_path = Path(session_path) self._db_name = db_name diff --git a/src/ndi/element/__init__.py b/src/ndi/element/__init__.py index 246e294..6b81d2c 100644 --- a/src/ndi/element/__init__.py +++ b/src/ndi/element/__init__.py @@ -114,21 +114,19 @@ def _load_from_document(self, session: Any, document: Any) -> None: """Load element from a document.""" props = getattr(document, "document_properties", document) - # Get basic properties - if hasattr(props, "element"): - self._name = getattr(props.element, "name", "") - self._reference = getattr(props.element, "reference", 0) - self._type = getattr(props.element, "type", "") - self._direct = getattr(props.element, "direct", True) - else: - self._name = "" - self._reference = 0 - self._type = "" - self._direct = True + # Get basic properties from the element dict + elem = props.get("element", {}) if isinstance(props, dict) else {} + self._name = elem.get("name", "") + ref = elem.get("reference", 0) + self._reference = int(ref) if ref != "" else 0 + self._type = elem.get("type", "") + direct_val = elem.get("direct", True) + self._direct = bool(int(direct_val)) if direct_val != "" else True # Get ID from base - if hasattr(props, "base") and hasattr(props.base, "id"): - self.identifier = props.base.id + base = props.get("base", {}) if isinstance(props, dict) else {} + if "id" in base: + self._id = base["id"] self._session = session self._subject_id = document.dependency_value("subject_id", error_if_not_found=False) or "" @@ -185,6 +183,20 @@ def dependencies(self) -> dict[str, str]: """Get additional dependencies.""" return self._dependencies + def ndi_element_class(self) -> str: + """Return the NDI element class name for document storage. + + MATLAB equivalent: ``class(ndi_element_obj)`` + + Returns the MATLAB-compatible class name string used in the + ``element.ndi_element_class`` document field. Subclasses + override this to return their own class name. + + Returns: + MATLAB class name string, e.g. ``'ndi.element'``. + """ + return "ndi.element" + def elementstring(self) -> str: """ Format element as human-readable string. @@ -395,6 +407,7 @@ def newdocument(self) -> Any: doc = Document( "element", **{ + "element.ndi_element_class": self.ndi_element_class(), "element.name": self._name, "element.reference": self._reference, "element.type": self._type, @@ -423,12 +436,22 @@ def searchquery(self) -> Any: """ Create a query to find this element. + MATLAB equivalent: ``ndi.element/searchquery`` + + Builds a query matching session_id, element name, type, + ndi_element_class, and reference — the same fields used + by the MATLAB implementation. + Returns: Query matching this element's document """ from ..query import Query - q = Query("base.id") == self.id + q = self._session.searchquery() if self._session is not None else Query("") + q = q & (Query("element.name") == self._name) + q = q & (Query("element.type") == self._type) + q = q & (Query("element.ndi_element_class") == self.ndi_element_class()) + q = q & (Query("element.reference") == self._reference) return q # ========================================================================= diff --git a/src/ndi/ido.py b/src/ndi/ido.py index f10a44d..ba6a5d0 100644 --- a/src/ndi/ido.py +++ b/src/ndi/ido.py @@ -64,10 +64,10 @@ def unique_id() -> str: """ # Get current time in microseconds since epoch time_us = int(time.time() * 1000000) - time_hex = format(time_us, "x") + time_hex = format(time_us, "016x") # Generate random component - random_hex = format(random.getrandbits(48), "012x") + random_hex = format(random.getrandbits(64), "016x") return f"{time_hex}_{random_hex}" diff --git a/src/ndi/probe/__init__.py b/src/ndi/probe/__init__.py index b1f2f8a..7cf2309 100644 --- a/src/ndi/probe/__init__.py +++ b/src/ndi/probe/__init__.py @@ -305,38 +305,22 @@ def getchanneldevinfo( } # ========================================================================= - # DocumentService Override + # Element overrides # ========================================================================= - def newdocument(self) -> Any: - """ - Create a new document for this probe. + def ndi_element_class(self) -> str: + """Return the NDI element class name for document storage. - Returns: - Document representing this probe - """ - from ..document import Document - - doc = Document( - "element", - **{ - "element.name": self._name, - "element.reference": self._reference, - "element.type": self._type, - "element.direct": True, # MATLAB sets direct=1 for probes - "base.id": self.id, - }, - ) + MATLAB equivalent: ``class(ndi_probe_obj)`` - # Set session ID - if self._session is not None: - doc.set_session_id(self._session.id) + Returns ``'ndi.probe'``. Subclasses (e.g. + ``ProbeTimeseriesMFDAQ``) override this to return their own + MATLAB class name. - # Set subject dependency - if self._subject_id: - doc.set_dependency_value("subject_id", self._subject_id) - - return doc + Returns: + MATLAB class name string. + """ + return "ndi.probe" # ========================================================================= # Static Methods diff --git a/src/ndi/probe/timeseries.py b/src/ndi/probe/timeseries.py index 362a719..67c4a02 100644 --- a/src/ndi/probe/timeseries.py +++ b/src/ndi/probe/timeseries.py @@ -32,6 +32,10 @@ class ProbeTimeseries(Probe): >>> data, t, timeref = probe.readtimeseries(epoch=1, t0=0, t1=10) """ + def ndi_element_class(self) -> str: + """Return ``'ndi.probe.timeseries'``.""" + return "ndi.probe.timeseries" + def readtimeseries( self, epoch: int | str | Any = None, diff --git a/src/ndi/probe/timeseries_mfdaq.py b/src/ndi/probe/timeseries_mfdaq.py index 18fa495..a46a82c 100644 --- a/src/ndi/probe/timeseries_mfdaq.py +++ b/src/ndi/probe/timeseries_mfdaq.py @@ -35,6 +35,10 @@ class ProbeTimeseriesMFDAQ(ProbeTimeseries): >>> sr = probe.samplerate(1) """ + def ndi_element_class(self) -> str: + """Return ``'ndi.probe.timeseries.mfdaq'``.""" + return "ndi.probe.timeseries.mfdaq" + def read_epochsamples( self, epoch: int | str, diff --git a/src/ndi/probe/timeseries_stimulator.py b/src/ndi/probe/timeseries_stimulator.py index ae0fbcf..1d9aa0b 100644 --- a/src/ndi/probe/timeseries_stimulator.py +++ b/src/ndi/probe/timeseries_stimulator.py @@ -58,14 +58,20 @@ def __init__( name: str = "", reference: int = 1, type: str = "stimulator", + **kwargs: Any, ): super().__init__( session=session, name=name, reference=reference, type=type, + **kwargs, ) + def ndi_element_class(self) -> str: + """Return ``'ndi.probe.timeseries.stimulator'``.""" + return "ndi.probe.timeseries.stimulator" + def readtimeseriesepoch( self, epoch: int | str, diff --git a/src/ndi/session/dir.py b/src/ndi/session/dir.py index 7277e32..af513ab 100644 --- a/src/ndi/session/dir.py +++ b/src/ndi/session/dir.py @@ -95,7 +95,10 @@ def __init__( # Try to load session info from database read_from_database = False if should_read_from_database: - session_docs = self.database_search(Query("").isa("session")) + # Search without the session_id filter so we can discover the + # existing session_id from documents already in the database + # (e.g. artifacts produced by MATLAB). + session_docs = self._database.search(Query("").isa("session")) if session_docs: # Use the oldest session document oldest_doc = session_docs[0] @@ -108,10 +111,10 @@ def __init__( oldest_doc = session_docs[0] props = oldest_doc.document_properties - if hasattr(props, "base"): - self._identifier = getattr(props.base, "session_id", self._identifier) - if hasattr(props, "session"): - self._reference = getattr(props.session, "reference", self._reference) + if "base" in props and isinstance(props["base"], dict): + self._identifier = props["base"].get("session_id", self._identifier) + if "session" in props and isinstance(props["session"], dict): + self._reference = props["session"].get("reference", self._reference) read_from_database = True # If not read from database, try reference file or create new diff --git a/src/ndi/session/session_base.py b/src/ndi/session/session_base.py index 982cac1..c1ed1d1 100644 --- a/src/ndi/session/session_base.py +++ b/src/ndi/session/session_base.py @@ -1077,6 +1077,15 @@ def _document_to_object(self, document: Document) -> Any: """ Convert a document to its corresponding NDI object. + MATLAB equivalent: ``ndi.database.fun.ndi_document2ndi_object`` + + For element documents MATLAB reads + ``element.ndi_element_class`` (e.g. + ``'ndi.probe.timeseries.mfdaq'``) and calls + ``eval([class_string '(session, doc)'])``. Python mirrors + this by maintaining a registry that maps MATLAB class name + strings to the corresponding Python classes. + Args: document: Document to convert @@ -1088,15 +1097,36 @@ def _document_to_object(self, document: Document) -> Any: from ..daq.system import DAQSystem return DAQSystem(session=self, document=document) - elif document.doc_isa("probe"): - from ..probe import Probe - return Probe(session=self, document=document) - elif document.doc_isa("element"): + if document.doc_isa("element"): + # Mirror MATLAB ndi_document2ndi_object: read the + # ndi_element_class field and construct the right class. from ..element import Element + from ..probe import Probe + from ..probe.timeseries import ProbeTimeseries + from ..probe.timeseries_mfdaq import ProbeTimeseriesMFDAQ + from ..probe.timeseries_stimulator import ProbeTimeseriesStimulator + + _NDI_CLASS_REGISTRY: dict[str, type] = { + "ndi.element": Element, + "ndi.probe": Probe, + "ndi.probe.timeseries": ProbeTimeseries, + "ndi.probe.timeseries.mfdaq": ProbeTimeseriesMFDAQ, + "ndi.probe.timeseries.stimulator": ProbeTimeseriesStimulator, + } + props = document.document_properties + ndi_class = props.get("element", {}).get("ndi_element_class", "") + cls = _NDI_CLASS_REGISTRY.get(ndi_class) + if cls is not None: + return cls(session=self, document=document) + # Fallback: if ndi_element_class contains "probe" but + # isn't a known key (e.g. "ndi.probe.timage"), use Probe. + if "probe" in ndi_class: + return Probe(session=self, document=document) return Element(session=self, document=document) - elif document.doc_isa("syncgraph"): + + if document.doc_isa("syncgraph"): return SyncGraph(session=self, document=document) return None diff --git a/src/ndi/util/__init__.py b/src/ndi/util/__init__.py index fa076d6..9616283 100644 --- a/src/ndi/util/__init__.py +++ b/src/ndi/util/__init__.py @@ -13,6 +13,7 @@ ``ndi.openminds_convert``. """ +from .compare_session_summary import compareSessionSummary from .datestamp2datetime import datestamp2datetime from .downsampleTimeseries import downsampleTimeseries from .getHexDiffFromFileObj import getHexDiffFromFileObj @@ -20,9 +21,11 @@ from .hexDiffBytes import hexDiffBytes from .hexDump import hexDump from .rehydrateJSONNanNull import rehydrateJSONNanNull +from .session_summary import sessionSummary from .unwrapTableCellContent import unwrapTableCellContent __all__ = [ + "compareSessionSummary", "datestamp2datetime", "downsampleTimeseries", "getHexDiffFromFileObj", @@ -30,5 +33,6 @@ "hexDiffBytes", "hexDump", "rehydrateJSONNanNull", + "sessionSummary", "unwrapTableCellContent", ] diff --git a/src/ndi/util/compare_session_summary.py b/src/ndi/util/compare_session_summary.py new file mode 100644 index 0000000..3b80369 --- /dev/null +++ b/src/ndi/util/compare_session_summary.py @@ -0,0 +1,146 @@ +"""Compare two session summaries and return a report of differences. + +MATLAB equivalent: ``ndi.util.compareSessionSummary`` + +Compares two summary dicts (as produced by :func:`sessionSummary`) and +returns a list of human-readable difference strings. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any + + +def compareSessionSummary( + summary1: dict[str, Any], + summary2: dict[str, Any], + *, + excludeFiles: list[str] | None = None, +) -> list[str]: + """Compare two session summaries and return a report. + + MATLAB equivalent: ``ndi.util.compareSessionSummary(s1, s2, ...)`` + + Args: + summary1: First session summary dict. + summary2: Second session summary dict. + excludeFiles: Filenames to ignore when comparing ``files`` + and ``filesInDotNDI`` fields. + + Returns: + List of difference strings. Empty list means summaries match. + """ + if excludeFiles is None: + excludeFiles = [] + + report: list[str] = [] + + # 1. Fields check + fields1 = set(summary1.keys()) + fields2 = set(summary2.keys()) + + for f in sorted(fields1 - fields2): + report.append(f"Field {f} is in summary1 but not summary2") + for f in sorted(fields2 - fields1): + report.append(f"Field {f} is in summary2 but not summary1") + + common_fields = sorted(fields1 & fields2) + + # 2. Compare common fields + for field in common_fields: + val1 = summary1[field] + val2 = summary2[field] + + # Filter excluded files + if excludeFiles and field in ("files", "filesInDotNDI"): + if isinstance(val1, list): + val1 = [v for v in val1 if v not in excludeFiles] + if isinstance(val2, list): + val2 = [v for v in val2 if v not in excludeFiles] + + # Handle empty values + _empty1 = val1 is None or (isinstance(val1, (list, dict)) and len(val1) == 0) + _empty2 = val2 is None or (isinstance(val2, (list, dict)) and len(val2) == 0) + if _empty1 and _empty2: + continue + + # Unwrap single-element lists for comparison + if isinstance(val1, list) and not isinstance(val2, list) and len(val1) == 1: + val1 = val1[0] + elif isinstance(val2, list) and not isinstance(val1, list) and len(val2) == 1: + val2 = val2[0] + + if isinstance(val1, list) and isinstance(val2, list): + if len(val1) != len(val2): + report.append( + f"Field {field} has different lengths in summary1 ({len(val1)}) " + f"and summary2 ({len(val2)})" + ) + continue + + for j, (item1, item2) in enumerate(zip(val1, val2)): + if isinstance(item1, str) and isinstance(item2, str): + s1 = re.sub(r"[\r\n]+", "", item1) + s2 = re.sub(r"[\r\n]+", "", item2) + if s1 != s2: + # Check if it's just an absolute path difference + n1 = os.path.basename(s1) + n2 = os.path.basename(s2) + if not (n1 and n1 == n2): + report.append(f'Field {field}[{j}] differs: "{s1}" vs "{s2}"') + elif isinstance(item1, dict) and isinstance(item2, dict): + sub = compareSessionSummary(item1, item2) + for s in sub: + report.append(f"Field {field}[{j}] struct diff: {s}") + else: + if item1 != item2: + report.append(f"Field {field}[{j}] differs in content") + + elif isinstance(val1, dict) and isinstance(val2, dict): + # Both are dicts — could be a single struct or a struct array + sub = compareSessionSummary(val1, val2) + for s in sub: + report.append(f"Field {field} struct diff: {s}") + + elif isinstance(val1, str) and isinstance(val2, str): + s1 = re.sub(r"[\r\n\t]+", "", val1) + s2 = re.sub(r"[\r\n\t]+", "", val2) + if s1 != s2: + n1 = os.path.basename(s1) + n2 = os.path.basename(s2) + if not (n1 and n1 == n2): + report.append(f'Field {field} differs: "{s1}" vs "{s2}"') + + else: + is_same = False + if isinstance(val1, (int, float)) and isinstance(val2, (int, float)): + is_same = val1 == val2 + elif isinstance(val1, bool) and isinstance(val2, bool): + is_same = val1 == val2 + else: + is_same = val1 == val2 + + # Fallback: compare JSON representations + if not is_same: + try: + j1 = re.sub(r"[\r\n\t]+", "", json.dumps(val1, sort_keys=True)) + j2 = re.sub(r"[\r\n\t]+", "", json.dumps(val2, sort_keys=True)) + is_same = j1 == j2 + except (TypeError, ValueError): + pass + + if not is_same: + try: + v1s = json.dumps(val1) + v2s = json.dumps(val2) + except (TypeError, ValueError): + v1s = "" + v2s = "" + report.append( + f"Field {field} differs in value/object:\n" f" Val1: {v1s}\n Val2: {v2s}" + ) + + return report diff --git a/src/ndi/util/session_summary.py b/src/ndi/util/session_summary.py new file mode 100644 index 0000000..2b6fd86 --- /dev/null +++ b/src/ndi/util/session_summary.py @@ -0,0 +1,110 @@ +"""Session summary utility for symmetry testing. + +MATLAB equivalent: ``ndi.util.sessionSummary`` + +Creates a summary dict of an ``ndi.session`` object containing key fields +and properties, intended for symmetry testing between NDI language +implementations. +""" + +from __future__ import annotations + +import os +from typing import Any + + +def sessionSummary(session_obj: Any) -> dict[str, Any]: + """Create a summary structure of an ndi.session object. + + MATLAB equivalent: ``ndi.util.sessionSummary(session_obj)`` + + Args: + session_obj: An NDI session object. + + Returns: + Dict with keys: reference, sessionId, files, filesInDotNDI, + daqSystemNames, daqSystemDetails, probes. + """ + summary: dict[str, Any] = {} + + # 1. Session basics + summary["reference"] = session_obj.reference + summary["sessionId"] = session_obj.id() + + # 2. Files in session path + session_path = str(session_obj.path) + if os.path.isdir(session_path): + entries = os.listdir(session_path) + summary["files"] = sorted(entries) + else: + summary["files"] = [] + + # 3. Files in .ndi folder + dot_ndi_path = os.path.join(session_path, ".ndi") + if os.path.isdir(dot_ndi_path): + entries = os.listdir(dot_ndi_path) + summary["filesInDotNDI"] = sorted(entries) + else: + summary["filesInDotNDI"] = [] + + # 4. DAQ Systems + daqs = session_obj.daqsystem_load(name="(.*)") + if daqs is None: + daqs = [] + elif not isinstance(daqs, list): + daqs = [daqs] + + daq_names: list[str] = [] + daq_details: list[dict[str, Any]] = [] + + for sys in daqs: + daq_names.append(getattr(sys, "name", "")) + + details: dict[str, Any] = {} + + # Get filenavigator class + fn = getattr(sys, "filenavigator", None) + if fn is not None: + details["filenavigator_class"] = type(fn).__qualname__ + try: + details["epochNodes_filenavigator"] = fn.epochnodes() + except Exception: + details["epochNodes_filenavigator"] = [] + else: + details["filenavigator_class"] = "" + details["epochNodes_filenavigator"] = [] + + # Get daqreader class + dr = getattr(sys, "daqreader", None) + if dr is not None: + details["daqreader_class"] = type(dr).__qualname__ + else: + details["daqreader_class"] = "" + + # Get epoch nodes of daq system + try: + details["epochNodes_daqsystem"] = sys.epochnodes() + except Exception: + details["epochNodes_daqsystem"] = [] + + daq_details.append(details) + + summary["daqSystemNames"] = daq_names + summary["daqSystemDetails"] = daq_details + + # 5. Probes + probes = session_obj.getprobes() + probe_structs: list[dict[str, Any]] = [] + for p in probes: + probe_structs.append( + { + "name": p.name, + "reference": p.reference, + "type": p.type, + "subject_id": getattr(p, "subject_id", ""), + } + ) + + summary["probes"] = probe_structs + + return summary diff --git a/tests/symmetry/__init__.py b/tests/symmetry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/conftest.py b/tests/symmetry/conftest.py new file mode 100644 index 0000000..fd4a16f --- /dev/null +++ b/tests/symmetry/conftest.py @@ -0,0 +1,17 @@ +"""Shared fixtures and configuration for NDI symmetry tests. + +Symmetry tests verify cross-language parity between NDI-python and NDI-matlab. +They are separated from the regular test suite because they depend on artifacts +that may have been generated by a prior MATLAB (or Python) run. +""" + +import tempfile +from pathlib import Path + +# Base directory where all symmetry artifacts live: +# /NDI/symmetryTest///// +SYMMETRY_BASE = Path(tempfile.gettempdir()) / "NDI" / "symmetryTest" +PYTHON_ARTIFACTS = SYMMETRY_BASE / "pythonArtifacts" +MATLAB_ARTIFACTS = SYMMETRY_BASE / "matlabArtifacts" + +SOURCE_TYPES = ["matlabArtifacts", "pythonArtifacts"] diff --git a/tests/symmetry/make_artifacts/INSTRUCTIONS.md b/tests/symmetry/make_artifacts/INSTRUCTIONS.md new file mode 100644 index 0000000..b3bc9f8 --- /dev/null +++ b/tests/symmetry/make_artifacts/INSTRUCTIONS.md @@ -0,0 +1,37 @@ +# NDI Symmetry Artifacts Instructions (Python — makeArtifacts) + +This folder contains Python tests whose purpose is to generate standard NDI artifacts for symmetry testing with other NDI language ports (e.g., MATLAB). + +## Rules for `make_artifacts` tests: + +1. **Artifact Location**: Tests must store their generated artifacts in the system's temporary directory (`tempfile.gettempdir()`). +2. **Directory Structure**: Inside the temporary directory, artifacts must be placed in a specific nested folder structure: + `NDI/symmetryTest/pythonArtifacts////` + + - ``: The sub-package name under `make_artifacts`. For example, for a test located at `tests/symmetry/make_artifacts/session/`, the namespace is `session`. + - ``: The name of the test class (e.g., `buildSession`), written in camelCase to match MATLAB conventions. + - ``: The specific name of the test method being executed (e.g., `testBuildSessionArtifacts`), also in camelCase. + +3. **Persistent Teardown**: The generated artifacts and the underlying NDI session database must persist in the temporary directory so that the MATLAB test suite can read them. Do **not** use `tmp_path` for the artifact output directory — only use it for the ephemeral session that is later *copied* to the artifact directory. + +4. **Artifact Contents**: Every `makeArtifacts` test should produce at minimum: + - A copy of the NDI session directory (including the `.ndi/` database folder). + - A `jsonDocuments/` sub-directory containing one `.json` file per document in the session. + - A `probes.json` file listing all probes as an array of `{"name", "reference", "type", "subject_id"}` objects. + +5. **Imports**: Use the shared constant `PYTHON_ARTIFACTS` from `tests/symmetry/conftest.py` to build the artifact path. + +## Example: + +For a test class `TestBuildSession` in `tests/symmetry/make_artifacts/session/test_build_session.py` with a test method `test_build_session_artifacts`, the artifacts should be saved to: + +``` +/NDI/symmetryTest/pythonArtifacts/session/buildSession/testBuildSessionArtifacts/ +``` + +## Adding a new symmetry test: + +1. Create a sub-package under `make_artifacts/` named after the NDI domain (e.g., `session/`, `document/`, `probe/`). +2. Add a `test_.py` file with a test class that builds an NDI session, populates it, and exports artifacts to the path described above. +3. Mirror the directory naming in MATLAB: `tests/+ndi/+symmetry/+makeArtifacts/+/.m`. +4. Add a corresponding `readArtifacts` test that can verify the generated artifacts (see `tests/symmetry/read_artifacts/INSTRUCTIONS.md`). diff --git a/tests/symmetry/make_artifacts/__init__.py b/tests/symmetry/make_artifacts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/make_artifacts/session/__init__.py b/tests/symmetry/make_artifacts/session/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/make_artifacts/session/test_build_session.py b/tests/symmetry/make_artifacts/session/test_build_session.py new file mode 100644 index 0000000..a31b1d6 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_build_session.py @@ -0,0 +1,113 @@ +"""Generate symmetry artifacts for a basic NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/buildSession.m + +This test creates an NDI DirSession with a subject and a subjectmeasurement +document, then persists the session database, a ``sessionSummary.json`` +manifest, and individual JSON representations of every document into: + + /NDI/symmetryTest/pythonArtifacts/session/buildSession/ + testBuildSessionArtifacts/ + +The artifacts are left on disk so that the MATLAB ``readArtifacts`` suite +(and the Python ``read_artifacts`` suite) can load and verify them. +""" + +import json +import shutil + +import pytest + +from ndi.query import Query +from ndi.session.dir import DirSession +from ndi.subject import Subject +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "session" / "buildSession" / "testBuildSessionArtifacts" + + +class TestBuildSession: + """Mirror of ndi.symmetry.makeArtifacts.session.buildSession.""" + + # -- setup / teardown ---------------------------------------------------- + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Build an example session in a temporary directory. + + Mirrors the MATLAB ``buildSessionSetup`` method. Because Python + does not ship the Intan example binary, the session is built + without raw acquisition files — only structured NDI documents. + """ + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = DirSession("exp1", session_dir) + session.database_clear("yes") + session.cache.clear() + + # Add a subject + subject = Subject("anteater27@nosuchlab.org", "") + session.database_add(subject.newdocument()) + + # Add a subjectmeasurement document + doc = session.newdocument( + "subjectmeasurement", + **{ + "base.name": "Animal statistics", + "subjectmeasurement.measurement": "age", + "subjectmeasurement.value": 30, + "subjectmeasurement.datestamp": "2017-03-17T19:53:57.066Z", + }, + ) + doc = doc.set_dependency_value("subject_id", subject.id, error_if_not_found=False) + session.database_add(doc) + + self.session = session + # No teardown — artifacts must persist for readArtifacts. + + # -- tests --------------------------------------------------------------- + + def test_build_session_artifacts(self): + """Export the session to the shared symmetry artifact directory.""" + artifact_dir = ARTIFACT_DIR + + # Clear previous artifacts + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Call getprobes() BEFORE copying to generate any internal documents. + self.session.getprobes() + + # Re-open the session to ensure all internal documents are flushed. + session_path = self.session.path + self.session = DirSession("exp1", session_path) + + # Create comprehensive session summary + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + # Copy the entire session directory to the persistent artifact dir. + shutil.copytree(str(session_path), str(artifact_dir)) + + # Write individual JSON documents. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(Query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text(json.dumps(props, indent=2, allow_nan=True), encoding="utf-8") + + # Write sessionSummary.json + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + # Verify artifacts were created + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() + assert len(list(json_docs_dir.glob("*.json"))) > 0 diff --git a/tests/symmetry/read_artifacts/INSTRUCTIONS.md b/tests/symmetry/read_artifacts/INSTRUCTIONS.md new file mode 100644 index 0000000..6a7453a --- /dev/null +++ b/tests/symmetry/read_artifacts/INSTRUCTIONS.md @@ -0,0 +1,67 @@ +# NDI Read Artifacts Instructions (Python — readArtifacts) + +This folder contains Python tests whose purpose is to read standard NDI artifacts generated by both the Python NDI test suite and the MATLAB test suite, comparing them against expected values. + +## Process overview: + +1. **Artifact Location**: The test suites place their generated artifacts in the system's temporary directory (`tempfile.gettempdir()`). +2. **Directory Structure**: Inside the temporary directory, artifacts can be found in a specific nested folder structure: + `NDI/symmetryTest/////` + + - ``: Either `matlabArtifacts` or `pythonArtifacts`. + - ``: The module/package location of the corresponding test (e.g., `session`). + - ``: The name of the test class (e.g., `buildSession`). + - ``: The specific name of the test method that was executed (e.g., `testBuildSessionArtifacts`). + +3. **Testing Goals**: The Python tests located in this `read_artifacts` package should define assertions that: + - Load the JSON representations created by the target suite. + - Load the actual NDI session produced by the target suite. + - Assert that the NDI objects (documents, probes, etc.) retrieved by Python match the expected JSON structure dumped by the test suite. + - Run across both `pythonArtifacts` and `matlabArtifacts` using parameterized testing to ensure parity. + +4. **Skipping**: When the artifact directory for a given `SourceType` does not exist, the test must **skip** (not fail). This allows the suite to run on machines that only have one language installed. + +## Example: + +Use pytest's `@pytest.fixture(params=...)` to dynamically pass the `SourceType` to your tests: + +```python +import json +import pytest +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from ndi.query import Query +from ndi.session.dir import DirSession + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + + +class TestBuildSession: + def test_read_session(self, source_type): + artifact_dir = ( + SYMMETRY_BASE / source_type / "session" / "buildSession" + / "testBuildSessionArtifacts" + ) + if not artifact_dir.exists(): + pytest.skip(f"Artifact directory from {source_type} does not exist.") + + # 1. Load the NDI session + session = DirSession("exp1", artifact_dir) + + # 2. Load the ground truth file (probes.json) + probes_json = artifact_dir / "probes.json" + expected_probes = json.loads(probes_json.read_text()) + + # 3. Verify they agree + actual_probes = session.getprobes() + assert len(actual_probes) == len(expected_probes) +``` + +## Adding a new symmetry test: + +1. Create a sub-package under `read_artifacts/` matching the namespace used in `make_artifacts/` (e.g., `session/`, `document/`). +2. Add a `test_.py` file with a parameterized test class that reads from both `matlabArtifacts` and `pythonArtifacts`. +3. Always use `pytest.skip()` when the expected artifact directory does not exist. +4. Mirror the MATLAB counterpart: `tests/+ndi/+symmetry/+readArtifacts/+/.m`. diff --git a/tests/symmetry/read_artifacts/__init__.py b/tests/symmetry/read_artifacts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/read_artifacts/session/__init__.py b/tests/symmetry/read_artifacts/session/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/read_artifacts/session/test_build_session.py b/tests/symmetry/read_artifacts/session/test_build_session.py new file mode 100644 index 0000000..723dcff --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_build_session.py @@ -0,0 +1,109 @@ +"""Read and verify symmetry artifacts for a basic NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/buildSession.m + +This test loads artifacts produced by *either* the MATLAB or the Python +``makeArtifacts`` suite and verifies that the Python NDI stack can: + +1. Open the copied session database. +2. Generate a live session summary that matches the stored ``sessionSummary.json``. +3. Load every document whose JSON was exported to ``jsonDocuments/``. + +The test is parameterized over ``source_type`` so that a single test class +covers both ``matlabArtifacts`` and ``pythonArtifacts``. +""" + +import json + +import pytest + +from ndi.query import Query +from ndi.session.dir import DirSession +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestBuildSession: + """Mirror of ndi.symmetry.readArtifacts.session.buildSession.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE / source_type / "session" / "buildSession" / "testBuildSessionArtifacts" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, DirSession("exp1", artifact_dir) + + # -- tests --------------------------------------------------------------- + + def test_build_session_summary(self, source_type): + """Verify that the live session summary matches sessionSummary.json.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.exists(): + pytest.skip(f"sessionSummary.json not found in {source_type} artifact directory.") + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + + assert ( + len(report) == 0 + ), f"Session summary mismatch against {source_type} generated artifacts:\n" + "\n".join( + report + ) + + def test_build_session_documents(self, source_type): + """Verify that every exported JSON document can be loaded from the session DB.""" + artifact_dir, session = self._open_session(source_type) + + json_docs_dir = artifact_dir / "jsonDocuments" + if not json_docs_dir.exists(): + pytest.skip(f"jsonDocuments directory not found in {source_type}.") + + json_files = list(json_docs_dir.glob("*.json")) + + actual_docs = session.database_search(Query("base.id").match("(.*)")) + + assert len(actual_docs) == len(json_files), ( + f"Number of documents in session ({len(actual_docs)}) does not match " + f"{source_type} JSON artifacts ({len(json_files)})." + ) + + for jf in json_files: + expected_doc = json.loads(jf.read_text(encoding="utf-8")) + expected_id = expected_doc.get("base", {}).get("id", "") + + found = False + for actual in actual_docs: + if actual.id == expected_id: + found = True + actual_props = actual.document_properties + assert actual_props.get("document_class", {}).get( + "class_name" + ) == expected_doc.get("document_class", {}).get("class_name"), ( + f"Document class mismatch for id: {expected_id} " f"in {source_type}" + ) + break + assert found, ( + f"Document from {source_type} artifact not found in session: " f"{expected_id}" + )