Port utility functions, validators, and enforce strict MATLAB naming

MATLAB_MAPPING.md

@@ -56,8 +56,8 @@ Complete reference mapping every MATLAB NDI function/class to its Python equival
 | `ndi.probe.timeseries` | `ndi.probe.ProbeTimeseries` | `ndi.probe.timeseries` |
 | `ndi.probe.timeseries.mfdaq` | `ndi.probe.ProbeTimeseriesMFDAQ` | `ndi.probe.timeseries_mfdaq` |
 | `ndi.probe.timeseries.stimulator` | `ndi.probe.ProbeTimeseriesStimulator` | `ndi.probe.timeseries_stimulator` |
-| `ndi.probe.fun.initProbeTypeMap` | `ndi.probe.init_probe_type_map()` | `ndi.probe` |
-| `ndi.probe.fun.getProbeTypeMap` | `ndi.probe.get_probe_type_map()` | `ndi.probe` |
+| `ndi.probe.fun.initProbeTypeMap` | `ndi.probe.initProbeTypeMap()` | `ndi.probe` |
+| `ndi.probe.fun.getProbeTypeMap` | `ndi.probe.getProbeTypeMap()` | `ndi.probe` |
 
 ## Session & Dataset
 
@@ -290,8 +290,8 @@ Auth functions are re-exported from `ndi.cloud.__init__` so `from ndi.cloud impo
 | `submitDataset` | `submitDataset(dataset_id)` | |
 | `createDatasetBranch` | `createDatasetBranch(dataset_id)` | |
 | `getBranches` | `getBranches(dataset_id)` | |
-| — | `undeleteDataset(dataset_id)` | Soft-delete API |
-| — | `listDeletedDatasets(...)` | Soft-delete API |
+| `undeleteDataset` | `undeleteDataset(dataset_id)` | |
+| `listDeletedDatasets` | `listDeletedDatasets(...)` | |
 
 ### Documents API (`ndi.cloud.api.documents`)
 
@@ -311,7 +311,7 @@ Auth functions are re-exported from `ndi.cloud.__init__` so `from ndi.cloud impo
 | `ndiquery` | `ndiquery(scope, search_structure, ...)` | |
 | `ndiqueryAll` | `ndiqueryAll(scope, search_structure, ...)` | |
 | — | `bulkUpload(dataset_id, zip_path)` | Python-only |
-| — | `listDeletedDocuments(dataset_id, ...)` | Soft-delete API |
+| `listDeletedDocuments` | `listDeletedDocuments(dataset_id, ...)` | |
 
 ### Files API (`ndi.cloud.api.files`)
 
@@ -416,7 +416,7 @@ from ndi.cloud import downloadDataset, uploadDataset, syncDataset, uploadSingleF
 | `+internal/getUploadedDocumentIds` | — | Via `listRemoteDocumentIds()` |
 | `+internal/getUploadedFileIds` | — | Via `listFiles()` |
 | `+internal/dropDuplicateDocsFromJsonDecode` | — | Not needed (Python JSON is exact) |
-| `+internal/duplicateDocuments` | — | Not yet ported |
+| `+internal/duplicateDocuments` | `internal.duplicateDocuments()` | |
 | `+sync/+internal/listLocalDocuments` | `internal.listLocalDocuments()` | |
 | `+sync/+internal/getFileUidsFromDocuments` | `internal.getFileUidsFromDocuments()` | |
 | `+sync/+internal/filesNotYetUploaded` | `internal.filesNotYetUploaded()` | |
@@ -453,7 +453,45 @@ from ndi.cloud import downloadDataset, uploadDataset, syncDataset, uploadSingleF
 | `ndi.cloud.ui.dialog.selectCloudDataset` | MATLAB GUI dialog |
 | `ndi.cloud.utility.createCloudMetadataStruct` | MATLAB struct validator; `CloudConfig` replaces |
 | `ndi.cloud.utility.mustBeValidMetadata` | MATLAB struct validator; type hints replace |
-| `+internal/duplicateDocuments` | Not yet ported |
+
+## Validators (`ndi.validators.*`)
+
+| MATLAB | Python | Module |
+|--------|--------|--------|
+| `ndi.validators.mustBeCellArrayOfClass` | `ndi.validators.mustBeCellArrayOfClass()` | `ndi.validators.mustBeCellArrayOfClass` |
+| `ndi.validators.mustBeCellArrayOfNdiSessions` | `ndi.validators.mustBeCellArrayOfNdiSessions()` | `ndi.validators.mustBeCellArrayOfNdiSessions` |
+| `ndi.validators.mustBeCellArrayOfNonEmptyCharacterArrays` | `ndi.validators.mustBeCellArrayOfNonEmptyCharacterArrays()` | `ndi.validators.mustBeCellArrayOfNonEmptyCharacterArrays` |
+| `ndi.validators.mustBeClassnameOfType` | `ndi.validators.mustBeClassnameOfType()` | `ndi.validators.mustBeClassnameOfType` |
+| `ndi.validators.mustBeEpochInput` | `ndi.validators.mustBeEpochInput()` | `ndi.validators.mustBeEpochInput` |
+| `ndi.validators.mustBeID` | `ndi.validators.mustBeID()` | `ndi.validators.mustBeID` |
+| `ndi.validators.mustBeNumericClass` | `ndi.validators.mustBeNumericClass()` | `ndi.validators.mustBeNumericClass` |
+| `ndi.validators.mustBeTextLike` | `ndi.validators.mustBeTextLike()` | `ndi.validators.mustBeTextLike` |
+| `ndi.validators.mustHaveFields` | `ndi.validators.mustHaveFields()` | `ndi.validators.mustHaveFields` |
+| `ndi.validators.mustHaveRequiredColumns` | `ndi.validators.mustHaveRequiredColumns()` | `ndi.validators.mustHaveRequiredColumns` |
+| `ndi.validators.mustMatchRegex` | `ndi.validators.mustMatchRegex()` | `ndi.validators.mustMatchRegex` |
+
+## Utilities (`ndi.util.*`)
+
+| MATLAB | Python | Module |
+|--------|--------|--------|
+| `ndi.util.datestamp2datetime` | `ndi.util.datestamp2datetime()` | `ndi.util.datestamp2datetime` |
+| `ndi.util.downsampleTimeseries` | `ndi.util.downsampleTimeseries()` | `ndi.util.downsampleTimeseries` |
+| `ndi.util.hexDiff` | `ndi.util.hexDiff()` | `ndi.util.hexDiff` |
+| `ndi.util.hexDiffBytes` | `ndi.util.hexDiffBytes()` | `ndi.util.hexDiffBytes` |
+| `ndi.util.getHexDiffFromFileObj` | `ndi.util.getHexDiffFromFileObj()` | `ndi.util.getHexDiffFromFileObj` |
+| `ndi.util.hexDump` | `ndi.util.hexDump()` | `ndi.util.hexDump` |
+| `ndi.util.rehydrateJSONNanNull` | `ndi.util.rehydrateJSONNanNull()` | `ndi.util.rehydrateJSONNanNull` |
+| `ndi.util.unwrapTableCellContent` | `ndi.util.unwrapTableCellContent()` | `ndi.util.unwrapTableCellContent` |
+| `ndi.util.openminds.find_instance_name` | `ndi.openminds_convert.find_controlled_instance()` | `ndi.openminds_convert` |
+| `ndi.util.openminds.find_techniques_names` | `ndi.openminds_convert.find_technique_names()` | `ndi.openminds_convert` |
+
+### Util: Not Ported
+
+| MATLAB | Reason |
+|--------|--------|
+| `ndi.util.choosefile` | MATLAB GUI dialog |
+| `ndi.util.choosefileordir` | MATLAB GUI dialog |
+| `ndi.util.toolboxdir` | MATLAB-specific path resolution |
 
 ## Ontology
 
@@ -500,3 +538,6 @@ The following MATLAB components were intentionally not ported (GUI, MATLAB-speci
 | `ndi.fun.convertoldnsd2ndi` | Legacy NSD→NDI migration |
 | `ndi.fun.run_Linux_checks` | MATLAB Linux environment checks |
 | `ndi.fun.plot_extracellular_spikeshapes` | MATLAB GUI plotting |
+| `ndi.util.choosefile` | MATLAB GUI dialog |
+| `ndi.util.choosefileordir` | MATLAB GUI dialog |
+| `ndi.util.toolboxdir` | MATLAB-specific path resolution |

PYTHON_PORTING_GUIDE.md

@@ -6,11 +6,12 @@ The MATLAB codebase is the **Source of Truth**. The Python version is a "faithfu
 
 ## 2. Function & Variable Naming (The "Strict Mirror" Rule)
 
-Do **not** attempt to "translate" MATLAB names into Python PEP 8 (`snake_case`).
+**MATLAB function names are the source of truth.** Do **not** attempt to "translate" MATLAB names into Python PEP 8 (`snake_case`).
 
 - **Exact Match:** Every function name in Python must be an identical string match to the MATLAB function name.
-- **Case Sensitivity:** If the MATLAB function is `ListAllDocuments`, the Python function must be `ListAllDocuments`. If the MATLAB function is `get_dataset_id`, the Python function must be `get_dataset_id`.
+- **Case Sensitivity:** If the MATLAB function is `ListAllDocuments`, the Python function must be `ListAllDocuments`. If the MATLAB function is `get_dataset_id`, the Python function must be `get_dataset_id`. If the MATLAB function is `savetofile`, the Python method must be `savetofile` (not `save_to_file`).
 - **No Aliasing:** Do not create `snake_case` aliases unless explicitly requested. The user should be able to copy-paste function names between environments.
+- **Verification:** When porting or reviewing code, always check the MATLAB source at [VH-Lab/NDI-matlab](https://github.com/VH-Lab/NDI-matlab) to confirm the exact function name. The MATLAB `.m` file's `function` line is the canonical reference.
 
 ## 3. Namespace and Directory Structure
 
@@ -30,13 +31,116 @@ To replicate the robustness of the MATLAB `arguments` block, use Pydantic for al
   - MATLAB `char` or `string` → Python `str`
   - MATLAB `{member1, member2}` → Python `Literal["member1", "member2"]`
 - **Coercion:** Allow Pydantic's default behavior of casting (e.g., allowing a string `"1"` or integer `1` to satisfy a `bool` type).
+- **Arbitrary Types:** When a function accepts types that Pydantic cannot serialise natively (e.g. `numpy.ndarray`, file-like `IO[bytes]`), pass `config=ConfigDict(arbitrary_types_allowed=True)` to the decorator.
+- **Constraints:** Use `Annotated[type, pydantic.Field(...)]` to express MATLAB `arguments`-block constraints such as `mustBePositive` (`gt=0`), `mustBeNonnegative` (`ge=0`), and `mustBeInteger`.
 
-## 5. Error Handling
+### 4a. Reusable Validators in `ndi.validators`
+
+MATLAB centralises custom validation functions in the `+ndi/+validators/` namespace.  Python must do the same in the `ndi/validators/` package.
+
+- **When to create a validator:** Any type check, format check, or constraint that appears (or is likely to appear) in more than one function should be extracted into its own module under `ndi/validators/` instead of being written inline.
+- **Naming convention:** MATLAB-originated validators keep their exact MATLAB name (e.g. `mustBeID`).  Python-specific validators that have no MATLAB counterpart use `snake_case` prefixed with a descriptive verb (e.g. `is_ndarray`, `is_iso8601`).
+- **Signature pattern:** Each validator takes a single value, raises `ValueError` on failure, and returns the validated value unchanged:
+  ```python
+  def is_ndarray(val: object) -> np.ndarray:
+      if not isinstance(val, np.ndarray):
+          raise ValueError("Input must be a numpy.ndarray")
+      return val
+  ```
+- **Registration:** Every new validator must be imported and listed in `ndi/validators/__init__.py` so it is accessible as `ndi.validators.<name>`.
+
+## 5. Code Style & Linting
+
+All Python code must pass **`black`** formatting and **`ruff`** linting before being committed.
+
+- **Formatter:** [Black](https://black.readthedocs.io/) is the project's sole code formatter.
+- **Linter:** [Ruff](https://docs.astral.sh/ruff/) is the project's linter.
+- **Before committing:** Always run both checks and fix any issues:
+  1. `ruff check src/ tests/` — fix any lint errors (unused imports, undefined names, etc.). Use `ruff check --fix src/ tests/` for auto-fixable issues.
+  2. `black --check src/ tests/` — verify formatting. Run `black src/ tests/` to fix.
+- **Line length:** Use Black's default (88 characters).
+- **No manual formatting overrides:** Do not use `# fmt: off` / `# fmt: on` or `# noqa` unless absolutely necessary.
+
+## 6. Error Handling
 
 - If a MATLAB function throws an error for a specific condition, the Python version must raise a corresponding Exception (`ValueError`, `TypeError`, or a custom `NDIError`).
 - The goal is to ensure that a user providing bad input gets a **"Hard Fail"** at the function entry point in both languages.
 
-## 6. Documentation (Docstring Symmetry)
+## 7. Documentation (Docstring Symmetry)
 
 - Include the original MATLAB documentation in the Python docstring.
 - Note any Python-specific requirements (like specific library dependencies) at the bottom of the docstring.
+
+---
+
+## Namespace Coverage Status
+
+Verified coverage of each MATLAB namespace against the Python port. See
+[MATLAB_MAPPING.md](MATLAB_MAPPING.md) for the full function-by-function mapping.
+
+### `ndi.cloud.api` — Fully Ported
+
+**Verified:** 2026-03-11 against `VH-Lab/NDI-matlab` branch `main`.
+
+| Submodule | MATLAB funcs | Python funcs | Coverage | Notes |
+|-----------|:------------:|:------------:|:--------:|-------|
+| `+datasets` (14) | 14 | 14 + 2 | **100 %** | Python adds `listAllDatasets` (auto-paginator), `listDeletedDatasets` |
+| `+documents` (15) | 15 | 15 + 1 | **100 %** | `countDocuments` subsumes MATLAB's `documentCount`; Python adds `bulkUpload` |
+| `+files` (6) | 6 | 6 + 2 | **100 %** | Python adds `putFileBytes`, `getBulkUploadURL` |
+| `+users` (3) | 3 | 3 | **100 %** | |
+| `+compute` (6) | 6 | 6 | **100 %** | |
+| `+auth` (8) | 8 | 8 | **100 %** | `loginOriginal`/`logoutOriginal` (legacy) intentionally skipped; auth funcs live in `ndi.cloud.auth` |
+| `call.m` / `url.m` | 2 | — | **N/A** | Replaced by `CloudClient` + `CloudConfig` (architectural change) |
+| `+implementation/*` (50 classes) | 50 | — | **N/A** | Eliminated; single `CloudClient` replaces all impl classes |
+
+**Architectural differences from MATLAB:**
+
+- MATLAB uses an abstract `call` base class with 50 concrete implementation classes (one per endpoint). Python replaces this with `CloudClient`, a thin `requests.Session` wrapper with `get`/`post`/`put`/`delete` methods.
+- MATLAB `url.m` builds endpoint URLs from a name→template dictionary. Python uses `CloudConfig.api_url` + path templates in each function.
+- All Python API functions use `@pydantic.validate_call` for input validation (matching MATLAB `arguments` blocks) and `@_auto_client` to make the `client` parameter optional.
+
+**Not ported (intentional):**
+
+| MATLAB | Reason |
+|--------|--------|
+| `ndi.cloud.uilogin` | MATLAB GUI |
+| `ndi.cloud.ui.dialog.selectCloudDataset` | MATLAB GUI dialog |
+| `ndi.cloud.utility.createCloudMetadataStruct` | MATLAB struct validator; `CloudConfig` replaces |
+| `ndi.cloud.utility.mustBeValidMetadata` | MATLAB struct validator; type hints replace |
+
+### `ndi.validators` — Fully Ported
+
+**Verified:** 2026-03-11 against `VH-Lab/NDI-matlab` branch `main`.
+
+| MATLAB | Python | Coverage |
+|--------|--------|:--------:|
+| 11 functions | 11 + 2 | **100 %** |
+
+All 11 MATLAB `arguments`-block validators ported 1:1 with matching
+function names.  Python equivalents accept Python types (``list`` for
+cell array, ``dict`` for struct, ``pd.DataFrame`` for table).
+
+Python adds two reusable validators with no direct MATLAB counterpart:
+
+| Python | Purpose |
+|--------|---------|
+| `is_ndarray` | Validates value is a `numpy.ndarray` |
+| `is_iso8601` | Validates string is parseable ISO 8601 |
+
+### `ndi.util` — Fully Ported
+
+**Verified:** 2026-03-11 against `VH-Lab/NDI-matlab` branch `main`.
+
+| Category | MATLAB funcs | Python funcs | Coverage | Notes |
+|----------|:------------:|:------------:|:--------:|-------|
+| Data/time utilities (8) | 8 | 8 | **100 %** | |
+| `+openminds` (2) | 2 | 2 | **100 %** | Ported in `ndi.openminds_convert` |
+| GUI / MATLAB-specific (3) | 3 | — | **N/A** | `choosefile`, `choosefileordir`, `toolboxdir` |
+
+**Not ported (intentional):**
+
+| MATLAB | Reason |
+|--------|--------|
+| `ndi.util.choosefile` | MATLAB GUI dialog (`inputdlg`) |
+| `ndi.util.choosefileordir` | MATLAB GUI dialog (`inputdlg`) |
+| `ndi.util.toolboxdir` | MATLAB-specific path resolution |

src/ndi/__init__.py

@@ -32,7 +32,7 @@
 # Import epoch module (Phase 6)
 # Import Phase 10: Cloud API client
 # Import Phase 11: Schema validation
-from . import calc, cloud, daq, epoch, file, session, time, validate
+from . import calc, cloud, daq, epoch, file, session, time, util, validate, validators
 
 # Import Phase 9: App framework and calculators
 from .app import App
@@ -123,6 +123,8 @@ def version() -> tuple:
     "Calculator",
     "calc",
     "cloud",
+    "util",
     "validate",
+    "validators",
     "version",
 ]

src/ndi/element/__init__.py

@@ -356,7 +356,7 @@ def addepoch(
         self._session.database_add(doc)
 
         # Clear cache
-        self.clear_cache()
+        self.resetepochtable()
 
         return self, doc
 

src/ndi/element/functions.py

@@ -102,7 +102,7 @@ def oneepoch(
     return elem_out
 
 
-def spikes_for_probe(
+def spikesForProbe(
     session: Any,
     probe: Any,
     name: str,
@@ -129,7 +129,7 @@ def spikes_for_probe(
         ...     {'epochid': 'epoch_001', 'spiketimes': [0.1, 0.5, 1.2]},
         ...     {'epochid': 'epoch_002', 'spiketimes': [0.3, 0.8]},
         ... ]
-        >>> neuron = spikes_for_probe(session, probe, 'unit1', 1, spikes)
+        >>> neuron = spikesForProbe(session, probe, 'unit1', 1, spikes)
 
     Note:
         This creates the structural framework for a neuron element.

src/ndi/epoch/__init__.py

@@ -14,13 +14,13 @@
 from .epochprobemap import EpochProbeMap
 from .epochprobemap_daqsystem import EpochProbeMapDAQSystem
 from .epochset import EpochSet
-from .functions import epochrange, find_epoch_node
+from .functions import epochrange, findepochnode
 
 __all__ = [
     "Epoch",
     "EpochSet",
     "EpochProbeMap",
     "EpochProbeMapDAQSystem",
     "epochrange",
-    "find_epoch_node",
+    "findepochnode",
 ]

src/ndi/epoch/epochprobemap_daqsystem.py

@@ -84,7 +84,7 @@ def serialize(self) -> str:
             ]
         )
 
-    def save_to_file(self, filename: str) -> None:
+    def savetofile(self, filename: str) -> None:
         """
         Write this epoch probe map to a file.
 
@@ -122,7 +122,7 @@ def decode(cls, s: str) -> EpochProbeMapDAQSystem:
         )
 
     @classmethod
-    def load_from_file(cls, filename: str) -> list[EpochProbeMapDAQSystem]:
+    def loadfromfile(cls, filename: str) -> list[EpochProbeMapDAQSystem]:
         """
         Load epoch probe maps from a file.
 

src/ndi/epoch/epochset.py

@@ -312,7 +312,10 @@ def epochgraph(self) -> list[dict[str, Any]]:
 
         return nodes
 
-    def clear_cache(self) -> None:
-        """Clear the epoch table cache."""
+    def resetepochtable(self) -> None:
+        """Reset (clear) the epoch table cache.
+
+        MATLAB equivalent: ndi.epoch.epochset.resetepochtable
+        """
         self._epochtable_cache = None
         self._epochtable_hash = None

src/ndi/epoch/functions.py

@@ -118,7 +118,7 @@ def _resolve_epoch_index(
     raise ValueError(f"Epoch ID '{epoch}' not found")
 
 
-def find_epoch_node(
+def findepochnode(
     epoch_node: dict[str, Any],
     epoch_node_array: list[dict[str, Any]],
 ) -> list[int]:
@@ -153,9 +153,9 @@ def find_epoch_node(
         ...     {'epoch_id': 'e1', 'objectname': 'probe1'},
         ...     {'epoch_id': 'e2', 'objectname': 'probe2'},
         ... ]
-        >>> find_epoch_node({'epoch_id': 'e1'}, nodes)
+        >>> findepochnode({'epoch_id': 'e1'}, nodes)
         [0]
-        >>> find_epoch_node({}, nodes)  # empty = match all
+        >>> findepochnode({}, nodes)  # empty = match all
         [0, 1]
     """
     STRING_FIELDS = (

src/ndi/probe/__init__.py

@@ -234,6 +234,17 @@ def issyncgraphroot(self) -> bool:
     # Probe-specific Methods
     # =========================================================================
 
+    def probestring(self) -> str:
+        """
+        Format probe as human-readable string.
+
+        MATLAB equivalent: ndi.probe.probestring
+
+        Returns:
+            String in format "name | reference | type"
+        """
+        return f"{self._name} | {self._reference} | {self._type}"
+
     def epochprobemapmatch(
         self,
         epochprobemap: Any,
@@ -428,7 +439,7 @@ def __repr__(self) -> str:
 _PROBE_TYPE_MAP: dict[str, str] | None = None
 
 
-def init_probe_type_map() -> dict[str, str]:
+def initProbeTypeMap() -> dict[str, str]:
     """Load the probe type→class mapping from ``probetype2object.json``.
 
     MATLAB equivalent: ndi.probe.fun.initProbeTypeMap
@@ -462,7 +473,7 @@ def init_probe_type_map() -> dict[str, str]:
     return result
 
 
-def get_probe_type_map() -> dict[str, str]:
+def getProbeTypeMap() -> dict[str, str]:
     """Return the cached probe type→class mapping.
 
     MATLAB equivalent: ndi.probe.fun.getProbeTypeMap
@@ -474,5 +485,5 @@ def get_probe_type_map() -> dict[str, str]:
     """
     global _PROBE_TYPE_MAP
     if _PROBE_TYPE_MAP is None:
-        _PROBE_TYPE_MAP = init_probe_type_map()
+        _PROBE_TYPE_MAP = initProbeTypeMap()
     return _PROBE_TYPE_MAP