Refactor Dataset to mirror MATLAB architecture with base/subclass pattern

src/ndi/__init__.py

@@ -43,7 +43,7 @@
 from .calculator import Calculator
 from .common import PathConstants, get_logger, getLogger, timestamp
 from .database import Database, open_database
-from .dataset import Dataset
+from .dataset import Dataset, DatasetDir
 from .document import Document
 from .documentservice import DocumentService
 from .element import Element
@@ -117,6 +117,7 @@ def version() -> tuple:
     "ElementTimeseries",
     "Neuron",
     "Dataset",
+    "DatasetDir",
     "App",
     "AppDoc",
     "DocExistsAction",

src/ndi/cloud/orchestration.py

@@ -78,24 +78,24 @@ def downloadDataset(
         for dj in doc_jsons:
             updateFileInfoForRemoteFiles(dj, cloud_dataset_id)
 
-    # Convert to Document objects and add to a local Dataset
-    from ndi.dataset import Dataset
+    # Convert to Document objects and create Dataset with them.
+    # Mirrors MATLAB: ndi.dataset.dir([], datasetFolder, ndiDocuments)
+    from ndi.dataset import DatasetDir
 
-    dataset = Dataset(target)
     documents = jsons2documents(doc_jsons)
-    for doc in documents:
+    dataset = DatasetDir("", target, documents=documents)
+
+    # Create remote link document if not already present
+    from ndi.query import Query
+
+    existing = dataset.database_search(Query("").isa("dataset_remote"))
+    if not existing:
+        remote_doc = createRemoteDatasetDoc(cloud_dataset_id, dataset)
         try:
-            dataset._session._database.add(doc)
+            dataset._session._database.add(remote_doc)
         except Exception:
             pass
 
-    # Create remote link document
-    remote_doc = createRemoteDatasetDoc(cloud_dataset_id, dataset)
-    try:
-        dataset._session._database.add(remote_doc)
-    except Exception:
-        pass
-
     # Store cloud client for on-demand file fetching
     dataset.cloud_client = client
 
@@ -174,20 +174,21 @@ def load_dataset_from_json_dir(
             updateFileInfoForRemoteFiles(dj, cloud_dataset_id)
 
     # Create Dataset
-    from ndi.dataset import Dataset
+    from ndi.dataset import DatasetDir
 
     if target_folder is None:
         target = json_path.parent / f"{json_path.name}_dataset"
     else:
         target = Path(target_folder)
     target.mkdir(parents=True, exist_ok=True)
 
-    dataset = Dataset(target)
+    # Convert JSON dicts to Document objects and create dataset with them
+    from .download import jsons2documents as _j2d
 
-    # Use bulk_add for fast insertion (single transaction, no per-doc
-    # duplicate checks).  This bypasses session_id enforcement since
-    # documents come from multiple remote sessions.
-    added, skipped = dataset._session._database._driver.bulk_add(doc_jsons)
+    all_documents = _j2d(doc_jsons)
+    dataset = DatasetDir("", target, documents=all_documents)
+    added = len(all_documents)
+    skipped = 0
 
     # Wire cloud client for on-demand file fetching
     if client is not None:

src/ndi/dataset/__init__.py

@@ -10,13 +10,19 @@
     ndi.dataset.dir  -> ndi.dataset.dir (constructor for directory-based datasets)
 """
 
-from ._dataset import Dataset
+from ._dataset import Dataset as _DatasetBase  # noqa: F401
+from ._dataset import DatasetDir
+
+# For backward compatibility, ``ndi.dataset.Dataset`` is ``DatasetDir``.
+# The base class is available as ``ndi.dataset._DatasetBase`` if needed.
+Dataset = DatasetDir
 
 # MATLAB compatibility: ``ndi.dataset.dir(path)`` creates a directory-based
 # dataset, mirroring the MATLAB constructor ``ndi.dataset.dir``.
-dir = Dataset
+dir = DatasetDir
 
 __all__ = [
     "Dataset",
+    "DatasetDir",
     "dir",
 ]