From 178cc54956fefb19b4c8f4d136484784f32bd1a6 Mon Sep 17 00:00:00 2001 From: audriB <138315415+audriB@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:57:36 -0400 Subject: [PATCH] fix(V_delta): sync truncated index.json superclasses (14 classes) + integrity test + all_ancestors map schemas/V_delta/index.json carried a per-entry `superclasses` mirror that had dropped every non-base parent for 14 classes (neuron_extracellular missing app, tuningcurve_calc missing stimulus_tuningcurve, image_stack missing image_stack_parameters, the openminds_* missing openminds, stimulus_presentation missing app+epochid, etc.). Any consumer resolving isa()/inheritance from the index rather than each class file's document_class.superclasses got silent false negatives. - Re-sync the 14 entries to their class files (only entries whose superclass set differed are touched; minimal diff). - tests/test_index_integrity.py: assert index.superclasses == file superclasses for every entry, that every referenced parent resolves, and that all_ancestors.json equals the transitive closure. First V_delta-tier test (the suite previously parametrized V_beta/V_gamma only). - scripts/build_v_delta_aux.py: reproducible fixer (--fix-index) + builder of schemas/V_delta/all_ancestors.json, the flattened isa() closure map (107 classes) so consumers stop re-deriving the inheritance chain. --- schemas/V_delta/all_ancestors.json | 379 +++++++++++++++++++++++++++++ schemas/V_delta/index.json | 45 ++-- scripts/build_v_delta_aux.py | 104 ++++++++ tests/test_index_integrity.py | 84 +++++++ 4 files changed, 598 insertions(+), 14 deletions(-) create mode 100644 schemas/V_delta/all_ancestors.json create mode 100644 scripts/build_v_delta_aux.py create mode 100644 tests/test_index_integrity.py diff --git a/schemas/V_delta/all_ancestors.json b/schemas/V_delta/all_ancestors.json new file mode 100644 index 0000000..a341d0e --- /dev/null +++ b/schemas/V_delta/all_ancestors.json @@ -0,0 +1,379 @@ +{ + "CURIE_lookups_meta": [], + "app": [ + "base" + ], + "base": [], + "binaryseries_parameters": [ + "base" + ], + "binnedspikeratevm": [ + "base" + ], + "calculator": [ + "app", + "base" + ], + "contrast_sensitivity_calc": [ + "app", + "base", + "calculator" + ], + "contrast_tuning": [ + "base" + ], + "contrast_tuning_calc": [ + "app", + "base", + "calculator", + "contrast_tuning", + "tuning_fit" + ], + "control_stimulus_ids": [ + "app", + "base" + ], + "daqmetadatareader": [ + "base" + ], + "daqmetadatareader_epochdata_ingested": [ + "base", + "epochid" + ], + "daqreader": [ + "base" + ], + "daqreader_epochdata_ingested": [ + "base" + ], + "daqreader_mfdaq_epochdata_ingested": [ + "base", + "daqreader_epochdata_ingested", + "epochid" + ], + "daqreader_ndr": [ + "base", + "daqreader" + ], + "daqsystem": [ + "base" + ], + "dataset_remote": [ + "base" + ], + "dataset_session_info": [ + "base" + ], + "demo_ndi": [ + "base" + ], + "demo_ndi_mock": [ + "base" + ], + "did_schema_meta": [], + "directory": [ + "base" + ], + "distance_metadata": [ + "base" + ], + "electrode_offset_voltage": [ + "base" + ], + "element": [ + "base" + ], + "element_epoch": [ + "base", + "epochid" + ], + "ephys_zarr": [ + "base", + "zarr" + ], + "epochclocktimes": [ + "base", + "epochid" + ], + "epochfiles_ingested": [ + "base" + ], + "epochid": [ + "base" + ], + "filenavigator": [ + "base" + ], + "filter": [ + "base" + ], + "fitcurve": [ + "base" + ], + "generic_file": [ + "base" + ], + "hartley_calc": [ + "app", + "base", + "calculator", + "hartley_reverse_correlation", + "ngrid", + "reverse_correlation" + ], + "hartley_reverse_correlation": [ + "base", + "ngrid", + "reverse_correlation" + ], + "image": [ + "base" + ], + "image_collection": [ + "base" + ], + "image_stack": [ + "base", + "image_stack_parameters" + ], + "image_stack_parameters": [ + "base" + ], + "image_zarr": [ + "base", + "zarr" + ], + "jrclust_clusters": [ + "app", + "base" + ], + "measurement": [ + "base", + "epochid" + ], + "metadata_editor": [ + "base" + ], + "mock": [ + "base" + ], + "ndi_reserved_keys": [], + "neuron_extracellular": [ + "app", + "base" + ], + "ngrid": [ + "base" + ], + "oneepoch": [ + "base", + "epochid" + ], + "ontology_image": [ + "base" + ], + "ontology_label": [ + "base" + ], + "ontology_table_row": [ + "base" + ], + "openminds": [ + "base" + ], + "openminds_element": [ + "base", + "openminds" + ], + "openminds_stimulus": [ + "base", + "epochid", + "openminds" + ], + "openminds_subject": [ + "base", + "openminds" + ], + "oridirtuning_calc": [ + "app", + "base", + "calculator", + "orientation_direction_tuning", + "tuning_fit" + ], + "orientation_direction_tuning": [ + "base" + ], + "position_metadata": [ + "base" + ], + "probe_geometry": [ + "base" + ], + "probe_location": [ + "base" + ], + "projectvar": [ + "base" + ], + "pyraview": [ + "base", + "epochclocktimes", + "epochid", + "filter" + ], + "reverse_correlation": [ + "base", + "ngrid" + ], + "session": [ + "base" + ], + "session_in_a_dataset": [ + "base" + ], + "simple_calc": [ + "app", + "base", + "calculator" + ], + "site2channelmap": [ + "base" + ], + "sorting_parameters": [ + "base" + ], + "spatial_frequency_tuning": [ + "base" + ], + "spatial_frequency_tuning_calc": [ + "app", + "base", + "calculator", + "spatial_frequency_tuning", + "tuning_fit" + ], + "speed_tuning": [ + "base" + ], + "speed_tuning_calc": [ + "app", + "base", + "calculator", + "speed_tuning", + "tuning_fit" + ], + "spike_clusters": [ + "base" + ], + "spike_extraction_parameters": [ + "base" + ], + "spike_extraction_parameters_modification": [ + "base" + ], + "spike_interface_sorting_outputs": [ + "base" + ], + "spikewaves": [ + "base" + ], + "stimulus_bath": [ + "base", + "epochid" + ], + "stimulus_parameter": [ + "base" + ], + "stimulus_parameter_table": [ + "base" + ], + "stimulus_presentation": [ + "app", + "base", + "epochid" + ], + "stimulus_response": [ + "base" + ], + "stimulus_response_scalar": [ + "base", + "stimulus_response" + ], + "stimulus_response_scalar_parameters": [ + "base" + ], + "stimulus_response_scalar_parameters_basic": [ + "base", + "stimulus_response_scalar_parameters" + ], + "stimulus_tuningcurve": [ + "base" + ], + "subject": [ + "base" + ], + "subject_group": [ + "base" + ], + "syncgraph": [ + "base" + ], + "syncrule": [ + "base" + ], + "syncrule_mapping": [ + "base" + ], + "temporal_frequency_tuning": [ + "base" + ], + "temporal_frequency_tuning_calc": [ + "app", + "base", + "calculator", + "temporal_frequency_tuning", + "tuning_fit" + ], + "treatment": [ + "base" + ], + "treatment_drug": [ + "base" + ], + "treatment_transfer": [ + "base" + ], + "tuning_fit": [ + "app", + "base", + "calculator" + ], + "tuningcurve_calc": [ + "app", + "base", + "calculator", + "stimulus_tuningcurve" + ], + "valid_interval": [ + "base" + ], + "virus_injection": [ + "base" + ], + "vmneuralresponseresiduals": [ + "base" + ], + "vmspikefilteringparameters": [ + "base" + ], + "vmspikefit": [ + "base" + ], + "vmspikesummary": [ + "base" + ], + "zarr": [ + "base" + ] +} diff --git a/schemas/V_delta/index.json b/schemas/V_delta/index.json index 6b1f08e..4b8d9a3 100644 --- a/schemas/V_delta/index.json +++ b/schemas/V_delta/index.json @@ -117,7 +117,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "app" ], "path": "schemas/V_delta/stable/control_stimulus_ids.json", "is_meta": false @@ -139,7 +140,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "epochid" ], "path": "schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json", "is_meta": false @@ -172,7 +174,9 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "daqreader_epochdata_ingested", + "epochid" ], "path": "schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json", "is_meta": false @@ -451,7 +455,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "image_stack_parameters" ], "path": "schemas/V_delta/stable/image_stack.json", "is_meta": false @@ -484,7 +489,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "app" ], "path": "schemas/V_delta/stable/jrclust_clusters.json", "is_meta": false @@ -538,7 +544,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "app" ], "path": "schemas/V_delta/stable/neuron_extracellular.json", "is_meta": false @@ -616,7 +623,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "openminds" ], "path": "schemas/V_delta/stable/openminds_element.json", "is_meta": false @@ -627,7 +635,9 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "openminds", + "epochid" ], "path": "schemas/V_delta/stable/openminds_stimulus.json", "is_meta": false @@ -638,7 +648,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "openminds" ], "path": "schemas/V_delta/stable/openminds_subject.json", "is_meta": false @@ -902,7 +913,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "epochid" ], "path": "schemas/V_delta/stable/stimulus_bath.json", "is_meta": false @@ -935,7 +947,9 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "app", + "epochid" ], "path": "schemas/V_delta/stable/stimulus_presentation.json", "is_meta": false @@ -957,7 +971,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "stimulus_response" ], "path": "schemas/V_delta/stable/stimulus_response_scalar.json", "is_meta": false @@ -979,7 +994,8 @@ "class_version": "1.0.0", "maturity_level": "stable", "superclasses": [ - "base" + "base", + "stimulus_response_scalar_parameters" ], "path": "schemas/V_delta/stable/stimulus_response_scalar_parameters_basic.json", "is_meta": false @@ -1125,7 +1141,8 @@ "maturity_level": "stable", "superclasses": [ "base", - "calculator" + "calculator", + "stimulus_tuningcurve" ], "path": "schemas/V_delta/stable/tuningcurve_calc.json", "is_meta": false diff --git a/scripts/build_v_delta_aux.py b/scripts/build_v_delta_aux.py new file mode 100644 index 0000000..2202ed6 --- /dev/null +++ b/scripts/build_v_delta_aux.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Build/verify the V_delta index auxiliaries. + +Both jobs are sourced from the authoritative per-class schema files (each +class's ``document_class.superclasses``), never from ``index.json`` itself: + +1. ``--fix-index`` rewrites ``schemas/V_delta/index.json`` so every entry's + ``superclasses`` mirror equals its class file's + ``document_class.superclasses`` (in file order). Only entries whose + superclass *set* differs are rewritten, so the diff stays minimal. + +2. (always) (re)generates ``schemas/V_delta/all_ancestors.json`` -- the + flattened transitive-closure inheritance map ``{class_name: [ancestors]}`` + -- so consumers can look up the full isa() chain without re-deriving it + (and without diverging from it). + +The index ``superclasses`` mirror had silently dropped all non-``base`` parents +for 14 classes (e.g. ``neuron_extracellular`` was missing ``app``, +``tuningcurve_calc`` was missing ``stimulus_tuningcurve``). Any consumer that +resolved inheritance from the index rather than the class files got false +negatives. ``tests/test_index_integrity.py`` is the guard that keeps this from +recurring; this script is the reproducible fixer + closure builder. + +Run from anywhere; paths resolve relative to the repo root. +""" +import argparse +import json +import os + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +INDEX = os.path.join(ROOT, "schemas", "V_delta", "index.json") +ALL_ANCESTORS = os.path.join(ROOT, "schemas", "V_delta", "all_ancestors.json") + + +def superclass_names(superclasses): + """Class-name strings from a superclasses list (handles {class_name} or str).""" + names = [] + for sc in superclasses or []: + if isinstance(sc, dict) and "class_name" in sc: + names.append(sc["class_name"]) + elif isinstance(sc, str): + names.append(sc) + return names + + +def load_index(): + with open(INDEX, encoding="utf-8") as fh: + return json.load(fh) + + +def file_superclasses(entry): + """The class's own document_class.superclasses (file order, as strings).""" + with open(os.path.join(ROOT, entry["path"]), encoding="utf-8") as fh: + doc = json.load(fh) + return superclass_names((doc.get("document_class") or {}).get("superclasses")) + + +def _closure(cls, direct, seen): + for parent in direct.get(cls, []): + if parent not in seen: + seen.add(parent) + _closure(parent, direct, seen) + return seen + + +def write_json(path, obj): + with open(path, "w", encoding="utf-8") as fh: + json.dump(obj, fh, indent=2, ensure_ascii=False) + fh.write("\n") + + +def fix_index(): + index = load_index() + changed = [] + for entry in index["schemas"]: + want = file_superclasses(entry) + have = superclass_names(entry.get("superclasses")) + if set(have) != set(want): + entry["superclasses"] = want + changed.append(entry["class_name"]) + write_json(INDEX, index) + print(f"fix-index: rewrote {len(changed)} entries: {sorted(changed)}") + return changed + + +def build_all_ancestors(): + index = load_index() + direct = {e["class_name"]: file_superclasses(e) for e in index["schemas"]} + out = {cn: sorted(_closure(cn, direct, set())) for cn in sorted(direct)} + write_json(ALL_ANCESTORS, out) + print(f"all_ancestors: wrote {len(out)} classes -> {os.path.relpath(ALL_ANCESTORS, ROOT)}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fix-index", + action="store_true", + help="rewrite index.json superclasses from the per-class schema files", + ) + args = parser.parse_args() + if args.fix_index: + fix_index() + build_all_ancestors() diff --git a/tests/test_index_integrity.py b/tests/test_index_integrity.py new file mode 100644 index 0000000..d56b01f --- /dev/null +++ b/tests/test_index_integrity.py @@ -0,0 +1,84 @@ +"""Integrity of ``schemas/V_delta/index.json`` against the per-class schema files. + +``index.json`` carries a per-entry ``superclasses`` mirror that MUST equal each +class's own ``document_class.superclasses``. If it drifts, any consumer that +resolves ``isa()`` / inheritance from the index (rather than from the class +files) silently gets false negatives -- which is exactly what happened: the +mirror had dropped every non-``base`` parent for 14 classes. + +This is also the first V_delta-tier test (``test_schemas.py`` parametrizes only +V_beta / V_gamma), so it doubles as basic coverage that V_delta exists and +every indexed class file is loadable. +""" +import json +import os + +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +INDEX_PATH = os.path.join(ROOT, "schemas", "V_delta", "index.json") +ALL_ANCESTORS_PATH = os.path.join(ROOT, "schemas", "V_delta", "all_ancestors.json") + + +def superclass_names(superclasses): + """Class-name strings from a superclasses list (handles {class_name} or str).""" + names = [] + for sc in superclasses or []: + if isinstance(sc, dict) and "class_name" in sc: + names.append(sc["class_name"]) + elif isinstance(sc, str): + names.append(sc) + return names + + +with open(INDEX_PATH, encoding="utf-8") as _fh: + _INDEX = json.load(_fh) +_ENTRIES = _INDEX["schemas"] +_IDS = [e["class_name"] for e in _ENTRIES] +_CLASS_NAMES = {e["class_name"] for e in _ENTRIES} + + +def _file_superclasses(entry): + with open(os.path.join(ROOT, entry["path"]), encoding="utf-8") as fh: + doc = json.load(fh) + return superclass_names((doc.get("document_class") or {}).get("superclasses")) + + +@pytest.mark.parametrize("entry", _ENTRIES, ids=_IDS) +def test_index_superclasses_match_file(entry): + file_sc = set(_file_superclasses(entry)) + index_sc = set(superclass_names(entry.get("superclasses"))) + assert index_sc == file_sc, ( + f"{entry['class_name']}: index.json superclasses {sorted(index_sc)} " + f"!= document_class.superclasses {sorted(file_sc)} " + f"(missing from index: {sorted(file_sc - index_sc)}; " + f"extra in index: {sorted(index_sc - file_sc)})" + ) + + +@pytest.mark.parametrize("entry", _ENTRIES, ids=_IDS) +def test_index_superclasses_resolve(entry): + for parent in superclass_names(entry.get("superclasses")): + assert parent in _CLASS_NAMES, ( + f"{entry['class_name']}: superclass '{parent}' is not a class_name in index.json" + ) + + +def test_all_ancestors_matches_transitive_closure(): + """The published flattened ancestor map must equal the closure of the files.""" + if not os.path.exists(ALL_ANCESTORS_PATH): + pytest.skip("all_ancestors.json not present") + with open(ALL_ANCESTORS_PATH, encoding="utf-8") as fh: + published = json.load(fh) + + direct = {e["class_name"]: _file_superclasses(e) for e in _ENTRIES} + + def closure(cls, seen): + for parent in direct.get(cls, []): + if parent not in seen: + seen.add(parent) + closure(parent, seen) + return seen + + expected = {cn: sorted(closure(cn, set())) for cn in direct} + assert published == expected