From dbd78e012e67c8922488550e70836c8baab26df7 Mon Sep 17 00:00:00 2001 From: espg Date: Mon, 20 Jul 2026 18:11:53 -0700 Subject: [PATCH 1/8] feat: moczarr-backed hive virtual store under /gridlook/hive/ (phase 6d) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N32Mx4rWhbTK8xXm7Mt2ax --- jupyter/gridlook_jupyter/config.py | 38 ++- jupyter/gridlook_jupyter/extension.py | 10 +- jupyter/gridlook_jupyter/handlers.py | 12 +- jupyter/gridlook_jupyter/hive.py | 342 ++++++++++++++++++++++++++ jupyter/pyproject.toml | 11 + 5 files changed, 405 insertions(+), 8 deletions(-) create mode 100644 jupyter/gridlook_jupyter/hive.py diff --git a/jupyter/gridlook_jupyter/config.py b/jupyter/gridlook_jupyter/config.py index 414d4598..d25e28a8 100644 --- a/jupyter/gridlook_jupyter/config.py +++ b/jupyter/gridlook_jupyter/config.py @@ -3,7 +3,7 @@ import os from pathlib import Path -from traitlets import List, Unicode, default +from traitlets import Bool, Int, List, Unicode, default from traitlets.config import Configurable #: Bounded per-process cache of bucket -> store (buckets come from the allowlist, @@ -47,6 +47,42 @@ class GridlookProxy(Configurable): ), ).tag(config=True) + allow_local_hive_stores = Bool( + False, + help=( + "Allow /gridlook/hive/open to open hive stores from local filesystem " + "paths (development/tests). s3:// stores are always gated by " + "allowed_buckets. Env fallback: GRIDLOOK_ALLOW_LOCAL_HIVE_STORES=1." + ), + ).tag(config=True) + + hive_max_views = Int( + 8, + help=( + "Maximum materialized hive views held per server process; opening " + "beyond it evicts the least-recently-used view (its /gridlook/hive/ " + "URLs then 404 until re-opened)." + ), + ).tag(config=True) + + hive_max_cells = Int( + 500_000, + help=( + "Maximum cells a single hive view may materialize; /gridlook/hive/open " + "returns 413 beyond it. Views are held in memory (~100 B/cell at " + "typical zagg variable counts), so hive_max_views * hive_max_cells " + "bounds the cache footprint." + ), + ).tag(config=True) + + @default("allow_local_hive_stores") + def _default_allow_local_hive_stores(self): + return os.environ.get("GRIDLOOK_ALLOW_LOCAL_HIVE_STORES", "").strip().lower() in ( + "1", + "true", + "yes", + ) + @default("allowed_buckets") def _default_allowed_buckets(self): env = os.environ.get("GRIDLOOK_ALLOWED_BUCKETS", "") diff --git a/jupyter/gridlook_jupyter/extension.py b/jupyter/gridlook_jupyter/extension.py index a8a2ea3f..e53e2295 100644 --- a/jupyter/gridlook_jupyter/extension.py +++ b/jupyter/gridlook_jupyter/extension.py @@ -9,6 +9,7 @@ from .config import GridlookProxy from .handlers import HealthHandler, S3ProxyHandler +from .hive import HiveOpenHandler, HiveViewCache, HiveViewHandler def load_extension(serverapp): @@ -26,9 +27,11 @@ def load_extension(serverapp): handlers = [ (escaped + r"/api/health", HealthHandler), (escaped + r"/s3/([^/]+)/(.+)", S3ProxyHandler), - # /gridlook/hive/... is RESERVED: phase 6 mounts the moczarr virtual-store - # endpoint there (an open_hive() AOI served as one flat zarr store). - # Do not claim that namespace with other routes. + # Phase 6d: the moczarr virtual store on the namespace phase 4 reserved — + # an open_hive() product/AOI/window selection served as ONE flat zarr + # store (fabricated NESTED cell_ids included; see hive.py). + (escaped + r"/hive/open", HiveOpenHandler), + (escaped + r"/hive/([0-9a-f]{16})/(.+)", HiveViewHandler), (escaped + r"$", web.RedirectHandler, {"url": base + "/"}), ( escaped + r"/(.*)", @@ -37,6 +40,7 @@ def load_extension(serverapp): ), ] serverapp.web_app.settings["gridlook_proxy"] = proxy + serverapp.web_app.settings["gridlook_hive_views"] = HiveViewCache(proxy) serverapp.web_app.add_handlers(".*$", handlers) serverapp.log.info( "gridlook-jupyter loaded: app at %s/, proxy %s", diff --git a/jupyter/gridlook_jupyter/handlers.py b/jupyter/gridlook_jupyter/handlers.py index afc1e7e3..9e3f4626 100644 --- a/jupyter/gridlook_jupyter/handlers.py +++ b/jupyter/gridlook_jupyter/handlers.py @@ -77,10 +77,8 @@ async def get(self): ) -class S3ProxyHandler(JupyterHandler): - """Streaming byte-range proxy: GET/HEAD only, no LIST, hub-side credentials only.""" - - SUPPORTED_METHODS = ("GET", "HEAD") +class PlainTextErrorMixin: + """Surface HTTPError log messages as plain-text bodies (pointed, curl-readable).""" def write_error(self, status_code, **kwargs): exc = kwargs.get("exc_info", (None, None, None))[1] @@ -90,6 +88,12 @@ def write_error(self, status_code, **kwargs): self.set_header("Content-Type", "text/plain; charset=utf-8") self.finish(message or f"error {status_code}") + +class S3ProxyHandler(PlainTextErrorMixin, JupyterHandler): + """Streaming byte-range proxy: GET/HEAD only, no LIST, hub-side credentials only.""" + + SUPPORTED_METHODS = ("GET", "HEAD") + def _store_for(self, bucket: str): proxy: GridlookProxy = self.settings["gridlook_proxy"] if not proxy.enabled: diff --git a/jupyter/gridlook_jupyter/hive.py b/jupyter/gridlook_jupyter/hive.py new file mode 100644 index 00000000..ce156205 --- /dev/null +++ b/jupyter/gridlook_jupyter/hive.py @@ -0,0 +1,342 @@ +"""``/gridlook/hive/``: moczarr-backed virtual store — one flat zarr per ``open_hive()`` view. + +Phase 6d of the viewer plan (espg/gridlook#1): the hub-side answer to "a hive +store is many leaves, but gridlook expects ONE zarr source". ``GET +/gridlook/hive/open`` runs moczarr's ``open_hive()`` (product/AOI/window +selection, fabricated NESTED ``cell_ids`` — post-englacial/zagg#314 stores are +morton-only, so the fabrication is what makes them renderable at all) and +MATERIALIZES the result into an in-memory zarr v3 store; ``GET +/gridlook/hive//`` then serves that store's objects (metadata and +whole chunks — no Range support needed) to zarrita in the browser. + +Materialize-on-open, deliberately: views are AOI-scale and bounded +(``GridlookProxy.hive_max_cells``, 413 beyond), materializing keeps this module +free of zarr chunk/codec arithmetic (xarray writes the store; we serve opaque +objects), and every subsequent request is a cache lookup. A streaming/virtual +encoding — computing zarr objects on demand from the open dataset — is the +future optimization if views ever outgrow memory; the URL contract here would +not change. + +moczarr (and its xarray/zarr stack) is an extras-gated dependency +(``gridlook-jupyter[hive]``); everything module-level here imports without it. +""" + +import functools +import hashlib +import json +import re +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any + +from jupyter_server.base.handlers import JupyterHandler +from jupyter_server.utils import url_path_join +from tornado import web +from tornado.ioloop import IOLoop + +from .config import GridlookProxy +from .handlers import PlainTextErrorMixin + +#: hex digest prefix length for view ids (deterministic per request tuple). +_VIEW_ID_HEX = 16 + +#: Product names are single path segments under the store root (zagg D19 named +#: product roots): no separators, no leading dot — nothing traversal-shaped. +_PRODUCT_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + +#: AOI tokens are morton decimals (mortie spec §2; negative for southern base +#: cells). The terminal-``p`` point form never names an area cover. +_AOI_TOKEN_RE = re.compile(r"-?\d+") + + +class ViewTooLargeError(Exception): + """Raised when an open would materialize more cells than ``hive_max_cells``.""" + + def __init__(self, cells: int): + self.cells = cells + + +@dataclass +class HiveView: + """One materialized view: an in-memory zarr store plus its provenance.""" + + store: Any # zarr.storage.MemoryStore + cells: int + cell_order: int + store_url: str + product: str | None + window: str | None + aoi: tuple[str, ...] | None = field(default=None) + + +class HiveViewCache: + """LRU-bounded ``view-id -> HiveView`` map, one per server process. + + View ids are deterministic over the request tuple, so re-opening the same + selection refreshes (LRU-bumps) the existing view instead of duplicating + it; serving a view's objects bumps it too, so actively rendered views + survive. Views are re-materialized only after eviction. + """ + + def __init__(self, proxy: GridlookProxy): + self._proxy = proxy + self._views: OrderedDict[str, HiveView] = OrderedDict() + + @staticmethod + def view_id( + store_url: str, product: str | None, window: str | None, aoi: tuple[str, ...] | None + ) -> str: + payload = json.dumps( + [store_url, product, window, list(aoi) if aoi else None], separators=(",", ":") + ) + return hashlib.sha256(payload.encode()).hexdigest()[:_VIEW_ID_HEX] + + def get(self, view_id: str) -> HiveView | None: + view = self._views.get(view_id) + if view is not None: + self._views.move_to_end(view_id) + return view + + def put(self, view_id: str, view: HiveView) -> None: + self._views[view_id] = view + self._views.move_to_end(view_id) + while len(self._views) > max(1, self._proxy.hive_max_views): + self._views.popitem(last=False) + + +#: The zarr-conventions envelope entry for the dggs convention, added only when +#: a store somehow lacks one (gridlook's detector keys on the envelope's presence). +_DGGS_CONVENTION_ENTRY = { + "schema_url": "https://raw.githubusercontent.com/zarr-conventions/dggs/refs/tags/v1/schema.json", + "spec_url": "https://github.com/zarr-conventions/dggs/blob/v1/README.md", + "uuid": "7b255807-140c-42ca-97f6-7a1cfecdbc38", + "name": "dggs", + "description": "Discrete Global Grid Systems convention for zarr", +} + + +def _shim_dggs_attrs(ds) -> None: + """Rewrite the served ``dggs`` attrs to the healpix-shaped block gridlook reads TODAY. + + PRE-6C COMPATIBILITY SHIM. A morton-only hive's stored convention block is + ``{name: "morton", coordinate: "morton"}`` (mortie spec §5); gridlook's + ``gridTypeDetector.ts`` currently rejects any ``dggs.name != "healpix"``, + and ``Healpix.vue`` reads ``refinement_level`` (→ nside) plus + ``coordinate`` (→ the cell-id coordinate). The fabricated NESTED + ``cell_ids`` ARE plain HEALPix NESTED indices at ``refinement_level``, so + advertising ``{name: "healpix", coordinate: "cell_ids"}`` feeds the + existing sparse limited-area HEALPix path bit-for-bit what it already + consumes — no frontend change needed. Phase 6c teaches the detector the + morton convention entry natively; when it lands, this shim can serve the + stored block unmodified. + """ + dggs = dict(ds.attrs.get("dggs") or {}) + level = dggs.get("refinement_level") + if level is None: + level = ds.attrs["morton_hive"]["cell_order"] + dggs.update({"name": "healpix", "refinement_level": int(level), "coordinate": "cell_ids"}) + dggs.setdefault("spatial_dimension", "cells") + ds.attrs["dggs"] = dggs + ds.attrs.setdefault("zarr_conventions", [_DGGS_CONVENTION_ENTRY]) + + +def build_view( + root: str, + *, + store_url: str, + product: str | None, + aoi: tuple[str, ...] | None, + window: str | None, + max_cells: int, +) -> HiveView: + """Open a hive selection and materialize it as an in-memory zarr store. + + Synchronous and potentially slow (S3 GETs, concat) — the handler runs it + on the executor, off the event loop. + """ + import zarr + from moczarr import open_hive + + ds = open_hive( + root, + aoi=list(aoi) if aoi else None, + window=window, + # Load-bearing: post-zagg#314 stores carry only the packed-u64 morton + # coordinate; "auto" fabricates the exact NESTED cell_ids view the + # browser-side HEALPix path consumes (and keeps stored bytes on any + # remaining dual-written store). + fabricate_cell_ids="auto", + ) + dim = ds["morton"].dims[0] if "morton" in ds.coords else "cells" + cells = int(ds.sizes.get(dim, 0)) + if cells > max_cells: + raise ViewTooLargeError(cells) + _shim_dggs_attrs(ds) + mem = zarr.storage.MemoryStore() + # No compression: objects are served whole over hub-local HTTP, views are + # session-scoped, and codec-free chunks keep the served bytes trivially + # predictable (the tests compare them raw). + encoding = {name: {"compressors": None} for name in list(ds.data_vars) + list(ds.coords)} + ds.to_zarr(mem, mode="w", consolidated=False, zarr_format=3, encoding=encoding) + return HiveView( + store=mem, + cells=cells, + cell_order=int(ds.attrs["morton_hive"]["cell_order"]), + store_url=store_url, + product=product, + window=window, + aoi=aoi, + ) + + +def _parse_aoi(raw: str | None) -> tuple[str, ...] | None: + if raw is None: + return None + tokens = tuple(t.strip() for t in raw.split(",") if t.strip()) + if not tokens: + raise web.HTTPError(400, "aoi= is empty — expected comma-separated morton decimals") + for token in tokens: + if not _AOI_TOKEN_RE.fullmatch(token): + raise web.HTTPError(400, f"aoi token {token!r} is not a morton decimal") + return tokens + + +def _authorize_store_root(proxy: GridlookProxy, store_url: str, product: str | None) -> str: + """Allowlist gate (same posture as the S3 proxy) and product-root resolution.""" + if product is not None and not _PRODUCT_RE.fullmatch(product): + raise web.HTTPError(400, f"invalid product name: {product!r}") + if store_url.startswith("s3://"): + bucket = store_url[len("s3://") :].split("/", 1)[0] + if not bucket: + raise web.HTTPError(400, f"invalid store URL: {store_url!r}") + if not proxy.enabled: + raise web.HTTPError( + 403, + "gridlook hive endpoint is disabled for S3: no allowed buckets are " + "configured. Set GridlookProxy.allowed_buckets (or GRIDLOOK_ALLOWED_BUCKETS).", + ) + if bucket not in proxy.allowed_buckets: + raise web.HTTPError( + 403, + f"bucket '{bucket}' is not in the gridlook proxy allowlist " + f"(GridlookProxy.allowed_buckets)", + ) + elif "://" in store_url: + raise web.HTTPError( + 403, f"unsupported store scheme in {store_url!r}: use s3://… or a local path" + ) + elif not proxy.allow_local_hive_stores: + raise web.HTTPError( + 403, + "local-path hive stores are disabled — set " + "GridlookProxy.allow_local_hive_stores = True (development only)", + ) + root = store_url.rstrip("/") + return f"{root}/{product}" if product else root + + +class HiveOpenHandler(PlainTextErrorMixin, JupyterHandler): + """``GET /gridlook/hive/open?store=…[&product=…][&aoi=…][&window=…]``. + + Creates (or LRU-refreshes) a view and returns its id plus the entry URL to + paste into gridlook as a zarr dataset source. + """ + + @web.authenticated + async def get(self): + proxy: GridlookProxy = self.settings["gridlook_proxy"] + cache: HiveViewCache = self.settings["gridlook_hive_views"] + store_url = self.get_query_argument("store", None) + if not store_url: + raise web.HTTPError( + 400, "missing required query parameter: store=" + ) + product = self.get_query_argument("product", None) or None + window = self.get_query_argument("window", None) or None + aoi = _parse_aoi(self.get_query_argument("aoi", None) or None) + root = _authorize_store_root(proxy, store_url, product) + try: + import moczarr # noqa: F401 + except ImportError as e: + raise web.HTTPError( + 500, + "the /gridlook/hive/ endpoints need moczarr — install gridlook-jupyter[hive]", + ) from e + + view_id = cache.view_id(store_url, product, window, aoi) + view = cache.get(view_id) + cached = view is not None + if view is None: + build = functools.partial( + build_view, + root, + store_url=store_url, + product=product, + aoi=aoi, + window=window, + max_cells=proxy.hive_max_cells, + ) + try: + view = await IOLoop.current().run_in_executor(None, build) + except ViewTooLargeError as e: + raise web.HTTPError( + 413, + f"hive view would materialize {e.cells} cells, over the " + f"{proxy.hive_max_cells}-cell limit — narrow the aoi= selection " + f"(or raise GridlookProxy.hive_max_cells)", + ) from e + except FileNotFoundError as e: + raise web.HTTPError(404, f"no hive store at {store_url!r}: {e}") from e + except ValueError as e: + # moczarr's NoCoverageError (nothing committed anywhere) is a + # ValueError subclass: the store exists but has nothing to + # serve — 404, not a bad request. + status = 404 if type(e).__name__ == "NoCoverageError" else 400 + raise web.HTTPError(status, f"cannot open {store_url!r}: {e}") from e + cache.put(view_id, view) + + self.set_header("Content-Type", "application/json") + self.finish( + json.dumps( + { + "view": view_id, + "url": url_path_join(self.base_url, "gridlook", "hive", view_id), + "cells": view.cells, + "cell_order": view.cell_order, + "cached": cached, + } + ) + ) + + +class HiveViewHandler(PlainTextErrorMixin, JupyterHandler): + """``GET /gridlook/hive//``: serve one zarr object of a view. + + Objects are metadata documents and whole (small) chunks — no Range + support, mirroring how zarrita fetches them. + """ + + @web.authenticated + async def get(self, view_id: str, key: str): + cache: HiveViewCache = self.settings["gridlook_hive_views"] + view = cache.get(view_id) + if view is None: + raise web.HTTPError( + 404, + f"no hive view '{view_id}' (never opened, or evicted from the LRU " + f"cache) — (re)open it via /gridlook/hive/open", + ) + from zarr.core.buffer import default_buffer_prototype + + buf = await view.store.get(key, prototype=default_buffer_prototype()) + if buf is None: + raise web.HTTPError(404, f"no object '{key}' in hive view '{view_id}'") + data = buf.to_bytes() + self.set_header( + "Content-Type", + "application/json" if key.endswith(".json") else "application/octet-stream", + ) + self.set_header("Content-Length", str(len(data))) + # View URLs die on eviction; keep intermediaries from pinning stale objects. + self.set_header("Cache-Control", "no-store") + self.finish(data) diff --git a/jupyter/pyproject.toml b/jupyter/pyproject.toml index 56b1f6e3..1b75cad6 100644 --- a/jupyter/pyproject.toml +++ b/jupyter/pyproject.toml @@ -17,11 +17,22 @@ dependencies = [ ] [project.optional-dependencies] +# The /gridlook/hive/ virtual store (phase 6d): moczarr opens morton-hive +# stores hub-side and fabricates the NESTED cell_ids view. Not on PyPI yet — +# the git source is the interim shape (tests run against a local checkout). +hive = [ + "moczarr @ git+https://github.com/espg/moczarr", +] test = [ "pytest>=8", "pytest-jupyter[server]>=0.10", ] +[tool.hatch.metadata] +# The [hive] extra's moczarr pin is a git direct reference until moczarr +# publishes to PyPI; drop this (and the pin's @ git+…) when it does. +allow-direct-references = true + [tool.hatch.build.targets.wheel] packages = ["gridlook_jupyter"] # The static SPA is copied in by the build hook and is gitignored; `artifacts` From ffbaa90d376740a9c0cbab80504ad69f3a4afca3 Mon Sep 17 00:00:00 2001 From: espg Date: Mon, 20 Jul 2026 18:11:53 -0700 Subject: [PATCH 2/8] test: hive virtual-store suite against the moczarr SERC fixture Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N32Mx4rWhbTK8xXm7Mt2ax --- jupyter/tests/conftest.py | 9 +- jupyter/tests/test_hive.py | 204 +++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 jupyter/tests/test_hive.py diff --git a/jupyter/tests/conftest.py b/jupyter/tests/conftest.py index fad469de..a97c0269 100644 --- a/jupyter/tests/conftest.py +++ b/jupyter/tests/conftest.py @@ -27,12 +27,19 @@ def allowed_buckets(): @pytest.fixture -def jp_server_config(static_root, allowed_buckets): +def hive_config(): + """GridlookProxy hive traits; override per test class (LRU/size knobs).""" + return {"allow_local_hive_stores": True} + + +@pytest.fixture +def jp_server_config(static_root, allowed_buckets, hive_config): return { "ServerApp": {"jpserver_extensions": {"gridlook_jupyter": True}}, "GridlookProxy": { "allowed_buckets": allowed_buckets, "static_dir": str(static_root), + **hive_config, }, } diff --git a/jupyter/tests/test_hive.py b/jupyter/tests/test_hive.py new file mode 100644 index 00000000..816d12aa --- /dev/null +++ b/jupyter/tests/test_hive.py @@ -0,0 +1,204 @@ +"""``/gridlook/hive/`` virtual-store views against moczarr's committed SERC fixture. + +The fixture (``tests/data/serc_hive`` in the moczarr repo) is zagg-written and +morton-only (post englacial/zagg#314), so every ``cell_ids`` assertion here +exercises moczarr's NESTED fabrication — the load-bearing 6d piece. The suite +needs moczarr importable from a repo checkout (e.g. ``uv pip install -e +/path/to/moczarr``) so the fixture is reachable next to the package; a +plain-``[test]`` environment skips it wholesale. +""" + +import json +import os +from pathlib import Path + +import pytest +from tornado.httpclient import HTTPClientError + +moczarr = pytest.importorskip("moczarr", reason="hive tests need gridlook-jupyter[hive]") +np = pytest.importorskip("numpy") + + +def _moczarr_testdata() -> Path: + env = os.environ.get("GRIDLOOK_MOCZARR_TESTDATA") + if env: + return Path(env) + # src layout: src/moczarr/__init__.py -> parents[2] is the repo root. + return Path(moczarr.__file__).resolve().parents[2] / "tests" / "data" + + +TESTDATA = _moczarr_testdata() +SERC = TESTDATA / "serc_hive" +#: The stored cell_ids of the last dual-written SERC fixture (pre-zagg#314), +#: whole-store concat order — moczarr's fabrication-parity golden. +GOLDEN = TESTDATA / "serc_cell_ids_golden.npy" +#: An order-6 stamped shard of the fixture (the SERC site itself). +SERC_SHARD = "4331422" + +pytestmark = pytest.mark.skipif( + not (SERC / "morton_hive.json").exists(), + reason=f"moczarr SERC fixture not found at {SERC} (needs a moczarr repo checkout; " + "set GRIDLOOK_MOCZARR_TESTDATA to its tests/data)", +) + + +async def _open(jp_fetch, **params): + resp = await jp_fetch("gridlook", "hive", "open", params={"store": str(SERC), **params}) + assert resp.code == 200 + return json.loads(resp.body) + + +async def _fetch_array(jp_fetch, view, name, dtype=None): + meta = json.loads((await jp_fetch("gridlook", "hive", view, f"{name}/zarr.json")).body) + # Uncompressed by design: the raw chunk bytes ARE the array. + assert [c["name"] for c in meta["codecs"]] == ["bytes"] + if dtype is None: + dtype = np.dtype(meta["data_type"]).newbyteorder("<") + chunk = await jp_fetch("gridlook", "hive", view, f"{name}/c/0") + assert chunk.headers["Content-Type"] == "application/octet-stream" + return np.frombuffer(chunk.body, dtype=dtype) + + +async def test_open_returns_view_and_entry_url(jp_fetch): + out = await _open(jp_fetch) + assert set(out) >= {"view", "url", "cells", "cell_order", "cached"} + assert out["url"].endswith(f"/gridlook/hive/{out['view']}") + assert out["cells"] == len(np.load(GOLDEN)) + assert out["cell_order"] == 8 + assert out["cached"] is False + + +async def test_reopen_same_selection_reuses_view(jp_fetch): + first = await _open(jp_fetch) + again = await _open(jp_fetch) + assert again["view"] == first["view"] + assert again["cached"] is True + + +async def test_zarr_json_carries_healpix_shim_attrs(jp_fetch): + out = await _open(jp_fetch) + resp = await jp_fetch("gridlook", "hive", out["view"], "zarr.json") + assert resp.code == 200 + assert resp.headers["Content-Type"] == "application/json" + attrs = json.loads(resp.body)["attributes"] + # Pre-6c compatibility shim: gridlook's detector accepts only + # dggs.name == "healpix" today; the served block points its existing + # sparse-HEALPix path at the fabricated NESTED cell_ids. + assert "zarr_conventions" in attrs + dggs = attrs["dggs"] + assert dggs["name"] == "healpix" + assert dggs["coordinate"] == "cell_ids" + assert dggs["refinement_level"] == 8 + assert attrs["morton_hive"]["cell_order"] == 8 + + +async def test_cell_ids_match_moczarr_fabrication_golden(jp_fetch): + out = await _open(jp_fetch) + ids = await _fetch_array(jp_fetch, out["view"], "cell_ids", "/` | streaming S3 proxy — GET/HEAD only, `Range` pass-through (206), no LIST | -| `/gridlook/hive/…` | **reserved** for the phase-6 moczarr virtual-store endpoint (an `open_hive()` AOI served as one flat zarr store) — not implemented here | +| `/gridlook/hive/open` | open (or LRU-refresh) a **morton-hive virtual-store view** via moczarr — see below | +| `/gridlook/hive//` | serve one zarr object (metadata / whole chunk) of an open view | ## Configuration @@ -58,6 +59,11 @@ Via traitlets (`jupyter_server_config.py`, or `--GridlookProxy.…` on the comma c.GridlookProxy.allowed_buckets = ["my-zagg-outputs"] c.GridlookProxy.region = "us-west-2" # optional; ambient AWS config otherwise c.GridlookProxy.static_dir = "/path/to/dist" # optional; dev override for the SPA files + +# /gridlook/hive/ knobs (phase 6d; defaults shown) +c.GridlookProxy.hive_max_views = 8 # LRU bound on materialized views +c.GridlookProxy.hive_max_cells = 500_000 # per-view cell bound; 413 beyond +c.GridlookProxy.allow_local_hive_stores = False # local-path stores (dev only) ``` Or environment variables (used only when the trait is not configured): @@ -72,6 +78,58 @@ the standard hub setup. The proxy streams responses chunk-by-chunk and never buf objects; there are no presigned URLs, so nothing credential-shaped is ever exposed to the browser. +## Morton-hive virtual store (`/gridlook/hive/`) + +Phase 6d of the viewer plan: a zagg **morton-hive** store is many leaf zarrs, but gridlook +expects one zarr source — and post-englacial/zagg#314 stores are **morton-only**, so gridlook's +existing HEALPix path (which consumes NESTED `cell_ids`) cannot read a leaf directly. The hive +endpoint closes both gaps hub-side with [moczarr](https://github.com/espg/moczarr): +`open_hive()` selects a product/AOI/window, **fabricates the exact NESTED `cell_ids` +coordinate**, and the extension serves the result as **one flat zarr v3 store** the browser's +unmodified zarrita reads. + +``` +GET /gridlook/hive/open?store=[&product=][&aoi=][&window=