diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml new file mode 100644 index 00000000..5bdfe0b0 --- /dev/null +++ b/.github/workflows/jupyter.yml @@ -0,0 +1,58 @@ +name: Jupyter + +on: + push: + branches: + - main + paths: + - "jupyter/**" + - "src/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/jupyter.yml" + pull_request: + paths: + - "jupyter/**" + - "src/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/jupyter.yml" + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up node + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Build the wheel (runs the vite build hook) + run: uv build --wheel jupyter -o jupyter/dist-py + + - name: Assert the wheel carries the SPA + run: unzip -l jupyter/dist-py/*.whl | grep -q "gridlook_jupyter/static/index.html" + + - name: Assert the wheel ships no source maps + run: "! unzip -l jupyter/dist-py/*.whl | grep -q '\\.map$'" + + - name: Install the wheel with test deps + # setup-uv's python-version input already created and activated .venv + run: uv pip install "$(ls jupyter/dist-py/*.whl)[test]" ruff + + - name: Ruff + run: | + ruff check jupyter/gridlook_jupyter jupyter/tests jupyter/hatch_build.py + ruff format --check jupyter/gridlook_jupyter jupyter/tests jupyter/hatch_build.py + + - name: Pytest + run: pytest jupyter/tests -v diff --git a/.gitignore b/.gitignore index b7ec07d5..c67e60e1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ tsconfig.tsbuildinfo *.njsproj *.sln *.sw? + +# uv lockfile from jupyter test runs (not committed for this package) +jupyter/uv.lock diff --git a/jupyter/.gitignore b/jupyter/.gitignore new file mode 100644 index 00000000..86489ae2 --- /dev/null +++ b/jupyter/.gitignore @@ -0,0 +1,9 @@ +# Built SPA copied in by the wheel build hook (hatch_build.py) +gridlook_jupyter/static/ +__pycache__/ +*.egg-info/ +.venv/ +dist-py/ +build/ +.pytest_cache/ +.ruff_cache/ diff --git a/jupyter/README.md b/jupyter/README.md new file mode 100644 index 00000000..08e3bdd8 --- /dev/null +++ b/jupyter/README.md @@ -0,0 +1,95 @@ +# gridlook-jupyter + +A jupyter-server extension that serves the built [gridlook](../README.md) viewer inside a +JupyterHub/Jupyter environment, plus a **streaming S3 byte-range proxy** so the browser can read +private buckets through the hub's credentials — credentials never reach the browser, and the +buckets never need to be public. Phase 4 of the viewer plan +([espg/gridlook#1](https://github.com/espg/gridlook/issues/1)). + +The primary deployment target is [CryoCloud](https://cryointhecloud.com/) (a 2i2c-operated +JupyterHub): the hub role holds the S3 read credentials, and gridlook runs against buckets that +are hub-viewable but not public (e.g. zagg output stores — see englacial/zagg#301 for the +cost-visualization use case). + +## Install + +```bash +pip install ./jupyter # from a gridlook checkout — needs node >= 24 on PATH +# or +pip install gridlook_jupyter--py3-none-any.whl +``` + +The wheel embeds the built Vite app as package data. **Building the wheel requires node/npm** +(the build hook runs `npm ci && npm run build` at the repo root and copies `dist/` into the +wheel); installing a pre-built wheel does not. The extension auto-enables on install via +`jupyter_server_config.d`. + +Editable installs (`pip install -e ./jupyter`) skip the frontend build — point +`GridlookProxy.static_dir` at a locally built `dist/` instead (see below). + +## Launch + +There is no launcher card yet (that is phase 5). Open the app by URL: + +``` +/gridlook/ +``` + +e.g. on a hub: `https://hub.example.org/user//gridlook/`. + +## Routes + +| Route | What | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `/gridlook/` | the static gridlook SPA | +| `/gridlook/api/health` | tiny JSON probe (`{"extension": "gridlook-jupyter", ...}`) | +| `/gridlook/s3//` | streaming S3 proxy — GET/HEAD only, `Range` pass-through (206), no LIST | +| `/gridlook/hive/…` | **reserved** for the phase-6 moczarr virtual-store endpoint (an `open_hive()` AOI served as one flat zarr store) — not implemented here | + +## Configuration + +Allowlist-only auth: **the proxy is disabled until you configure buckets.** Requests for +non-allowlisted buckets get a 403 naming the bucket; with an empty allowlist every proxy request +gets a 403 saying the proxy is disabled. + +Via traitlets (`jupyter_server_config.py`, or `--GridlookProxy.…` on the command line): + +```python +c.GridlookProxy.allowed_buckets = ["my-zagg-outputs"] +c.GridlookProxy.region = "us-west-2" # optional; ambient AWS config otherwise +c.GridlookProxy.static_dir = "/path/to/dist" # optional; dev override for the SPA files +``` + +Or environment variables (used only when the trait is not configured): + +```bash +export GRIDLOOK_ALLOWED_BUCKETS="my-zagg-outputs,another-bucket" +export GRIDLOOK_S3_REGION="us-west-2" +``` + +S3 credentials come from the ambient chain (instance/pod role, `AWS_*` env, shared config) — +the standard hub setup. The proxy streams responses chunk-by-chunk and never buffers whole +objects; there are no presigned URLs, so nothing credential-shaped is ever exposed to the +browser. + +## `s3://` inputs in the app + +When the SPA is served under `/gridlook/` (it detects this by probing `api/health`), pasting an +`s3://bucket/prefix` dataset URL rewrites it to the proxy path (`…/gridlook/s3/bucket/prefix`) +automatically. Served standalone (dev server, static hosting), `s3://` inputs are left +untouched. Pasting the proxy URL directly always works. + +## Development + +```bash +cd jupyter +uv venv && uv pip install -e ".[test]" ruff +.venv/bin/pytest tests -v +.venv/bin/ruff check gridlook_jupyter tests hatch_build.py +# run against a dev-built frontend: +npm run build # at the repo root +jupyter server --GridlookProxy.static_dir="$(pwd)/../dist" --GridlookProxy.allowed_buckets='["my-bucket"]' +``` + +Tests exercise the proxy against an obstore `LocalStore` via the `GridlookProxy.store_factory` +seam — no real S3 needed. diff --git a/jupyter/gridlook_jupyter/__init__.py b/jupyter/gridlook_jupyter/__init__.py new file mode 100644 index 00000000..150793a0 --- /dev/null +++ b/jupyter/gridlook_jupyter/__init__.py @@ -0,0 +1,17 @@ +"""gridlook-jupyter: serve the gridlook viewer and an S3 byte-range proxy from jupyter-server.""" + +__version__ = "0.1.0" + + +def _jupyter_server_extension_points(): + return [{"module": "gridlook_jupyter"}] + + +def _load_jupyter_server_extension(serverapp): + from .extension import load_extension + + load_extension(serverapp) + + +# Alias kept for older jupyter-server enable paths. +load_jupyter_server_extension = _load_jupyter_server_extension diff --git a/jupyter/gridlook_jupyter/config.py b/jupyter/gridlook_jupyter/config.py new file mode 100644 index 00000000..414d4598 --- /dev/null +++ b/jupyter/gridlook_jupyter/config.py @@ -0,0 +1,80 @@ +"""Traitlets config surface for the gridlook extension.""" + +import os +from pathlib import Path + +from traitlets import List, Unicode, default +from traitlets.config import Configurable + +#: Bounded per-process cache of bucket -> store (buckets come from the allowlist, +#: so this stays tiny; the bound is belt-and-braces). +_MAX_CACHED_STORES = 64 + + +def default_store_factory(bucket: str, region: str | None): + """Build an obstore store for *bucket* using the ambient AWS credential chain.""" + from obstore.store import S3Store + + kwargs = {"region": region} if region else {} + return S3Store(bucket, **kwargs) + + +class GridlookProxy(Configurable): + """Config for the gridlook S3 proxy (jupyter_server_config / CLI / env).""" + + allowed_buckets = List( + Unicode(), + help=( + "Buckets the proxy may read from. Empty (the default) disables the " + "proxy entirely. Env fallback: GRIDLOOK_ALLOWED_BUCKETS (comma-separated), " + "used only when this trait is not configured." + ), + ).tag(config=True) + + region = Unicode( + "", + help=( + "AWS region for the S3 stores. Empty defers to the ambient AWS " + "configuration. Env fallback: GRIDLOOK_S3_REGION." + ), + ).tag(config=True) + + static_dir = Unicode( + "", + help=( + "Directory holding the built gridlook SPA. Defaults to the static/ " + "directory packaged in the wheel; point it at a repo dist/ for development." + ), + ).tag(config=True) + + @default("allowed_buckets") + def _default_allowed_buckets(self): + env = os.environ.get("GRIDLOOK_ALLOWED_BUCKETS", "") + return [b.strip() for b in env.split(",") if b.strip()] + + @default("region") + def _default_region(self): + return os.environ.get("GRIDLOOK_S3_REGION", "") + + @default("static_dir") + def _default_static_dir(self): + return str(Path(__file__).parent / "static") + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # Seam for tests: swap in e.g. an obstore LocalStore factory so the + # proxy can be exercised without real S3. Plain attribute, not a trait. + self.store_factory = default_store_factory + self._stores: dict[str, object] = {} + + @property + def enabled(self) -> bool: + return bool(self.allowed_buckets) + + def get_store(self, bucket: str): + """Return (building if needed) the store for an allowlisted bucket.""" + if bucket not in self._stores: + if len(self._stores) >= _MAX_CACHED_STORES: + self._stores.clear() + self._stores[bucket] = self.store_factory(bucket, self.region or None) + return self._stores[bucket] diff --git a/jupyter/gridlook_jupyter/extension.py b/jupyter/gridlook_jupyter/extension.py new file mode 100644 index 00000000..a8a2ea3f --- /dev/null +++ b/jupyter/gridlook_jupyter/extension.py @@ -0,0 +1,45 @@ +"""Wires the gridlook routes into a running jupyter-server.""" + +import re +from pathlib import Path + +from jupyter_server.base.handlers import AuthenticatedFileHandler +from jupyter_server.utils import url_path_join +from tornado import web + +from .config import GridlookProxy +from .handlers import HealthHandler, S3ProxyHandler + + +def load_extension(serverapp): + proxy = GridlookProxy(parent=serverapp) + static_dir = proxy.static_dir + if not (Path(static_dir) / "index.html").exists(): + serverapp.log.warning( + "gridlook-jupyter: no built SPA at %s — /gridlook/ will 404. " + "Install from a wheel, or set GridlookProxy.static_dir to a gridlook dist/.", + static_dir, + ) + + base = url_path_join(serverapp.base_url, "gridlook") + escaped = re.escape(base) + handlers = [ + (escaped + r"/api/health", HealthHandler), + (escaped + r"/s3/([^/]+)/(.+)", S3ProxyHandler), + # /gridlook/hive/... is RESERVED: phase 6 mounts the moczarr virtual-store + # endpoint there (an open_hive() AOI served as one flat zarr store). + # Do not claim that namespace with other routes. + (escaped + r"$", web.RedirectHandler, {"url": base + "/"}), + ( + escaped + r"/(.*)", + AuthenticatedFileHandler, + {"path": static_dir, "default_filename": "index.html"}, + ), + ] + serverapp.web_app.settings["gridlook_proxy"] = proxy + serverapp.web_app.add_handlers(".*$", handlers) + serverapp.log.info( + "gridlook-jupyter loaded: app at %s/, proxy %s", + base, + f"enabled for buckets {proxy.allowed_buckets}" if proxy.enabled else "disabled", + ) diff --git a/jupyter/gridlook_jupyter/handlers.py b/jupyter/gridlook_jupyter/handlers.py new file mode 100644 index 00000000..afc1e7e3 --- /dev/null +++ b/jupyter/gridlook_jupyter/handlers.py @@ -0,0 +1,185 @@ +"""Tornado handlers: health endpoint and the streaming S3 byte-range proxy.""" + +import json +import mimetypes +import re + +import obstore +from jupyter_server.base.handlers import JupyterHandler +from obstore.exceptions import BaseError, NotFoundError +from tornado import web + +from . import __version__ +from .config import GridlookProxy + +#: Single-range "bytes=" header; multi-range is not supported (RFC 9110 permits +#: ignoring the Range header, in which case the full object is served with 200). +_RANGE_RE = re.compile(r"^bytes=(\d*)-(\d*)$") + +_STREAM_CHUNK = 256 * 1024 + + +class _RangeNotSatisfiableError(Exception): + """Raised by range parsing for a well-formed but unsatisfiable range (e.g. ``bytes=-0``).""" + + +def parse_range_header(header: str): + """Map a single-range ``Range`` header to obstore get options, or None to ignore it. + + Raises :class:`_RangeNotSatisfiableError` for a syntactically valid range that can never + be satisfied (a zero-length suffix, ``bytes=-0``). + """ + m = _RANGE_RE.match(header.strip()) + if m is None: + return None + start, end = m.groups() + if start and end: + if int(end) < int(start): + return None + return (int(start), int(end) + 1) # inclusive -> exclusive + if start: + return {"offset": int(start)} + if end: + suffix = int(end) + if suffix == 0: # bytes=-0 selects the last zero bytes: unsatisfiable + raise _RangeNotSatisfiableError + return {"suffix": suffix} + return None + + +def _content_type(key: str, attributes=None) -> str: + """Pass through the object's content-type when the store reports one, else guess.""" + try: + for k, v in dict(attributes or {}).items(): + if str(k).lower().replace("_", "-") == "content-type": + return str(v) + except (TypeError, ValueError): + pass + guess, _ = mimetypes.guess_type(key) + return guess or "application/octet-stream" + + +class HealthHandler(JupyterHandler): + """Tiny probe endpoint; the SPA uses it to detect extension-served context.""" + + @web.authenticated + async def get(self): + proxy: GridlookProxy = self.settings["gridlook_proxy"] + self.set_header("Content-Type", "application/json") + self.finish( + json.dumps( + { + "extension": "gridlook-jupyter", + "version": __version__, + "proxy_enabled": proxy.enabled, + } + ) + ) + + +class S3ProxyHandler(JupyterHandler): + """Streaming byte-range proxy: GET/HEAD only, no LIST, hub-side credentials only.""" + + SUPPORTED_METHODS = ("GET", "HEAD") + + def write_error(self, status_code, **kwargs): + exc = kwargs.get("exc_info", (None, None, None))[1] + message = "" + if isinstance(exc, web.HTTPError) and exc.log_message: + message = exc.log_message + self.set_header("Content-Type", "text/plain; charset=utf-8") + self.finish(message or f"error {status_code}") + + def _store_for(self, bucket: str): + proxy: GridlookProxy = self.settings["gridlook_proxy"] + if not proxy.enabled: + raise web.HTTPError( + 403, + "gridlook S3 proxy is disabled: no allowed buckets are configured. " + "Set GridlookProxy.allowed_buckets (or GRIDLOOK_ALLOWED_BUCKETS).", + ) + if bucket not in proxy.allowed_buckets: + raise web.HTTPError( + 403, + f"bucket '{bucket}' is not in the gridlook proxy allowlist " + f"(GridlookProxy.allowed_buckets)", + ) + return proxy.get_store(bucket) + + async def _send_range_not_satisfiable(self, store, key): + """Emit a 416 with ``Content-Range: bytes */``; head the object for the size. + + RFC 9110 §14.4 requires a numeric complete-length in the unsatisfied-range form + (``*/*`` is not valid there), so we head the object — a single request on this cold + error path — and only fall back to ``*/*`` if that head itself fails. + """ + size = None + try: + meta = await obstore.head_async(store, key) + size = meta["size"] + except (BaseError, OSError, ValueError): + pass + self.set_status(416) + self.set_header("Content-Type", "text/plain; charset=utf-8") + self.set_header("Content-Range", f"bytes */{size}" if size is not None else "bytes */*") + self.finish("range not satisfiable") + + @web.authenticated + async def get(self, bucket, key): + store = self._store_for(bucket) + range_header = self.request.headers.get("Range") + try: + rng = parse_range_header(range_header) if range_header else None + except _RangeNotSatisfiableError: + await self._send_range_not_satisfiable(store, key) + return + options = {"range": rng} if rng is not None else {} + try: + result = await obstore.get_async(store, key, options=options) + except (NotFoundError, FileNotFoundError) as e: + raise web.HTTPError(404, f"no such object: s3://{bucket}/{key}") from e + except ValueError as e: + # obstore's path parser rejects malformed keys (``..``, empty segments) + # before any I/O; that is a bad client request, not a server fault. + raise web.HTTPError(400, f"invalid key: {key}") from e + except (BaseError, OSError) as e: + # A range that runs past EOF surfaces as a generic "range invalid" error; + # that is a client range fault (416), not an upstream/gateway fault (502). + if rng is not None and "range" in str(e).lower(): + await self._send_range_not_satisfiable(store, key) + return + raise web.HTTPError(502, f"S3 error for s3://{bucket}/{key}: {e}") from e + + size = result.meta["size"] + start, end = result.range + self.set_header("Accept-Ranges", "bytes") + self.set_header("Content-Type", _content_type(key, result.attributes)) + self.set_header("Content-Length", str(end - start)) + etag = result.meta.get("e_tag") + if etag: + self.set_header("Etag", etag) + if rng is not None: + self.set_status(206) + self.set_header("Content-Range", f"bytes {start}-{end - 1}/{size}") + # Stream chunk-by-chunk; never buffer whole objects. + async for chunk in result.stream(min_chunk_size=_STREAM_CHUNK): + self.write(bytes(chunk)) + await self.flush() + + @web.authenticated + async def head(self, bucket, key): + store = self._store_for(bucket) + try: + meta = await obstore.head_async(store, key) + except (NotFoundError, FileNotFoundError) as e: + raise web.HTTPError(404, f"no such object: s3://{bucket}/{key}") from e + except ValueError as e: + raise web.HTTPError(400, f"invalid key: {key}") from e + except (BaseError, OSError) as e: + raise web.HTTPError(502, f"S3 error for s3://{bucket}/{key}: {e}") from e + self.set_header("Accept-Ranges", "bytes") + self.set_header("Content-Type", _content_type(key)) + self.set_header("Content-Length", str(meta["size"])) + etag = meta.get("e_tag") + if etag: + self.set_header("Etag", etag) diff --git a/jupyter/hatch_build.py b/jupyter/hatch_build.py new file mode 100644 index 00000000..b5cd3d5c --- /dev/null +++ b/jupyter/hatch_build.py @@ -0,0 +1,57 @@ +"""Hatchling build hook: build the Vite SPA at the repo root and package dist/ as static/. + +Wheel builds need either a pre-populated gridlook_jupyter/static/ (index.html present) +or node/npm on PATH plus the frontend sources one directory up. Editable installs skip +the hook — point GridlookProxy.static_dir at a dev dist/ instead. +""" + +import shutil +import subprocess +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class ViteBuildHook(BuildHookInterface): + PLUGIN_NAME = "custom" + + def initialize(self, version, build_data): + if self.target_name != "wheel" or version == "editable": + return + pkg_root = Path(self.root) + static = pkg_root / "gridlook_jupyter" / "static" + if (static / "index.html").exists(): + self.app.display_info(f"gridlook-jupyter: packaging pre-built SPA from {static}") + return + + repo_root = pkg_root.parent + if not (repo_root / "package.json").exists(): + raise RuntimeError( + "gridlook-jupyter: gridlook_jupyter/static/ is empty and the frontend " + "sources are not present one directory up (no package.json). Build from " + "the gridlook repo checkout, or pre-populate gridlook_jupyter/static/ " + "with a built dist/. Refusing to ship a wheel without the SPA." + ) + npm = shutil.which("npm") + if npm is None: + raise RuntimeError( + "gridlook-jupyter: gridlook_jupyter/static/ is empty and npm is not on " + "PATH. Install node >= 24 and retry, or pre-populate " + "gridlook_jupyter/static/ with a built dist/. Refusing to ship a wheel " + "without the SPA." + ) + + self.app.display_info("gridlook-jupyter: running npm ci && npm run build") + subprocess.run([npm, "ci"], cwd=repo_root, check=True) + subprocess.run([npm, "run", "build"], cwd=repo_root, check=True) + dist = repo_root / "dist" + if not (dist / "index.html").exists(): + raise RuntimeError(f"gridlook-jupyter: npm run build produced no {dist}/index.html") + if static.exists(): + shutil.rmtree(static) + # vite.config.ts keeps sourcemaps on for dev/standalone builds; the wheel is a + # server-extension artifact that never needs them, so drop *.map on the way in. + shutil.copytree(dist, static, ignore=shutil.ignore_patterns("*.map")) + self.app.display_info( + f"gridlook-jupyter: packaged {dist} -> {static} (source maps excluded)" + ) diff --git a/jupyter/jupyter-config/jupyter_server_config.d/gridlook_jupyter.json b/jupyter/jupyter-config/jupyter_server_config.d/gridlook_jupyter.json new file mode 100644 index 00000000..7f6c7b1e --- /dev/null +++ b/jupyter/jupyter-config/jupyter_server_config.d/gridlook_jupyter.json @@ -0,0 +1,7 @@ +{ + "ServerApp": { + "jpserver_extensions": { + "gridlook_jupyter": true + } + } +} diff --git a/jupyter/pyproject.toml b/jupyter/pyproject.toml new file mode 100644 index 00000000..56b1f6e3 --- /dev/null +++ b/jupyter/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "gridlook-jupyter" +# Static version: this fork's tags belong to upstream gridlook releases, so no hatch-vcs. +version = "0.1.0" +description = "Jupyter server extension serving the gridlook viewer with a streaming S3 byte-range proxy" +readme = "README.md" +license = "MIT" +requires-python = ">=3.12" +authors = [{ name = "Shane Grigsby", email = "refuge@rocktalus.com" }] +dependencies = [ + "jupyter-server>=2", + "obstore>=0.10", +] + +[project.optional-dependencies] +test = [ + "pytest>=8", + "pytest-jupyter[server]>=0.10", +] + +[tool.hatch.build.targets.wheel] +packages = ["gridlook_jupyter"] +# The static SPA is copied in by the build hook and is gitignored; `artifacts` +# re-includes VCS-ignored paths. +artifacts = ["gridlook_jupyter/static/**"] + +[tool.hatch.build.targets.wheel.shared-data] +"jupyter-config/jupyter_server_config.d" = "etc/jupyter/jupyter_server_config.d" + +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[tool.hatch.build.targets.sdist] +include = [ + "gridlook_jupyter", + "jupyter-config", + "hatch_build.py", + "README.md", +] + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "N"] +ignore = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" diff --git a/jupyter/tests/conftest.py b/jupyter/tests/conftest.py new file mode 100644 index 00000000..fad469de --- /dev/null +++ b/jupyter/tests/conftest.py @@ -0,0 +1,60 @@ +import pytest +from obstore.store import LocalStore + +pytest_plugins = ["pytest_jupyter.jupyter_server"] + +ALLOWED_BUCKET = "test-bucket" + + +@pytest.fixture +def static_root(tmp_path): + d = tmp_path / "static" + d.mkdir() + (d / "index.html").write_text("gridlook test index") + return d + + +@pytest.fixture +def bucket_root(tmp_path): + d = tmp_path / "buckets" + (d / ALLOWED_BUCKET).mkdir(parents=True) + return d + + +@pytest.fixture +def allowed_buckets(): + return [ALLOWED_BUCKET] + + +@pytest.fixture +def jp_server_config(static_root, allowed_buckets): + return { + "ServerApp": {"jpserver_extensions": {"gridlook_jupyter": True}}, + "GridlookProxy": { + "allowed_buckets": allowed_buckets, + "static_dir": str(static_root), + }, + } + + +@pytest.fixture +def proxy(jp_serverapp, bucket_root): + """The live GridlookProxy, with its store factory pointed at a LocalStore tree.""" + p = jp_serverapp.web_app.settings["gridlook_proxy"] + + def local_factory(bucket, region): + return LocalStore(prefix=str(bucket_root / bucket)) + + p.store_factory = local_factory + p._stores.clear() + return p + + +@pytest.fixture +def put_object(bucket_root): + def _put(bucket: str, key: str, data: bytes): + path = bucket_root / bucket / key + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + return _put diff --git a/jupyter/tests/test_extension.py b/jupyter/tests/test_extension.py new file mode 100644 index 00000000..e2c03f4a --- /dev/null +++ b/jupyter/tests/test_extension.py @@ -0,0 +1,84 @@ +import json + +import pytest +from tornado.httpclient import HTTPClientError + + +async def test_health(jp_fetch, proxy): + resp = await jp_fetch("gridlook", "api", "health") + assert resp.code == 200 + body = json.loads(resp.body) + assert body["extension"] == "gridlook-jupyter" + assert body["proxy_enabled"] is True + + +async def test_static_index_served(jp_fetch, proxy): + resp = await jp_fetch("gridlook/") + assert resp.code == 200 + assert b"gridlook test index" in resp.body + + +async def test_static_asset_served(jp_fetch, proxy, static_root): + (static_root / "app.js").write_text("console.log('gridlook');") + resp = await jp_fetch("gridlook", "app.js") + assert resp.code == 200 + assert b"console.log" in resp.body + + +async def test_bare_route_redirects_to_slash(jp_fetch, jp_base_url, proxy): + # follow_redirects=False: the follow would need DNS, which sandboxed + # environments may block; the Location assertion is what matters. + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", follow_redirects=False) + assert e.value.code == 301 + assert e.value.response.headers["Location"] == f"{jp_base_url}gridlook/" + + +class TestEnvFallback: + def test_allowlist_from_env(self, monkeypatch): + from gridlook_jupyter.config import GridlookProxy + + monkeypatch.setenv("GRIDLOOK_ALLOWED_BUCKETS", "bucket-a, bucket-b") + proxy = GridlookProxy() + assert proxy.allowed_buckets == ["bucket-a", "bucket-b"] + assert proxy.enabled + + def test_region_from_env(self, monkeypatch): + from gridlook_jupyter.config import GridlookProxy + + monkeypatch.setenv("GRIDLOOK_S3_REGION", "us-west-2") + assert GridlookProxy().region == "us-west-2" + + def test_traitlet_wins_over_env(self, monkeypatch): + from gridlook_jupyter.config import GridlookProxy + + monkeypatch.setenv("GRIDLOOK_ALLOWED_BUCKETS", "env-bucket") + proxy = GridlookProxy(allowed_buckets=["cfg-bucket"]) + assert proxy.allowed_buckets == ["cfg-bucket"] + + def test_empty_by_default(self, monkeypatch): + from gridlook_jupyter.config import GridlookProxy + + monkeypatch.delenv("GRIDLOOK_ALLOWED_BUCKETS", raising=False) + proxy = GridlookProxy() + assert proxy.allowed_buckets == [] + assert not proxy.enabled + + +@pytest.mark.parametrize( + "header,expected", + [ + ("bytes=10-19", (10, 20)), + ("bytes=95-", {"offset": 95}), + ("bytes=-5", {"suffix": 5}), + ("bytes=0-0", (0, 1)), + ("bytes=20-10", None), + ("bytes=1-2,5-6", None), + ("chars=1-2", None), + ("bytes=", None), + ], +) +def test_parse_range_header(header, expected): + from gridlook_jupyter.handlers import parse_range_header + + assert parse_range_header(header) == expected diff --git a/jupyter/tests/test_proxy.py b/jupyter/tests/test_proxy.py new file mode 100644 index 00000000..14981555 --- /dev/null +++ b/jupyter/tests/test_proxy.py @@ -0,0 +1,138 @@ +import pytest +from tornado.httpclient import HTTPClientError + +ALLOWED_BUCKET = "test-bucket" # matches conftest.ALLOWED_BUCKET + +PAYLOAD = bytes(range(256)) * 4 # 1024 bytes, position-identifiable + + +@pytest.fixture +def seeded(proxy, put_object): + put_object(ALLOWED_BUCKET, "data/chunk.bin", PAYLOAD) + return proxy + + +async def test_full_get(jp_fetch, seeded): + resp = await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, "data/chunk.bin") + assert resp.code == 200 + assert resp.body == PAYLOAD + assert resp.headers["Accept-Ranges"] == "bytes" + assert resp.headers["Content-Type"] == "application/octet-stream" + + +async def test_range_get_exact_slice(jp_fetch, seeded): + resp = await jp_fetch( + "gridlook", "s3", ALLOWED_BUCKET, "data/chunk.bin", headers={"Range": "bytes=10-19"} + ) + assert resp.code == 206 + assert resp.body == PAYLOAD[10:20] + assert resp.headers["Content-Range"] == f"bytes 10-19/{len(PAYLOAD)}" + assert resp.headers["Content-Length"] == "10" + + +async def test_open_ended_range(jp_fetch, seeded): + resp = await jp_fetch( + "gridlook", "s3", ALLOWED_BUCKET, "data/chunk.bin", headers={"Range": "bytes=1000-"} + ) + assert resp.code == 206 + assert resp.body == PAYLOAD[1000:] + assert resp.headers["Content-Range"] == f"bytes 1000-1023/{len(PAYLOAD)}" + + +async def test_suffix_range(jp_fetch, seeded): + resp = await jp_fetch( + "gridlook", "s3", ALLOWED_BUCKET, "data/chunk.bin", headers={"Range": "bytes=-16"} + ) + assert resp.code == 206 + assert resp.body == PAYLOAD[-16:] + + +async def test_out_of_range_416(jp_fetch, seeded): + with pytest.raises(HTTPClientError) as e: + await jp_fetch( + "gridlook", + "s3", + ALLOWED_BUCKET, + "data/chunk.bin", + headers={"Range": "bytes=999999-1000000"}, + ) + assert e.value.code == 416 + assert e.value.response.headers["Content-Range"] == f"bytes */{len(PAYLOAD)}" + + +async def test_zero_suffix_range_416(jp_fetch, seeded): + with pytest.raises(HTTPClientError) as e: + await jp_fetch( + "gridlook", "s3", ALLOWED_BUCKET, "data/chunk.bin", headers={"Range": "bytes=-0"} + ) + assert e.value.code == 416 + assert e.value.response.headers["Content-Range"] == f"bytes */{len(PAYLOAD)}" + + +async def test_head(jp_fetch, seeded): + resp = await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, "data/chunk.bin", method="HEAD") + assert resp.code == 200 + assert resp.headers["Content-Length"] == str(len(PAYLOAD)) + assert resp.headers["Accept-Ranges"] == "bytes" + assert resp.body == b"" + + +async def test_content_type_guess(jp_fetch, seeded, put_object): + put_object(ALLOWED_BUCKET, "meta/zarr.json", b'{"zarr_format": 3}') + resp = await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, "meta/zarr.json") + assert resp.headers["Content-Type"] == "application/json" + + +async def test_missing_key_404(jp_fetch, seeded): + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, "nope/missing.bin") + assert e.value.code == 404 + assert b"no such object" in e.value.response.body + + +@pytest.mark.parametrize("key", ["../secret/x", "a/../b", "....//x"]) +async def test_malformed_key_400_get(jp_fetch, seeded, key): + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, key) + assert e.value.code == 400 + assert b"invalid key" in e.value.response.body + + +@pytest.mark.parametrize("key", ["../secret/x", "a/../b", "....//x"]) +async def test_malformed_key_400_head(jp_fetch, seeded, key): + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, key, method="HEAD") + assert e.value.code == 400 + + +async def test_non_allowlisted_bucket_403(jp_fetch, seeded): + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", "s3", "sneaky-bucket", "any/key") + assert e.value.code == 403 + assert b"sneaky-bucket" in e.value.response.body + assert b"allowlist" in e.value.response.body + + +async def test_head_non_allowlisted_403(jp_fetch, seeded): + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", "s3", "sneaky-bucket", "any/key", method="HEAD") + assert e.value.code == 403 + + +class TestDisabledProxy: + @pytest.fixture + def allowed_buckets(self): + return [] + + async def test_empty_allowlist_disables_proxy(self, jp_fetch, proxy): + with pytest.raises(HTTPClientError) as e: + await jp_fetch("gridlook", "s3", ALLOWED_BUCKET, "any/key") + assert e.value.code == 403 + assert b"disabled" in e.value.response.body + assert b"GRIDLOOK_ALLOWED_BUCKETS" in e.value.response.body + + async def test_health_reports_disabled(self, jp_fetch, proxy): + import json + + resp = await jp_fetch("gridlook", "api", "health") + assert json.loads(resp.body)["proxy_enabled"] is False diff --git a/src/ui/overlays/controls/DataInput.vue b/src/ui/overlays/controls/DataInput.vue index aa224b03..f278858f 100644 --- a/src/ui/overlays/controls/DataInput.vue +++ b/src/ui/overlays/controls/DataInput.vue @@ -6,6 +6,7 @@ import CatalogPanel from "./CatalogPanel.vue"; import { useGlobeControlStore } from "@/store/store.ts"; import Modal from "@/ui/common/Modal.vue"; import { fetchCatalog, type TCatalogEntry } from "@/utils/catalog.ts"; +import { maybeRewriteS3Uri } from "@/utils/proxyRewrite.ts"; const props = defineProps<{ currentSource: string }>(); @@ -41,10 +42,13 @@ function close() { } async function setLocationHash() { - const next = dataPath.value.trim(); + let next = dataPath.value.trim(); if (!next) { return; } + // Served by gridlook-jupyter, s3:// inputs route through its proxy; + // standalone, they pass through untouched. + next = await maybeRewriteS3Uri(next); const filenameToCheck = next.endsWith("/") ? next.slice(0, -1) : next; // Catalogs are expected to be JSON files, so if the input ends with .json, we // can try to fetch it as a catalog before setting the location hash diff --git a/src/utils/proxyRewrite.ts b/src/utils/proxyRewrite.ts new file mode 100644 index 00000000..2b0ca7f6 --- /dev/null +++ b/src/utils/proxyRewrite.ts @@ -0,0 +1,82 @@ +/** + * Rewrites s3:// dataset URIs to the gridlook-jupyter proxy path when the app + * is served by the jupyter server extension. Standalone serving (dev server, + * static hosting) leaves inputs untouched. + */ + +const S3_URI_RE = /^s3:\/\/([^/]+)\/?(.*)$/; + +/** + * Base URL of the served app: the current location without hash/query, + * truncated to its containing directory. + */ +export function appBaseFromHref(href: string): string { + const url = new URL(href); + url.hash = ""; + url.search = ""; + if (!url.pathname.endsWith("/")) { + url.pathname = url.pathname.slice(0, url.pathname.lastIndexOf("/") + 1); + } + return url.toString(); +} + +/** + * Pure rewrite rule: s3://bucket/prefix -> s3/bucket/prefix. + * Returns null for anything that is not an s3:// URI. + */ +export function rewriteS3Uri(input: string, appBase: string): string | null { + const m = S3_URI_RE.exec(input.trim()); + if (!m) { + return null; + } + const [, bucket, key] = m; + return new URL(`s3/${bucket}/${key}`, appBase).toString(); +} + +let probe: Promise | null = null; + +/** Test seam: forget the cached health probe result. */ +export function resetExtensionProbe(): void { + probe = null; +} + +/** + * True when the app is served by gridlook-jupyter, detected once by probing + * the extension's health endpoint relative to the app base. The body check + * guards against SPA-fallback hosts that answer 200 to any path. + */ +export function isExtensionServed(appBase: string): Promise { + probe ??= fetch(new URL("api/health", appBase).toString(), { + credentials: "same-origin", + }) + .then(async (r) => { + if (!r.ok) { + return false; + } + const body: unknown = await r.json(); + return ( + typeof body === "object" && + body !== null && + (body as { extension?: unknown }).extension === "gridlook-jupyter" + ); + }) + .catch(() => false); + return probe; +} + +/** + * Rewrite an s3:// dataset input to the proxy URL when extension-served; + * otherwise (non-s3 input, or standalone serving) return it unchanged. + */ +export async function maybeRewriteS3Uri( + input: string, + appBase: string = appBaseFromHref(window.location.href) +): Promise { + if (!input.startsWith("s3://")) { + return input; + } + if (!(await isExtensionServed(appBase))) { + return input; + } + return rewriteS3Uri(input, appBase) ?? input; +} diff --git a/tests/unit/utils/proxyRewrite.test.ts b/tests/unit/utils/proxyRewrite.test.ts new file mode 100644 index 00000000..c2ae07cb --- /dev/null +++ b/tests/unit/utils/proxyRewrite.test.ts @@ -0,0 +1,112 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + appBaseFromHref, + isExtensionServed, + maybeRewriteS3Uri, + resetExtensionProbe, + rewriteS3Uri, +} from "@/utils/proxyRewrite.ts"; + +const BASE = "https://hub.example.org/user/me/gridlook/"; + +function mockHealth(ok: boolean, body?: unknown): void { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok, + json: () => Promise.resolve(body ?? {}), + }) + ); +} + +beforeEach(() => { + resetExtensionProbe(); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("rewriteS3Uri", () => { + it("rewrites bucket/prefix to the proxy path", () => { + expect(rewriteS3Uri("s3://my-bucket/some/prefix.zarr", BASE)).toBe( + `${BASE}s3/my-bucket/some/prefix.zarr` + ); + }); + + it("handles a bare bucket", () => { + expect(rewriteS3Uri("s3://my-bucket", BASE)).toBe(`${BASE}s3/my-bucket/`); + }); + + it("returns null for non-s3 inputs", () => { + expect(rewriteS3Uri("https://example.org/x.zarr", BASE)).toBeNull(); + expect(rewriteS3Uri("gs://bucket/key", BASE)).toBeNull(); + }); +}); + +describe("appBaseFromHref", () => { + it("keeps directory paths, drops hash and query", () => { + expect(appBaseFromHref(`${BASE}#s3://b/k::catalog=x`)).toBe(BASE); + }); + + it("truncates file paths to their directory", () => { + expect(appBaseFromHref(`${BASE}index.html#foo`)).toBe(BASE); + }); +}); + +describe("maybeRewriteS3Uri", () => { + it("rewrites when served by the extension", async () => { + mockHealth(true, { extension: "gridlook-jupyter" }); + await expect(maybeRewriteS3Uri("s3://bucket/key.zarr", BASE)).resolves.toBe( + `${BASE}s3/bucket/key.zarr` + ); + expect(fetch).toHaveBeenCalledWith(`${BASE}api/health`, { + credentials: "same-origin", + }); + }); + + it("leaves input untouched when standalone (health 404)", async () => { + mockHealth(false); + await expect(maybeRewriteS3Uri("s3://bucket/key.zarr", BASE)).resolves.toBe( + "s3://bucket/key.zarr" + ); + }); + + it("leaves input untouched when the probe fetch rejects", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline"))); + await expect(maybeRewriteS3Uri("s3://bucket/key.zarr", BASE)).resolves.toBe( + "s3://bucket/key.zarr" + ); + }); + + it("is not fooled by SPA-fallback hosts answering 200 with index.html", async () => { + mockHealth(true, undefined); // json() resolves to {} + await expect(maybeRewriteS3Uri("s3://bucket/key.zarr", BASE)).resolves.toBe( + "s3://bucket/key.zarr" + ); + }); + + it("passes non-s3 inputs through without probing", async () => { + const spy = vi.fn(); + vi.stubGlobal("fetch", spy); + await expect( + maybeRewriteS3Uri("https://example.org/x.zarr", BASE) + ).resolves.toBe("https://example.org/x.zarr"); + expect(spy).not.toHaveBeenCalled(); + }); + + it("caches the probe across calls", async () => { + mockHealth(true, { extension: "gridlook-jupyter" }); + await maybeRewriteS3Uri("s3://a/k", BASE); + await maybeRewriteS3Uri("s3://b/k", BASE); + expect(fetch).toHaveBeenCalledTimes(1); + }); +}); + +describe("isExtensionServed", () => { + it("rejects a health response missing the extension marker", async () => { + mockHealth(true, { something: "else" }); + await expect(isExtensionServed(BASE)).resolves.toBe(false); + }); +});