From f2c8dc7a91177974f653bfb0f9da767b267542ec Mon Sep 17 00:00:00 2001 From: espg Date: Mon, 20 Jul 2026 18:11:09 -0700 Subject: [PATCH 1/4] feat: bigint morton word codec and decimal grammar Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N32Mx4rWhbTK8xXm7Mt2ax --- src/lib/morton/decimal.ts | 117 ++++++++++++++++ src/lib/morton/word.ts | 185 ++++++++++++++++++++++++++ tests/unit/lib/morton/decimal.test.ts | 130 ++++++++++++++++++ tests/unit/lib/morton/word.test.ts | 159 ++++++++++++++++++++++ 4 files changed, 591 insertions(+) create mode 100644 src/lib/morton/decimal.ts create mode 100644 src/lib/morton/word.ts create mode 100644 tests/unit/lib/morton/decimal.test.ts create mode 100644 tests/unit/lib/morton/word.test.ts diff --git a/src/lib/morton/decimal.ts b/src/lib/morton/decimal.ts new file mode 100644 index 00000000..8de9f851 --- /dev/null +++ b/src/lib/morton/decimal.ts @@ -0,0 +1,117 @@ +/** + * Decimal morton string grammar (mortie spec v1.0 sections 2 and 4). + * + * Grammar: ["-"] base-digit *order-digit ["p"], base-digit 1..6, order + * digits 1..4, one per order. The sign renders the southern base cells + * (base cells 6..11); the string length past the sign+base component IS the + * order; a string prefix is a spatial ancestor. The terminal "p" kind + * suffix is legal only on a full order-29 string (points exist only at + * order 29) and is render/interchange-only -- paths never carry it. The + * section-4 tie-break: a p-marked string yields the POINT word, an unmarked + * order-29 string always yields the AREA word. + */ + +import { + BODY_TUPLES, + MAX_ORDER, + areaSuffix, + baseCellOf, + isPointWord, + pointSuffix, + wordToNested, +} from "@/lib/morton/word.ts"; + +const DECIMAL_RE = /^(-?)([1-6])([1-4]*)(p?)$/; + +const PREFIX_SHIFT = 60n; +const SUFFIX_BITS = 6n; + +/** HEALPix order of a decimal id: one digit per level past the base. */ +export function decimalOrder(id: string): number { + return id.length - (id.startsWith("-") ? 2 : 1); +} + +/** The {sign+base} component of a decimal id ("-5" of "-5112333"). */ +export function decimalBase(id: string): string { + return id.slice(0, id.startsWith("-") ? 2 : 1); +} + +/** + * Base-4 value of a decimal id's digit tail (digits 1..4 -> 0..3): the + * rank convention of the coverage encodings -- ascending packed-word + * (Z-)order within one base cell at a fixed order. Number-valued, so only + * valid through order 24 tails (4**24 < 2**53) -- coverage endpoints live + * at shard orders, far below that. + */ +export function decimalRank(id: string): number { + let rank = 0; + for (const ch of id.slice(decimalBase(id).length)) { + rank = rank * 4 + (Number(ch) - 1); + } + return rank; +} + +/** Inverse of decimalRank: the width-`depth` digit tail of a rank. */ +export function rankTail(rank: number, depth: number): string { + const digits: string[] = []; + for (let i = 0; i < depth; i++) { + digits.push(String((rank % 4) + 1)); + rank = Math.floor(rank / 4); + } + return digits.reverse().join(""); +} + +/** + * Parse a decimal morton string to its packed word (BigInt). Enforces the + * grammar, the order cap (29), and the p rules: "p" is legal solely on a + * full order-29 string and selects the POINT word; an unmarked order-29 + * string parses as the AREA word (the section-4 tie-break). + */ +export function parseMortonDecimal(id: string): bigint { + const m = DECIMAL_RE.exec(id); + if (!m) { + throw new Error(`malformed decimal morton id ${JSON.stringify(id)}`); + } + const [, sign, baseDigit, digits, mark] = m; + const order = digits.length; + if (order > MAX_ORDER) { + throw new Error(`decimal morton id ${id} exceeds order ${MAX_ORDER}`); + } + if (mark === "p" && order !== MAX_ORDER) { + throw new Error( + `the "p" kind suffix is legal only at order ${MAX_ORDER} (got order ${order})` + ); + } + // North bases 0..5 render 1..6; south bases 6..11 render -1..-6. + const baseCell = sign === "-" ? Number(baseDigit) + 5 : Number(baseDigit) - 1; + let word = BigInt(baseCell + 1) << PREFIX_SHIFT; + const bodyOrders = Math.min(order, BODY_TUPLES); + for (let n = 1; n <= bodyOrders; n++) { + const stored = BigInt(digits.charCodeAt(n - 1) - 0x31); // "1".."4" -> 0..3 + word |= stored << (SUFFIX_BITS + 2n * BigInt(BODY_TUPLES - n)); + } + const t28 = order >= 28 ? digits.charCodeAt(27) - 0x31 : 0; + const t29 = order >= 29 ? digits.charCodeAt(28) - 0x31 : 0; + const suffix = + mark === "p" ? pointSuffix(t28, t29) : areaSuffix(order, t28, t29); + return word | BigInt(suffix); +} + +/** + * Render a packed word as its decimal string (decode-through-kernel). Area + * words render unmarked; POINT words carry the terminal "p", making the + * decimal round-trip lossless for both kinds. Throws on the empty sentinel + * or an invalid prefix. + */ +export function renderMortonDecimal(word: bigint): string { + const baseCell = baseCellOf(word); + const { order, nested } = wordToNested(word); + const within = nested - BigInt(baseCell) * (1n << (2n * BigInt(order))); + const southern = baseCell >= 6; + let s = + (southern ? "-" : "") + String(southern ? baseCell - 5 : baseCell + 1); + for (let n = order - 1; n >= 0; n--) { + s += String(Number((within >> (2n * BigInt(n))) & 3n) + 1); + } + return isPointWord(word) ? s + "p" : s; +} diff --git a/src/lib/morton/word.ts b/src/lib/morton/word.ts new file mode 100644 index 00000000..5f1b0d3c --- /dev/null +++ b/src/lib/morton/word.ts @@ -0,0 +1,185 @@ +/** + * Packed 64-bit morton word codec (mortie spec v1.0 section 1, BigInt). + * + * Layout (MSB -> LSB): [4-bit prefix][54-bit body (27 x 2-bit)][6-bit suffix]. + * The prefix stores the HEALPix base cell as base_id + 1 (0 is the empty + * sentinel, 13..15 invalid); the body holds one stored 0..3 tuple per order + * 1..27 (order 1 highest); the suffix is a preorder numbering of the path + * tail past tuple 27 -- 0..27 area (order == suffix), 28..47 order-28/29 + * area (r = t28*5 + (t29 present ? t29 + 1 : 0)), 48..63 order-29 point + * (r2 = t28*4 + t29). Kind is carried by the suffix range alone (spec + * section 4) -- never by metadata. Words exceed 2**53, so everything here is + * BigInt; only NESTED ids at order <= 24 are handed out as Numbers. + */ + +/** Highest HEALPix order the packed word encodes. */ +export const MAX_ORDER = 29; +/** Number of two-bit body tuples (orders 1..27). */ +export const BODY_TUPLES = 27; +/** + * Highest order whose NESTED ids stay below 2**53 (12 * 4**24 < 2**53) and + * are therefore exact as JS Numbers -- the viewer-side cast bound of spec + * section 4. + */ +export const FLOAT64_EXACT_MAX_ORDER = 24; + +const PREFIX_SHIFT = 60n; +const SUFFIX_BITS = 6n; +const SUFFIX_MASK = 63n; +/** First suffix value of the order-28/29 area preorder region. */ +const AREA_TAIL_BASE = 28; +/** First suffix value of the order-29 point region. */ +const POINT_BASE = 28 + 20; + +/** The 6-bit suffix as a plain number (0..63). */ +function suffixOf(word: bigint): number { + return Number(word & SUFFIX_MASK); +} + +/** + * HEALPix base cell (0..11) from the prefix. Throws on the empty sentinel + * (prefix 0) and invalid prefixes (13..15) -- a corrupt word must not + * silently place data. + */ +export function baseCellOf(word: bigint): number { + const prefix = Number((word >> PREFIX_SHIFT) & 0xfn); + if (prefix < 1 || prefix > 12) { + throw new RangeError( + `invalid morton word ${word}: base-cell prefix ${prefix} (valid 1..12)` + ); + } + return prefix - 1; +} + +/** + * HEALPix order (0..29) carried by the suffix. Total over all 64 suffix + * values: 0..27 is the order itself; in 28..47 a 5-block parent (r % 5 == 0) + * is order 28, its children order 29; 48..63 is the order-29 point region. + */ +export function orderOf(word: bigint): number { + const suffix = suffixOf(word); + if (suffix <= BODY_TUPLES) { + return suffix; + } + if (suffix < POINT_BASE) { + return (suffix - AREA_TAIL_BASE) % 5 === 0 ? 28 : 29; + } + return MAX_ORDER; +} + +/** + * Kind predicate (spec section 4): suffix 48..63 is an order-29 point (no + * area claim); everything below is an area cell at its encoded order. + */ +export function isPointWord(word: bigint): boolean { + return suffixOf(word) >= POINT_BASE; +} + +/** + * Stored 0..3 tuples for orders 28/29 out of a tail suffix (28..63): + * [t28, t29 | null]. Suffixes 0..27 carry no tail (caller handles). + */ +function decodeTail(suffix: number): [number, number | null] { + if (suffix >= POINT_BASE) { + const r2 = suffix - POINT_BASE; + return [r2 >> 2, r2 & 3]; + } + const r = suffix - AREA_TAIL_BASE; + const pos = r % 5; + return [Math.floor(r / 5), pos === 0 ? null : pos - 1]; +} + +/** The area suffix for order k with stored tail tuples (spec section 1). */ +export function areaSuffix(order: number, t28: number, t29: number): number { + if (order <= BODY_TUPLES) { + return order; + } + if (order === 28) { + return AREA_TAIL_BASE + t28 * 5; + } + return AREA_TAIL_BASE + t28 * 5 + t29 + 1; +} + +/** The point suffix for stored order-28/29 tuples (spec section 1). */ +export function pointSuffix(t28: number, t29: number): number { + return POINT_BASE + t28 * 4 + t29; +} + +/** + * Clip (coarsen) a word to order k -- mortie's clip2order: keep the base + * cell and the first k tuples, zero-fill below, rewrite the suffix. Returns + * the word unchanged when k >= its native order (a point stays a point); + * any real clip yields an area word (spec section 4: membership in a + * coarser cell is a transient truncation). + */ +export function clipToOrder(word: bigint, k: number): bigint { + baseCellOf(word); // reject empty/invalid prefixes + if (!Number.isInteger(k) || k < 0 || k > MAX_ORDER) { + throw new RangeError(`clip order ${k} outside 0..${MAX_ORDER}`); + } + const native = orderOf(word); + if (k >= native) { + return word; + } + const prefix = word & (0xfn << PREFIX_SHIFT); + const keptBody = BigInt(Math.min(k, BODY_TUPLES)); + const keepBits = 2n * keptBody; + const bodyMask = + keptBody === 0n + ? 0n + : (((1n << keepBits) - 1n) << (54n - keepBits)) << SUFFIX_BITS; + // k == 28 keeps the order-28 tuple out of the source tail (native is 29 + // here); every lower target uses the variable-length form. + const suffix = + k === 28 + ? areaSuffix(28, decodeTail(suffixOf(word))[0], 0) + : areaSuffix(k, 0, 0); + return prefix | (word & bodyMask) | BigInt(suffix); +} + +/** + * Morton -> HEALPix NESTED conversion (body-bit arithmetic, exact at every + * order): nested = base * 4**order + within, where within packs the stored + * tuples with order 1 in the most significant pair. Points convert like + * area cells (the bare NESTED id does not carry kind). + */ +export function wordToNested(word: bigint): { order: number; nested: bigint } { + const base = BigInt(baseCellOf(word)); + const order = orderOf(word); + const bodyOrders = Math.min(order, BODY_TUPLES); + let within = 0n; + for (let n = 1; n <= bodyOrders; n++) { + const pair = (word >> (SUFFIX_BITS + 2n * BigInt(BODY_TUPLES - n))) & 3n; + within |= pair << (2n * BigInt(order - n)); + } + if (order >= 28) { + const [t28, t29] = decodeTail(suffixOf(word)); + within |= BigInt(t28) << (2n * BigInt(order - 28)); + if (t29 !== null) { + within |= BigInt(t29); + } + } + return { order, nested: base * (1n << (2n * BigInt(order))) + within }; +} + +/** + * The viewer-side NESTED cast (spec section 4, "viewer-side float64 + * casts"): POINT-kind words clip 29 -> 24 first (Number-safe by + * construction); AREA words never clip -- coarsening an area cell changes + * the labelled thing -- so an area word above order 24 throws (that data + * takes the proxy/virtual-store path, issue #1 phase 6d). + */ +export function viewNestedId(word: bigint): { order: number; cellId: number } { + const cast = isPointWord(word) + ? clipToOrder(word, FLOAT64_EXACT_MAX_ORDER) + : word; + const { order, nested } = wordToNested(cast); + if (order > FLOAT64_EXACT_MAX_ORDER) { + throw new RangeError( + `area cell at order ${order}: NESTED ids above order ` + + `${FLOAT64_EXACT_MAX_ORDER} exceed 2**53 and are unsafe as Numbers; ` + + `render finer-than-24 area data through the hive virtual store instead` + ); + } + return { order, cellId: Number(nested) }; +} diff --git a/tests/unit/lib/morton/decimal.test.ts b/tests/unit/lib/morton/decimal.test.ts new file mode 100644 index 00000000..ea2107e7 --- /dev/null +++ b/tests/unit/lib/morton/decimal.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; + +import { + decimalBase, + decimalOrder, + decimalRank, + parseMortonDecimal, + rankTail, + renderMortonDecimal, +} from "@/lib/morton/decimal.ts"; +import { isPointWord } from "@/lib/morton/word.ts"; + +// Golden provenance: the "-5112333" family of moczarr tests/conftest.py / +// tests/test_convention.py (the bitmap-encoding SERC-suite shard and its +// northern mirror). Packed words derived once with mortie 0.9.0. +const FAMILY: [string, bigint][] = [ + ["-5112333", 12711972898206646278n], + ["5112333", 5794443870565564422n], + ["-511", 12682136550675316738n], + ["-5", 12682136550675316736n], +]; + +// Golden provenance: moczarr tests/test_ranges.py +// TestSouthernAndMultiBaseGoldens.SOUTH_WORDS -- the eight order-8 cells of +// the rank-adjacent southern shards -51111111 / -51111112, ascending. +const SOUTH_WORDS: [string, bigint][] = [ + ["-511111111", 12682136550675316744n], + ["-511111112", 12682154142861361160n], + ["-511111113", 12682171735047405576n], + ["-511111114", 12682189327233449992n], + ["-511111121", 12682206919419494408n], + ["-511111122", 12682224511605538824n], + ["-511111123", 12682242103791583240n], + ["-511111124", 12682259695977627656n], +]; + +// Golden provenance: the cross-base shard ids of moczarr tests/test_ranges.py +// TestSouthernAndMultiBaseGoldens (SEAM_SHARDS + MULTI_SHARDS); packed words +// derived once with mortie 0.9.0. Unsigned word order sorts northern bases +// (1..6) before southern (-1..-6) -- the raw-sort Z-order property. +const CROSS_BASE: [string, bigint][] = [ + ["3444444", 4611404543450677254n], + ["6111111", 6917529027641081862n], + ["6444444", 8070169057271218182n], + ["-1111111", 8070450532247928838n], + ["-4444444", 12681855075698606086n], +]; + +const O29_AREA = "3" + "23142314231423142314231423142"; + +describe("parseMortonDecimal", () => { + it("round-trips the -5112333 family", () => { + for (const [id, word] of FAMILY) { + expect(parseMortonDecimal(id)).toBe(word); + expect(renderMortonDecimal(word)).toBe(id); + } + }); + + it("round-trips the southern order-8 range goldens", () => { + for (const [id, word] of SOUTH_WORDS) { + expect(parseMortonDecimal(id)).toBe(word); + expect(renderMortonDecimal(word)).toBe(id); + } + }); + + it("round-trips the cross-base goldens in raw-sort order", () => { + const words = CROSS_BASE.map(([id, word]) => { + expect(parseMortonDecimal(id)).toBe(word); + return word; + }); + expect([...words].sort((a, b) => (a < b ? -1 : 1))).toEqual(words); + }); + + it("parses the SERC and order-19 fabrication goldens", () => { + // moczarr tests/test_fabricate.py GOLDEN_WORD / GOLDEN_WORD_O19_SOUTH. + expect(parseMortonDecimal("433142211")).toBe(5347180132572332040n); + expect(parseMortonDecimal("-41132132113423444234")).toBe( + 11570383905173274643n + ); + }); + + it("rejects malformed ids", () => { + for (const bad of ["", "-", "7", "05", "4331520", "433142x", "5112333 "]) { + expect(() => parseMortonDecimal(bad)).toThrow(/malformed/); + } + expect(() => parseMortonDecimal("1" + "1".repeat(30))).toThrow(/order 29/); + }); +}); + +describe("order-29 kind marking (spec section 4)", () => { + it("unmarked order-29 strings parse AREA (the tie-break)", () => { + const word = parseMortonDecimal(O29_AREA); + expect(word).toBe(3906369333256140333n); + expect(isPointWord(word)).toBe(false); + expect(renderMortonDecimal(word)).toBe(O29_AREA); + }); + + it("p-marked order-29 strings parse POINT and render back marked", () => { + const word = parseMortonDecimal(O29_AREA + "p"); + expect(word).toBe(3906369333256140349n); // spec band arithmetic golden + expect(isPointWord(word)).toBe(true); + expect(renderMortonDecimal(word)).toBe(O29_AREA + "p"); + }); + + it("p below order 29 is illegal (points exist only at order 29)", () => { + expect(() => parseMortonDecimal("-5112333p")).toThrow(/order 29/); + expect(() => parseMortonDecimal("3p")).toThrow(/order 29/); + }); +}); + +describe("decimal helpers", () => { + it("computes order and base component", () => { + expect(decimalOrder("-5112333")).toBe(6); + expect(decimalOrder("433142211")).toBe(8); + expect(decimalOrder("-5")).toBe(0); + expect(decimalBase("-5112333")).toBe("-5"); + expect(decimalBase("433142211")).toBe("4"); + }); + + it("ranks digit tails in base 4 and inverts via rankTail", () => { + expect(decimalRank("4331422")).toBe( + 2 * 4 ** 5 + 2 * 4 ** 4 + 0 * 4 ** 3 + 3 * 4 ** 2 + 1 * 4 + 1 + ); + expect(decimalRank("-5")).toBe(0); + for (const [id] of SOUTH_WORDS) { + const tail = id.slice(2); + expect(rankTail(decimalRank(id), tail.length)).toBe(tail); + } + }); +}); diff --git a/tests/unit/lib/morton/word.test.ts b/tests/unit/lib/morton/word.test.ts new file mode 100644 index 00000000..04e2efbc --- /dev/null +++ b/tests/unit/lib/morton/word.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; + +import { + parseMortonDecimal, + renderMortonDecimal, +} from "@/lib/morton/decimal.ts"; +import { + FLOAT64_EXACT_MAX_ORDER, + baseCellOf, + clipToOrder, + isPointWord, + orderOf, + viewNestedId, + wordToNested, +} from "@/lib/morton/word.ts"; + +// Golden provenance: moczarr tests/test_fabricate.py GOLDEN_WORD / +// GOLDEN_NESTED -- the first cell of the SERC fixture's shard 4331422 +// (decimal "433142211", order 8) and its stored NESTED id. +const SERC_WORD = 5347180132572332040n; +const SERC_NESTED = 238416; + +// Golden provenance: moczarr tests/test_fabricate.py GOLDEN_WORD_O19_SOUTH / +// GOLDEN_NESTED_O19_SOUTH -- production order (19), southern hemisphere, +// negative polar base (lat ~ -80). Decimal derived once with mortie 0.9.0. +const SOUTH_O19_WORD = 11570383905173274643n; +const SOUTH_O19_NESTED = 2483716583387n; +const SOUTH_O19_DECIMAL = "-41132132113423444234"; + +// Suffix-band family: one order-29 path ("3" + tail, t28=3 t29=1) encoded +// at orders 26..29. Words derived once with mortie 0.9.0; the expected +// suffixes are the spec section-1 band formulas (26, 27, 28+3*5=43, +// 28+3*5+1+1=45). The POINT twin (suffix 48+3*4+1=61) is derived from the +// same frozen formulas -- mortie 0.9.0 predates point emission (PR #121). +const TAIL29 = "23142314231423142314231423142"; +const BAND_WORDS: [number, bigint, number][] = [ + [26, 3906369333256140314n, 26], + [27, 3906369333256140315n, 27], + [28, 3906369333256140331n, 43], + [29, 3906369333256140333n, 45], +]; +const POINT_WORD = 3906369333256140349n; // area o29 word - 45 + 61 + +describe("orderOf / baseCellOf / kind", () => { + it("decodes the SERC golden word", () => { + expect(orderOf(SERC_WORD)).toBe(8); + expect(baseCellOf(SERC_WORD)).toBe(3); // base digit "4", northern + expect(isPointWord(SERC_WORD)).toBe(false); + }); + + it("decodes the southern order-19 golden word", () => { + expect(orderOf(SOUTH_O19_WORD)).toBe(19); + expect(baseCellOf(SOUTH_O19_WORD)).toBe(9); // base "-4" -> cell 9 + expect(renderMortonDecimal(SOUTH_O19_WORD)).toBe(SOUTH_O19_DECIMAL); + }); + + it("walks the suffix bands (area 0..27, 28..47, point 48..63)", () => { + for (const [order, word, suffix] of BAND_WORDS) { + expect(Number(word & 63n)).toBe(suffix); + expect(orderOf(word)).toBe(order); + expect(isPointWord(word)).toBe(false); + } + expect(Number(POINT_WORD & 63n)).toBe(61); + expect(orderOf(POINT_WORD)).toBe(29); + expect(isPointWord(POINT_WORD)).toBe(true); + }); + + it("rejects the empty sentinel and invalid prefixes", () => { + expect(() => baseCellOf(0n)).toThrow(/prefix 0/); + expect(() => baseCellOf(13n << 60n)).toThrow(/prefix 13/); + expect(() => orderOf(5n)).not.toThrow(); // order is prefix-independent + }); +}); + +describe("wordToNested", () => { + it("matches the SERC fabrication golden", () => { + expect(wordToNested(SERC_WORD)).toEqual({ order: 8, nested: 238416n }); + }); + + it("matches the southern order-19 fabrication golden", () => { + expect(wordToNested(SOUTH_O19_WORD)).toEqual({ + order: 19, + nested: SOUTH_O19_NESTED, + }); + }); + + it("converts the deep suffix bands (goldens from mortie 0.9.0)", () => { + expect(wordToNested(BAND_WORDS[3][1]).nested).toBe(688361957162323341n); + expect(wordToNested(BAND_WORDS[2][1]).nested).toBe(172090489290580835n); + // A point converts like its area twin: NESTED carries no kind. + expect(wordToNested(POINT_WORD).nested).toBe(688361957162323341n); + }); + + it("handles order 0 (goldens from mortie 0.9.0)", () => { + expect(wordToNested(parseMortonDecimal("3"))).toEqual({ + order: 0, + nested: 2n, + }); + expect(wordToNested(parseMortonDecimal("-3"))).toEqual({ + order: 0, + nested: 8n, + }); + }); +}); + +describe("clipToOrder", () => { + it("clips 29 -> 28 onto the encoded order-28 golden", () => { + expect(clipToOrder(BAND_WORDS[3][1], 28)).toBe(BAND_WORDS[2][1]); + }); + + it("clips 29 -> 24 (mortie 0.9.0 clip2order golden)", () => { + const clipped = clipToOrder(BAND_WORDS[3][1], 24); + expect(clipped).toBe(3906369333256138776n); + expect(renderMortonDecimal(clipped)).toBe("3" + TAIL29.slice(0, 24)); + expect(wordToNested(clipped).nested).toBe(672228473791331n); + }); + + it("returns the word unchanged at or above its native order", () => { + expect(clipToOrder(SERC_WORD, 8)).toBe(SERC_WORD); + expect(clipToOrder(SERC_WORD, 29)).toBe(SERC_WORD); + expect(clipToOrder(POINT_WORD, 29)).toBe(POINT_WORD); // point stays point + }); + + it("clipping a point yields its containing area cell", () => { + const clipped = clipToOrder(POINT_WORD, 24); + expect(isPointWord(clipped)).toBe(false); + expect(clipped).toBe(clipToOrder(BAND_WORDS[3][1], 24)); // same path + }); + + it("rejects out-of-range targets", () => { + expect(() => clipToOrder(SERC_WORD, 30)).toThrow(RangeError); + expect(() => clipToOrder(SERC_WORD, -1)).toThrow(RangeError); + }); +}); + +describe("viewNestedId (the viewer-side float64 cast)", () => { + it("passes area cells at order <= 24 through as exact Numbers", () => { + expect(viewNestedId(SERC_WORD)).toEqual({ order: 8, cellId: SERC_NESTED }); + expect(viewNestedId(SOUTH_O19_WORD)).toEqual({ + order: 19, + cellId: Number(SOUTH_O19_NESTED), + }); + }); + + it("clips POINT words 29 -> 24 first", () => { + expect(viewNestedId(POINT_WORD)).toEqual({ + order: FLOAT64_EXACT_MAX_ORDER, + cellId: 672228473791331, + }); + }); + + it("never clips area cells: order > 24 throws toward the proxy path", () => { + for (const [order, word] of BAND_WORDS) { + expect(() => viewNestedId(word)).toThrow( + new RegExp(`order ${order}.*virtual store`, "s") + ); + } + }); +}); From 87ef66a694684b5aa3652c9fd158e2fe390520f0 Mon Sep 17 00:00:00 2001 From: espg Date: Mon, 20 Jul 2026 18:17:36 -0700 Subject: [PATCH 2/4] feat: morton-hive manifest, path, and coverage arithmetic Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N32Mx4rWhbTK8xXm7Mt2ax --- src/lib/morton/coverage.ts | 148 +++++++++++++++++++ src/lib/morton/hive.ts | 189 +++++++++++++++++++++++++ src/lib/morton/manifest.ts | 136 ++++++++++++++++++ tests/unit/lib/morton/coverage.test.ts | 156 ++++++++++++++++++++ tests/unit/lib/morton/hive.test.ts | 151 ++++++++++++++++++++ tests/unit/lib/morton/manifest.test.ts | 127 +++++++++++++++++ 6 files changed, 907 insertions(+) create mode 100644 src/lib/morton/coverage.ts create mode 100644 src/lib/morton/hive.ts create mode 100644 src/lib/morton/manifest.ts create mode 100644 tests/unit/lib/morton/coverage.test.ts create mode 100644 tests/unit/lib/morton/hive.test.ts create mode 100644 tests/unit/lib/morton/manifest.test.ts diff --git a/src/lib/morton/coverage.ts b/src/lib/morton/coverage.ts new file mode 100644 index 00000000..214efcd7 --- /dev/null +++ b/src/lib/morton/coverage.ts @@ -0,0 +1,148 @@ +/** + * Root coverage-MOC parsing (mortie spec v1.0 section 7.3): the `ranges` + * envelope a store/product root's coverage.moc declares -- inclusive + * [first, last] runs of same-order shard cells within one base cell, + * consecutive in base-4 digit-tail rank, endpoints as decimal STRINGS + * (packed words exceed 2**53; JSON numbers would be float-mangled). + * + * Readers are MOC-first, always: the manifest plus this envelope yield + * every leaf path arithmetically, and recursive enumeration is + * out-of-contract for the viewer (issue #1 phase-6 amendment) -- there is + * deliberately no LIST fallback here. The leaf bitmap tier (zstd) stays + * server-side; the browser consumes the box/ranges tiers only. + * + * Posture split, mirroring moczarr: the envelope is a regenerable cache, so + * an unusable payload PARSES as null (the caller reports no-coverage); + * malformed ranges inside a well-formed envelope THROW (a corrupt cache + * must never yield a plausible partial answer). + */ + +import { + decimalBase, + decimalOrder, + decimalRank, + rankTail, +} from "@/lib/morton/decimal.ts"; +import { leafPath } from "@/lib/morton/hive.ts"; +import type { HiveManifest } from "@/lib/morton/manifest.ts"; + +/** Convention version of coverage envelopes. */ +export const COVERAGE_SPEC = "morton-moc/1"; +/** Root coverage object name at a store/product root. */ +export const ROOT_COVERAGE_NAME = "coverage.moc"; + +export interface RootCoverage { + spec: string; + encoding: "ranges"; + /** Common HEALPix order of every range endpoint. */ + order: number; + /** Inclusive [first, last] decimal-string runs. */ + ranges: [string, string][]; +} + +/** + * A usable store-root coverage envelope, or null. Tolerant by design: a + * non-object payload, an unknown spec, a non-"ranges" encoding, or missing + * required keys read as absent (the root MOC is a regenerable cache). + */ +export function parseRootCoverage(payload: unknown): RootCoverage | null { + if ( + typeof payload !== "object" || + payload === null || + Array.isArray(payload) + ) { + return null; + } + const raw = payload as Record; + const order = raw["order"]; + const ranges = raw["ranges"]; + const usable = + raw["spec"] === COVERAGE_SPEC && + raw["encoding"] === "ranges" && + typeof order === "number" && + Number.isInteger(order) && + Array.isArray(ranges); + if (!usable) { + return null; + } + return { + spec: COVERAGE_SPEC, + encoding: "ranges", + order: order as number, + ranges: ranges as [string, string][], + }; +} + +/** Validated (base, loRank, hiRank) of one range; throws when malformed. */ +function checkRange( + lo: unknown, + hi: unknown, + order: number +): [string, number, number] { + if (typeof lo !== "string" || typeof hi !== "string") { + // Spec section 7.3: endpoints are decimal strings, never JSON numbers. + throw new Error( + `coverage range endpoints must be decimal strings (got [${lo}, ${hi}])` + ); + } + const base = decimalBase(lo); + const loRank = decimalRank(lo); + const hiRank = decimalRank(hi); + const ok = + decimalBase(hi) === base && + loRank <= hiRank && + decimalOrder(lo) === order && + decimalOrder(hi) === order; + if (!ok) { + throw new Error( + `malformed coverage range [${lo}, ${hi}] at order ${order}` + ); + } + return [base, loRank, hiRank]; +} + +/** + * The covered shard ids, expanded exactly from the envelope's ranges -- + * ascending within each range by construction (consecutive digit-tail + * rank). O(covered shards); containment checks should use rangesContain. + */ +export function rangesShardIds(envelope: RootCoverage): string[] { + const ids: string[] = []; + for (const [lo, hi] of envelope.ranges) { + const [base, loRank, hiRank] = checkRange(lo, hi, envelope.order); + for (let r = loRank; r <= hiRank; r++) { + ids.push(base + rankTail(r, envelope.order)); + } + } + return [...new Set(ids)]; +} + +/** Whether the envelope's ranges list one shard id -- O(ranges). */ +export function rangesContain( + envelope: RootCoverage, + shardId: string +): boolean { + if (decimalOrder(shardId) !== envelope.order) { + return false; + } + const base = decimalBase(shardId); + const rank = decimalRank(shardId); + return envelope.ranges.some(([lo, hi]) => { + const [rangeBase, loRank, hiRank] = checkRange(lo, hi, envelope.order); + return rangeBase === base && loRank <= rank && rank <= hiRank; + }); +} + +/** + * The store-relative leaf path of every covered shard -- the MOC-first + * arithmetic enumeration (manifest + root MOC, zero LISTs) the phase-6c + * data path consumes. `window` selects the windowed leaf under /2 and /3 + * manifests. + */ +export function coveredLeafPaths( + manifest: HiveManifest, + envelope: RootCoverage, + window?: string | null +): string[] { + return rangesShardIds(envelope).map((id) => leafPath(manifest, id, window)); +} diff --git a/src/lib/morton/hive.ts b/src/lib/morton/hive.ts new file mode 100644 index 00000000..b3bcce93 --- /dev/null +++ b/src/lib/morton/hive.ts @@ -0,0 +1,189 @@ +/** + * Morton-hive store arithmetic (mortie spec v1.0 section 6): leaf paths, + * leaf-name splitting, and D19 product-root discovery -- pure functions + * over ids, manifests, and listings; the fetch layer arrives with phase 6c. + * + * Path components chunk the digit tail per the manifest's `path_grouping` + * as the ONLY path-construction code path (issue #1 phase-6 amendment): + * the constant-width {sign+base} component first, then `path_grouping` + * digits per component with leading components full-width and the LAST + * carrying the remainder -- component boundaries are ancestor prefixes + * shared by deeper shards regardless of their order. + */ + +import type { HiveManifest } from "@/lib/morton/manifest.ts"; +import { MANIFEST_NAME } from "@/lib/morton/manifest.ts"; + +/** Frozen window-label charset (no `_`, so leaf names split unambiguously). */ +const LABEL_RE = /^[0-9A-Za-z-]{1,32}$/; + +/** Unmarked decimal-id grammar -- paths never carry the "p" kind suffix. */ +const PATH_ID_RE = /^(-?[1-6])([1-4]*)$/; + +/** + * Product-name grammar (spec section 6.5): [a-z0-9_-]{1,192}, minus the + * base-component exclusion (-?[1-6]) so child classification at a product + * root stays unambiguous. + */ +const PRODUCT_NAME_RE = /^[a-z0-9_-]{1,192}$/; + +/** Whether `name` is a {sign+base}-shaped hive root child. */ +export function isBaseComponent(name: string): boolean { + return /^-?[1-6]$/.test(name); +} + +/** Whether `name` is a legal product-root child name (spec section 6.5). */ +export function isProductName(name: string): boolean { + return PRODUCT_NAME_RE.test(name) && !isBaseComponent(name); +} + +/** Validate a window label against the frozen charset; returns it. */ +export function validateLabel(label: string): string { + if (!LABEL_RE.test(label)) { + throw new Error( + `window label ${JSON.stringify(label)} does not match the frozen ` + + `grammar (${LABEL_RE.source}; morton-hive/2)` + ); + } + return label; +} + +/** + * The digit-tree path components of a shard id: {sign+base}, then the digit + * tail chunked `pathGrouping` per component (leading full-width, last + * remainder). Rejects marked ("p") or malformed ids -- paths carry the + * unmarked area grammar only. + */ +export function hiveComponents( + shardId: string, + pathGrouping: number +): string[] { + const m = PATH_ID_RE.exec(shardId); + if (!m) { + throw new Error( + `malformed shard id ${JSON.stringify(shardId)} for a hive path` + ); + } + if (!Number.isInteger(pathGrouping) || pathGrouping < 1) { + throw new Error( + `path_grouping must be a positive integer (got ${pathGrouping})` + ); + } + const [, base, tail] = m; + const components = [base]; + for (let i = 0; i < tail.length; i += pathGrouping) { + components.push(tail.slice(i, i + pathGrouping)); + } + return components; +} + +/** + * The leaf zarr basename for a shard under a manifest: version /3 names by + * window alone (`{window}.zarr`, the reserved `all` for schedule none); /1 + * and /2 carry the full id (`{full_id}.zarr` / `{full_id}_{window}.zarr`). + */ +export function leafName( + manifest: HiveManifest, + shardId: string, + window?: string | null +): string { + if (manifest.version === 3) { + return `${window === undefined || window === null ? "all" : validateLabel(window)}.zarr`; + } + if (window === undefined || window === null) { + return `${shardId}.zarr`; + } + return `${shardId}_${validateLabel(window)}.zarr`; +} + +/** + * Store-relative hive path of a shard's leaf zarr, computed arithmetically + * from the manifest (path_grouping chunking + version-specific basename). + */ +export function leafPath( + manifest: HiveManifest, + shardId: string, + window?: string | null +): string { + const components = hiveComponents(shardId, manifest.pathGrouping); + return `${components.join("/")}/${leafName(manifest, shardId, window)}`; +} + +/** + * `{fullId, window}` from a /1 or /2 leaf basename -- split on the FIRST + * `_` (morton ids never contain one and window labels cannot, by charset). + * Throws on a non-.zarr name or a malformed label. /3 basenames carry no + * id -- recover those with shardIdFromPath. + */ +export function splitLeafName(name: string): { + fullId: string; + window: string | null; +} { + if (!name.endsWith(".zarr")) { + throw new Error(`${JSON.stringify(name)} is not a leaf zarr name`); + } + const stem = name.slice(0, -".zarr".length); + const sep = stem.indexOf("_"); + if (sep === -1) { + return { fullId: stem, window: null }; + } + return { + fullId: stem.slice(0, sep), + window: validateLabel(stem.slice(sep + 1)), + }; +} + +/** + * The shard id a leaf path encodes, recovered from its digit components + * (the concatenation invariant; how /3 leaves, named by window alone, get + * their id back). Works for any path_grouping. + */ +export function shardIdFromPath(relPath: string): string { + const parts = relPath.replace(/^\/+|\/+$/g, "").split("/"); + if (parts.length < 2 || !parts[parts.length - 1].endsWith(".zarr")) { + throw new Error(`${JSON.stringify(relPath)} is not a hive leaf path`); + } + const [head, ...digits] = parts.slice(0, -1); + const id = head + digits.join(""); + if ( + !isBaseComponent(head) || + !PATH_ID_RE.test(id) || + digits.some((d) => d === "") + ) { + throw new Error( + `path ${JSON.stringify(relPath)} violates the hive node invariant` + ); + } + return id; +} + +/** D19 product-root discovery result: a bare store, or a product directory. */ +export type RootClassification = + | { kind: "store" } + | { kind: "products"; products: string[] }; + +/** + * Classify a store-root listing (D19, spec section 6.5). Readers decide by + * content: a manifest object at the root means a bare single-product store; + * otherwise name-shaped child prefixes are the product entries (each a + * complete morton-hive store). Non-conforming names are ignored per the + * node invariant; a root with neither form is not a morton-hive root. + */ +export function classifyStoreRoot( + objectNames: string[], + childPrefixes: string[] +): RootClassification { + if (objectNames.includes(MANIFEST_NAME)) { + return { kind: "store" }; + } + const products = childPrefixes + .map((name) => name.replace(/\/+$/, "")) + .filter(isProductName) + .sort(); + if (products.length === 0) { + throw new Error( + `not a morton-hive root: no ${MANIFEST_NAME} object and no product-shaped children` + ); + } + return { kind: "products", products }; +} diff --git a/src/lib/morton/manifest.ts b/src/lib/morton/manifest.ts new file mode 100644 index 00000000..4fcad858 --- /dev/null +++ b/src/lib/morton/manifest.ts @@ -0,0 +1,136 @@ +/** + * morton_hive.json manifest parsing (mortie spec v1.0 section 6). + * + * The manifest is the reader's bootstrap: with it every shard path is + * computable arithmetically with zero requests, so malformed content is a + * loud error, never a degradable cache. Versions are discriminated by the + * `spec` string only (never by sniffing names): /1 bare leaves, /2 + * time-windowed leaves (a /1 store is a /2 store with schedule none), /3 + * window-only leaf naming (the reserved `all` token names schedule none). + */ + +/** Manifest object name at a store/product root. */ +export const MANIFEST_NAME = "morton_hive.json"; + +const HIVE_SPECS = ["morton-hive/1", "morton-hive/2", "morton-hive/3"] as const; + +export type HiveSpec = (typeof HIVE_SPECS)[number]; + +export interface HiveManifest { + spec: HiveSpec; + /** 1, 2 or 3 -- the numeric half of `spec`. */ + version: number; + /** HEALPix order of the cell coordinate inside each leaf. */ + cellOrder: number; + /** HEALPix order the shard tree splits down to. */ + shardOrder: number; + /** Digits per path component (spec section 6.1; absent reads as 1). */ + pathGrouping: number; + /** Window schedule ("none" when the store is unwindowed). */ + schedule: string; + /** Explicit window labels, when the temporal block declares them. */ + windows: string[] | null; + /** The raw manifest payload, for fields this reader does not model. */ + raw: Record; +} + +function asInt(value: unknown, key: string): number { + if (typeof value !== "number" || !Number.isInteger(value)) { + throw new Error( + `manifest ${key} must be an integer (got ${JSON.stringify(value)})` + ); + } + return value; +} + +/** + * The `path_grouping` chunk width. Absent/null reads as 1 (zagg D21: + * existing stores are retroactively 1); anything but a positive integer is + * malformed -- readers chunk per the manifest, never by assumption. + */ +function pathGroupingOf(raw: Record): number { + const value = raw["path_grouping"]; + if (value === undefined || value === null) { + return 1; + } + const grouping = asInt(value, "path_grouping"); + if (grouping < 1) { + throw new Error(`manifest path_grouping must be >= 1 (got ${grouping})`); + } + return grouping; +} + +/** The temporal block's schedule and explicit windows, version-checked. */ +function temporalOf( + raw: Record, + version: number +): { schedule: string; windows: string[] | null } { + const temporal = raw["temporal"]; + if (version === 1) { + if (temporal !== undefined && temporal !== null) { + throw new Error( + "a morton-hive/1 manifest must not carry a temporal block" + ); + } + return { schedule: "none", windows: null }; + } + const block = (temporal ?? {}) as Record; + const schedule = block["schedule"]; + if (typeof schedule !== "string" || schedule === "") { + // /3 separates time into the basename; an absent block is schedule none. + if (version === 3 && (temporal === undefined || temporal === null)) { + return { schedule: "none", windows: null }; + } + throw new Error( + `a morton-hive/${version} manifest requires a temporal block with a schedule` + ); + } + const windows = block["windows"]; + if (windows !== undefined && !Array.isArray(windows)) { + throw new Error("manifest temporal.windows must be a list of labels"); + } + return { schedule, windows: (windows as string[] | undefined) ?? null }; +} + +/** + * Validate a morton_hive.json payload into a typed manifest. Loud on + * malformed input: unknown spec, missing/non-integer orders, inverted + * orders, a bad path_grouping, or a /2 manifest without its temporal block + * all throw. + */ +export function parseHiveManifest(payload: unknown): HiveManifest { + if ( + typeof payload !== "object" || + payload === null || + Array.isArray(payload) + ) { + throw new Error("morton_hive.json is not a JSON object"); + } + const raw = payload as Record; + const spec = raw["spec"]; + if (!HIVE_SPECS.includes(spec as HiveSpec)) { + throw new Error( + `unknown manifest spec ${JSON.stringify(spec)} (expected one of ${HIVE_SPECS.join(", ")})` + ); + } + const version = Number((spec as string).split("/")[1]); + const cellOrder = asInt(raw["cell_order"], "cell_order"); + const shardOrder = asInt(raw["shard_order"], "shard_order"); + if (cellOrder < shardOrder) { + throw new Error( + `manifest cell_order ${cellOrder} is above shard_order ${shardOrder} ` + + "(cells nest inside shards)" + ); + } + const { schedule, windows } = temporalOf(raw, version); + return { + spec: spec as HiveSpec, + version, + cellOrder, + shardOrder, + pathGrouping: pathGroupingOf(raw), + schedule, + windows, + raw, + }; +} diff --git a/tests/unit/lib/morton/coverage.test.ts b/tests/unit/lib/morton/coverage.test.ts new file mode 100644 index 00000000..44ee8bf9 --- /dev/null +++ b/tests/unit/lib/morton/coverage.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest"; + +import { + coveredLeafPaths, + parseRootCoverage, + rangesContain, + rangesShardIds, +} from "@/lib/morton/coverage.ts"; +import { parseHiveManifest } from "@/lib/morton/manifest.ts"; + +// Wire-format keys (snake_case, per the mortie spec sections 6 and 7). +const Key = { + CELL_ORDER: "cell_order", + SHARD_ORDER: "shard_order", + PATH_GROUPING: "path_grouping", + GENERATED_AT: "generated_at", +} as const; + +// Golden provenance: moczarr tests/data/serc_hive/coverage.moc -- the +// zagg-written root ranges MOC of the SERC fixture (shard order 6). +const SERC_COVERAGE = { + spec: "morton-moc/1", + encoding: "ranges", + order: 6, + source: "dispatcher", + [Key.GENERATED_AT]: "2026-07-21T00:53:56+00:00", + ranges: [ + ["4331244", "4331244"], + ["4331421", "4331422"], + ["4331424", "4331424"], + ["4332133", "4332133"], + ["4332311", "4332311"], + ], +}; + +const SERC_SHARDS = [ + "4331244", + "4331421", + "4331422", + "4331424", + "4332133", + "4332311", +]; + +function envelope(overrides: Record = {}) { + const parsed = parseRootCoverage({ ...SERC_COVERAGE, ...overrides }); + if (parsed === null) { + throw new Error("fixture envelope must parse"); + } + return parsed; +} + +describe("parseRootCoverage", () => { + it("parses the SERC fixture envelope", () => { + const parsed = envelope(); + expect(parsed.order).toBe(6); + expect(parsed.ranges).toHaveLength(5); + }); + + it("reads unusable payloads as absent (regenerable-cache posture)", () => { + expect(parseRootCoverage(null)).toBeNull(); + expect(parseRootCoverage([])).toBeNull(); + expect(parseRootCoverage("moc")).toBeNull(); + expect( + parseRootCoverage({ ...SERC_COVERAGE, spec: "morton-moc/2" }) + ).toBeNull(); + expect( + parseRootCoverage({ ...SERC_COVERAGE, encoding: "bitmap" }) + ).toBeNull(); + expect(parseRootCoverage({ ...SERC_COVERAGE, order: "6" })).toBeNull(); + expect( + parseRootCoverage({ ...SERC_COVERAGE, ranges: undefined }) + ).toBeNull(); + }); +}); + +describe("rangesShardIds", () => { + it("expands the SERC envelope to its covered shards", () => { + expect(rangesShardIds(envelope())).toEqual(SERC_SHARDS); + }); + + it("expands a multi-cell run in ascending digit-tail rank", () => { + const ids = rangesShardIds(envelope({ ranges: [["4331421", "4331424"]] })); + expect(ids).toEqual(["4331421", "4331422", "4331423", "4331424"]); + }); + + it("throws on malformed ranges (corrupt cache, never a partial answer)", () => { + const cases: [string, string][][] = [ + [["4331423", "4331422"]], // reversed + [["4331422", "-4331423"]], // base-crossing + [["4331422", "43314231"]], // order mismatch + ]; + for (const ranges of cases) { + expect(() => rangesShardIds(envelope({ ranges }))).toThrow( + /malformed coverage range/ + ); + } + // Spec 7.3: endpoints are strings, never JSON numbers (2**53 mangling). + expect(() => + rangesShardIds(envelope({ ranges: [[4331422, 4331423]] })) + ).toThrow(/decimal strings/); + }); +}); + +describe("rangesContain", () => { + it("answers membership without expansion", () => { + for (const id of SERC_SHARDS) { + expect(rangesContain(envelope(), id)).toBe(true); + } + expect(rangesContain(envelope(), "4331423")).toBe(false); + expect(rangesContain(envelope(), "-4331422")).toBe(false); + }); + + it("a wrong-order id is never contained", () => { + expect(rangesContain(envelope(), "43314221")).toBe(false); + expect(rangesContain(envelope(), "433142")).toBe(false); + }); +}); + +describe("coveredLeafPaths (MOC-first arithmetic enumeration)", () => { + const manifest = parseHiveManifest({ + spec: "morton-hive/1", + [Key.CELL_ORDER]: 8, + [Key.SHARD_ORDER]: 6, + [Key.PATH_GROUPING]: 1, + }); + + it("computes the SERC fixture's leaf paths with zero requests", () => { + // Golden provenance: the on-disk tree of moczarr tests/data/serc_hive. + expect(coveredLeafPaths(manifest, envelope())).toEqual([ + "4/3/3/1/2/4/4/4331244.zarr", + "4/3/3/1/4/2/1/4331421.zarr", + "4/3/3/1/4/2/2/4331422.zarr", + "4/3/3/1/4/2/4/4331424.zarr", + "4/3/3/2/1/3/3/4332133.zarr", + "4/3/3/2/3/1/1/4332311.zarr", + ]); + }); + + it("threads path_grouping and windows through", () => { + const grouped = parseHiveManifest({ + spec: "morton-hive/2", + [Key.CELL_ORDER]: 8, + [Key.SHARD_ORDER]: 6, + [Key.PATH_GROUPING]: 3, + temporal: { schedule: "yearly" }, + }); + expect( + coveredLeafPaths( + grouped, + envelope({ ranges: [["4331422", "4331422"]] }), + "2019" + ) + ).toEqual(["4/331/422/4331422_2019.zarr"]); + }); +}); diff --git a/tests/unit/lib/morton/hive.test.ts b/tests/unit/lib/morton/hive.test.ts new file mode 100644 index 00000000..96496d67 --- /dev/null +++ b/tests/unit/lib/morton/hive.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from "vitest"; + +import { + classifyStoreRoot, + hiveComponents, + isProductName, + leafPath, + shardIdFromPath, + splitLeafName, +} from "@/lib/morton/hive.ts"; +import { parseHiveManifest } from "@/lib/morton/manifest.ts"; + +// Manifest wire-format keys (snake_case, per the mortie spec section 6). +const Key = { + CELL_ORDER: "cell_order", + SHARD_ORDER: "shard_order", + PATH_GROUPING: "path_grouping", +} as const; + +function manifest(overrides: Record = {}) { + return parseHiveManifest({ + spec: "morton-hive/1", + [Key.CELL_ORDER]: 8, + [Key.SHARD_ORDER]: 6, + [Key.PATH_GROUPING]: 1, + ...overrides, + }); +} + +const V2 = { + spec: "morton-hive/2", + temporal: { schedule: "yearly" }, +}; + +describe("leafPath (path_grouping as the only construction path)", () => { + it("matches the moczarr leaf-path goldens at path_grouping 1", () => { + // Golden provenance: moczarr tests/test_convention.py + // TestLeafPath::test_south_and_north (the "-5112333" family). + expect(leafPath(manifest(), "-5112333")).toBe( + "-5/1/1/2/3/3/3/-5112333.zarr" + ); + expect(leafPath(manifest(), "5112333")).toBe("5/1/1/2/3/3/3/5112333.zarr"); + }); + + it("matches the SERC fixture tree layout", () => { + // Golden provenance: moczarr tests/data/serc_hive (zagg-written). + expect(leafPath(manifest(), "4331422")).toBe("4/3/3/1/4/2/2/4331422.zarr"); + }); + + it("chunks per path_grouping: 3, last component carrying the remainder", () => { + const grouped = manifest({ [Key.PATH_GROUPING]: 3 }); + expect(leafPath(grouped, "-5112333")).toBe("-5/112/333/-5112333.zarr"); + expect(leafPath(grouped, "43314224")).toBe("4/331/422/4/43314224.zarr"); + expect(hiveComponents("43314224", 3)).toEqual(["4", "331", "422", "4"]); + }); + + it("keeps component boundaries as shared ancestor prefixes", () => { + // A deeper shard's leading components equal its ancestor's components. + const parent = hiveComponents("4331422", 3); + const child = hiveComponents("433142211", 3); + expect(child.slice(0, 2)).toEqual(parent.slice(0, 2)); + }); + + it("names /2 windowed leaves and /3 window-only leaves", () => { + // Golden provenance: moczarr tests/test_convention.py leaf_name/2019. + expect(leafPath(manifest(V2), "-5112333", "2019")).toBe( + "-5/1/1/2/3/3/3/-5112333_2019.zarr" + ); + const v3 = manifest({ spec: "morton-hive/3" }); + expect(leafPath(v3, "-5112333", "2019")).toBe("-5/1/1/2/3/3/3/2019.zarr"); + expect(leafPath(v3, "-5112333")).toBe("-5/1/1/2/3/3/3/all.zarr"); + }); + + it("rejects marked, malformed, or negative-int-shaped ids", () => { + for (const bad of ["-5112333p", "7112333", "-51123335", ""]) { + expect(() => leafPath(manifest(), bad)).toThrow(/malformed shard id/); + } + expect(() => leafPath(manifest(V2), "-5112333", "20_19")).toThrow(/frozen/); + }); +}); + +describe("splitLeafName (first-underscore split)", () => { + it("matches the moczarr split goldens", () => { + // Golden provenance: moczarr tests/test_convention.py TestLeafNames. + expect(splitLeafName("-5112333.zarr")).toEqual({ + fullId: "-5112333", + window: null, + }); + expect(splitLeafName("-5112333_2019.zarr")).toEqual({ + fullId: "-5112333", + window: "2019", + }); + }); + + it("rejects non-zarr names and malformed labels", () => { + expect(() => splitLeafName("-5112333")).toThrow(/not a leaf zarr name/); + expect(() => splitLeafName("-5112333_20_19.zarr")).toThrow(/frozen/); + }); +}); + +describe("shardIdFromPath", () => { + it("recovers ids across groupings and versions", () => { + expect(shardIdFromPath("-5/1/1/2/3/3/3/-5112333.zarr")).toBe("-5112333"); + expect(shardIdFromPath("-5/112/333/2019.zarr")).toBe("-5112333"); + expect(shardIdFromPath("4/331/422/4/all.zarr")).toBe("43314224"); + }); + + it("rejects paths violating the node invariant", () => { + for (const bad of [ + "-5112333.zarr", + "x/1/leaf.zarr", + "-5/15/x.zarr", + "-5/1/leaf", + ]) { + expect(() => shardIdFromPath(bad)).toThrow( + /node invariant|not a hive leaf path/ + ); + } + }); +}); + +describe("classifyStoreRoot (D19 product discovery)", () => { + it("a root manifest means a bare store", () => { + expect( + classifyStoreRoot(["morton_hive.json", "coverage.moc"], ["4"]) + ).toEqual({ kind: "store" }); + }); + + it("name-shaped children are the product entries, sorted", () => { + expect( + classifyStoreRoot( + ["coverage.moc"], + ["atl06_q50/", "atl03_counts/", "-5/", "README", "4/"] + ) + ).toEqual({ kind: "products", products: ["atl03_counts", "atl06_q50"] }); + }); + + it("neither form is not a morton-hive root", () => { + expect(() => classifyStoreRoot([], ["-5/", "4/"])).toThrow( + /not a morton-hive root/ + ); + }); + + it("excludes base components and enforces the grammar", () => { + expect(isProductName("atl06_q50")).toBe(true); + expect(isProductName("a".repeat(192))).toBe(true); + for (const bad of ["-5", "4", "Upper", "dots.", "a".repeat(193), ""]) { + expect(isProductName(bad)).toBe(false); + } + }); +}); diff --git a/tests/unit/lib/morton/manifest.test.ts b/tests/unit/lib/morton/manifest.test.ts new file mode 100644 index 00000000..d5d3e9ff --- /dev/null +++ b/tests/unit/lib/morton/manifest.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; + +import { parseHiveManifest } from "@/lib/morton/manifest.ts"; + +// Manifest wire-format keys (snake_case, per the mortie spec section 6). +const Key = { + CELL_ORDER: "cell_order", + SHARD_ORDER: "shard_order", + PATH_GROUPING: "path_grouping", + SPLIT_SCHEDULE: "split_schedule", + SEMANTIC_HASH: "semantic_hash", + SHORT_NAME: "short_name", + TIME_FIELD: "time_field", + GENERATED_AT: "generated_at", +} as const; + +// Golden provenance: moczarr tests/data/serc_hive/morton_hive.json (the +// zagg-written SERC fixture manifest, post englacial/zagg#314), trimmed to +// the fields this reader models plus the fields it must carry through raw. +const SERC_MANIFEST = { + spec: "morton-hive/1", + dataset: { [Key.SHORT_NAME]: "ATL06", version: "007" }, + [Key.SEMANTIC_HASH]: + "ab87989b37adbce3797f73fa1fd5d9b2a634538764e180aef05f5c6a7106870b", + [Key.CELL_ORDER]: 8, + [Key.SHARD_ORDER]: 6, + [Key.SPLIT_SCHEDULE]: [1, 1, 1, 1, 1, 1], + [Key.PATH_GROUPING]: 1, + pyramid: { orders: [], aggregation: {} }, + [Key.GENERATED_AT]: "2026-07-21T00:53:56+00:00", +}; + +describe("parseHiveManifest", () => { + it("parses the SERC fixture manifest", () => { + const manifest = parseHiveManifest(SERC_MANIFEST); + expect(manifest.spec).toBe("morton-hive/1"); + expect(manifest.version).toBe(1); + expect(manifest.cellOrder).toBe(8); + expect(manifest.shardOrder).toBe(6); + expect(manifest.pathGrouping).toBe(1); + expect(manifest.schedule).toBe("none"); + expect(manifest.windows).toBeNull(); + expect(manifest.raw[Key.SEMANTIC_HASH]).toBe( + SERC_MANIFEST[Key.SEMANTIC_HASH] + ); + }); + + it("path_grouping absent reads as 1 (zagg D21 retroactive default)", () => { + const rest: Record = { ...SERC_MANIFEST }; + delete rest[Key.PATH_GROUPING]; + expect(parseHiveManifest(rest).pathGrouping).toBe(1); + }); + + it("accepts a grouped manifest (the path_grouping: 3 fixture case)", () => { + const grouped = { ...SERC_MANIFEST, [Key.PATH_GROUPING]: 3 }; + expect(parseHiveManifest(grouped).pathGrouping).toBe(3); + }); + + it("rejects a non-positive or non-integer path_grouping", () => { + for (const bad of [0, -1, 1.5, "3"]) { + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, [Key.PATH_GROUPING]: bad }) + ).toThrow(/path_grouping/); + } + }); +}); + +describe("parseHiveManifest temporal blocks", () => { + it("parses a /2 manifest's temporal block", () => { + const manifest = parseHiveManifest({ + ...SERC_MANIFEST, + spec: "morton-hive/2", + temporal: { + schedule: "yearly", + [Key.TIME_FIELD]: "delta_time", + epoch: "2018-01-01T00:00:00Z", + }, + }); + expect(manifest.version).toBe(2); + expect(manifest.schedule).toBe("yearly"); + expect(manifest.windows).toBeNull(); + }); + + it("carries a /2 explicit window list through", () => { + const manifest = parseHiveManifest({ + ...SERC_MANIFEST, + spec: "morton-hive/2", + temporal: { schedule: "explicit", windows: ["melt-2019", "melt-2020"] }, + }); + expect(manifest.windows).toEqual(["melt-2019", "melt-2020"]); + }); + + it("requires a schedule on /2 and refuses temporal on /1", () => { + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, spec: "morton-hive/2" }) + ).toThrow(/temporal block with a schedule/); + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, temporal: { schedule: "yearly" } }) + ).toThrow(/must not carry a temporal block/); + }); + + it("treats an absent /3 temporal block as schedule none", () => { + const manifest = parseHiveManifest({ + ...SERC_MANIFEST, + spec: "morton-hive/3", + }); + expect(manifest.version).toBe(3); + expect(manifest.schedule).toBe("none"); + }); +}); + +describe("parseHiveManifest validation", () => { + it("rejects unknown specs, bad orders, and non-objects", () => { + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, spec: "morton-hive/4" }) + ).toThrow(/unknown manifest spec/); + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, [Key.CELL_ORDER]: "8" }) + ).toThrow(/cell_order/); + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, [Key.CELL_ORDER]: 5 }) + ).toThrow(/cells nest inside shards/); + for (const bad of [null, [], "spec", 7]) { + expect(() => parseHiveManifest(bad)).toThrow(/not a JSON object/); + } + }); +}); From cb4adf9cf2238cd8e209af4cde91e7d7d3386be0 Mon Sep 17 00:00:00 2001 From: espg Date: Mon, 20 Jul 2026 18:37:24 -0700 Subject: [PATCH 3/4] fix: reconcile coverage parse posture to moczarr Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N32Mx4rWhbTK8xXm7Mt2ax --- src/lib/morton/coverage.ts | 85 ++++++++++++++++++-------- tests/unit/lib/morton/coverage.test.ts | 39 ++++++++++-- 2 files changed, 93 insertions(+), 31 deletions(-) diff --git a/src/lib/morton/coverage.ts b/src/lib/morton/coverage.ts index 214efcd7..b4c329ec 100644 --- a/src/lib/morton/coverage.ts +++ b/src/lib/morton/coverage.ts @@ -11,10 +11,14 @@ * deliberately no LIST fallback here. The leaf bitmap tier (zstd) stays * server-side; the browser consumes the box/ranges tiers only. * - * Posture split, mirroring moczarr: the envelope is a regenerable cache, so - * an unusable payload PARSES as null (the caller reports no-coverage); - * malformed ranges inside a well-formed envelope THROW (a corrupt cache - * must never yield a plausible partial answer). + * Posture split, mirroring moczarr's parse_root_coverage vs ranges_words + * exactly: the tolerant (-> null) bucket gates on `spec` + `encoding` ONLY -- + * a missing/unknown envelope is a regenerable-cache miss, so the caller reports + * no-coverage. Everything structural (the `order` shape, the `ranges` shape, + * endpoint validity) is LOUD: a well-formed envelope carrying a corrupt body + * must never silently read empty, because there is no LIST fallback to recover + * the true coverage. A numeric-string `order` (e.g. "6") reads like a JSON + * number, as moczarr's ranges_words does. */ import { @@ -41,9 +45,32 @@ export interface RootCoverage { } /** - * A usable store-root coverage envelope, or null. Tolerant by design: a - * non-object payload, an unknown spec, a non-"ranges" encoding, or missing - * required keys read as absent (the root MOC is a regenerable cache). + * The envelope's common order as an integer. Mirrors moczarr's ranges_words, + * which reads a numeric-string `order` (e.g. "6") as readily as a JSON number; + * a missing, non-numeric, or non-integer order is malformed and loud -- a + * corrupt order inside a well-formed envelope must never silently drop + * coverage. + */ +function coverageOrder(value: unknown): number { + if (typeof value === "number" && Number.isInteger(value)) { + return value; + } + if (typeof value === "string" && /^\d+$/.test(value)) { + return Number(value); + } + throw new Error( + `coverage order must be an integer (got ${JSON.stringify(value)})` + ); +} + +/** + * A usable store-root coverage envelope, or null. Tolerant ONLY at the + * envelope gate (moczarr parse_root_coverage): a non-object payload, an + * unknown `spec`, or a non-"ranges" `encoding` reads as absent -- the root MOC + * is a regenerable cache. A well-formed envelope with a malformed `order` or a + * non-list `ranges` is LOUD (moczarr ranges_words): a corrupt body must never + * read as no-coverage. Per-range structural and endpoint checks stay loud at + * expansion (checkRange). */ export function parseRootCoverage(payload: unknown): RootCoverage | null { if ( @@ -54,35 +81,39 @@ export function parseRootCoverage(payload: unknown): RootCoverage | null { return null; } const raw = payload as Record; - const order = raw["order"]; - const ranges = raw["ranges"]; - const usable = - raw["spec"] === COVERAGE_SPEC && - raw["encoding"] === "ranges" && - typeof order === "number" && - Number.isInteger(order) && - Array.isArray(ranges); - if (!usable) { + if (raw["spec"] !== COVERAGE_SPEC || raw["encoding"] !== "ranges") { return null; } + const order = coverageOrder(raw["order"]); + const ranges = raw["ranges"]; + if (!Array.isArray(ranges)) { + throw new Error( + `coverage ranges must be a list of [first, last] pairs ` + + `(got ${JSON.stringify(ranges)})` + ); + } return { spec: COVERAGE_SPEC, encoding: "ranges", - order: order as number, + order, ranges: ranges as [string, string][], }; } /** Validated (base, loRank, hiRank) of one range; throws when malformed. */ -function checkRange( - lo: unknown, - hi: unknown, - order: number -): [string, number, number] { +function checkRange(range: unknown, order: number): [string, number, number] { + if (!Array.isArray(range) || range.length !== 2) { + // A mangled cache must be loud, never a plausible partial answer. + throw new Error( + `coverage range must be a [first, last] pair (got ${JSON.stringify(range)})` + ); + } + const [lo, hi] = range; if (typeof lo !== "string" || typeof hi !== "string") { // Spec section 7.3: endpoints are decimal strings, never JSON numbers. throw new Error( - `coverage range endpoints must be decimal strings (got [${lo}, ${hi}])` + `coverage range endpoints must be decimal strings ` + + `(got [${JSON.stringify(lo)}, ${JSON.stringify(hi)}])` ); } const base = decimalBase(lo); @@ -108,8 +139,8 @@ function checkRange( */ export function rangesShardIds(envelope: RootCoverage): string[] { const ids: string[] = []; - for (const [lo, hi] of envelope.ranges) { - const [base, loRank, hiRank] = checkRange(lo, hi, envelope.order); + for (const range of envelope.ranges) { + const [base, loRank, hiRank] = checkRange(range, envelope.order); for (let r = loRank; r <= hiRank; r++) { ids.push(base + rankTail(r, envelope.order)); } @@ -127,8 +158,8 @@ export function rangesContain( } const base = decimalBase(shardId); const rank = decimalRank(shardId); - return envelope.ranges.some(([lo, hi]) => { - const [rangeBase, loRank, hiRank] = checkRange(lo, hi, envelope.order); + return envelope.ranges.some((range) => { + const [rangeBase, loRank, hiRank] = checkRange(range, envelope.order); return rangeBase === base && loRank <= rank && rank <= hiRank; }); } diff --git a/tests/unit/lib/morton/coverage.test.ts b/tests/unit/lib/morton/coverage.test.ts index 44ee8bf9..b86e0300 100644 --- a/tests/unit/lib/morton/coverage.test.ts +++ b/tests/unit/lib/morton/coverage.test.ts @@ -57,7 +57,8 @@ describe("parseRootCoverage", () => { expect(parsed.ranges).toHaveLength(5); }); - it("reads unusable payloads as absent (regenerable-cache posture)", () => { + it("gates the tolerant-null bucket on spec/encoding only (moczarr)", () => { + // Envelope-level miss reads as no-coverage (regenerable cache). expect(parseRootCoverage(null)).toBeNull(); expect(parseRootCoverage([])).toBeNull(); expect(parseRootCoverage("moc")).toBeNull(); @@ -67,10 +68,29 @@ describe("parseRootCoverage", () => { expect( parseRootCoverage({ ...SERC_COVERAGE, encoding: "bitmap" }) ).toBeNull(); - expect(parseRootCoverage({ ...SERC_COVERAGE, order: "6" })).toBeNull(); - expect( + }); + + it("reads a numeric-string order like moczarr's ranges_words", () => { + const parsed = parseRootCoverage({ ...SERC_COVERAGE, order: "6" }); + expect(parsed?.order).toBe(6); + expect(rangesShardIds(parsed!)).toEqual(SERC_SHARDS); + }); + + it("is loud on a structurally-corrupt body (never silent-empty)", () => { + // A well-formed envelope with a mangled order/ranges must throw, not + // read as no-coverage -- there is no LIST fallback to recover coverage. + expect(() => parseRootCoverage({ ...SERC_COVERAGE, order: "six" })).toThrow( + /coverage order/ + ); + expect(() => + parseRootCoverage({ ...SERC_COVERAGE, order: undefined }) + ).toThrow(/coverage order/); + expect(() => parseRootCoverage({ ...SERC_COVERAGE, ranges: undefined }) - ).toBeNull(); + ).toThrow(/coverage ranges must be a list/); + expect(() => + parseRootCoverage({ ...SERC_COVERAGE, ranges: "4331244" }) + ).toThrow(/coverage ranges must be a list/); }); }); @@ -99,6 +119,17 @@ describe("rangesShardIds", () => { expect(() => rangesShardIds(envelope({ ranges: [[4331422, 4331423]] })) ).toThrow(/decimal strings/); + // A non-array / wrong-arity range entry is loud, not a silent skip. + expect(() => + rangesShardIds( + envelope({ ranges: ["4331422"] as unknown as [string, string][] }) + ) + ).toThrow(/\[first, last\] pair/); + expect(() => + rangesShardIds( + envelope({ ranges: [["4331422"]] as unknown as [string, string][] }) + ) + ).toThrow(/\[first, last\] pair/); }); }); From 3c6f7b24e613bc613aa44dbce01d2db4c7fcf910 Mon Sep 17 00:00:00 2001 From: espg Date: Mon, 20 Jul 2026 18:37:36 -0700 Subject: [PATCH 4/4] fix: reject explicit path_grouping null for moczarr parity Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N32Mx4rWhbTK8xXm7Mt2ax --- src/lib/morton/manifest.ts | 10 ++++++---- tests/unit/lib/morton/manifest.test.ts | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lib/morton/manifest.ts b/src/lib/morton/manifest.ts index 4fcad858..f38825b9 100644 --- a/src/lib/morton/manifest.ts +++ b/src/lib/morton/manifest.ts @@ -44,13 +44,15 @@ function asInt(value: unknown, key: string): number { } /** - * The `path_grouping` chunk width. Absent/null reads as 1 (zagg D21: - * existing stores are retroactively 1); anything but a positive integer is - * malformed -- readers chunk per the manifest, never by assumption. + * The `path_grouping` chunk width. An ABSENT key reads as 1 (zagg D21: + * existing stores are retroactively 1), matching moczarr's + * `payload.get("path_grouping", 1)`; an explicit `null` -- like anything but a + * positive integer -- is malformed and loud, so a serializer-nulled or + * hand-edited manifest can't read differently in the browser than in Python. */ function pathGroupingOf(raw: Record): number { const value = raw["path_grouping"]; - if (value === undefined || value === null) { + if (value === undefined) { return 1; } const grouping = asInt(value, "path_grouping"); diff --git a/tests/unit/lib/morton/manifest.test.ts b/tests/unit/lib/morton/manifest.test.ts index d5d3e9ff..bca4b32d 100644 --- a/tests/unit/lib/morton/manifest.test.ts +++ b/tests/unit/lib/morton/manifest.test.ts @@ -51,6 +51,12 @@ describe("parseHiveManifest", () => { expect(parseHiveManifest(rest).pathGrouping).toBe(1); }); + it("rejects an explicit path_grouping: null (only absent defaults, moczarr)", () => { + expect(() => + parseHiveManifest({ ...SERC_MANIFEST, [Key.PATH_GROUPING]: null }) + ).toThrow(/path_grouping/); + }); + it("accepts a grouped manifest (the path_grouping: 3 fixture case)", () => { const grouped = { ...SERC_MANIFEST, [Key.PATH_GROUPING]: 3 }; expect(parseHiveManifest(grouped).pathGrouping).toBe(3);