diff --git a/packages/core/src/catalog/expr.test.ts b/packages/core/src/catalog/expr.test.ts new file mode 100644 index 0000000..d0ee980 --- /dev/null +++ b/packages/core/src/catalog/expr.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect } from "vitest"; +import { evaluateExpr } from "./expr.js"; +import { applyTableDerivations, TableAuthorPropsSchema } from "./lenses/table.js"; + +const NOW = Date.parse("2026-07-09T12:00:00Z"); + +describe("evaluateExpr", () => { + const row = { score: "0.62", posted: "2026-07-06", empty: "" }; + + it("does arithmetic with precedence and parens", () => { + expect(evaluateExpr("1 + 2 * 3", { row, nowMs: NOW })).toBe(7); + expect(evaluateExpr("(1 + 2) * 3", { row, nowMs: NOW })).toBe(9); + expect(evaluateExpr("-2 * 3", { row, nowMs: NOW })).toBe(-6); + }); + + it("num() reads a row field as a number", () => { + expect(evaluateExpr('num("score") * 2', { row, nowMs: NOW })).toBeCloseTo(1.24); + }); + + it("days_since() measures from a bare date (UTC midnight) to now", () => { + // 2026-07-06T00:00 → 2026-07-09T12:00 = 3.5 days + expect(evaluateExpr('days_since("posted")', { row, nowMs: NOW })).toBeCloseTo(3.5); + }); + + it("days_since() accepts Z-less datetimes as UTC", () => { + const r = { at: "2026-07-08T12:00:00" }; + expect(evaluateExpr('days_since("at")', { row: r, nowMs: NOW })).toBeCloseTo(1); + }); + + it("tier() picks the first threshold the value fits under", () => { + const src = "tier(num(\"x\"), 1, 1.0, 3, 0.75, 7, 0.55, 0.05)"; + expect(evaluateExpr(src, { row: { x: 0.5 }, nowMs: NOW })).toBe(1.0); + expect(evaluateExpr(src, { row: { x: 3 }, nowMs: NOW })).toBe(0.75); + expect(evaluateExpr(src, { row: { x: 5 }, nowMs: NOW })).toBe(0.55); + expect(evaluateExpr(src, { row: { x: 99 }, nowMs: NOW })).toBe(0.05); + }); + + it("tier() blanks instead of returning the default when x is not numeric", () => { + const src = "tier(num(\"x\"), 1, 1.0, 3, 0.75, 0.05)"; + expect(evaluateExpr(src, { row: { x: true }, nowMs: NOW })).toBeNull(); + expect(evaluateExpr(src, { row: {}, nowMs: NOW })).toBeNull(); + }); + + it("returns null instead of NaN/throwing on bad input", () => { + expect(evaluateExpr('num("missing")', { row, nowMs: NOW })).toBeNull(); + expect(evaluateExpr('num("empty")', { row, nowMs: NOW })).toBeNull(); + expect(evaluateExpr('num("flag")', { row: { flag: true }, nowMs: NOW })).toBeNull(); + expect(evaluateExpr('num("blank")', { row: { blank: " " }, nowMs: NOW })).toBeNull(); + expect(evaluateExpr('days_since("empty")', { row, nowMs: NOW })).toBeNull(); + expect(evaluateExpr("1 +", { row, nowMs: NOW })).toBeNull(); + expect(evaluateExpr("window.alert(1)", { row, nowMs: NOW })).toBeNull(); + expect(evaluateExpr('nope("score")', { row, nowMs: NOW })).toBeNull(); + expect(evaluateExpr("", { row, nowMs: NOW })).toBeNull(); + }); +}); + +describe("applyTableDerivations", () => { + const author = TableAuthorPropsSchema.parse({ + columns: [ + { key: "name", label: "Name" }, + { + key: "current", + label: "Current", + expr: '0.35 * tier(days_since("posted"), 1, 1.0, 3, 0.75, 7, 0.55, 0.05) + 0.65 * num("affinity")', + precision: 2, + }, + ], + sort: { key: "current", dir: "desc" }, + }); + + it("injects derived values and sorts by them, nulls last", () => { + const rows = [ + { name: "stale-but-loved", posted: "2026-06-01", affinity: "0.9" }, + { name: "fresh", posted: "2026-07-09", affinity: "0.5" }, + { name: "broken", posted: null, affinity: null }, + ]; + const out = applyTableDerivations(author, rows, NOW); + expect(out.map((r) => r["name"])).toEqual([ + "fresh", // 0.35*1.0 + 0.65*0.5 = 0.68 + "stale-but-loved", // 0.35*0.05 + 0.65*0.9 = 0.60 + "broken", // unparseable → null → last + ]); + expect(out[0]!["current"]).toBe(0.68); + expect(out[1]!["current"]).toBe(0.6); + expect(out[2]!["current"]).toBeNull(); + }); + + it("leaves rows untouched when no expr/sort is authored", () => { + const plain = TableAuthorPropsSchema.parse({ + columns: [{ key: "name", label: "Name" }], + }); + const rows = [{ name: "b" }, { name: "a" }]; + expect(applyTableDerivations(plain, rows, NOW)).toEqual(rows); + }); + + it("sorts string columns lexically and respects asc", () => { + const byName = TableAuthorPropsSchema.parse({ + columns: [{ key: "name", label: "Name" }], + sort: { key: "name", dir: "asc" }, + }); + const out = applyTableDerivations(byName, [{ name: "b" }, { name: "a" }], NOW); + expect(out.map((r) => r["name"])).toEqual(["a", "b"]); + }); +}); diff --git a/packages/core/src/catalog/expr.ts b/packages/core/src/catalog/expr.ts new file mode 100644 index 0000000..90e5a85 --- /dev/null +++ b/packages/core/src/catalog/expr.ts @@ -0,0 +1,240 @@ +/** + * Tiny arithmetic expression evaluator for derived table columns. + * + * Some column values are *derived facts* that decay with time — e.g. a lead's + * freshness, which is a function of a stored post date and "now". Storing the + * decayed value in the graph would mean rewriting every row every day, so the + * graph stores the durable fact and the notebook computes the derived value + * at read time. This module is that computation. + * + * Grammar (deliberately small — no variables, no assignment, no property + * access, no ambient environment; a row and a clock are the only inputs): + * + * expr := term (("+" | "-") term)* + * term := unary (("*" | "/") unary)* + * unary := "-" unary | primary + * primary := number | call | "(" expr ")" + * call := ident "(" (arg ("," arg)*)? ")" + * arg := expr | string + * + * Functions (the whole builtin surface): + * num("col") — the row field parsed as a number + * days_since("col") — fractional days between the row field (a date or + * datetime string / epoch-ms number) and now + * tier(x, t1, v1, t2, v2, ..., vDefault) + * — first threshold t where x <= t yields its v; no + * threshold matches → vDefault. Encodes step/decay + * functions without needing array literals. + * + * Anything unparseable, a missing field, or a NaN result evaluates to null — + * derived cells degrade to blank rather than rendering "NaN" or throwing + * mid-render. + */ + +export interface ExprContext { + row: Record; + /** Epoch ms "now" — injected so callers control the clock (and tests can). */ + nowMs: number; +} + +type Token = + | { kind: "num"; value: number } + | { kind: "str"; value: string } + | { kind: "ident"; value: string } + | { kind: "punct"; value: "(" | ")" | "," | "+" | "-" | "*" | "/" }; + +function tokenize(src: string): Token[] | null { + const tokens: Token[] = []; + let i = 0; + while (i < src.length) { + const ch = src[i]!; + if (ch === " " || ch === "\t" || ch === "\n") { + i++; + continue; + } + if ( + ch === "(" || ch === ")" || ch === "," || + ch === "+" || ch === "-" || ch === "*" || ch === "/" + ) { + tokens.push({ kind: "punct", value: ch }); + i++; + continue; + } + if (ch === '"' || ch === "'") { + const close = src.indexOf(ch, i + 1); + if (close === -1) return null; + tokens.push({ kind: "str", value: src.slice(i + 1, close) }); + i = close + 1; + continue; + } + if (/[0-9.]/.test(ch)) { + const match = /^\d*\.?\d+/.exec(src.slice(i)); + if (!match) return null; + tokens.push({ kind: "num", value: Number(match[0]) }); + i += match[0].length; + continue; + } + if (/[a-z_]/i.test(ch)) { + const match = /^[a-z_][a-z0-9_]*/i.exec(src.slice(i)); + tokens.push({ kind: "ident", value: match![0] }); + i += match![0].length; + continue; + } + return null; // any character outside the grammar rejects the whole expr + } + return tokens; +} + +/** Parse a field's value as a timestamp; bare dates read as UTC midnight. */ +function toEpochMs(value: unknown): number { + if (typeof value === "number") return value; + if (typeof value !== "string" || value === "") return NaN; + // Date-only and Z-less datetime strings both parse; Z-less is treated as + // UTC (omnigraph DateTimes are stored Z-less by convention). + const iso = /^\d{4}-\d{2}-\d{2}$/.test(value) + ? `${value}T00:00:00Z` + : /[zZ]$|[+-]\d{2}:\d{2}$/.test(value) + ? value + : `${value}Z`; + return Date.parse(iso); +} + +class Parser { + private pos = 0; + constructor( + private tokens: Token[], + private ctx: ExprContext, + ) {} + + private peek(): Token | undefined { + return this.tokens[this.pos]; + } + private takePunct(value: string): boolean { + const t = this.peek(); + if (t?.kind === "punct" && t.value === value) { + this.pos++; + return true; + } + return false; + } + + expr(): number { + let left = this.term(); + for (;;) { + if (this.takePunct("+")) left += this.term(); + else if (this.takePunct("-")) left -= this.term(); + else return left; + } + } + + private term(): number { + let left = this.unary(); + for (;;) { + if (this.takePunct("*")) left *= this.unary(); + else if (this.takePunct("/")) left /= this.unary(); + else return left; + } + } + + private unary(): number { + if (this.takePunct("-")) return -this.unary(); + return this.primary(); + } + + private primary(): number { + const t = this.peek(); + if (t === undefined) throw new Error("unexpected end of expression"); + if (t.kind === "num") { + this.pos++; + return t.value; + } + if (t.kind === "punct" && t.value === "(") { + this.pos++; + const v = this.expr(); + if (!this.takePunct(")")) throw new Error("expected )"); + return v; + } + if (t.kind === "ident") { + this.pos++; + return this.call(t.value); + } + throw new Error(`unexpected token`); + } + + /** An argument is either a string literal (a column ref) or a sub-expr. */ + private arg(): number | string { + const t = this.peek(); + if (t?.kind === "str") { + this.pos++; + return t.value; + } + return this.expr(); + } + + private call(name: string): number { + if (!this.takePunct("(")) throw new Error(`${name} is not a value`); + const args: Array = []; + if (!this.takePunct(")")) { + do { + args.push(this.arg()); + } while (this.takePunct(",")); + if (!this.takePunct(")")) throw new Error("expected )"); + } + + switch (name) { + case "num": { + if (args.length !== 1 || typeof args[0] !== "string") + throw new Error('num() takes one column name, e.g. num("score")'); + const raw = this.ctx.row[args[0]]; + if (typeof raw === "number") return raw; + if (typeof raw !== "string" || raw.trim() === "") return NaN; + return Number(raw); + } + case "days_since": { + if (args.length !== 1 || typeof args[0] !== "string") + throw new Error('days_since() takes one column name'); + const ms = toEpochMs(this.ctx.row[args[0]]); + return (this.ctx.nowMs - ms) / 86_400_000; + } + case "tier": { + // tier(x, t1, v1, ..., vDefault): odd arg count >= 2, all numeric. + if (args.length < 2 || args.length % 2 !== 0) + throw new Error( + "tier() wants tier(x, t1, v1, ..., default) — pairs plus a default", + ); + if (args.some((a) => typeof a !== "number" || !Number.isFinite(a))) + throw new Error("tier() arguments must be finite numeric values"); + const nums = args as number[]; + const x = nums[0]!; + for (let i = 1; i < nums.length - 1; i += 2) { + if (x <= nums[i]!) return nums[i + 1]!; + } + return nums[nums.length - 1]!; + } + default: + throw new Error(`unknown function ${name}()`); + } + } + + done(): boolean { + return this.pos === this.tokens.length; + } +} + +/** + * Evaluate an expression against one row. Returns null (never throws, never + * NaN) on any lexical, syntactic, or data problem — a bad expr or a missing + * field blanks the cell instead of breaking the table. + */ +export function evaluateExpr(src: string, ctx: ExprContext): number | null { + const tokens = tokenize(src); + if (tokens === null || tokens.length === 0) return null; + try { + const parser = new Parser(tokens, ctx); + const value = parser.expr(); + if (!parser.done()) return null; + return Number.isFinite(value) ? value : null; + } catch { + return null; + } +} diff --git a/packages/core/src/catalog/index.ts b/packages/core/src/catalog/index.ts index 76636ed..6293e50 100644 --- a/packages/core/src/catalog/index.ts +++ b/packages/core/src/catalog/index.ts @@ -5,6 +5,7 @@ import { TableAuthorPropsSchema, TableRuntimePropsSchema, TableDescription, + applyTableDerivations, type TableAuthorProps, type TableRuntimeProps, } from "./lenses/table.js"; @@ -269,7 +270,12 @@ function buildRuntimeProps( switch (lens) { case "Table": { const author: TableAuthorProps = TableAuthorPropsSchema.parse(authorProps); - const runtime: TableRuntimeProps = { ...author, rows: result.rows }; + // Derived (expr) columns and author sort are materialized here, per + // query refresh, so every renderer (web, TUI) shows identical values. + const runtime: TableRuntimeProps = { + ...author, + rows: applyTableDerivations(author, result.rows, Date.now()), + }; return runtime as unknown as Record; } case "Tree": { diff --git a/packages/core/src/catalog/lenses/table.ts b/packages/core/src/catalog/lenses/table.ts index 2c168b8..73aff4f 100644 --- a/packages/core/src/catalog/lenses/table.ts +++ b/packages/core/src/catalog/lenses/table.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { evaluateExpr } from "../expr.js"; export const TableAuthorPropsSchema = z.object({ columns: z @@ -15,10 +16,36 @@ export const TableAuthorPropsSchema = z.object({ badge: z.boolean().optional(), /** Cell text alignment; numeric columns default to "right" (web only). */ align: z.enum(["left", "right"]).optional(), + /** + * Derived column: an expression evaluated per row at read time, whose + * result is injected into the row under this column's `key`. For + * values that decay with time (freshness scores, ages, SLAs) the + * source stores the durable fact (a date) and the notebook derives + * the current value on every render — no daily recompute of stored + * scores. Grammar and builtins (`num`, `days_since`, `tier`): + * see catalog/expr.ts. Example — current lead rank from a stored + * post date plus stored component scores: + * 0.35 * tier(days_since("s.observed_at"), + * 1,1.0, 3,0.75, 7,0.55, 14,0.35, 30,0.2, 0.05) + * + 0.25 * num("eq.summary") + 0.40 * num("ea.summary") + */ + expr: z.string().min(1).optional(), + /** Round a derived value to N decimal places (expr columns only). */ + precision: z.number().int().min(0).max(6).optional(), }), ) .min(1), dense: z.boolean().optional(), + /** + * Sort rows before rendering — needed when the display order depends on a + * derived column the source query cannot order by. Nulls sort last. + */ + sort: z + .object({ + key: z.string().min(1), + dir: z.enum(["asc", "desc"]).optional(), + }) + .optional(), /** * Make rows clickable: on click, write the row's `select_column` value to * this JSON-pointer state path (e.g. "/selected"). Another cell (a Card) @@ -35,5 +62,60 @@ export const TableRuntimePropsSchema = TableAuthorPropsSchema.extend({ }); export type TableRuntimeProps = z.infer; +/** Fixed locale so string sort order is identical in the Node TUI and any browser. */ +const collator = new Intl.Collator("en"); + +/** + * Materialize `expr` columns into the rows and apply the author's `sort`. + * Runs once per query refresh (in buildRuntimeProps), so web and TUI render + * identical derived values; renderers stay presentation-only. + */ +export function applyTableDerivations( + author: TableAuthorProps, + rows: ReadonlyArray>, + nowMs: number, +): Array> { + const derived = author.columns.filter((c) => c.expr !== undefined); + let out: Array> = + derived.length === 0 + ? [...rows] + : rows.map((row) => { + const extended = { ...row }; + for (const col of derived) { + const value = evaluateExpr(col.expr!, { row, nowMs }); + extended[col.key] = + value === null + ? null + : col.precision !== undefined + ? Number(value.toFixed(col.precision)) + : value; + } + return extended; + }); + + if (author.sort !== undefined) { + const { key, dir } = author.sort; + const sign = dir === "desc" ? -1 : 1; + out = out + .map((row, index) => ({ row, index })) // stable sort w/ nulls last + .sort((a, b) => { + const av = a.row[key]; + const bv = b.row[key]; + const aNull = av === null || av === undefined || av === ""; + const bNull = bv === null || bv === undefined || bv === ""; + if (aNull || bNull) return aNull === bNull ? a.index - b.index : aNull ? 1 : -1; + const an = Number(av); + const bn = Number(bv); + const cmp = + !Number.isNaN(an) && !Number.isNaN(bn) + ? an - bn + : collator.compare(String(av), String(bv)); + return cmp !== 0 ? sign * cmp : a.index - b.index; + }) + .map((entry) => entry.row); + } + return out; +} + export const TableDescription = - "Tabular display of typed rows from a query. Author specifies columns; rows come from the query result."; + "Tabular display of typed rows from a query. Author specifies columns (including derived expr columns computed at read time); rows come from the query result.";