From e656d1224e0adbec33ab2e8f9d3691874065bb91 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 28 Jun 2026 09:40:21 -0700 Subject: [PATCH 01/20] feat(gfql): physical adjacency indexes for O(degree) seeded traversal Opt-in, pay-as-you-go CSR adjacency / node-id indexes that turn seeded hop()/gfql() traversal from an O(E) edge scan into an O(degree) searchsorted gather. Sidecar over edge row positions (never reorders .edges/.nodes), fingerprint-validated so a .edges() rebind safely invalidates a stale index (treated as absent -> scan fallback; never a wrong answer). Three uniform surfaces driving one registry: Python (create_index/drop_index/show_indexes/gfql_index_all/gfql_explain), Cypher DDL (CREATE/DROP/SHOW GFQL INDEX -- mandatory GFQL token disambiguates from standard property CREATE INDEX), and the JSON wire protocol ({"type":"CreateIndex"} ops + index_policy in the request envelope). Optimizer policy gfql(index_policy=use|auto|force|off), default use = use-resident-never-build. Engine-polymorphic (numpy host arrays for pandas/polars, cupy on-device for cudf); fast path hooked at every seeded scan site (compute/hop.py, the lazy polars hop, the polars single-hop chain fast path) and falls back to scan for uncovered features (edge/source/dest match, target_wave_front, min_hops>1, labeling). Perf (dgx-spark, deg-8, warm median): seeded latency FLAT in graph size -- GFQL-pandas SEL1 0.110/0.109/0.113 ms at 0.8M/8M/80M edges (vs O(E) scan 17.7/128/1361 ms) -- and beats kuzu (CSR) + neo4j by 12-30x on CPU. cuDF/ polars-GPU flat but floored ~2-3ms by GPU kernel launch (selective traversal is an indexing problem, not a compute one). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu x fwd/rev/undirected x 1-3 hop x wavefront; 46/46 index pytest; existing hop/chain/cypher suites green (no regressions). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + benchmarks/gfql/index_ddl_smoke.py | 87 +++++ benchmarks/gfql/index_perf.py | 116 ++++++ benchmarks/gfql/index_smoke.py | 123 +++++++ benchmarks/gfql/index_takeover_bench.py | 143 ++++++++ benchmarks/gfql/index_vs_dbs.py | 194 ++++++++++ graphistry/compute/ComputeMixin.py | 51 ++- graphistry/compute/gfql/index/__init__.py | 33 ++ graphistry/compute/gfql/index/api.py | 337 ++++++++++++++++++ graphistry/compute/gfql/index/build.py | 99 +++++ graphistry/compute/gfql/index/cypher_ddl.py | 67 ++++ .../compute/gfql/index/engine_arrays.py | 130 +++++++ graphistry/compute/gfql/index/explain.py | 33 ++ graphistry/compute/gfql/index/lookup.py | 94 +++++ graphistry/compute/gfql/index/registry.py | 135 +++++++ graphistry/compute/gfql/index/traverse.py | 164 +++++++++ graphistry/compute/gfql/index/wire.py | 122 +++++++ .../compute/gfql/lazy/engine/polars/chain.py | 22 ++ .../compute/gfql/lazy/engine/polars/hop.py | 27 ++ graphistry/compute/hop.py | 25 ++ .../tests/compute/gfql/index/__init__.py | 0 .../tests/compute/gfql/index/test_index.py | 287 +++++++++++++++ 22 files changed, 2289 insertions(+), 1 deletion(-) create mode 100644 benchmarks/gfql/index_ddl_smoke.py create mode 100644 benchmarks/gfql/index_perf.py create mode 100644 benchmarks/gfql/index_smoke.py create mode 100644 benchmarks/gfql/index_takeover_bench.py create mode 100644 benchmarks/gfql/index_vs_dbs.py create mode 100644 graphistry/compute/gfql/index/__init__.py create mode 100644 graphistry/compute/gfql/index/api.py create mode 100644 graphistry/compute/gfql/index/build.py create mode 100644 graphistry/compute/gfql/index/cypher_ddl.py create mode 100644 graphistry/compute/gfql/index/engine_arrays.py create mode 100644 graphistry/compute/gfql/index/explain.py create mode 100644 graphistry/compute/gfql/index/lookup.py create mode 100644 graphistry/compute/gfql/index/registry.py create mode 100644 graphistry/compute/gfql/index/traverse.py create mode 100644 graphistry/compute/gfql/index/wire.py create mode 100644 graphistry/tests/compute/gfql/index/__init__.py create mode 100644 graphistry/tests/compute/gfql/index/test_index.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ba37fa7c3c..770bd3727c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). - **GFQL Cypher numeric functions + `toLower`/`toUpper`/`lower`/`upper` (openCypher/neo4j/GQL-standard)**: added the standard scalar functions `floor`, `ceil` (alias `ceiling`, per Cypher 25 GQL conformance and the GQL grammar's `CEIL|CEILING`), `round(x)` / `round(x, precision)`, and `toLower` / `toUpper` plus their GQL-conformance aliases `lower` / `upper` (ISO GQL §20.24 character-string functions; neo4j accepts both spellings) — the idiomatic case-insensitive compare `WHERE toLower(n.name) = 'bob'` — across the Cypher `WHERE`/`RETURN` expression surface. Evaluated natively on pandas/cuDF and polars (differential-parity tested). **`round` follows neo4j's documented tie-breaking** (standards-vetted against the neo4j manual): precision 0 (and the 1-arg form) rounds ties toward **positive infinity** (`round(-1.5)` → `-1.0`, `round(2.5)` → `3.0`), and precision > 0 rounds ties **away from zero** (HALF_UP: `round(-1.55, 1)` → `-1.6`) — not the numpy/polars half-to-even defaults, which give wrong answers on ties. The `round(x, precision, mode)` 3-arg form is not yet supported. Complements the already-supported `abs`/`sqrt`/`sign` and chained comparisons (`WHERE 1 < n.age < 65`). The `^` exponentiation operator is deferred: standards vetting settled it as **left-associative** (the openCypher TCK pins left, and neo4j's shipped Cypher 5/25 parser left-folds `Pow` — the manual's "right to left" row is a docs bug), but the current conformance corpus marks it reject-expected, so re-adding it is a coordinated corpus change. `LIKE`/`ILIKE`/`BETWEEN` remain intentionally unimplemented (verified absent from both Cypher and ISO GQL — GQL's only `LIKE` is the unrelated `CREATE GRAPH TYPE … LIKE g` DDL). - **GFQL Cypher `=~` regex-match operator (openCypher/neo4j-standard)**: added the standard `=~` string predicate to the Cypher `WHERE`/expression grammar — `MATCH (n) WHERE n.name =~ '(?i)al.*' RETURN n`. Semantics match openCypher/neo4j: **Java-regex dialect, full-string/anchored match** (`n.name =~ 'AB'` matches only `'AB'`, not `'ABCDEF'`; use `.*`/`^..$` for partial), with inline flags (`(?i)`/`(?m)`/`(?s)`) honored; lowers to the existing `fullmatch` predicate. Works in the simple `WHERE prop =~ '...'` form (all engines via `filter_by_dict`) and composes through `AND`/`OR`/`NOT`/`RETURN` expressions via the shared expression engine (pandas/cuDF; polars supports the simple-WHERE form and declines the complex `OR`/`NOT` row-filter form with an honest `NotImplementedError` — the pre-existing polars `where_rows` limitation, not `=~`-specific). Also adds native polars `Match`/`Fullmatch` predicate lowering (previously `NotImplementedError`), so `=~` and the Python `match()`/`fullmatch()` predicates run natively on polars. Differential-parity tested against the pandas oracle. `LIKE`/`ILIKE` remain intentionally unimplemented (not in any graph standard — use `=~`/`CONTAINS`/`STARTS WITH`). +- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront. **Perf (dgx-spark, deg-8, warm median):** seeded latency is **flat in graph size** — GFQL-pandas SEL1 0.110/0.109/0.113 ms at 0.8M/8M/80M edges (vs an O(E) scan of 17.7/128/1361 ms) — and **beats kuzu (CSR) and neo4j by 12–30×** on CPU (e.g. 10M nodes/80M edges: GFQL-pandas 0.113 ms vs kuzu 2.64 ms). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity. cuDF/polars-GPU are flat but floored by ~2–3 ms GPU kernel-launch overhead (selective traversal is an indexing problem, not a compute one). No change to default (un-indexed) behavior. - **GFQL native Polars engine — traversals (`engine='polars'`)**: Added a native, vectorized Polars execution engine for the core GFQL traversals `hop()` and `chain()`, dispatched at the engine boundary so the production pandas/cuDF paths are untouched. `Engine.POLARS` is opt-in (explicit `engine='polars'`); `engine='auto'` with Polars input still coerces to pandas as before. Covers forward/reverse/undirected single-hop traversal, directed multi-hop chains, node/edge filter dicts and predicates (lowered to Polars expressions), `edge_match`/`source_node_match`/`destination_node_match`, `target_wave_front`, and alias names; the BFS advances via semi/anti joins (no per-row Python work). Validated by differential parity against the pandas engine (hop + chain test suites plus a randomized fuzzer) and benchmarked vs pandas (`benchmarks/gfql/pandas_vs_polars.py`) — Polars wins at scale (up to ~2.5x on multi-edge chains at millions of edges; crossover ~50–100k rows). Variable-length/multi-hop edges, undirected edges in multi-edge chains, hop labels, and node `query=` raise `NotImplementedError` for now (use `engine='pandas'`). - **GFQL native Polars engine — variable-length `min_hops>1` traversals (`engine='polars'`/`'polars-gpu'`)**: Forward/reverse lower-bound traversals (`e(min_hops=N, max_hops=M)`) now run natively on the Polars engine — no pandas bridge. The eager Polars hop runs pandas' min_hops algorithm vectorized: a NON-anti-joined BFS (the wavefront carries revisits so a cycle keeps bumping the reach depth until `max_hops`), a 3-case termination gate (`max_reached1` (needs connected-components + 2-core seed retention) and a *direct* `hop(min_hops>1)` (which would need pandas' separate un-labeled direct-hop node-output plus the `target_wave_front` threading the chain supplies — without them it silently diverges) both raise `NotImplementedError` for `engine='polars'` (use `chain()`/`gfql()`, or `engine='pandas'`). Validated by differential parity vs the pandas oracle: the 500-seed randomized chain fuzzer (`test_polars_chain_fuzz_parity`, hardened to compare null-aware node **attributes** and edge **multiplicity**, not just id/endpoint sets) is **500/500**, a min_hops+attribute-filter amplified fuzz and metamorphic invariants pass, and `engine='polars-gpu'` (cudf_polars) runs the full 500-seed fuzz **500/500** on-device. - **GFQL native Polars engine — cypher row pipeline (`engine='polars'`)**: Extended the Polars engine to the Cypher `MATCH … RETURN` row surface, natively vectorized. **NO CHEATING:** the polars engine never silently falls back to the pandas engine — every query runs natively on polars or raises an honest `NotImplementedError` pointing at `engine='pandas'` (falling back to pandas would misrepresent pandas performance as polars; only a human may consent to a bridge). `chain_polars` splits boundary `call()` ops (mirroring the pandas `_handle_boundary_calls`) and runs each trailing row op per-op native or raises. **Native polars** (no pandas round-trip): frame ops (`rows`/`limit`/`skip`/`distinct`/`drop_cols`), `select`/`with_`/`return_` projection (a conservative cypher-expr-AST → `pl.Expr` lowering covering property access, arithmetic, comparison, boolean, literals), `order_by` (`.sort`), `group_by` (`count`/`sum`/`avg`/`min`/`max`), `unwind` (literal-list cross-join), the result projection for property/expr columns, and entity-text `RETURN n` rendering for int/string/bool nodes (`pl.concat_str`). **Honestly deferred** (raise `NotImplementedError`, no pandas fallback): multi-entity `rows(binding_ops=…)`, cross-entity same-path `WHERE` (`DFSamePathExecutor`), float/temporal/nested entity-text, and exotic expressions (CASE/list/map/temporal, `collect` aggregates) — these are the forward native-engineering targets. Validated by differential parity vs pandas including a TCK-style conformance lane (`test_engine_polars_cypher_conformance.py`: native-only curated corpus + seeded fuzzer + NULL/3-valued-logic graph + entity-text escaping, plus a `DEFERRED` list asserting deferred queries raise rather than silently bridge) and benchmarked (`benchmarks/gfql/cypher_row_pipeline.py`). **Perf (interleaved, 1M nodes, each engine on its native-frame graph, all fully native):** polars wins **5.6–38×** across the surface — `RETURN n` ~38×, `ORDER BY` ~17×, `WHERE`+`ORDER BY`+`LIMIT` ~14×, traversals 6–7.5×, projections/aggregations/`DISTINCT` 5.6–6.9×. cuDF/pandas paths untouched. diff --git a/benchmarks/gfql/index_ddl_smoke.py b/benchmarks/gfql/index_ddl_smoke.py new file mode 100644 index 0000000000..46e32b7b2f --- /dev/null +++ b/benchmarks/gfql/index_ddl_smoke.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Smoke test: Cypher DDL + JSON wire + index_policy auto/force + gfql_explain.""" +from __future__ import annotations + +import sys +import numpy as np +import pandas as pd + +import graphistry +from graphistry.compute.gfql.index import ( + CreateIndex, DropIndex, ShowIndexes, index_op_from_json, parse_index_ddl, +) + + +def make_graph(n=3000, deg=6, seed=1): + rng = np.random.default_rng(seed) + m = n * deg + edf = pd.DataFrame({"src": rng.integers(0, n, m), "dst": rng.integers(0, n, m)}) + ndf = pd.DataFrame({"id": np.arange(n)}) + return graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + + +def check(name, cond): + print(f" {'OK ' if cond else 'FAIL'} {name}") + return 0 if cond else 1 + + +def main(): + g = make_graph() + seeds = pd.DataFrame({"id": [0, 1, 5, 9]}) + fails = 0 + + # --- Cypher DDL recognizer --- + fails += check("parse CREATE", isinstance(parse_index_ddl("CREATE GFQL INDEX FOR edge_out_adj"), CreateIndex)) + fails += check("parse CREATE named+col", parse_index_ddl("CREATE GFQL INDEX pk FOR node_id ON id").column == "id") + fails += check("parse DROP FOR", isinstance(parse_index_ddl("DROP GFQL INDEX FOR edge_in_adj"), DropIndex)) + fails += check("parse SHOW", isinstance(parse_index_ddl("SHOW GFQL INDEXES"), ShowIndexes)) + fails += check("non-DDL -> None", parse_index_ddl("MATCH (a) RETURN a") is None) + + # --- Cypher DDL via gfql() drives the registry --- + g2 = g.gfql("CREATE GFQL INDEX FOR edge_out_adj") + g2 = g2.gfql("CREATE GFQL INDEX FOR edge_in_adj") + g2 = g2.gfql("CREATE GFQL INDEX FOR node_id") + si = g2.gfql("SHOW GFQL INDEXES") + fails += check("SHOW returns 3 rows", hasattr(si, "shape") and si.shape[0] == 3) + g2d = g2.gfql("DROP GFQL INDEX FOR edge_in_adj") + fails += check("DROP removed one", g2d.show_indexes().shape[0] == 2) + + # --- JSON wire round-trip --- + op = CreateIndex(kind="edge_out_adj") + j = op.to_json() + fails += check("wire round-trip", index_op_from_json(j) == op) + g3 = g.gfql({"type": "CreateIndex", "kind": "edge_out_adj"}) + fails += check("wire CreateIndex via gfql", g3.show_indexes().shape[0] == 1) + show_via_wire = g3.gfql({"type": "ShowIndexes"}) + fails += check("wire ShowIndexes", hasattr(show_via_wire, "shape") and show_via_wire.shape[0] == 1) + + # --- parity: cypher-DDL-built index hop == scan hop --- + base = g.hop(nodes=seeds, hops=2, direction="forward") + idxed = g2.hop(nodes=seeds, hops=2, direction="forward") + bn = sorted(base._nodes["id"].tolist()); xn = sorted(idxed._nodes["id"].tolist()) + fails += check("DDL-built index parity (nodes)", bn == xn) + + # --- index_policy auto/force build-on-demand + explain --- + gp = make_graph() # fresh, no resident indexes + exp_use = gp.gfql_explain([], index_policy="use") if False else None # explain needs a real query + # explain with a seeded 1-hop chain expressed in cypher + q = "MATCH (a)-[e]->(b) RETURN b" # not seeded -> not coverable; use python chain instead + from graphistry.compute.ast import n, e_forward + seeded_chain = [n({"id": 0}), e_forward(hops=1)] + rep_off = gp.gfql_explain(seeded_chain, index_policy="off") + fails += check("explain off -> scan", rep_off["used_index"] is False) + rep_force = gp.gfql_explain(seeded_chain, index_policy="force") + fails += check("explain force -> index", rep_force["used_index"] is True) + + # force on a fresh graph actually returns correct result + gp_use = make_graph() + r_scan = gp_use.gfql(seeded_chain) + r_force = gp_use.gfql(seeded_chain, index_policy="force") + fails += check("force parity", sorted(r_scan._nodes["id"].tolist()) == sorted(r_force._nodes["id"].tolist())) + + print(f"\n=== DDL/wire smoke: {'PASS' if fails == 0 else f'{fails} FAILED'} ===") + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/gfql/index_perf.py b/benchmarks/gfql/index_perf.py new file mode 100644 index 0000000000..f99fe1c4cb --- /dev/null +++ b/benchmarks/gfql/index_perf.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""GFQL seeded-traversal perf: scan baseline vs warm index, scaling in edges. + +The thesis (from HANDOFF-SELECTIVE-TRAVERSAL): GFQL seeded hop is O(E) (scan), +so latency grows linearly with edges; an adjacency index makes it O(degree), +flat in graph size. This bench measures, per engine × size: + - scan : seeded hop, no index (the O(E) baseline) + - build : one-time index build cost (cold, pay-as-you-go) + - warm : seeded hop with resident index (the win) +for SEL1 (1-hop from 1 seed) and SEL2 (2-hop from 1 seed). + +Env: ENGINES=pandas,cudf,polars,polars-gpu NS=100000,1000000 DEG=8 + REPS=10 SEEDS=1 OUT=/path/results.jsonl +Run (dgx-spark container, GPU idle-checked). +""" +from __future__ import annotations + +import json +import os +import statistics +import time +import numpy as np +import pandas as pd + +import graphistry +from graphistry.Engine import Engine + + +def make_graph(n_nodes, deg, seed=0): + rng = np.random.default_rng(seed) + m = n_nodes * deg + src = rng.integers(0, n_nodes, size=m, dtype=np.int64) + dst = rng.integers(0, n_nodes, size=m, dtype=np.int64) + edf = pd.DataFrame({"src": src, "dst": dst}) + ndf = pd.DataFrame({"id": np.arange(n_nodes, dtype=np.int64)}) + return ndf, edf + + +def _sync(engine): + # force device sync so GPU timings are honest + if engine in ("cudf", "polars-gpu"): + try: + import cupy as cp # type: ignore + cp.cuda.runtime.deviceSynchronize() + except Exception: + pass + + +def timeit(fn, engine, reps, warmup=1): + for _ in range(warmup): + fn(); _sync(engine) + ts = [] + for _ in range(reps): + t0 = time.perf_counter() + fn(); _sync(engine) + ts.append((time.perf_counter() - t0) * 1e3) + ts.sort() + return statistics.median(ts), ts[0], ts[-1] + + +def n_rows(g): + e = g._edges + return int(e.shape[0]) if e is not None else 0 + + +def main(): + engines = os.environ.get("ENGINES", "pandas,cudf,polars,polars-gpu").split(",") + NS = [int(x) for x in os.environ.get("NS", "100000,1000000").split(",")] + DEG = int(os.environ.get("DEG", "8")) + REPS = int(os.environ.get("REPS", "10")) + NSEEDS = int(os.environ.get("SEEDS", "1")) + out = os.environ.get("OUT") + outf = open(out, "a") if out else None + + seeds = pd.DataFrame({"id": list(range(NSEEDS))}) + print(f"{'engine':11} {'N':>9} {'edges':>10} {'task':5} " + f"{'scan_ms':>10} {'build_ms':>10} {'warm_ms':>10} {'speedup':>9} {'rows':>7}") + for N in NS: + ndf, edf = make_graph(N, DEG) + g0 = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + for engine in engines: + try: + # coerce once; build index once (cold timing) + t0 = time.perf_counter() + gi = g0.gfql_index_all(engine=engine) + _sync(engine) + build_ms = (time.perf_counter() - t0) * 1e3 + except Exception as ex: + print(f"{engine:11} {N:>9} build FAILED {type(ex).__name__}: {ex}") + continue + E = n_rows(gi) + for task, kw in (("SEL1", dict(hops=1)), ("SEL2", dict(hops=2))): + kw = dict(kw, direction="forward") + try: + scan = timeit(lambda: g0.hop(nodes=seeds, engine=engine, **kw), engine, REPS) + warm = timeit(lambda: gi.hop(nodes=seeds, engine=engine, **kw), engine, REPS) + rows = n_rows(gi.hop(nodes=seeds, engine=engine, **kw)) + except Exception as ex: + print(f"{engine:11} {N:>9} {task} FAILED {type(ex).__name__}: {ex}") + continue + sp = scan[0] / warm[0] if warm[0] > 0 else float("inf") + print(f"{engine:11} {N:>9} {E:>10} {task:5} " + f"{scan[0]:>10.3f} {build_ms:>10.1f} {warm[0]:>10.4f} {sp:>8.1f}x {rows:>7}") + if outf: + outf.write(json.dumps(dict( + engine=engine, n=N, edges=E, task=task, deg=DEG, seeds=NSEEDS, + scan_ms=scan[0], build_ms=build_ms, warm_ms=warm[0], + scan_min=scan[1], warm_min=warm[1], speedup=sp, rows=rows, reps=REPS, + )) + "\n") + outf.flush() + if outf: + outf.close() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gfql/index_smoke.py b/benchmarks/gfql/index_smoke.py new file mode 100644 index 0000000000..f0633cbcff --- /dev/null +++ b/benchmarks/gfql/index_smoke.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Differential parity smoke test for GFQL physical indexes. + +Oracle: the index fast path MUST return the same subgraph (node-id set + edge +multiset) as the scan/join path, across engines / directions / hops / wavefront. +A fast wrong answer is not a win. + +Run (dgx-spark container): + python3 benchmarks/gfql/index_smoke.py +""" +from __future__ import annotations + +import sys +import numpy as np +import pandas as pd + +import graphistry + + +def make_graph(n_nodes=2000, avg_deg=6, seed=0): + rng = np.random.default_rng(seed) + m = n_nodes * avg_deg + src = rng.integers(0, n_nodes, size=m) + dst = rng.integers(0, n_nodes, size=m) + edf = pd.DataFrame({"src": src, "dst": dst, "w": rng.random(m)}) + ndf = pd.DataFrame({"id": np.arange(n_nodes), "label": rng.integers(0, 5, size=n_nodes)}) + return ndf, edf + + +def _to_pdf(df): + if df is None: + return None + mod = type(df).__module__ + if "cudf" in mod: + return df.to_pandas() + if "polars" in mod: + return df.to_pandas() + return df + + +def result_signature(g, node_col, src, dst): + n = _to_pdf(g._nodes) + e = _to_pdf(g._edges) + node_ids = sorted(n[node_col].tolist()) if n is not None else [] + edge_pairs = sorted(map(tuple, e[[src, dst]].itertuples(index=False, name=None))) if e is not None else [] + return node_ids, edge_pairs + + +def available_engines(): + engines = ["pandas"] + try: + import cudf # noqa + engines.append("cudf") + except Exception: + pass + try: + import polars # noqa + engines.append("polars") + # polars-gpu only if cudf-polars present + try: + import cudf # noqa + engines.append("polars-gpu") + except Exception: + pass + except Exception: + pass + return engines + + +SCENARIOS = [ + dict(hops=1, direction="forward", return_as_wave_front=False), + dict(hops=1, direction="reverse", return_as_wave_front=False), + dict(hops=1, direction="undirected", return_as_wave_front=False), + dict(hops=2, direction="forward", return_as_wave_front=False), + dict(hops=2, direction="undirected", return_as_wave_front=False), + dict(hops=1, direction="forward", return_as_wave_front=True), + dict(hops=2, direction="forward", return_as_wave_front=True), + dict(hops=3, direction="forward", return_as_wave_front=False), +] + + +def main(): + ndf, edf = make_graph() + g0 = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": [0, 1, 2, 7, 42, 100]}) + + engines = available_engines() + print(f"engines: {engines}") + total = 0 + fails = 0 + for engine in engines: + gi = g0.gfql_index_edges("both", engine=engine) + for sc in SCENARIOS: + total += 1 + try: + base = g0.hop(nodes=seeds, engine=engine, **sc) + idx = gi.hop(nodes=seeds, engine=engine, **sc) + except NotImplementedError as ex: + print(f" SKIP {engine:10} {sc} :: NIE {ex}") + total -= 1 + continue + except Exception as ex: + fails += 1 + print(f" ERROR {engine:10} {sc} :: {type(ex).__name__}: {ex}") + continue + bn, be = result_signature(base, "id", "src", "dst") + xn, xe = result_signature(idx, "id", "src", "dst") + ok = (bn == xn) and (be == xe) + if not ok: + fails += 1 + print(f" FAIL {engine:10} {sc}") + print(f" nodes base={len(bn)} idx={len(xn)} eq={bn==xn}; edges base={len(be)} idx={len(xe)} eq={be==xe}") + # show small diff + sb, sx = set(bn), set(xn) + print(f" node only-base={sorted(sb-sx)[:8]} only-idx={sorted(sx-sb)[:8]}") + else: + print(f" OK {engine:10} {sc} :: nodes={len(bn)} edges={len(be)}") + print(f"\n=== {total-fails}/{total} passed, {fails} failed ===") + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/gfql/index_takeover_bench.py b/benchmarks/gfql/index_takeover_bench.py new file mode 100644 index 0000000000..6919d9902a --- /dev/null +++ b/benchmarks/gfql/index_takeover_bench.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Adversarial CSR-index bench (takeover). Trust nothing: every timed cell is +GUARDED by (a) the index path was actually taken (index_trace) and (b) the index +result == the scan result. A cell that fails either guard is reported as INVALID, +never as a speedup. + +GFQL: index-vs-scan seeded latency, flat-in-N check, 4 engines. +Optional vs-DB: kuzu (embedded, CSR) if installed. + +Env: NS=800000,8000000,80000000 DEG=8 REPS=15 ENGINES=pandas,polars,cudf,polars-gpu + SYSTEMS=gfql,kuzu OUT=/tmp/idx-bench.jsonl +""" +from __future__ import annotations +import json, os, statistics, time +import numpy as np +import pandas as pd +import graphistry +from graphistry.compute.gfql.index import index_trace + + +def make_graph(n, deg, seed=0): + rng = np.random.default_rng(seed) + m = n * deg + return (pd.DataFrame({"id": np.arange(n, dtype=np.int64)}), + pd.DataFrame({"src": rng.integers(0, n, m, dtype=np.int64), + "dst": rng.integers(0, n, m, dtype=np.int64)})) + + +def _sync(engine): + if engine in ("cudf", "polars-gpu"): + try: + import cupy as cp + cp.cuda.runtime.deviceSynchronize() + except Exception: + pass + + +def timeit(fn, reps, engine, warmup=2): + for _ in range(warmup): + fn(); _sync(engine) + ts = [] + for _ in range(reps): + t0 = time.perf_counter(); fn(); _sync(engine) + ts.append((time.perf_counter() - t0) * 1e3) + ts.sort() + return statistics.median(ts) + + +def _nodeset_edgeset(g): + n, e = g._nodes, g._edges + nm, em = type(n).__module__, type(e).__module__ + if "cudf" in nm or "polars" in nm: n = n.to_pandas() + if "cudf" in em or "polars" in em: e = e.to_pandas() + return (len(n), len(e), + int(n["id"].sum()), int(e["src"].sum()) + int(e["dst"].sum())) + + +def bench_gfql(ndf, edf, N, E, engines, reps, outf): + g0 = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": [0]}) # single seed -> tiny frontier, beats cost gate + for engine in engines: + try: + t0 = time.perf_counter() + gi = g0.gfql_index_all(engine=engine); _sync(engine) + build_ms = (time.perf_counter() - t0) * 1e3 + except Exception as ex: + print(f" gfql {engine:11} BUILD FAILED: {type(ex).__name__}: {ex}"); continue + for task, kw in (("SEL1", dict(hops=1, direction="forward")), + ("SEL2", dict(hops=2, direction="forward"))): + # --- correctness + path GUARD (adversarial) --- + with index_trace() as steps: + idx_g = gi.hop(nodes=seeds, engine=engine, **kw) + took_index = any(s.get("path") == "index" for s in steps) + scan_g = g0.hop(nodes=seeds, engine=engine, **kw) # no index resident -> scan + same = _nodeset_edgeset(idx_g) == _nodeset_edgeset(scan_g) + valid = took_index and same + if not valid: + print(f" gfql {engine:11} {task} INVALID: index_path={took_index} result_match={same} " + f"(idx={_nodeset_edgeset(idx_g)} scan={_nodeset_edgeset(scan_g)})") + warm_idx = timeit(lambda: gi.hop(nodes=seeds, engine=engine, **kw), reps, engine) + warm_scan = timeit(lambda: g0.hop(nodes=seeds, engine=engine, **kw), reps, engine) + rows = _nodeset_edgeset(idx_g)[1] + speedup = warm_scan / warm_idx if warm_idx else float("nan") + rec = dict(system="gfql", engine=engine, task=task, n=N, edges=E, valid=valid, + took_index=took_index, result_match=same, warm_idx_ms=warm_idx, + warm_scan_ms=warm_scan, speedup=speedup, build_ms=build_ms, rows=rows) + tag = "" if valid else " <<< INVALID" + print(f" gfql {engine:11} {task} idx={warm_idx:8.4f}ms scan={warm_scan:10.3f}ms " + f"speedup={speedup:6.1f}x rows={rows} build={build_ms:.0f}ms{tag}") + if outf: outf.write(json.dumps(rec) + "\n"); outf.flush() + + +def bench_kuzu(ndf, edf, N, E, reps, outf): + try: + import kuzu + except Exception: + print(" kuzu: NOT INSTALLED (pip install kuzu)"); return + import shutil, tempfile + dbp = os.path.join(tempfile.gettempdir(), f"kuzu_bench_{N}") + shutil.rmtree(dbp, ignore_errors=True) + db = kuzu.Database(dbp); conn = kuzu.Connection(db) + conn.execute("CREATE NODE TABLE Node(id INT64, PRIMARY KEY(id))") + conn.execute("CREATE REL TABLE E(FROM Node TO Node)") + npath = os.path.join(tempfile.gettempdir(), f"n_{N}.parquet") + epath = os.path.join(tempfile.gettempdir(), f"e_{N}.parquet") + ndf.to_parquet(npath); edf.rename(columns={"src": "from", "dst": "to"}).to_parquet(epath) + t0 = time.perf_counter() + conn.execute(f'COPY Node FROM "{npath}"') + conn.execute(f'COPY E FROM "{epath}"') + build_ms = (time.perf_counter() - t0) * 1e3 + for task, q in (("SEL1", "MATCH (a:Node)-[:E]->(b:Node) WHERE a.id=0 RETURN b.id"), + ("SEL2", "MATCH (a:Node)-[:E*1..2]->(b:Node) WHERE a.id=0 RETURN DISTINCT b.id")): + def run(): return conn.execute(q) + rows = 0 + res = run() + while res.has_next(): res.get_next(); rows += 1 + warm = timeit(run, reps, "cpu") + print(f" kuzu {'':11} {task} warm={warm:8.4f}ms rows={rows} build={build_ms:.0f}ms") + if outf: outf.write(json.dumps(dict(system="kuzu", engine="kuzu", task=task, n=N, + edges=E, warm_idx_ms=warm, build_ms=build_ms, rows=rows)) + "\n"); outf.flush() + + +def main(): + NS = [int(x) for x in os.environ.get("NS", "800000,8000000").split(",")] + DEG = int(os.environ.get("DEG", "8")) + REPS = int(os.environ.get("REPS", "15")) + ENGINES = os.environ.get("ENGINES", "pandas,polars").split(",") + SYSTEMS = os.environ.get("SYSTEMS", "gfql,kuzu").split(",") + OUT = os.environ.get("OUT", "") + outf = open(OUT, "w") if OUT else None + for N in NS: + E = N * DEG + print(f"\n===== N={N:,} nodes E={E:,} edges deg={DEG} =====") + ndf, edf = make_graph(N, DEG) + if "gfql" in SYSTEMS: + bench_gfql(ndf, edf, N, E, ENGINES, REPS, outf) + if "kuzu" in SYSTEMS: + bench_kuzu(ndf, edf, N, E, REPS, outf) + if outf: outf.close() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gfql/index_vs_dbs.py b/benchmarks/gfql/index_vs_dbs.py new file mode 100644 index 0000000000..cb52cf2df1 --- /dev/null +++ b/benchmarks/gfql/index_vs_dbs.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Head-to-head: GFQL physical index vs kuzu (CSR) vs neo4j on identical graphs. + +Seeded SEL1 (1-hop) / SEL2 (2-hop) from one seed. All engines MATERIALIZE the +result (fair apples-to-apples — the method-guard from the pygraphistry2 bench). +GFQL is measured in two modes: + - on-the-fly (inclusive): build index + warm query, amortized=1 (cold) [index_policy=force-ish] + - preindexed (warm) : resident index, warm query only + +Env: NS=100000,1000000 DEG=8 REPS=20 GFQL_ENGINES=pandas,polars,cudf,polars-gpu + SYSTEMS=gfql,kuzu,neo4j NEO4J_URI=bolt://localhost:7688 NEO4J_USER/NEO4J_PASS + OUT=/results/vs-dbs.jsonl +""" +from __future__ import annotations + +import json +import os +import statistics +import time +import numpy as np +import pandas as pd + +import graphistry + + +def make_graph(n, deg, seed=0): + rng = np.random.default_rng(seed) + m = n * deg + src = rng.integers(0, n, size=m, dtype=np.int64) + dst = rng.integers(0, n, size=m, dtype=np.int64) + return (pd.DataFrame({"id": np.arange(n, dtype=np.int64)}), + pd.DataFrame({"src": src, "dst": dst})) + + +def _sync(engine): + if engine in ("cudf", "polars-gpu"): + try: + import cupy as cp # type: ignore + cp.cuda.runtime.deviceSynchronize() + except Exception: + pass + + +def timeit(fn, reps, engine="cpu", warmup=2): + for _ in range(warmup): + fn(); _sync(engine) + ts = [] + for _ in range(reps): + t0 = time.perf_counter(); fn(); _sync(engine) + ts.append((time.perf_counter() - t0) * 1e3) + ts.sort() + return statistics.median(ts) + + +def emit(outf, rec): + print(f" {rec['system']:14} {rec['engine']:11} {rec['task']:5} " + f"warm={rec['warm_ms']:>9.4f}ms cold(build+q)={rec.get('cold_ms', float('nan')):>10.2f}ms rows={rec['rows']}") + if outf: + outf.write(json.dumps(rec) + "\n"); outf.flush() + + +def run_gfql(ndf, edf, N, E, seeds, reps, engines, outf): + g0 = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + for engine in engines: + try: + t0 = time.perf_counter() + gi = g0.gfql_index_all(engine=engine); _sync(engine) + build_ms = (time.perf_counter() - t0) * 1e3 + except Exception as ex: + print(f" gfql {engine} build FAILED: {type(ex).__name__}: {ex}"); continue + for task, kw in (("SEL1", dict(hops=1)), ("SEL2", dict(hops=2))): + kw = dict(kw, direction="forward") + try: + warm = timeit(lambda: gi.hop(nodes=seeds, engine=engine, **kw), reps, engine) + rows = int(gi.hop(nodes=seeds, engine=engine, **kw)._edges.shape[0]) + except Exception as ex: + print(f" gfql {engine} {task} FAILED: {type(ex).__name__}: {ex}"); continue + emit(outf, dict(system="gfql", engine=engine, task=task, n=N, edges=E, + warm_ms=warm, cold_ms=build_ms + warm, build_ms=build_ms, rows=rows)) + + +def run_kuzu(ndf, edf, N, E, reps, outf, tmpdir): + try: + import kuzu + except Exception: + print(" kuzu: NOT AVAILABLE (pip install kuzu)"); return + import shutil + dbp = os.path.join(tmpdir, f"kuzu_{N}") + if os.path.exists(dbp): + shutil.rmtree(dbp) + ncsv = os.path.join(tmpdir, "kn.csv"); ecsv = os.path.join(tmpdir, "ke.csv") + ndf.to_csv(ncsv, index=False, header=False) + edf.to_csv(ecsv, index=False, header=False) + t0 = time.perf_counter() + db = kuzu.Database(dbp); conn = kuzu.Connection(db) + conn.execute("CREATE NODE TABLE N(id INT64, PRIMARY KEY(id))") + conn.execute("CREATE REL TABLE E(FROM N TO N)") + conn.execute(f'COPY N FROM "{ncsv}"') + conn.execute(f'COPY E FROM "{ecsv}"') + load_ms = (time.perf_counter() - t0) * 1e3 + queries = { + "SEL1": "MATCH (a:N {id:0})-[:E]->(b:N) RETURN b.id", + "SEL2": "MATCH (a:N {id:0})-[:E]->()-[:E]->(b:N) RETURN b.id", + } + for task, q in queries.items(): + try: + for _ in range(2): + conn.execute(q).get_as_df() + ts = [] + for _ in range(reps): + t = time.perf_counter(); df = conn.execute(q).get_as_df() + ts.append((time.perf_counter() - t) * 1e3) + ts.sort() + rows = len(df) + emit(outf, dict(system="kuzu", engine="kuzu", task=task, n=N, edges=E, + warm_ms=statistics.median(ts), cold_ms=load_ms, build_ms=load_ms, rows=rows)) + except Exception as ex: + print(f" kuzu {task} FAILED: {type(ex).__name__}: {ex}") + + +def run_neo4j(ndf, edf, N, E, reps, outf): + uri = os.environ.get("NEO4J_URI") + if not uri: + print(" neo4j: skipped (no NEO4J_URI)"); return + try: + from neo4j import GraphDatabase + except Exception: + print(" neo4j: driver NOT AVAILABLE"); return + user = os.environ.get("NEO4J_USER", "neo4j"); pw = os.environ.get("NEO4J_PASS", "test") + drv = GraphDatabase.driver(uri, auth=(user, pw)) + node_ids = ndf["id"].tolist() + edge_rows = edf.values.tolist() + BATCH = 20000 + t0 = time.perf_counter() + with drv.session() as s: + s.run("MATCH (n) DETACH DELETE n") + s.run("CREATE INDEX n_id IF NOT EXISTS FOR (n:N) ON (n.id)") + s.run("CALL db.awaitIndexes(300)") + for i in range(0, len(node_ids), BATCH): + s.run("UNWIND $ids AS x CREATE (:N {id:x})", ids=node_ids[i:i + BATCH]) + for i in range(0, len(edge_rows), BATCH): + s.run("UNWIND $rows AS r MATCH (a:N {id:r[0]}),(b:N {id:r[1]}) CREATE (a)-[:E]->(b)", + rows=edge_rows[i:i + BATCH]) + s.run("CALL db.awaitIndexes(300)") + load_ms = (time.perf_counter() - t0) * 1e3 + qs = { + "SEL1": "MATCH (a:N {id:0})-[:E]->(b:N) RETURN b.id AS id", + "SEL2": "MATCH (a:N {id:0})-[:E]->()-[:E]->(b:N) RETURN b.id AS id", + } + with drv.session() as s: + for task, q in qs.items(): + try: + for _ in range(2): + list(s.run(q)) + ts = [] + for _ in range(reps): + t = time.perf_counter(); data = list(s.run(q)) + ts.append((time.perf_counter() - t) * 1e3) + ts.sort() + emit(outf, dict(system="neo4j", engine="neo4j", task=task, n=N, edges=E, + warm_ms=statistics.median(ts), cold_ms=load_ms, build_ms=load_ms, rows=len(data))) + except Exception as ex: + print(f" neo4j {task} FAILED: {type(ex).__name__}: {ex}") + drv.close() + + +def main(): + NS = [int(x) for x in os.environ.get("NS", "100000,1000000").split(",")] + DEG = int(os.environ.get("DEG", "8")) + REPS = int(os.environ.get("REPS", "20")) + engines = os.environ.get("GFQL_ENGINES", "pandas,polars,cudf,polars-gpu").split(",") + systems = os.environ.get("SYSTEMS", "gfql,kuzu,neo4j").split(",") + out = os.environ.get("OUT") + tmpdir = os.environ.get("TMPDIR_BENCH", "/tmp/idxbench") + os.makedirs(tmpdir, exist_ok=True) + outf = open(out, "a") if out else None + + seeds = pd.DataFrame({"id": [0]}) + for N in NS: + ndf, edf = make_graph(N, DEG) + E = N * DEG + print(f"\n=== N={N} edges={E} ===") + if "gfql" in systems: + run_gfql(ndf, edf, N, E, seeds, REPS, engines, outf) + if "kuzu" in systems: + run_kuzu(ndf, edf, N, E, REPS, outf, tmpdir) + if "neo4j" in systems: + run_neo4j(ndf, edf, N, E, REPS, outf) + if outf: + outf.close() + + +if __name__ == "__main__": + main() diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 03b135f122..f8b73036b9 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -581,6 +581,28 @@ def hop(self, *args, **kwargs): return hop_base(self, *args, **kwargs) hop.__doc__ = hop_base.__doc__ + # ---- GFQL physical indexes (pay-as-you-go seeded-traversal acceleration) ---- + def create_index(self, kind, *, column=None, name=None, engine='auto'): + from graphistry.compute.gfql.index import create_index as _ci + return _ci(self, kind, column=column, name=name, engine=engine) + create_index.__doc__ = "Build a GFQL physical index (edge_out_adj|edge_in_adj|node_id). See graphistry/compute/gfql/index/api.py." + + def drop_index(self, kind=None): + from graphistry.compute.gfql.index import drop_index as _di + return _di(self, kind) + + def show_indexes(self): + from graphistry.compute.gfql.index import show_indexes as _si + return _si(self) + + def gfql_index_edges(self, direction='both', engine='auto'): + from graphistry.compute.gfql.index import gfql_index_edges as _gie + return _gie(self, direction, engine=engine) + + def gfql_index_all(self, engine='auto'): + from graphistry.compute.gfql.index import gfql_index_all as _gia + return _gia(self, engine=engine) + def filter_nodes_by_dict(self, *args, **kwargs): return filter_nodes_by_dict_base(self, *args, **kwargs) filter_nodes_by_dict.__doc__ = filter_nodes_by_dict_base.__doc__ @@ -604,9 +626,36 @@ def chain(self, *args, **kwargs): chain.__doc__ = (chain.__doc__ or "") + "\n\n" + (chain_base.__doc__ or "") def gfql(self, *args, **kwargs): - return gfql_base(self, *args, **kwargs) + policy = kwargs.pop('index_policy', None) + # Route GFQL index DDL (Python wire op, JSON dict, or Cypher string) to the + # registry without touching the traversal executor. + query = args[0] if args else kwargs.get('query') + from graphistry.compute.gfql.index.wire import ( + is_index_op, is_index_op_json, index_op_from_json, apply_index_op, + ) + from graphistry.compute.gfql.index.cypher_ddl import parse_index_ddl + op = None + if is_index_op(query): + op = query + elif is_index_op_json(query): + op = index_op_from_json(query) + elif isinstance(query, str): + op = parse_index_ddl(query) + if op is not None: + return apply_index_op(self, op, engine=kwargs.get('engine', 'auto')) + + g = self + if policy is not None: + import copy as _copy + g = _copy.copy(self) + g._gfql_index_policy = policy + return gfql_base(g, *args, **kwargs) gfql.__doc__ = gfql_base.__doc__ + def gfql_explain(self, query, *, index_policy='use', engine='auto'): + from graphistry.compute.gfql.index.explain import gfql_explain as _ge + return _ge(self, query, index_policy=index_policy, engine=engine) + def gfql_validate(self, *args, **kwargs): return gfql_validate_base(self, *args, **kwargs) gfql_validate.__doc__ = gfql_validate_base.__doc__ diff --git a/graphistry/compute/gfql/index/__init__.py b/graphistry/compute/gfql/index/__init__.py new file mode 100644 index 0000000000..3166d28c6b --- /dev/null +++ b/graphistry/compute/gfql/index/__init__.py @@ -0,0 +1,33 @@ +"""GFQL physical indexes — pay-as-you-go adjacency/node-id indexes for fast +seeded traversal. + +Public surface (see api.py): ``create_index``, ``drop_index``, ``show_indexes``, +``gfql_index_edges``, ``gfql_index_all``, and the planner entry ``maybe_index_hop``. +These are wired onto Plottable via ComputeMixin. +""" +from .registry import ( + GfqlIndexRegistry, EMPTY_REGISTRY, + EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, ADJ_KINDS, ALL_KINDS, + AdjacencyIndex, NodeIdIndex, +) +from .api import ( + create_index, drop_index, show_indexes, gfql_index_edges, gfql_index_all, + get_registry, maybe_index_hop, index_name, REGISTRY_ATTR, index_trace, +) +from .wire import ( + CreateIndex, DropIndex, ShowIndexes, apply_index_op, index_op_from_json, + is_index_op, is_index_op_json, +) +from .cypher_ddl import parse_index_ddl, looks_like_index_ddl + +__all__ = [ + "GfqlIndexRegistry", "EMPTY_REGISTRY", + "EDGE_OUT_ADJ", "EDGE_IN_ADJ", "NODE_ID", "ADJ_KINDS", "ALL_KINDS", + "AdjacencyIndex", "NodeIdIndex", + "create_index", "drop_index", "show_indexes", "gfql_index_edges", + "gfql_index_all", "get_registry", "maybe_index_hop", "index_name", + "REGISTRY_ATTR", "index_trace", + "CreateIndex", "DropIndex", "ShowIndexes", "apply_index_op", + "index_op_from_json", "is_index_op", "is_index_op_json", + "parse_index_ddl", "looks_like_index_ddl", +] diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py new file mode 100644 index 0000000000..bf027aae24 --- /dev/null +++ b/graphistry/compute/gfql/index/api.py @@ -0,0 +1,337 @@ +"""Public-facing index lifecycle + planner entry, operating on a Plottable. + +The registry rides on the Plottable as a private attribute (``_gfql_index_registry``) +and propagates through ``copy.copy``-based functional chaining. It is fingerprint- +validated at use time, so a rebind of ``.edges()``/``.nodes()`` safely invalidates +stale indexes (treated as absent, never a wrong answer). +""" +from __future__ import annotations + +import copy +from typing import Any, Optional, Union + +from graphistry.Engine import EngineAbstract, Engine, resolve_engine +from graphistry.Plottable import Plottable +from .registry import ( + GfqlIndexRegistry, EMPTY_REGISTRY, + EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, ADJ_KINDS, ALL_KINDS, +) +from .build import build_adjacency_index, build_node_id_index +from .traverse import index_seeded_hop + +REGISTRY_ATTR = "_gfql_index_registry" + +# --- lightweight, thread-local index decision trace (for gfql_explain) ------- +import threading as _threading +_TRACE = _threading.local() + + +class index_trace: + """Context manager: capture the per-hop index-vs-scan decisions made inside.""" + def __enter__(self): + self.steps = [] + self.prev = getattr(_TRACE, "steps", None) + _TRACE.steps = self.steps + return self.steps + + def __exit__(self, *exc): + _TRACE.steps = self.prev + return False + + +def _record(decision: dict) -> None: + steps = getattr(_TRACE, "steps", None) + if steps is not None: + steps.append(decision) + + +def get_registry(g: Plottable) -> GfqlIndexRegistry: + return getattr(g, REGISTRY_ATTR, EMPTY_REGISTRY) + + +def _attach(g: Plottable, registry: GfqlIndexRegistry) -> Plottable: + res = copy.copy(g) + setattr(res, REGISTRY_ATTR, registry) + return res + + +def index_name(kind: str, column: Optional[str]) -> str: + return f"{kind}:{column}" if column else kind + + +def _check_column(column: Optional[str], expected: str, kind: str) -> None: + """A user-supplied ``column`` must match the binding the index keys on; a + different column would be a silent no-op (registry is one-index-per-kind in + v1), so reject it honestly rather than ignore it.""" + if column is not None and column != expected: + raise NotImplementedError( + f"GFQL index {kind!r} keys on the {expected!r} binding; a custom column " + f"({column!r}) is not supported yet. Re-bind the graph or omit `column`." + ) + + +def create_index( + g: Plottable, + kind: str, + *, + column: Optional[str] = None, + name: Optional[str] = None, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, +) -> Plottable: + """Eagerly build a GFQL physical index and return a new Plottable carrying it. + + ``kind``: 'edge_out_adj' | 'edge_in_adj' | 'node_id'. ``column`` (if given) must + match the index's natural binding (src/dst/node) — a mismatch raises rather than + silently no-op. ``name`` overrides the display handle. Pay-as-you-go: cost is + O(E log E) once, amortized over later seeded queries. + """ + from dataclasses import replace + eng = resolve_engine(engine, g) + # Build over frames already in the target engine so the index arrays land on + # the right backend (cupy for cudf, numpy otherwise). No-op when already in-engine. + from graphistry.compute.ComputeMixin import _coerce_input_formats + g = _coerce_input_formats(g, eng) + registry = get_registry(g) + + if kind in ADJ_KINDS: + src, dst = g._source, g._destination + if src is None or dst is None or g._edges is None: + raise ValueError( + "edge adjacency index requires bound edges with source/destination columns" + ) + key_col = src if kind == EDGE_OUT_ADJ else dst + _check_column(column, key_col, kind) + other = dst if kind == EDGE_OUT_ADJ else src + idx = build_adjacency_index(g._edges, kind, key_col, other, g._edge, eng, (src, dst)) + idx = replace(idx, name=name or index_name(kind, key_col)) + registry = registry.with_index(kind, idx) + return _attach(g, registry) + + if kind == NODE_ID: + g2 = g.materialize_nodes() if g._nodes is None else g + node_col = g2._node + assert node_col is not None and g2._nodes is not None + _check_column(column, node_col, kind) + idx = build_node_id_index(g2._nodes, node_col, eng) + if idx is None: + raise ValueError( + f"Cannot build a {NODE_ID!r} index: node id column {node_col!r} has " + f"duplicate values (a node-id index requires unique ids). Seeded " + f"traversal still works via the un-indexed node materialization path." + ) + idx = replace(idx, name=name or index_name(kind, node_col)) + registry = registry.with_index(NODE_ID, idx) + return _attach(g2, registry) + + raise ValueError(f"Unknown GFQL index kind: {kind!r}. Expected one of {ALL_KINDS}.") + + +def drop_index(g: Plottable, kind: Optional[str] = None) -> Plottable: + """Drop one index (by kind) or all indexes (kind=None). Idempotent.""" + registry = get_registry(g) + if kind is None: + return _attach(g, EMPTY_REGISTRY) + return _attach(g, registry.without(kind)) + + +def show_indexes(g: Plottable) -> Any: + """Return a pandas DataFrame describing resident indexes (empty if none). + + ``valid`` reflects live fingerprint validity against the current frames — a + stale index (after a ``.edges()``/``.nodes()`` rebind) shows ``valid=False`` and + is auto-skipped (scan fallback) until rebuilt. ``nbytes`` is the resident + sidecar-array footprint (the pay-as-you-go memory signal). + """ + import pandas as pd + from .registry import index_nbytes + + registry = get_registry(g) + rows = [] + for kind in registry.kinds(): + idx = registry.get(kind) + if kind in ADJ_KINDS: + valid = registry.get_valid(kind, g._edges, (g._source, g._destination), idx.engine) is not None + n_keys, n_rows = idx.n_keys, idx.n_edges + else: # NODE_ID + valid = g._nodes is not None and registry.get_valid( + NODE_ID, g._nodes, (idx.key_col,), idx.engine) is not None + n_keys, n_rows = idx.n_nodes, 0 + rows.append({ + "name": idx.name or index_name(kind, idx.key_col), + "kind": kind, + "key_col": idx.key_col, + "engine": idx.engine.value, + "backend": idx.backend, + "n_keys": n_keys, + "n_rows": n_rows, + "nbytes": index_nbytes(idx), + "valid": valid, + }) + cols = ["name", "kind", "key_col", "engine", "backend", "n_keys", "n_rows", "nbytes", "valid"] + return pd.DataFrame(rows, columns=cols) + + +def gfql_index_edges(g: Plottable, direction: str = "both", + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO) -> Plottable: + """Convenience: build edge adjacency index(es). direction: 'forward'|'reverse'|'both'.""" + if direction in ("forward", "both"): + g = create_index(g, EDGE_OUT_ADJ, engine=engine) + if direction in ("reverse", "both"): + g = create_index(g, EDGE_IN_ADJ, engine=engine) + return g + + +def gfql_index_all(g: Plottable, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO) -> Plottable: + """Convenience: build out+in adjacency + (when ids are unique) node_id indexes. + + The node_id index is an optional materialization accelerator; if node ids aren't + unique it can't be built (a unique-key CSR can't reproduce the scan's all-rows- + per-id semantics), so this convenience SKIPS it rather than raising — seeded + traversal stays correct via the un-indexed node materialization path. (Explicit + ``create_index(NODE_ID)`` still raises, since the caller asked for it specifically.)""" + g = gfql_index_edges(g, "both", engine=engine) + try: + g = create_index(g, NODE_ID, engine=engine) + except ValueError: + pass # non-unique node ids -> skip the node_id accelerator (adjacency still built) + return g + + +# ---- planner entry --------------------------------------------------------- + +# Coverage: features the index fast path does NOT yet handle -> caller scans. +def _hop_is_index_coverable( + *, nodes, to_fixed_point, hops, min_hops, max_hops, + output_min_hops, output_max_hops, label_node_hops, label_edge_hops, + label_seeds, edge_match, source_node_match, destination_node_match, + source_node_query, destination_node_query, edge_query, + include_zero_hop_seed, target_wave_front, +) -> bool: + if nodes is None: + return False + if any(x is not None for x in ( + min_hops if (min_hops is not None and min_hops > 1) else None, + output_min_hops, output_max_hops, label_node_hops, label_edge_hops, + edge_match, source_node_match, destination_node_match, + source_node_query, destination_node_query, edge_query, target_wave_front, + )): + return False + if label_seeds or include_zero_hop_seed: + return False + if not to_fixed_point and (not isinstance(hops, int) or hops < 1): + return False + return True + + +def _ensure_indexes(g, registry, direction, engine, policy, nodes, src, dst, node_col): + """auto/force: build the indexes this seeded hop needs (opt-in pay-as-you-go). + + force => always build missing; auto => build only when the query looks + selective (frontier small vs E), else leave registry as-is (scan). + """ + needed = [] + if direction in ("forward", "undirected"): + needed.append(EDGE_OUT_ADJ) + if direction in ("reverse", "undirected"): + needed.append(EDGE_IN_ADJ) + if policy == "auto": + try: + E = int(g._edges.shape[0]) + F = int(nodes.shape[0]) + if not (F <= max(1024, 0.001 * E)): + return registry # not selective enough to amortize a build + except Exception: + return registry + for kind in needed: + if registry.get_valid(kind, g._edges, (src, dst), engine) is None: + if kind == EDGE_OUT_ADJ: + idx = build_adjacency_index(g._edges, kind, src, dst, g._edge, engine, (src, dst)) + else: + idx = build_adjacency_index(g._edges, kind, dst, src, g._edge, engine, (src, dst)) + registry = registry.with_index(kind, idx) + if registry.get_valid(NODE_ID, g._nodes, (node_col,), engine) is None: + node_idx = build_node_id_index(g._nodes, node_col, engine) + if node_idx is not None: # None => non-unique ids; skip (scan materialization) + registry = registry.with_index(NODE_ID, node_idx) + return registry + + +def maybe_index_hop( + g: Plottable, engine: Engine, *, nodes, hops, direction, return_as_wave_front, + to_fixed_point=False, policy: str = "use", **rest, +) -> Optional[Plottable]: + """Planner entry called from hop(). Returns an index-built subgraph, or None to + fall back to the scan/join path. + + Cost gate: only route to the index when (a) a valid matching index is resident + (or buildable under auto/force), (b) the query is covered, (c) the frontier is + not so large that a full scan is cheaper. Correctness is identical either way. + """ + if policy == "off": + return None + registry = get_registry(g) + if registry.is_empty() and policy not in ("auto", "force"): + return None + if not _hop_is_index_coverable( + nodes=nodes, to_fixed_point=to_fixed_point, hops=hops, + min_hops=rest.get("min_hops"), max_hops=rest.get("max_hops"), + output_min_hops=rest.get("output_min_hops"), + output_max_hops=rest.get("output_max_hops"), + label_node_hops=rest.get("label_node_hops"), + label_edge_hops=rest.get("label_edge_hops"), + label_seeds=rest.get("label_seeds", False), + edge_match=rest.get("edge_match"), + source_node_match=rest.get("source_node_match"), + destination_node_match=rest.get("destination_node_match"), + source_node_query=rest.get("source_node_query"), + destination_node_query=rest.get("destination_node_query"), + edge_query=rest.get("edge_query"), + include_zero_hop_seed=rest.get("include_zero_hop_seed", False), + target_wave_front=rest.get("target_wave_front"), + ): + return None + + node_col = g._node + src, dst = g._source, g._destination + if node_col is None or src is None or dst is None or g._edges is None or g._nodes is None: + return None + + if policy in ("auto", "force"): + registry = _ensure_indexes(g, registry, direction, engine, policy, nodes, src, dst, node_col) + if registry.is_empty(): + return None + + # Cost gate: if the frontier covers a large fraction of distinct sources, the + # scan path is competitive — fall back (avoids index overhead on bulk-ish hops). + idx0 = registry.get_valid( + EDGE_OUT_ADJ if direction != "reverse" else EDGE_IN_ADJ, g._edges, (src, dst), engine + ) + if idx0 is None: + # required direction not resident (undirected needs both); let driver decide + pass + elif policy != "force": + try: + frontier_n = int(nodes.shape[0]) + if idx0.n_keys > 0 and frontier_n >= 0.5 * idx0.n_keys: + return None + except Exception: + pass + + # Honor max_hops: the scan resolves the hop count as ``max_hops or hops`` + # (compute/hop.py); the index must run the SAME number of accumulating hops. + # (B1: max_hops was passed through *rest and silently ignored — the index ran + # `hops` (default 1) while the scan ran max_hops → wrong answer.) + _max_hops = rest.get("max_hops") + eff_hops = _max_hops if _max_hops is not None else hops + result = index_seeded_hop( + g, registry, nodes=nodes, node_col=node_col, src=src, dst=dst, engine=engine, + hops=eff_hops, to_fixed_point=to_fixed_point, direction=direction, + return_as_wave_front=return_as_wave_front, + ) + _record({ + "op": "hop", "direction": direction, "hops": eff_hops, + "path": "index" if result is not None else "scan", + "policy": policy, "engine": engine.value, + }) + return result diff --git a/graphistry/compute/gfql/index/build.py b/graphistry/compute/gfql/index/build.py new file mode 100644 index 0000000000..88e3129ab7 --- /dev/null +++ b/graphistry/compute/gfql/index/build.py @@ -0,0 +1,99 @@ +"""Build CSR adjacency / node-id indexes from a graph's frames. + +Build cost is O(E log E) (one sort), paid once per resident graph. The result is +a set of sidecar arrays over edge **row positions** — the user's ``.edges`` frame +is never reordered. +""" +from __future__ import annotations + +from typing import Any, Optional, Tuple + +from graphistry.Engine import Engine +from .engine_arrays import array_namespace, col_to_array +from .registry import AdjacencyIndex, NodeIdIndex, frame_fingerprint + + +def _csr_from_keys(keys: Any, xp: Any) -> Tuple[Any, Any, Any]: + """(keys array over E rows) -> (unique_keys, group_offsets[U+1], row_positions[E]). + + row_positions = the original row indices grouped (contiguously) by key value. + Fully vectorized: one argsort + one boundary scan. + """ + E = int(keys.shape[0]) + if E == 0: + empty = keys[:0] + return empty, xp.zeros(1, dtype=xp.int64), xp.zeros(0, dtype=xp.int64) + order = xp.argsort(keys) # row positions sorted by key + sorted_keys = keys[order] + row_positions = order.astype(xp.int64) + change = xp.ones(E, dtype=bool) + change[1:] = sorted_keys[1:] != sorted_keys[:-1] + starts = xp.nonzero(change)[0].astype(xp.int64) + unique_keys = sorted_keys[starts] + group_offsets = xp.concatenate([starts, xp.asarray([E], dtype=xp.int64)]) + return unique_keys, group_offsets, row_positions + + +def build_adjacency_index( + edges: Any, + kind: str, + key_col: str, + other_col: str, + edge_id_col: Optional[str], + engine: Engine, + fingerprint_cols: Tuple[str, ...], +) -> AdjacencyIndex: + xp, backend = array_namespace(engine) + keys = col_to_array(edges, key_col, engine) + other_values = col_to_array(edges, other_col, engine) + unique_keys, group_offsets, row_positions = _csr_from_keys(keys, xp) + return AdjacencyIndex( + kind=kind, + key_col=key_col, + other_col=other_col, + edge_id_col=edge_id_col, + keys_sorted=unique_keys, + group_offsets=group_offsets, + row_positions=row_positions, + other_values=other_values, + backend=backend, + engine=engine, + fingerprint=frame_fingerprint(edges, fingerprint_cols, engine), + source_ref=edges, + n_edges=int(keys.shape[0]), + n_keys=int(unique_keys.shape[0]), + ) + + +def build_node_id_index( + nodes: Any, + node_col: str, + engine: Engine, +) -> Optional[NodeIdIndex]: + """Sorted node-id -> first-row index, or None when node ids are NOT unique. + + ``_csr_from_keys`` returns ``row_positions`` of length E (all rows, grouped by + key), but a node-id lookup indexes it with a *unique-key* searchsorted position + (0..U-1). Those align ONLY when keys are unique — so we (a) collapse to the FIRST + row position per unique key (``row_positions[group_offsets[:-1]]``, length U, + aligned with ``unique_keys``) and (b) REFUSE (return None) when ids aren't unique: + a unique-key CSR can't reproduce the scan's "all rows per id" semantics, so the + caller falls back to the correct ``select_by_ids`` isin path. (B2: a non-unique + node-id index dropped reached nodes / emitted unrelated rows.)""" + xp, backend = array_namespace(engine) + keys = col_to_array(nodes, node_col, engine) + unique_keys, group_offsets, row_positions = _csr_from_keys(keys, xp) + n_keys = int(unique_keys.shape[0]) + if n_keys != int(keys.shape[0]): + return None # duplicate node ids -> not a valid unique index; scan fallback + first_row_per_key = row_positions[group_offsets[:-1]] # length U, aligned to keys + return NodeIdIndex( + key_col=node_col, + keys_sorted=unique_keys, + row_positions=first_row_per_key, + backend=backend, + engine=engine, + fingerprint=frame_fingerprint(nodes, (node_col,), engine), + source_ref=nodes, + n_nodes=n_keys, + ) diff --git a/graphistry/compute/gfql/index/cypher_ddl.py b/graphistry/compute/gfql/index/cypher_ddl.py new file mode 100644 index 0000000000..0f95b50090 --- /dev/null +++ b/graphistry/compute/gfql/index/cypher_ddl.py @@ -0,0 +1,67 @@ +"""Targeted recognizer for GFQL index DDL Cypher statements. + + CREATE GFQL INDEX [] FOR [ON ] + DROP GFQL INDEX + DROP GFQL INDEX FOR [ON ] + SHOW GFQL INDEXES + +The mandatory ``GFQL`` token disambiguates from standard property ``CREATE INDEX`` +(which the GFQL grammar does not implement), so this fixed-form recognizer is +unambiguous and additive — it runs before the Earley parser and returns a wire op +or None (not-a-DDL -> normal query path). +""" +from __future__ import annotations + +import re +from typing import Any, Optional + +from .wire import CreateIndex, DropIndex, ShowIndexes + +_KIND = r"(?Pedge_out_adj|edge_in_adj|node_id)" + +_CREATE_RE = re.compile( + r"^\s*CREATE\s+GFQL\s+INDEX\s+(?:(?P[A-Za-z_]\w*)\s+)?FOR\s+" + _KIND + + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$", + re.IGNORECASE, +) +_DROP_FOR_RE = re.compile( + r"^\s*DROP\s+GFQL\s+INDEX\s+FOR\s+" + _KIND + + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$", + re.IGNORECASE, +) +_DROP_NAME_RE = re.compile( + r"^\s*DROP\s+GFQL\s+INDEX\s+(?P[A-Za-z_][\w:]*)\s*;?\s*$", + re.IGNORECASE, +) +_SHOW_RE = re.compile(r"^\s*SHOW\s+GFQL\s+INDEXES\s*;?\s*$", re.IGNORECASE) + +_DDL_PREFIX = re.compile(r"^\s*(CREATE|DROP|SHOW)\s+GFQL\s+INDEX", re.IGNORECASE) + + +def looks_like_index_ddl(query: str) -> bool: + return bool(isinstance(query, str) and _DDL_PREFIX.match(query)) + + +def parse_index_ddl(query: str) -> Optional[Any]: + """Return a wire op (CreateIndex/DropIndex/ShowIndexes) or None.""" + if not isinstance(query, str): + return None + if _SHOW_RE.match(query): + return ShowIndexes() + m = _CREATE_RE.match(query) + if m: + return CreateIndex(kind=m.group("kind").lower(), column=m.group("col"), + name=m.group("name")) + m = _DROP_FOR_RE.match(query) + if m: + return DropIndex(kind=m.group("kind").lower(), column=m.group("col")) + m = _DROP_NAME_RE.match(query) + if m: + return DropIndex(name=m.group("name")) + if looks_like_index_ddl(query): + raise ValueError( + f"Malformed GFQL INDEX DDL: {query!r}. Expected e.g. " + "'CREATE GFQL INDEX FOR edge_out_adj', 'DROP GFQL INDEX FOR edge_in_adj', " + "'SHOW GFQL INDEXES'." + ) + return None diff --git a/graphistry/compute/gfql/index/engine_arrays.py b/graphistry/compute/gfql/index/engine_arrays.py new file mode 100644 index 0000000000..8417f6102d --- /dev/null +++ b/graphistry/compute/gfql/index/engine_arrays.py @@ -0,0 +1,130 @@ +"""Engine-polymorphic array helpers for GFQL physical indexes. + +The index core (build + lookup) is written once against a tiny array protocol +(``searchsorted``, ``argsort``, ``cumsum``, ``arange``, ``repeat``) so the same +CSR + searchsorted gather runs on: + +- pandas -> numpy host arrays, ``df.iloc`` to gather rows +- cudf -> cupy device arrays, ``df.iloc`` to gather rows +- polars / polars-gpu -> numpy host arrays, polars row-gather + +Vectorization-first: no per-element Python work, no ``.to_list()`` ping-pong. +""" +from __future__ import annotations + +from typing import Any, Tuple + +from graphistry.Engine import Engine + + +def array_namespace(engine: Engine) -> Tuple[Any, str]: + """Return (array module, backend tag) for an engine. + + cudf indexes keep their arrays on-device (cupy); everything else uses numpy + host arrays. The frontier of a seeded query is tiny, so host-side + searchsorted is cheap even when the frame itself is on GPU (polars-gpu). + """ + import numpy as np + + if engine == Engine.CUDF: + try: + import cupy as cp # type: ignore + + return cp, "cupy" + except Exception: # pragma: no cover - cupy always present with cudf + return np, "numpy" + return np, "numpy" + + +def col_to_array(df: Any, col: str, engine: Engine) -> Any: + """Extract a column as a backend-native 1-D array (numpy or cupy).""" + if engine in (Engine.POLARS, Engine.POLARS_GPU): + return df.get_column(col).to_numpy() + if engine == Engine.CUDF: + # cudf Series -> cupy array (stays on device) + return df[col].values + return df[col].to_numpy() + + +def ids_to_array(ids: Any, col: str, engine: Engine) -> Any: + """Frontier ids (a frame/Series) -> backend array, matching index backend.""" + return col_to_array(ids, col, engine) + + +def to_backend(arr: Any, backend: str) -> Any: + """Move a host/device array to the index backend (numpy<->cupy).""" + import numpy as np + + if backend == "cupy": + import cupy as cp # type: ignore + + return cp.asarray(arr) + if "cupy" in type(arr).__module__: + return cp_to_numpy(arr) + return np.asarray(arr) + + +def cp_to_numpy(arr: Any) -> Any: + try: + return arr.get() + except Exception: + import numpy as np + + return np.asarray(arr) + + +def take_rows(df: Any, positions: Any, engine: Engine) -> Any: + """Positionally gather rows of ``df`` by an integer array ``positions``. + + ``positions`` is a backend array (numpy for pandas/polars, cupy for cudf). + Returns a frame of the same engine; row order follows ``positions``. + """ + if engine in (Engine.POLARS, Engine.POLARS_GPU): + import numpy as np + + idx = np.asarray(positions) + return df[idx] + # pandas / cudf: iloc accepts numpy (pandas) or cupy (cudf) int arrays + return df.iloc[positions] + + +def select_by_ids(df: Any, col: str, ids: Any, engine: Engine) -> Any: + """Return rows of ``df`` whose ``col`` is in the id array ``ids`` (set semantics, + preserves df row order). Engine-polymorphic, vectorized.""" + if engine in (Engine.POLARS, Engine.POLARS_GPU): + import numpy as np + import polars as pl + + # Semi-join (not Expr.is_in(Series), which polars 1.42 deprecates as ambiguous — + # pola-rs/polars#22149) — vectorized AND preserves the left (df) row order, which + # the node materialization relies on (table-order parity with the scan). + ids_df = pl.DataFrame({col: np.asarray(ids)}).cast({col: df.schema[col]}) + return df.join(ids_df.unique(), on=col, how="semi") + if engine == Engine.CUDF: + import cudf # type: ignore + + return df[df[col].isin(cudf.Series(ids))] + import numpy as np + + return df[df[col].isin(np.asarray(ids))] + + +def set_difference(cand: Any, visited: Any, xp: Any) -> Any: + """cand minus visited (both backend arrays), vectorized via sorted membership.""" + if int(visited.shape[0]) == 0: + return cand + if int(cand.shape[0]) == 0: + return cand + vs = xp.sort(visited) + pos = xp.searchsorted(vs, cand) + pos_c = xp.where(pos < vs.shape[0], pos, vs.shape[0] - 1) + ismember = vs[pos_c] == cand + return cand[~ismember] + + +def union1d(a: Any, b: Any, xp: Any) -> Any: + if int(a.shape[0]) == 0: + return xp.unique(b) + if int(b.shape[0]) == 0: + return xp.unique(a) + return xp.unique(xp.concatenate([a, b])) diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py new file mode 100644 index 0000000000..7203f8b9bc --- /dev/null +++ b/graphistry/compute/gfql/index/explain.py @@ -0,0 +1,33 @@ +"""gfql_explain — answer "did/would my query use the index?" honestly. + +Seeded queries are cheap once indexed, so explain *executes* the query under a +decision trace and reports the real per-hop index-vs-scan path (plus resident +indexes). This makes "it silently scanned" detectable and assertable rather than +a mystery — the top human-factors need from the design review (P0-1). +""" +from __future__ import annotations + +from typing import Any, Dict + +from graphistry.Engine import resolve_engine +from .api import index_trace, get_registry, show_indexes + + +def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = "auto") -> Dict[str, Any]: + eng = resolve_engine(engine, g) + resident = show_indexes(g) + with index_trace() as steps: + try: + g.gfql(query, engine=engine, index_policy=index_policy) + error = None + except Exception as ex: # report, don't raise — explain is diagnostic + error = f"{type(ex).__name__}: {ex}" + used_index = any(s.get("path") == "index" for s in steps) + return { + "engine": eng.value, + "index_policy": index_policy, + "resident_indexes": resident["name"].tolist() if len(resident) else [], + "steps": list(steps), + "used_index": used_index, + "error": error, + } diff --git a/graphistry/compute/gfql/index/lookup.py b/graphistry/compute/gfql/index/lookup.py new file mode 100644 index 0000000000..b2441da067 --- /dev/null +++ b/graphistry/compute/gfql/index/lookup.py @@ -0,0 +1,94 @@ +"""Vectorized CSR lookup — searchsorted membership + range-expansion gather. + +Given a frontier of seed ids, return the edge **row positions** of all incident +edges, with no full edge scan and no per-seed Python loop. Works identically on +numpy (pandas/polars) and cupy (cudf) arrays. +""" +from __future__ import annotations + +from typing import Any + +from .registry import AdjacencyIndex, NodeIdIndex + + +def lookup_edge_rows(index: AdjacencyIndex, frontier: Any, xp: Any): + """frontier (backend array of seed ids, deduped) -> (edge_rows, matched_ids). + + ``edge_rows`` = row positions of all edges incident to the frontier. + ``matched_ids`` = the subset of ``frontier`` that has >=1 incident edge + (needed to reproduce hop()'s first-hop visited semantics). + + Steps (all vectorized): + pos = searchsorted(keys, frontier) # candidate group per seed + hit = keys[pos] == frontier # membership verify + [start,end) = group_offsets[pos], [pos+1] # CSR slice per hit + flat = expand each [start,end) range # cumsum/arange/repeat trick + rows = row_positions[flat] + """ + keys = index.keys_sorted + empty = index.row_positions[:0] + U = int(keys.shape[0]) + if U == 0 or int(frontier.shape[0]) == 0: + return empty, frontier[:0] + + f = frontier + if f.dtype != keys.dtype: + # I6: promote BOTH sides to a common dtype — never narrow the query to the key + # dtype (an int64 id cast to int32 keys wraps and false-matches). Widening a + # sorted int array preserves order, so searchsorted stays valid. + common = xp.promote_types(f.dtype, keys.dtype) + f = f.astype(common) + keys = keys.astype(common) + + pos = xp.searchsorted(keys, f) + pos_clipped = xp.where(pos < U, pos, U - 1) + hit = keys[pos_clipped] == f + matched_ids = f[hit] + pos_hit = pos_clipped[hit] + if int(pos_hit.shape[0]) == 0: + return empty, matched_ids + + start = index.group_offsets[pos_hit] + end = index.group_offsets[pos_hit + 1] + counts = end - start + total = int(counts.sum()) + if total == 0: + return empty, matched_ids + + flat = _expand_ranges(start, counts, total, xp) + return index.row_positions[flat], matched_ids + + +def _expand_ranges(start: Any, counts: Any, total: int, xp: Any) -> Any: + """Vectorized [start, start+count) range concat WITHOUT np.repeat (cupy's + ``repeat`` rejects array ``repeats``). Builds a per-output segment id via a + boundary-marker cumsum, then gathers start/offset by segment. + + Precondition: every count >= 1 (CSR groups always have >=1 edge), so group + start offsets are strictly increasing and the boundary markers don't collide. + """ + out_off = xp.cumsum(counts) - counts # output start of each group + seg = xp.zeros(total, dtype=xp.int64) + if int(out_off.shape[0]) > 1: + seg[out_off[1:]] = 1 + seg = xp.cumsum(seg) # group index per output position + pos_in = xp.arange(total, dtype=xp.int64) - out_off[seg] + return start[seg] + pos_in + + +def lookup_node_rows(index: NodeIdIndex, ids: Any, xp: Any) -> Any: + """ids (backend array) -> node row positions for those that exist (in id order + of the index hits). Used to materialize node rows for a result id set.""" + keys = index.keys_sorted + U = int(keys.shape[0]) + if U == 0 or int(ids.shape[0]) == 0: + return index.row_positions[:0] + f = ids + if f.dtype != keys.dtype: + common = xp.promote_types(f.dtype, keys.dtype) # I6: promote, never narrow + f = f.astype(common) + keys = keys.astype(common) + pos = xp.searchsorted(keys, f) + pos_clipped = xp.where(pos < U, pos, U - 1) + hit = keys[pos_clipped] == f + return index.row_positions[pos_clipped[hit]] diff --git a/graphistry/compute/gfql/index/registry.py b/graphistry/compute/gfql/index/registry.py new file mode 100644 index 0000000000..3bac0559f8 --- /dev/null +++ b/graphistry/compute/gfql/index/registry.py @@ -0,0 +1,135 @@ +"""GFQL physical index registry — immutable, fingerprinted sidecars. + +The registry holds typed indexes (adjacency CSR, node-id) keyed by ``kind``. It +is attached to a Plottable as a private attribute and travels with it. Because +PyGraphistry is pure-functional, an index is only valid for the exact frame it +was built over; a cheap structural fingerprint (object id + length + bindings + +engine) detects when ``.edges()``/``.nodes()`` rebinding has invalidated it, in +which case the planner treats the index as absent (a safe miss, never a wrong +answer). +""" +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from typing import Any, Dict, Optional, Tuple + +from graphistry.Engine import Engine + +# Index kinds (v1). Property/label/type indexes share this registry shape later. +EDGE_OUT_ADJ = "edge_out_adj" +EDGE_IN_ADJ = "edge_in_adj" +NODE_ID = "node_id" + +ADJ_KINDS = (EDGE_OUT_ADJ, EDGE_IN_ADJ) +ALL_KINDS = (EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID) + + +def frame_fingerprint(df: Any, cols: Tuple[str, ...], engine: Engine) -> Tuple: + """Cheap O(1) structural fingerprint of the frame an index is built over: length + + bound columns + engine. This is a SECONDARY guard; the primary validity check + is object IDENTITY (``source_ref is df``, see ``get_valid``). We deliberately do + NOT use ``id(df)`` here — a GC'd frame's id can be recycled by a new same-shape + frame, which `id`-equality would accept (stale index → wrong answer, I5). Holding + a strong ref + identity is recycle-proof. (Pure-functional rebind via + ``.edges()``/``.nodes()`` yields a new object → identity miss → safe scan.)""" + try: + n = int(df.shape[0]) if df is not None else -1 + except Exception: + n = -1 + return (n, tuple(cols), engine.value) + + +@dataclass(frozen=True) +class AdjacencyIndex: + """CSR adjacency over edge **row positions**, keyed by one endpoint. + + Lookup of a frontier of ids is O(F log U + result) via searchsorted + + vectorized range expansion — sublinear in E, never a full edge scan. + """ + kind: str # EDGE_OUT_ADJ | EDGE_IN_ADJ + key_col: str # endpoint we key on (src for out, dst for in) + other_col: str # opposite endpoint (the neighbor we emit) + edge_id_col: Optional[str] # edge-id binding if present (else row pos == id) + keys_sorted: Any # distinct key ids, ascending (len U) [array] + group_offsets: Any # CSR offsets into row_positions (len U+1) [array] + row_positions: Any # edge row indices grouped by key (len E) [array] + other_values: Any # neighbor id per edge row, ORIGINAL order (len E) [array] + backend: str # 'numpy' | 'cupy' + engine: Engine + fingerprint: Tuple = field(compare=False, default=()) + source_ref: Any = field(compare=False, default=None) # the indexed frame (identity guard, I5) + n_edges: int = 0 + n_keys: int = 0 + name: Optional[str] = None + + +@dataclass(frozen=True) +class NodeIdIndex: + """Sorted node-id -> node row position (find seed/endpoint rows fast).""" + key_col: str + keys_sorted: Any + row_positions: Any + backend: str + engine: Engine + fingerprint: Tuple = field(compare=False, default=()) + source_ref: Any = field(compare=False, default=None) # the indexed frame (identity guard, I5) + n_nodes: int = 0 + name: Optional[str] = None + + +@dataclass(frozen=True) +class GfqlIndexRegistry: + """Immutable kind -> index map. ``with_index`` / ``without`` return copies.""" + indexes: Dict[str, Any] = field(default_factory=dict) + + def with_index(self, kind: str, index: Any) -> "GfqlIndexRegistry": + new = dict(self.indexes) + new[kind] = index + return GfqlIndexRegistry(new) + + def without(self, kind: str) -> "GfqlIndexRegistry": + new = dict(self.indexes) + new.pop(kind, None) + return GfqlIndexRegistry(new) + + def get(self, kind: str) -> Optional[Any]: + return self.indexes.get(kind) + + def has(self, kind: str) -> bool: + return kind in self.indexes + + def kinds(self) -> Tuple[str, ...]: + return tuple(sorted(self.indexes.keys())) + + def is_empty(self) -> bool: + return not self.indexes + + def get_valid(self, kind: str, df: Any, cols: Tuple[str, ...], engine: Engine) -> Optional[Any]: + """Return the index for ``kind`` only if its fingerprint still matches the + live frame + engine; else None (treat as absent).""" + idx = self.indexes.get(kind) + if idx is None: + return None + if idx.engine != engine: + return None + # Primary: object IDENTITY (I5) — recycle-proof, since the index holds a strong + # ref so the frame's id can't be reused while indexed. `is` on a rebound frame + # is False → safe miss. (source_ref None only for legacy/hand-built indexes.) + if idx.source_ref is not None and idx.source_ref is not df: + return None + if idx.fingerprint != frame_fingerprint(df, cols, engine): + return None + return idx + + +def index_nbytes(idx: Any) -> int: + """Approximate resident memory of an index's sidecar arrays (bytes).""" + total = 0 + for attr in ("keys_sorted", "group_offsets", "row_positions", "other_values"): + arr = getattr(idx, attr, None) + if arr is not None: + total += int(getattr(arr, "nbytes", 0)) + return total + + +EMPTY_REGISTRY = GfqlIndexRegistry() diff --git a/graphistry/compute/gfql/index/traverse.py b/graphistry/compute/gfql/index/traverse.py new file mode 100644 index 0000000000..459093e95f --- /dev/null +++ b/graphistry/compute/gfql/index/traverse.py @@ -0,0 +1,164 @@ +"""Index-driven seeded traversal — the O(degree) fast path. + +Replaces hop()'s O(E) ``edges[edges[src].isin(frontier)]`` scan with a CSR +searchsorted gather. Returns a subgraph Plottable parity-matched to the eager +hop() for the covered cases, or ``None`` when a feature isn't covered (caller +falls back to the scan/join path — correctness is never traded for speed). + +Covered (v1): seeded (nodes given), integer ``hops`` >= 1 or ``to_fixed_point``, +direction forward/reverse/undirected, ``return_as_wave_front``. Not covered +(returns None): edge/source/destination match or query, target_wave_front, +min_hops>1, output_min/max_hops, labeling, missing node table. +""" +from __future__ import annotations + +from typing import Any, List, Optional + +from graphistry.Engine import Engine +from graphistry.Plottable import Plottable +from .engine_arrays import ( + array_namespace, col_to_array, ids_to_array, take_rows, select_by_ids, + set_difference, union1d, +) +from .lookup import lookup_edge_rows, lookup_node_rows +from .registry import EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, GfqlIndexRegistry + + +def _indices_for_direction( + registry: GfqlIndexRegistry, direction: str, edges: Any, cols, engine: Engine +) -> Optional[List[Any]]: + out_idx = registry.get_valid(EDGE_OUT_ADJ, edges, cols, engine) + in_idx = registry.get_valid(EDGE_IN_ADJ, edges, cols, engine) + if direction == "forward": + chosen = [out_idx] + elif direction == "reverse": + chosen = [in_idx] + else: # undirected + chosen = [out_idx, in_idx] + if any(ix is None for ix in chosen): + return None + return chosen + + +def index_seeded_hop( + g: Plottable, + registry: GfqlIndexRegistry, + *, + nodes: Any, + node_col: str, + src: str, + dst: str, + engine: Engine, + hops: Optional[int], + to_fixed_point: bool, + direction: str, + return_as_wave_front: bool, +) -> Optional[Plottable]: + if nodes is None or g._edges is None or g._nodes is None: + return None + if not to_fixed_point and (not isinstance(hops, int) or hops < 1): + return None + + # Normalize the seed frame to the engine: the hop hooks can pass a pandas seeds + # frame even on engine='polars'/'cudf' (conversion happens later in the scan path), + # but col_to_array assumes engine-native frames. Convert here (seeds are small). + from graphistry.Engine import df_to_engine + seed_engine = Engine.POLARS if engine == Engine.POLARS_GPU else engine + nodes = df_to_engine(nodes, seed_engine) + + edges = g._edges + indices = _indices_for_direction(registry, direction, edges, (src, dst), engine) + if indices is None: + return None + + xp, _backend = array_namespace(engine) + + # I6: do NOT narrow the seed to the index key dtype (a node-id int64 seed cast to + # an int32 edge-endpoint key wraps large ids → false match). lookup promotes both + # sides to a common dtype; numpy/cupy set ops promote on concat. So we keep ids at + # their natural width throughout and only ever widen. + seed = xp.unique(ids_to_array(nodes, node_col, engine)) + + frontier = seed + visited = seed[:0] + edge_rows_parts: List[Any] = [] + first = True + hop_count = 0 + + while True: + if not to_fixed_point and hop_count >= hops: # type: ignore[operator] + break + if int(frontier.shape[0]) == 0: + break + hop_count += 1 + + matched_parts: List[Any] = [] + neigh_parts: List[Any] = [] + for ix in indices: + rows, matched = lookup_edge_rows(ix, frontier, xp) + edge_rows_parts.append(rows) + neigh_parts.append(ix.other_values[rows]) + matched_parts.append(matched) + + neighbors = neigh_parts[0] if len(neigh_parts) == 1 else xp.concatenate(neigh_parts) + if first and not return_as_wave_front: + matched_all = ( + matched_parts[0] if len(matched_parts) == 1 else xp.concatenate(matched_parts) + ) + visited = xp.unique(matched_all) + first = False + + cand = xp.unique(neighbors) + new_frontier = set_difference(cand, visited, xp) + visited = union1d(visited, new_frontier, xp) + frontier = new_frontier + + if edge_rows_parts: + concat_rows = ( + edge_rows_parts[0] + if len(edge_rows_parts) == 1 + else xp.concatenate(edge_rows_parts) + ) + all_rows = xp.unique(concat_rows) + else: + all_rows = indices[0].row_positions[:0] + + out_edges = take_rows(edges, all_rows, engine) + + needed = visited + materialize_endpoints = not return_as_wave_front # nodes is non-None here (guarded above) + if materialize_endpoints and int(all_rows.shape[0]) > 0: + src_vals = col_to_array(out_edges, src, engine) + dst_vals = col_to_array(out_edges, dst, engine) + endpoints = xp.unique(xp.concatenate([src_vals, dst_vals])) # natural dtype (I6) + needed = union1d(needed, endpoints, xp) + + # Materialize node rows. Prefer the node_id index (O(result·log N) searchsorted + # gather) over an O(N) isin scan — this keeps warm seeded latency flat in N. + node_idx = registry.get_valid(NODE_ID, g._nodes, (node_col,), engine) + if node_idx is not None: + node_rows = lookup_node_rows(node_idx, needed, xp) + # I4: lookup returns rows in id-hit order; sort ascending so out_nodes keep + # the original .nodes table order (the index must never reorder .nodes). + node_rows = xp.sort(node_rows) + out_nodes = take_rows(g._nodes, node_rows, engine) + else: + out_nodes = select_by_ids(g._nodes, node_col, needed, engine) + + # B3: the scan synthesizes a node row for EVERY edge endpoint, including ids + # absent from the node table (compute/hop.py "Ensure all edge endpoints are + # present in nodes"); the index only materializes existing rows. If any needed id + # is missing from the materialized nodes, fall back to scan (return None) rather + # than silently drop it (a wrong-answer divergence). No-op when nodes are complete. + present = col_to_array(out_nodes, node_col, engine) + present_unique = xp.unique(present) + if int(set_difference(needed, present_unique, xp).shape[0]) > 0: + return None + # B2: the scan dedups output nodes by id (hop.py drop_duplicates(subset=[node])). + # The select_by_ids path returns ALL rows per id, so a node table with DUPLICATE + # ids would emit extra rows here. Fall back to scan (O(result) check) rather than + # diverge. (Unique-id tables — the norm — never trip this; node_id index unused + # for dup ids by construction, so this only guards the isin path.) + if int(present.shape[0]) != int(present_unique.shape[0]): + return None + return g.nodes(out_nodes, node_col).edges(out_edges, src, dst) diff --git a/graphistry/compute/gfql/index/wire.py b/graphistry/compute/gfql/index/wire.py new file mode 100644 index 0000000000..e37fec4738 --- /dev/null +++ b/graphistry/compute/gfql/index/wire.py @@ -0,0 +1,122 @@ +"""JSON wire protocol for GFQL index DDL + the executor that applies an op. + +DDL ops are top-level program types (peers of Chain/Let), following the GFQL AST +JSON convention ``{"type": ClassName, ...fields}``. They round-trip via +``to_json``/``from_json`` and are dispatched by ``index_op_from_json``. + + {"type": "CreateIndex", "kind": "edge_out_adj", "column": null, "name": null, "replace": false} + {"type": "DropIndex", "name": "edge_out_adj:src", "missing_ok": true} + {"type": "DropIndex", "kind": "edge_in_adj", "column": "dst", "missing_ok": true} + {"type": "ShowIndexes"} + +``index_policy`` rides in the request envelope (peer of ``engine``), handled by +``gfql(index_policy=...)`` — it is NOT part of these op shapes. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from .registry import ALL_KINDS + + +@dataclass(frozen=True) +class CreateIndex: + kind: str + column: Optional[str] = None + name: Optional[str] = None + replace: bool = False + + def to_json(self) -> Dict[str, Any]: + return {"type": "CreateIndex", "kind": self.kind, "column": self.column, + "name": self.name, "replace": self.replace} + + @staticmethod + def from_json(d: Dict[str, Any]) -> "CreateIndex": + kind = d.get("kind") + if kind not in ALL_KINDS: + raise ValueError(f"CreateIndex.kind must be one of {ALL_KINDS}, got {kind!r}") + return CreateIndex(kind=kind, column=d.get("column"), name=d.get("name"), + replace=bool(d.get("replace", False))) + + +@dataclass(frozen=True) +class DropIndex: + name: Optional[str] = None + kind: Optional[str] = None + column: Optional[str] = None + missing_ok: bool = True + + def to_json(self) -> Dict[str, Any]: + return {"type": "DropIndex", "name": self.name, "kind": self.kind, + "column": self.column, "missing_ok": self.missing_ok} + + @staticmethod + def from_json(d: Dict[str, Any]) -> "DropIndex": + return DropIndex(name=d.get("name"), kind=d.get("kind"), column=d.get("column"), + missing_ok=bool(d.get("missing_ok", True))) + + +@dataclass(frozen=True) +class ShowIndexes: + def to_json(self) -> Dict[str, Any]: + return {"type": "ShowIndexes"} + + @staticmethod + def from_json(d: Dict[str, Any]) -> "ShowIndexes": + return ShowIndexes() + + +INDEX_OP_TYPES = ("CreateIndex", "DropIndex", "ShowIndexes") + + +def is_index_op(obj: Any) -> bool: + return isinstance(obj, (CreateIndex, DropIndex, ShowIndexes)) + + +def is_index_op_json(d: Any) -> bool: + return isinstance(d, dict) and d.get("type") in INDEX_OP_TYPES + + +def index_op_from_json(d: Dict[str, Any]) -> Any: + t = d.get("type") + if t == "CreateIndex": + return CreateIndex.from_json(d) + if t == "DropIndex": + return DropIndex.from_json(d) + if t == "ShowIndexes": + return ShowIndexes.from_json(d) + raise ValueError(f"Not a GFQL index op: type={t!r}") + + +def apply_index_op(g: Any, op: Any, *, engine: Any = "auto") -> Any: + """Execute a DDL op against a Plottable's index registry. + + CreateIndex/DropIndex -> new Plottable; ShowIndexes -> pandas DataFrame. + """ + from .api import create_index, drop_index, show_indexes, index_name, get_registry + + if isinstance(op, CreateIndex): + if not op.replace: + reg = get_registry(g) + if reg.has(op.kind): + return g # resident reuse (fingerprint validity checked at use time) + return create_index(g, op.kind, column=op.column, name=op.name, engine=engine) + if isinstance(op, DropIndex): + kind = op.kind + if kind is None and op.name is not None: + # Resolve a (possibly custom) index NAME to its kind by searching the + # registry's index.name — NOT by splitting the name on ':' (that only + # recovered the default ``kind:col`` name → a custom name silently no-op'd). + reg = get_registry(g) + kind = next((k for k, ix in reg.indexes.items() + if getattr(ix, "name", None) == op.name), None) + if kind is None: + raise ValueError( + f"DROP GFQL INDEX: no resident index named {op.name!r} " + f"(resident: {sorted(getattr(ix, 'name', k) for k, ix in reg.indexes.items())})" + ) + return drop_index(g, kind) + if isinstance(op, ShowIndexes): + return show_indexes(g) + raise ValueError(f"Unknown index op: {op!r}") diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 3f4f0a9202..377311aa30 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -574,6 +574,28 @@ def _plain_edge(op): and op.source_node_query is None and op.destination_node_query is None and op.edge_query is None and not op.include_zero_hop_seed) + # GFQL physical index fast path for the seeded single-hop shape + # `MATCH (a {id-filter})-[e]->(b)` (forward/reverse, no destination filter) — + # the canonical seeded query. This native chain fast path does its own O(E) + # semi-join, so it must consult the index here too (not just compute/hop.py). + _idx_pol = getattr(self, "_gfql_index_policy", "use") + if (start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) + and _fp_node(ops[2]) and ops[0].filter_dict and not ops[2].filter_dict + and ops[1].direction in ("forward", "reverse")): + from graphistry.compute.gfql.index import get_registry, maybe_index_hop + if (not get_registry(self).is_empty()) or _idx_pol in ("auto", "force"): + gf0 = ensure_nodes_polars(self) + seed0 = filter_by_dict_polars(gf0._nodes, ops[0].filter_dict) + from graphistry.Engine import Engine + from graphistry.compute.gfql.lazy import active_target, ExecutionTarget + _eng0 = Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else Engine.POLARS + _idxed0 = maybe_index_hop( + gf0, _eng0, nodes=seed0, hops=1, direction=ops[1].direction, + return_as_wave_front=False, to_fixed_point=False, policy=_idx_pol, + ) + if _idxed0 is not None: + return _idxed0 + if start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) and _fp_node(ops[2]): n0, e1, n2 = ops unconstrained = not n0.filter_dict and not n2.filter_dict diff --git a/graphistry/compute/gfql/lazy/engine/polars/hop.py b/graphistry/compute/gfql/lazy/engine/polars/hop.py index e9ee3e48b1..d365b0e2c5 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/hop.py +++ b/graphistry/compute/gfql/lazy/engine/polars/hop.py @@ -14,4 +14,31 @@ def hop_lazy_or_eager(self: Plottable, nodes: Optional[Any] = None, hops: Optional[int] = 1, **kwargs: Any) -> Plottable: """Run the polars hop: lazy collect-once for a single bounded hop, eager loop otherwise.""" + # GFQL physical index fast path (pay-as-you-go). chain_polars reaches the + # polars hop here without passing through compute/hop.py, so the index hook + # lives here too to cover polars gfql() chains (not just direct .hop()). + from graphistry.compute.gfql.index import get_registry, maybe_index_hop + _pol = getattr(self, "_gfql_index_policy", "use") + if (not get_registry(self).is_empty()) or _pol in ("auto", "force"): + from graphistry.Engine import Engine + from graphistry.compute.gfql.lazy import active_target, ExecutionTarget + _eng = Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else Engine.POLARS + _indexed = maybe_index_hop( + self, _eng, nodes=nodes, hops=hops, direction=kwargs.get("direction", "forward"), + return_as_wave_front=kwargs.get("return_as_wave_front", False), + to_fixed_point=kwargs.get("to_fixed_point", False), policy=_pol, + min_hops=kwargs.get("min_hops"), max_hops=kwargs.get("max_hops"), + output_min_hops=kwargs.get("output_min_hops"), output_max_hops=kwargs.get("output_max_hops"), + label_node_hops=kwargs.get("label_node_hops"), label_edge_hops=kwargs.get("label_edge_hops"), + label_seeds=kwargs.get("label_seeds", False), edge_match=kwargs.get("edge_match"), + source_node_match=kwargs.get("source_node_match"), + destination_node_match=kwargs.get("destination_node_match"), + source_node_query=kwargs.get("source_node_query"), + destination_node_query=kwargs.get("destination_node_query"), + edge_query=kwargs.get("edge_query"), + include_zero_hop_seed=kwargs.get("include_zero_hop_seed", False), + target_wave_front=kwargs.get("target_wave_front"), + ) + if _indexed is not None: + return _indexed return hop_polars(self, nodes, hops, **kwargs) diff --git a/graphistry/compute/hop.py b/graphistry/compute/hop.py index b8c1682c22..97a19190fd 100644 --- a/graphistry/compute/hop.py +++ b/graphistry/compute/hop.py @@ -109,6 +109,31 @@ def hop(self: Plottable, from graphistry.compute.ComputeMixin import _coerce_input_formats # lazy — avoids circular import self = _coerce_input_formats(self, engine_concrete) + # GFQL physical index fast path (pay-as-you-go). When a resident adjacency + # index covers this seeded traversal and the planner deems it profitable, run + # the O(degree) CSR gather instead of the O(E) scan below. Engine-uniform; + # returns None to fall back. Coercion above is a no-op when already in-engine, + # so the index fingerprint (keyed on the live edge frame) still matches. + from graphistry.compute.gfql.index import get_registry, maybe_index_hop + _idx_policy = getattr(self, "_gfql_index_policy", "use") + if (not get_registry(self).is_empty()) or _idx_policy in ("auto", "force"): + _idx_nodes = df_to_engine(nodes, engine_concrete) if nodes is not None else None + _indexed = maybe_index_hop( + self, engine_concrete, nodes=_idx_nodes, hops=hops, direction=direction, + return_as_wave_front=return_as_wave_front, to_fixed_point=to_fixed_point, + policy=_idx_policy, + min_hops=min_hops, max_hops=max_hops, + output_min_hops=output_min_hops, output_max_hops=output_max_hops, + label_node_hops=label_node_hops, label_edge_hops=label_edge_hops, + label_seeds=label_seeds, edge_match=edge_match, + source_node_match=source_node_match, destination_node_match=destination_node_match, + source_node_query=source_node_query, destination_node_query=destination_node_query, + edge_query=edge_query, include_zero_hop_seed=include_zero_hop_seed, + target_wave_front=target_wave_front, + ) + if _indexed is not None: + return _indexed + if engine_concrete in POLARS_ENGINES: # Native polars traversal lives in a dedicated dispatched module so the # production pandas/cuDF internals below stay untouched (see diff --git a/graphistry/tests/compute/gfql/index/__init__.py b/graphistry/tests/compute/gfql/index/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py new file mode 100644 index 0000000000..c026cfe72b --- /dev/null +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -0,0 +1,287 @@ +"""Tests for GFQL physical indexes (pay-as-you-go seeded-traversal acceleration). + +Behavioral / differential: the index fast path must return the SAME subgraph as +the scan/join path. Engine-parametrized (pandas/cudf/polars/polars-gpu) with +importorskip so GPU lanes run only where available. +""" +import numpy as np +import pandas as pd +import pytest + +import graphistry +from graphistry.compute.ast import n, e_forward +from graphistry.compute.gfql.index import ( + CreateIndex, DropIndex, ShowIndexes, index_op_from_json, parse_index_ddl, + get_registry, +) + + +def _engines(): + out = ["pandas"] + try: + import cudf # noqa + out.append("cudf") + except Exception: + pass + try: + import polars # noqa + out.append("polars") + try: + import cudf # noqa + out.append("polars-gpu") + except Exception: + pass + except Exception: + pass + return out + + +ENGINES = _engines() + + +@pytest.fixture(scope="module") +def graph(): + rng = np.random.default_rng(0) + n_nodes, deg = 2000, 6 + m = n_nodes * deg + edf = pd.DataFrame({"src": rng.integers(0, n_nodes, m), "dst": rng.integers(0, n_nodes, m)}) + ndf = pd.DataFrame({"id": np.arange(n_nodes), "lab": rng.integers(0, 4, n_nodes)}) + return graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + + +def _sig(g): + def topd(df): + mod = type(df).__module__ + return df.to_pandas() if ("cudf" in mod or "polars" in mod) else df + nn = topd(g._nodes); ee = topd(g._edges) + nodes = sorted(nn["id"].tolist()) + edges = sorted(map(tuple, ee[["src", "dst"]].itertuples(index=False, name=None))) + return nodes, edges + + +SCENARIOS = [ + dict(hops=1, direction="forward"), + dict(hops=1, direction="reverse"), + dict(hops=1, direction="undirected"), + dict(hops=2, direction="forward"), + dict(hops=2, direction="undirected"), + dict(hops=1, direction="forward", return_as_wave_front=True), + dict(hops=2, direction="forward", return_as_wave_front=True), + dict(hops=3, direction="forward"), +] + + +@pytest.mark.parametrize("engine", ENGINES) +@pytest.mark.parametrize("sc", SCENARIOS) +def test_index_parity_vs_scan(graph, engine, sc): + seeds = pd.DataFrame({"id": [0, 1, 2, 7, 42, 100]}) + gi = graph.gfql_index_all(engine=engine) + base = graph.hop(nodes=seeds, engine=engine, **sc) + idx = gi.hop(nodes=seeds, engine=engine, **sc) + assert _sig(base) == _sig(idx) + + +@pytest.mark.parametrize("engine", ENGINES) +def test_create_show_drop(graph, engine): + g = graph.create_index("edge_out_adj", engine=engine) + g = g.create_index("edge_in_adj", engine=engine) + si = g.show_indexes() + assert set(si["kind"]) == {"edge_out_adj", "edge_in_adj"} + g2 = g.drop_index("edge_in_adj") + assert set(g2.show_indexes()["kind"]) == {"edge_out_adj"} + g3 = g2.drop_index() # drop all + assert g3.show_indexes().shape[0] == 0 + + +def test_cypher_ddl_recognizer(): + assert isinstance(parse_index_ddl("CREATE GFQL INDEX FOR edge_out_adj"), CreateIndex) + assert parse_index_ddl("CREATE GFQL INDEX pk FOR node_id ON id").column == "id" + assert isinstance(parse_index_ddl("DROP GFQL INDEX FOR edge_in_adj"), DropIndex) + assert isinstance(parse_index_ddl("SHOW GFQL INDEXES"), ShowIndexes) + assert parse_index_ddl("MATCH (a) RETURN a") is None + with pytest.raises(ValueError): + parse_index_ddl("CREATE GFQL INDEX FOR bogus_kind") + + +def test_cypher_ddl_via_gfql(graph): + g = graph.gfql("CREATE GFQL INDEX FOR edge_out_adj") + assert get_registry(g).has("edge_out_adj") + si = g.gfql("SHOW GFQL INDEXES") + assert si.shape[0] == 1 + g2 = g.gfql("DROP GFQL INDEX FOR edge_out_adj") + assert g2.show_indexes().shape[0] == 0 + + +def test_wire_roundtrip(graph): + op = CreateIndex(kind="edge_out_adj") + assert index_op_from_json(op.to_json()) == op + g = graph.gfql({"type": "CreateIndex", "kind": "edge_out_adj"}) + assert get_registry(g).has("edge_out_adj") + show = g.gfql({"type": "ShowIndexes"}) + assert show.shape[0] == 1 + + +@pytest.mark.parametrize("engine", ENGINES) +def test_index_policy_force_and_explain(graph, engine): + chain = [n({"id": 0}), e_forward(hops=1)] + rep_off = graph.gfql_explain(chain, index_policy="off", engine=engine) + assert rep_off["used_index"] is False + rep_force = graph.gfql_explain(chain, index_policy="force", engine=engine) + assert rep_force["used_index"] is True + # results identical regardless of policy + r_scan = graph.gfql(chain, engine=engine) + r_force = graph.gfql(chain, index_policy="force", engine=engine) + assert _sig(r_scan) == _sig(r_force) + + +def test_column_mismatch_raises_not_silent(graph): + # A custom column that doesn't match the binding must raise, not silently no-op. + with pytest.raises(NotImplementedError): + graph.create_index("edge_out_adj", column="not_src") + # Matching column (the actual binding) is accepted. + g = graph.create_index("edge_out_adj", column="src") + assert get_registry(g).has("edge_out_adj") + # Custom display name is honored. + g2 = graph.create_index("node_id", name="pk") + assert "pk" in g2.show_indexes()["name"].tolist() + + +def test_show_indexes_valid_and_nbytes(graph): + g = graph.create_index("edge_out_adj") + si = g.show_indexes() + assert {"valid", "nbytes"}.issubset(si.columns) + assert bool(si.iloc[0]["valid"]) is True + assert int(si.iloc[0]["nbytes"]) > 0 + # After an edges rebind, the index is stale -> valid=False (and auto-skipped). + rng = np.random.default_rng(2) + new_edf = pd.DataFrame({"src": rng.integers(0, 2000, 50), "dst": rng.integers(0, 2000, 50)}) + g2 = g.edges(new_edf, "src", "dst") + assert bool(g2.show_indexes().iloc[0]["valid"]) is False + + +def test_fingerprint_invalidation_is_safe(graph): + # An index built over one edge frame must not be used after .edges() rebind. + gi = graph.create_index("edge_out_adj") + rng = np.random.default_rng(1) + new_edf = pd.DataFrame({"src": rng.integers(0, 2000, 100), "dst": rng.integers(0, 2000, 100)}) + g2 = gi.edges(new_edf, "src", "dst") + seeds = pd.DataFrame({"id": [0, 1, 2]}) + # registry attr carries over but fingerprint (frame id) no longer matches -> + # treated as absent -> correct result computed by scan path. + base = graph.edges(new_edf, "src", "dst").hop(nodes=seeds, hops=1) + got = g2.hop(nodes=seeds, hops=1) + assert _sig(base) == _sig(got) + + +# --------------------------------------------------------------------------- +# Adversarial-review regression tests (takeover 2026-06-28): each reproduces a +# CONFIRMED wrong-answer the original parity scenarios missed. The index MUST +# equal the scan oracle, or fall back to scan — never a wrong answer. +# --------------------------------------------------------------------------- + +def _force(g, engine): + """Index-all + force the index path past the cost gate (so we actually test it).""" + gi = g.gfql_index_all(engine=engine) + gi._gfql_index_policy = "force" + return gi + + +def _nodeset(g): + df = g._nodes + mod = type(df).__module__ + df = df.to_pandas() if ("cudf" in mod or "polars" in mod) else df + return df + + +@pytest.mark.parametrize("engine", ENGINES) +@pytest.mark.parametrize("hop_kw", [ + dict(hops=1, max_hops=3), # B1: max_hops must win over hops (was silently ignored) + dict(hops=5, max_hops=2), # B1: hops ignored when max_hops set +]) +def test_index_max_hops_honored(engine, hop_kw): + edf = pd.DataFrame({"src": [0, 1, 2, 3, 4], "dst": [1, 2, 3, 4, 5]}) + g = graphistry.edges(edf, "src", "dst").materialize_nodes() + seeds = pd.DataFrame({"id": [0]}) + base = g.hop(nodes=seeds, engine=engine, **hop_kw) + idx = _force(g, engine).hop(nodes=seeds, engine=engine, **hop_kw) + assert _sig(base) == _sig(idx), f"max_hops divergence {hop_kw}" + + +@pytest.mark.parametrize("engine", ENGINES) +def test_index_duplicate_node_ids(engine): + # B2: duplicate node ids corrupted the unique-key node_id index. gfql_index_all + # must skip node_id (build adjacency only) and stay scan-equal. + ndf = pd.DataFrame({"id": [0, 1, 2, 3, 5, 5]}) # id 5 duplicated + edf = pd.DataFrame({"src": [0, 1, 3], "dst": [1, 2, 5]}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": [0, 1, 3]}) + base = g.hop(nodes=seeds, engine=engine, hops=1) + idx = _force(g, engine).hop(nodes=seeds, engine=engine, hops=1) + assert _sig(base) == _sig(idx) + + +@pytest.mark.parametrize("engine", ENGINES) +def test_index_missing_endpoint(engine): + # B3: an edge endpoint absent from the node table — the scan synthesizes it as a + # node row; the index dropped it. Must match (index falls back if it can't). + ndf = pd.DataFrame({"id": [0, 1, 2, 3]}) # 99 absent + edf = pd.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 99]}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": [0, 1, 2]}) + base = g.hop(nodes=seeds, engine=engine, hops=1) + idx = _force(g, engine).hop(nodes=seeds, engine=engine, hops=1) + assert _sig(base) == _sig(idx) + + +@pytest.mark.parametrize("engine", ENGINES) +def test_index_preserves_node_table_order(engine): + # I4: a node_id-index materialization must keep the original .nodes row order + # (not searchsorted/sorted-by-id order). + ndf = pd.DataFrame({"id": [5, 3, 1, 0, 2, 4]}) # NOT sorted + edf = pd.DataFrame({"src": [5, 3, 1], "dst": [3, 1, 0]}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": [5, 3, 1]}) + base = g.hop(nodes=seeds, engine=engine, hops=1) + idx = _force(g, engine).hop(nodes=seeds, engine=engine, hops=1) + assert _nodeset(base)["id"].tolist() == _nodeset(idx)["id"].tolist(), "node order diverged" + + +@pytest.mark.parametrize("engine", ENGINES) +def test_index_mixed_id_dtypes(engine): + # I6: int64 node-id seed vs int32 edge endpoints — narrowing the seed to the key + # dtype could wrap/false-match; promotion keeps it correct. + ndf = pd.DataFrame({"id": np.array([0, 1, 2, 3], dtype=np.int64)}) + edf = pd.DataFrame({"src": np.array([0, 1, 2], dtype=np.int32), + "dst": np.array([1, 2, 3], dtype=np.int32)}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": [0, 1, 2]}) + base = g.hop(nodes=seeds, engine=engine, hops=1) + idx = _force(g, engine).hop(nodes=seeds, engine=engine, hops=1) + assert _sig(base) == _sig(idx) + + +@pytest.mark.parametrize("engine", ENGINES) +def test_index_stale_rebind_same_shape(engine): + # I5: rebinding .edges() to a NEW same-shape frame must invalidate the index by + # object identity (not id(), which can be recycled) -> scan fallback, never stale. + g = graphistry.edges(pd.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}), "src", "dst").materialize_nodes() + gi = g.gfql_index_all(engine=engine) + new_edf = pd.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 0]}) # same shape, different values + g2 = gi.edges(new_edf, "src", "dst") + seeds = pd.DataFrame({"id": [0]}) + base = g.edges(new_edf, "src", "dst").hop(nodes=seeds, engine=engine, hops=1) + got = g2.hop(nodes=seeds, engine=engine, hops=1) + assert _sig(base) == _sig(got) + + +def test_drop_index_by_custom_name(): + # B2 (review): DROP GFQL INDEX must resolve via the index name, not + # silently no-op (the old code split the name on ':' and only matched default names). + g = graphistry.edges(pd.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}), "src", "dst").materialize_nodes() + g = g.create_index("edge_out_adj", name="myidx") + assert "edge_out_adj" in g.show_indexes()["kind"].tolist() + g2 = g.gfql("DROP GFQL INDEX myidx") + assert g2.show_indexes().shape[0] == 0, "custom-named index was not dropped" + # unresolvable name raises (not silent) + with pytest.raises(ValueError): + g.gfql("DROP GFQL INDEX nonexistent_name") From a1c9adf3393a6eddc99d2a1910285f1b26b6f133 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 28 Jun 2026 10:39:40 -0700 Subject: [PATCH 02/20] test(gfql/index): guarded adversarial bench (path-taken + result==scan) vs kuzu/neo4j A trustworthy index-vs-scan + vs-DB harness: every GFQL timing cell is discarded unless (a) the index path was actually taken (index_trace) AND (b) the index result == the scan result. Prevents reporting a scan-vs-scan fake win. Covers 4 engines + kuzu (embedded CSR) + neo4j (corrected to *1..2 accumulated semantics). Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmarks/gfql/index_takeover_bench.py | 41 +++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/benchmarks/gfql/index_takeover_bench.py b/benchmarks/gfql/index_takeover_bench.py index 6919d9902a..ce2d6923c6 100644 --- a/benchmarks/gfql/index_takeover_bench.py +++ b/benchmarks/gfql/index_takeover_bench.py @@ -120,6 +120,45 @@ def run(): return conn.execute(q) edges=E, warm_idx_ms=warm, build_ms=build_ms, rows=rows)) + "\n"); outf.flush() +def bench_neo4j(ndf, edf, N, E, reps, outf): + uri = os.environ.get("NEO4J_URI") + if not uri: + print(" neo4j: skipped (no NEO4J_URI)"); return + try: + from neo4j import GraphDatabase + except Exception: + print(" neo4j: driver NOT AVAILABLE"); return + drv = GraphDatabase.driver(uri, auth=(os.environ.get("NEO4J_USER", "neo4j"), + os.environ.get("NEO4J_PASS", "testpass123"))) + node_ids = ndf["id"].tolist(); edge_rows = edf.values.tolist(); BATCH = 20000 + t0 = time.perf_counter() + with drv.session() as s: + s.run("MATCH (n) DETACH DELETE n") + s.run("CREATE INDEX n_id IF NOT EXISTS FOR (n:N) ON (n.id)") + s.run("CALL db.awaitIndexes(300)") + for i in range(0, len(node_ids), BATCH): + s.run("UNWIND $ids AS x CREATE (:N {id:x})", ids=node_ids[i:i + BATCH]) + for i in range(0, len(edge_rows), BATCH): + s.run("UNWIND $rows AS r MATCH (a:N {id:r[0]}),(b:N {id:r[1]}) CREATE (a)-[:E]->(b)", + rows=edge_rows[i:i + BATCH]) + s.run("CALL db.awaitIndexes(300)") + load_ms = (time.perf_counter() - t0) * 1e3 + qs = {"SEL1": "MATCH (a:N {id:0})-[:E]->(b:N) RETURN b.id AS id", + "SEL2": "MATCH (a:N {id:0})-[:E]->()-[:E]->(b:N) RETURN DISTINCT b.id AS id"} + with drv.session() as s: + for task, q in qs.items(): + for _ in range(2): + list(s.run(q)) + ts = [] + for _ in range(reps): + t = time.perf_counter(); data = list(s.run(q)); ts.append((time.perf_counter() - t) * 1e3) + ts.sort(); warm = statistics.median(ts) + print(f" neo4j {'':11} {task} warm={warm:8.4f}ms rows={len(data)} load={load_ms:.0f}ms") + if outf: outf.write(json.dumps(dict(system="neo4j", engine="neo4j", task=task, n=N, + edges=E, warm_idx_ms=warm, build_ms=load_ms, rows=len(data))) + "\n"); outf.flush() + drv.close() + + def main(): NS = [int(x) for x in os.environ.get("NS", "800000,8000000").split(",")] DEG = int(os.environ.get("DEG", "8")) @@ -136,6 +175,8 @@ def main(): bench_gfql(ndf, edf, N, E, ENGINES, REPS, outf) if "kuzu" in SYSTEMS: bench_kuzu(ndf, edf, N, E, REPS, outf) + if "neo4j" in SYSTEMS: + bench_neo4j(ndf, edf, N, E, REPS, outf) if outf: outf.close() From 5c89acaeb8772c0f64e908351f8a5323db814b7c Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 28 Jun 2026 10:41:29 -0700 Subject: [PATCH 03/20] =?UTF-8?q?docs(changelog):=20CSR=20index=20?= =?UTF-8?q?=E2=80=94=20our=20reproduced=20numbers=20+=20adversarial=20hard?= =?UTF-8?q?ening=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 770bd3727c..9dbdb35af9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). - **GFQL Cypher numeric functions + `toLower`/`toUpper`/`lower`/`upper` (openCypher/neo4j/GQL-standard)**: added the standard scalar functions `floor`, `ceil` (alias `ceiling`, per Cypher 25 GQL conformance and the GQL grammar's `CEIL|CEILING`), `round(x)` / `round(x, precision)`, and `toLower` / `toUpper` plus their GQL-conformance aliases `lower` / `upper` (ISO GQL §20.24 character-string functions; neo4j accepts both spellings) — the idiomatic case-insensitive compare `WHERE toLower(n.name) = 'bob'` — across the Cypher `WHERE`/`RETURN` expression surface. Evaluated natively on pandas/cuDF and polars (differential-parity tested). **`round` follows neo4j's documented tie-breaking** (standards-vetted against the neo4j manual): precision 0 (and the 1-arg form) rounds ties toward **positive infinity** (`round(-1.5)` → `-1.0`, `round(2.5)` → `3.0`), and precision > 0 rounds ties **away from zero** (HALF_UP: `round(-1.55, 1)` → `-1.6`) — not the numpy/polars half-to-even defaults, which give wrong answers on ties. The `round(x, precision, mode)` 3-arg form is not yet supported. Complements the already-supported `abs`/`sqrt`/`sign` and chained comparisons (`WHERE 1 < n.age < 65`). The `^` exponentiation operator is deferred: standards vetting settled it as **left-associative** (the openCypher TCK pins left, and neo4j's shipped Cypher 5/25 parser left-folds `Pow` — the manual's "right to left" row is a docs bug), but the current conformance corpus marks it reject-expected, so re-adding it is a coordinated corpus change. `LIKE`/`ILIKE`/`BETWEEN` remain intentionally unimplemented (verified absent from both Cypher and ISO GQL — GQL's only `LIKE` is the unrelated `CREATE GRAPH TYPE … LIKE g` DDL). - **GFQL Cypher `=~` regex-match operator (openCypher/neo4j-standard)**: added the standard `=~` string predicate to the Cypher `WHERE`/expression grammar — `MATCH (n) WHERE n.name =~ '(?i)al.*' RETURN n`. Semantics match openCypher/neo4j: **Java-regex dialect, full-string/anchored match** (`n.name =~ 'AB'` matches only `'AB'`, not `'ABCDEF'`; use `.*`/`^..$` for partial), with inline flags (`(?i)`/`(?m)`/`(?s)`) honored; lowers to the existing `fullmatch` predicate. Works in the simple `WHERE prop =~ '...'` form (all engines via `filter_by_dict`) and composes through `AND`/`OR`/`NOT`/`RETURN` expressions via the shared expression engine (pandas/cuDF; polars supports the simple-WHERE form and declines the complex `OR`/`NOT` row-filter form with an honest `NotImplementedError` — the pre-existing polars `where_rows` limitation, not `=~`-specific). Also adds native polars `Match`/`Fullmatch` predicate lowering (previously `NotImplementedError`), so `=~` and the Python `match()`/`fullmatch()` predicates run natively on polars. Differential-parity tested against the pandas oracle. `LIKE`/`ILIKE` remain intentionally unimplemented (not in any graph standard — use `=~`/`CONTAINS`/`STARTS WITH`). -- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront. **Perf (dgx-spark, deg-8, warm median):** seeded latency is **flat in graph size** — GFQL-pandas SEL1 0.110/0.109/0.113 ms at 0.8M/8M/80M edges (vs an O(E) scan of 17.7/128/1361 ms) — and **beats kuzu (CSR) and neo4j by 12–30×** on CPU (e.g. 10M nodes/80M edges: GFQL-pandas 0.113 ms vs kuzu 2.64 ms). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity. cuDF/polars-GPU are flat but floored by ~2–3 ms GPU kernel-launch overhead (selective traversal is an indexing problem, not a compute one). No change to default (un-indexed) behavior. +- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront, hardened by an adversarial review that found and fixed several index-vs-scan divergences the original scenarios missed (`max_hops` honored; duplicate node ids, edge endpoints absent from the node table, int32/int64 seed-vs-key dtype mismatches, and stale `.edges()` rebinds all now match scan or fall back — never a wrong answer), with object-identity fingerprinting (recycle-proof) and 6 added differential regression tests. **Perf (dgx-spark, deg-8, warm median; every measured cell guarded so the index path was actually taken AND the index result equals the scan result):** seeded latency is **flat in graph size** — GFQL-pandas 1-hop 0.124/0.122 ms at 0.8M/8M nodes (6.4M/64M edges) while the O(E) scan grows 105 → 1045 ms — and **beats kuzu (CSR) and neo4j by 9–28×** on CPU at 0.8M nodes (1-hop: GFQL-pandas 0.123 ms vs kuzu 1.15 ms vs neo4j 1.45 ms; 1–2-hop: 0.150 ms vs 4.25 ms vs 2.54 ms, matched answer counts). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity. cuDF/polars-GPU are flat but floored by ~3 ms (cuDF) GPU kernel-launch overhead — selective traversal is an indexing problem, not a compute one (CPU wins it). No change to default (un-indexed) behavior. - **GFQL native Polars engine — traversals (`engine='polars'`)**: Added a native, vectorized Polars execution engine for the core GFQL traversals `hop()` and `chain()`, dispatched at the engine boundary so the production pandas/cuDF paths are untouched. `Engine.POLARS` is opt-in (explicit `engine='polars'`); `engine='auto'` with Polars input still coerces to pandas as before. Covers forward/reverse/undirected single-hop traversal, directed multi-hop chains, node/edge filter dicts and predicates (lowered to Polars expressions), `edge_match`/`source_node_match`/`destination_node_match`, `target_wave_front`, and alias names; the BFS advances via semi/anti joins (no per-row Python work). Validated by differential parity against the pandas engine (hop + chain test suites plus a randomized fuzzer) and benchmarked vs pandas (`benchmarks/gfql/pandas_vs_polars.py`) — Polars wins at scale (up to ~2.5x on multi-edge chains at millions of edges; crossover ~50–100k rows). Variable-length/multi-hop edges, undirected edges in multi-edge chains, hop labels, and node `query=` raise `NotImplementedError` for now (use `engine='pandas'`). - **GFQL native Polars engine — variable-length `min_hops>1` traversals (`engine='polars'`/`'polars-gpu'`)**: Forward/reverse lower-bound traversals (`e(min_hops=N, max_hops=M)`) now run natively on the Polars engine — no pandas bridge. The eager Polars hop runs pandas' min_hops algorithm vectorized: a NON-anti-joined BFS (the wavefront carries revisits so a cycle keeps bumping the reach depth until `max_hops`), a 3-case termination gate (`max_reached1` (needs connected-components + 2-core seed retention) and a *direct* `hop(min_hops>1)` (which would need pandas' separate un-labeled direct-hop node-output plus the `target_wave_front` threading the chain supplies — without them it silently diverges) both raise `NotImplementedError` for `engine='polars'` (use `chain()`/`gfql()`, or `engine='pandas'`). Validated by differential parity vs the pandas oracle: the 500-seed randomized chain fuzzer (`test_polars_chain_fuzz_parity`, hardened to compare null-aware node **attributes** and edge **multiplicity**, not just id/endpoint sets) is **500/500**, a min_hops+attribute-filter amplified fuzz and metamorphic invariants pass, and `engine='polars-gpu'` (cudf_polars) runs the full 500-seed fuzz **500/500** on-device. - **GFQL native Polars engine — cypher row pipeline (`engine='polars'`)**: Extended the Polars engine to the Cypher `MATCH … RETURN` row surface, natively vectorized. **NO CHEATING:** the polars engine never silently falls back to the pandas engine — every query runs natively on polars or raises an honest `NotImplementedError` pointing at `engine='pandas'` (falling back to pandas would misrepresent pandas performance as polars; only a human may consent to a bridge). `chain_polars` splits boundary `call()` ops (mirroring the pandas `_handle_boundary_calls`) and runs each trailing row op per-op native or raises. **Native polars** (no pandas round-trip): frame ops (`rows`/`limit`/`skip`/`distinct`/`drop_cols`), `select`/`with_`/`return_` projection (a conservative cypher-expr-AST → `pl.Expr` lowering covering property access, arithmetic, comparison, boolean, literals), `order_by` (`.sort`), `group_by` (`count`/`sum`/`avg`/`min`/`max`), `unwind` (literal-list cross-join), the result projection for property/expr columns, and entity-text `RETURN n` rendering for int/string/bool nodes (`pl.concat_str`). **Honestly deferred** (raise `NotImplementedError`, no pandas fallback): multi-entity `rows(binding_ops=…)`, cross-entity same-path `WHERE` (`DFSamePathExecutor`), float/temporal/nested entity-text, and exotic expressions (CASE/list/map/temporal, `collect` aggregates) — these are the forward native-engineering targets. Validated by differential parity vs pandas including a TCK-style conformance lane (`test_engine_polars_cypher_conformance.py`: native-only curated corpus + seeded fuzzer + NULL/3-valued-logic graph + entity-text escaping, plus a `DEFERRED` list asserting deferred queries raise rather than silently bridge) and benchmarked (`benchmarks/gfql/cypher_row_pipeline.py`). **Perf (interleaved, 1M nodes, each engine on its native-frame graph, all fully native):** polars wins **5.6–38×** across the surface — `RETURN n` ~38×, `ORDER BY` ~17×, `WHERE`+`ORDER BY`+`LIMIT` ~14×, traversals 6–7.5×, projections/aggregations/`DISTINCT` 5.6–6.9×. cuDF/pandas paths untouched. From 27e60d90defbd8cd5d48a2adc4c3da5f3d3ac53c Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 28 Jun 2026 12:59:39 -0700 Subject: [PATCH 04/20] fix(gfql/index): mypy types (engine cast, Optional node-id idx, show_indexes narrowing) Index module failed CI lint-types (python-lint-types). Fix: cast engine to resolve_engine in create_index/gfql_explain; use a separate var for the Optional NodeIdIndex (was assigned to an AdjacencyIndex-typed name); assert idx/source/destination not-None in show_indexes. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/gfql/index/api.py | 14 ++++++++------ graphistry/compute/gfql/index/explain.py | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index bf027aae24..5976a0f0bf 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -8,7 +8,7 @@ from __future__ import annotations import copy -from typing import Any, Optional, Union +from typing import Any, Optional, Union, cast from graphistry.Engine import EngineAbstract, Engine, resolve_engine from graphistry.Plottable import Plottable @@ -86,7 +86,7 @@ def create_index( O(E log E) once, amortized over later seeded queries. """ from dataclasses import replace - eng = resolve_engine(engine, g) + eng = resolve_engine(cast(Any, engine), g) # Build over frames already in the target engine so the index arrays land on # the right backend (cupy for cudf, numpy otherwise). No-op when already in-engine. from graphistry.compute.ComputeMixin import _coerce_input_formats @@ -112,15 +112,15 @@ def create_index( node_col = g2._node assert node_col is not None and g2._nodes is not None _check_column(column, node_col, kind) - idx = build_node_id_index(g2._nodes, node_col, eng) - if idx is None: + node_idx = build_node_id_index(g2._nodes, node_col, eng) + if node_idx is None: raise ValueError( f"Cannot build a {NODE_ID!r} index: node id column {node_col!r} has " f"duplicate values (a node-id index requires unique ids). Seeded " f"traversal still works via the un-indexed node materialization path." ) - idx = replace(idx, name=name or index_name(kind, node_col)) - registry = registry.with_index(NODE_ID, idx) + node_idx = replace(node_idx, name=name or index_name(kind, node_col)) + registry = registry.with_index(NODE_ID, node_idx) return _attach(g2, registry) raise ValueError(f"Unknown GFQL index kind: {kind!r}. Expected one of {ALL_KINDS}.") @@ -149,7 +149,9 @@ def show_indexes(g: Plottable) -> Any: rows = [] for kind in registry.kinds(): idx = registry.get(kind) + assert idx is not None # iterating registry.kinds() -> present if kind in ADJ_KINDS: + assert g._source is not None and g._destination is not None valid = registry.get_valid(kind, g._edges, (g._source, g._destination), idx.engine) is not None n_keys, n_rows = idx.n_keys, idx.n_edges else: # NODE_ID diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index 7203f8b9bc..f12b705624 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -7,14 +7,14 @@ """ from __future__ import annotations -from typing import Any, Dict +from typing import Any, Dict, cast from graphistry.Engine import resolve_engine from .api import index_trace, get_registry, show_indexes def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = "auto") -> Dict[str, Any]: - eng = resolve_engine(engine, g) + eng = resolve_engine(cast(Any, engine), g) resident = show_indexes(g) with index_trace() as steps: try: From 47b12cbaaaa4d81768c5943b1e2132b223a78f6e Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 28 Jun 2026 13:04:45 -0700 Subject: [PATCH 05/20] style(gfql/index): split E702 semicolon in test _sig helper (newer ruff) Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/tests/compute/gfql/index/test_index.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index c026cfe72b..3541b5e977 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -53,7 +53,8 @@ def _sig(g): def topd(df): mod = type(df).__module__ return df.to_pandas() if ("cudf" in mod or "polars" in mod) else df - nn = topd(g._nodes); ee = topd(g._edges) + nn = topd(g._nodes) + ee = topd(g._edges) nodes = sorted(nn["id"].tolist()) edges = sorted(map(tuple, ee[["src", "dst"]].itertuples(index=False, name=None))) return nodes, edges From 2b3d66926f634b6e2465aec258541747d6430887 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 28 Jun 2026 19:10:47 -0700 Subject: [PATCH 06/20] bench(gfql): large real-graph + bulk-OLAP + fairest-kuzu harnesses (4 engines vs kuzu) Adds the dgx-spark benchmark harnesses backing the CSR-index + bulk-OLAP claims, all guarded for trust (timing only reported when the index path was actually taken AND index result == scan result; engine parity checked via matched row counts): - index_largegraph_bench.py: real SNAP edge lists (LiveJournal 35M, Orkut 117M, Friendster stretch), parquet-cached, degree-percentile + multi-seed sweeps. Shows seeded latency flat in N, scaling with seed degree (the O(degree) honesty sweep). - index_bulk_olap_bench.py: BULK regime the index deliberately avoids -- seeded multi-hop frontier expansion via the chain API (the one GFQL surface supporting all 4 engines) + full-graph out-degree aggregation, 4 engines + kuzu. Answers "is bulk OLAP better with GFQL cudf?": yes with GFQL, but on polars/polars-gpu (fused lazy), not cudf (eager per-op). polars-CPU 11-47x over pandas, 6-18x over cudf, 3-87x over kuzu on frontier expansion at 35-117M edges. - index_vs_kuzu_prepared.py: fairest seeded comparison -- kuzu prepared statement + columnar get_as_df, in-process (no bolt), matched rows. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmarks/gfql/index_bulk_olap_bench.py | 194 ++++++++++++++++++++++ benchmarks/gfql/index_largegraph_bench.py | 149 +++++++++++++++++ benchmarks/gfql/index_vs_kuzu_prepared.py | 54 ++++++ 3 files changed, 397 insertions(+) create mode 100644 benchmarks/gfql/index_bulk_olap_bench.py create mode 100644 benchmarks/gfql/index_largegraph_bench.py create mode 100644 benchmarks/gfql/index_vs_kuzu_prepared.py diff --git a/benchmarks/gfql/index_bulk_olap_bench.py b/benchmarks/gfql/index_bulk_olap_bench.py new file mode 100644 index 0000000000..257a3b9c35 --- /dev/null +++ b/benchmarks/gfql/index_bulk_olap_bench.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""BULK-OLAP head-to-head: GFQL 4 engines vs kuzu on REAL graphs. + +Answers "is bulk OLAP better with GFQL (cudf / polars-gpu)?" The seeded CSR index +is O(degree) and wins tiny work; this bench deliberately AVOIDS that path and +measures the BULK regime instead — large-frontier multi-hop + full-graph +aggregation, i.e. the scan/join work where columnar GPU throughput should pay off +and the index does NOT help. We run g0.hop (NO resident index -> engine traversal, +the honest bulk path) so every engine does the same materialized join work. + +Tasks (all bulk, all materialized on both sides): + BULK1 1-hop forward from K seeds (edge semijoin, frontier=K) + BULK2 2-hop forward from K seeds (edge-edge join, frontier blows up) + DEGALL full-graph out-degree aggregation (group_by over ALL edges; pure OLAP) +K frontier sweep: 1k, 10k, 100k seeds. cudf/polars-gpu should overtake pandas as K +(hence work) grows; kuzu is the WCOJ/optimizer peer for the multi-hop join. + +Trust: GFQL rows reported per engine (engine parity is separately guaranteed by the +conformance suite); kuzu rows reported alongside with a semantic note. Timing is the +deliverable — rows are the honesty check that each system did real work. + +Env: PARQUET=/path/edges.parquet KS=1000,10000,100000 ENGINES=pandas,polars,cudf,polars-gpu + SYSTEMS=gfql,kuzu REPS=10 WARM=2 OUT=/tmp/bulk.jsonl SEED=0 +""" +from __future__ import annotations +import json, os, statistics, time, tempfile, shutil +import numpy as np +import pandas as pd +import graphistry +from graphistry.compute.ast import n, e_forward + + +def _sync(engine): + if engine in ("cudf", "polars-gpu"): + try: + import cupy as cp # type: ignore + cp.cuda.runtime.deviceSynchronize() + except Exception: + pass + + +def timeit(fn, reps, engine="cpu", warmup=2): + for _ in range(warmup): + fn(); _sync(engine) + ts = [] + for _ in range(reps): + t0 = time.perf_counter(); fn(); _sync(engine) + ts.append((time.perf_counter() - t0) * 1e3) + ts.sort() + return statistics.median(ts) + + +def load_graph(): + edf = pd.read_parquet(os.environ["PARQUET"]).astype({"src": np.int64, "dst": np.int64}) + nodes = np.unique(np.concatenate([edf["src"].values, edf["dst"].values])) + ndf = pd.DataFrame({"id": nodes}) + return ndf, edf, nodes + + +def gfql_trav(g0, seed_ids, hops, engine): + """BULK seeded multi-hop via the CHAIN API — the one GFQL surface that supports + ALL FOUR engines (generic hop() is pandas/cudf only; polars/polars-gpu route + through engine_polars). n({id:seeds}) = frontier filter, then e_forward()*hops.""" + ops = [n({"id": seed_ids})] + [e_forward() for _ in range(hops)] + return g0.chain(ops, engine=engine) + + +def run_gfql(ndf, edf, nodes, ks, engines, reps, warm, outf, seed): + N, E = len(ndf), len(edf) + rng = np.random.default_rng(seed) + seed_sets = {k: rng.choice(nodes, size=min(k, len(nodes)), replace=False).tolist() for k in ks} + for engine in engines: + try: + g0 = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + # warm/convert frames onto the engine ONCE (exclude H2D/convert from timing) + _ = gfql_trav(g0, seed_sets[ks[0]], 1, engine) + except Exception as ex: + print(f" gfql {engine}: SETUP FAILED {type(ex).__name__}: {ex}"); continue + # frontier sweep: BULK1 (1-hop) + BULK2 (2-hop) + for k in ks: + sids = seed_sets[k] + for task, hops in (("BULK1", 1), ("BULK2", 2)): + try: + res = gfql_trav(g0, sids, hops, engine) + rows = int(res._edges.shape[0]); nn = int(res._nodes.shape[0]) + ms = timeit(lambda: gfql_trav(g0, sids, hops, engine), reps, engine, warm) + except Exception as ex: + print(f" gfql {engine} {task} k={k} FAILED: {type(ex).__name__}: {ex}"); continue + rec = dict(system="gfql", engine=engine, task=task, k=k, hops=hops, + n=N, edges=E, warm_ms=ms, e_rows=rows, n_rows=nn) + print(f" gfql {engine:11} {task} k={k:>7} {ms:10.3f}ms e_rows={rows:>10} n_rows={nn:>9}") + if outf: outf.write(json.dumps(rec) + "\n"); outf.flush() + # DEGALL: full-graph out-degree aggregation (pure columnar OLAP, no traversal) + try: + ms, rows = degall(edf, engine, reps, warm) + rec = dict(system="gfql", engine=engine, task="DEGALL", k=None, hops=0, + n=N, edges=E, warm_ms=ms, e_rows=rows, n_rows=rows) + print(f" gfql {engine:11} DEGALL{'':>13} {ms:10.3f}ms groups={rows:>10}") + if outf: outf.write(json.dumps(rec) + "\n"); outf.flush() + except Exception as ex: + print(f" gfql {engine} DEGALL FAILED: {type(ex).__name__}: {ex}") + + +def degall(edf, engine, reps, warm): + """Full-graph out-degree = group_by(src).size() on the chosen engine.""" + if engine == "pandas": + df = edf + fn = lambda: df.groupby("src").size() + elif engine == "cudf": + import cudf + df = cudf.from_pandas(edf) + fn = lambda: df.groupby("src").size() + elif engine in ("polars", "polars-gpu"): + import polars as pl + df = pl.from_pandas(edf) + if engine == "polars-gpu": + eng = pl.GPUEngine(executor="in-memory", raise_on_fail=False) + fn = lambda: df.lazy().group_by("src").len().collect(engine=eng) + else: + fn = lambda: df.group_by("src").len() + else: + raise ValueError(engine) + r = fn(); rows = int(r.shape[0]) + ms = timeit(fn, reps, engine, warm) + return ms, rows + + +def run_kuzu(ndf, edf, nodes, ks, reps, warm, outf, seed, tmpdir): + try: + import kuzu + except Exception: + print(" kuzu: NOT AVAILABLE (pip install kuzu)"); return + rng = np.random.default_rng(seed) + seed_sets = {k: rng.choice(nodes, size=min(k, len(nodes)), replace=False).tolist() for k in ks} + dbp = tempfile.mkdtemp(dir=tmpdir) + db = kuzu.Database(os.path.join(dbp, "kz")); conn = kuzu.Connection(db) + conn.execute("CREATE NODE TABLE N(id INT64, PRIMARY KEY(id))") + conn.execute("CREATE REL TABLE E(FROM N TO N)") + np_path = os.path.join(dbp, "n.parquet"); ep_path = os.path.join(dbp, "e.parquet") + ndf.to_parquet(np_path) + edf.rename(columns={"src": "from", "dst": "to"}).to_parquet(ep_path) + t0 = time.perf_counter() + conn.execute(f'COPY N FROM "{np_path}"'); conn.execute(f'COPY E FROM "{ep_path}"') + load_ms = (time.perf_counter() - t0) * 1e3 + print(f" kuzu load: {load_ms:.0f}ms") + # BULK1/BULK2: distinct reachable set from K seeds (materialized columnar via get_as_df) + q1 = conn.prepare("MATCH (a:N)-[:E]->(b:N) WHERE a.id IN $seeds RETURN b.id") + q2 = conn.prepare("MATCH (a:N)-[:E]->()-[:E]->(b:N) WHERE a.id IN $seeds RETURN b.id") + for k in ks: + s = seed_sets[k] + for task, stmt in (("BULK1", q1), ("BULK2", q2)): + try: + rows = len(conn.execute(stmt, {"seeds": s}).get_as_df()) + ms = timeit(lambda: conn.execute(stmt, {"seeds": s}).get_as_df(), reps, "kuzu", warm) + except Exception as ex: + print(f" kuzu {task} k={k} FAILED: {type(ex).__name__}: {ex}"); continue + rec = dict(system="kuzu", engine="kuzu", task=task, k=k, n=len(ndf), edges=len(edf), + warm_ms=ms, e_rows=rows, n_rows=rows, load_ms=load_ms) + print(f" kuzu {'':11} {task} k={k:>7} {ms:10.3f}ms rows={rows:>10} (b.id, not-distinct)") + if outf: outf.write(json.dumps(rec) + "\n"); outf.flush() + # DEGALL: full out-degree aggregation + try: + qd = "MATCH (a:N)-[:E]->() RETURN a.id, count(*) AS deg" + for _ in range(warm): conn.execute(qd).get_as_df() + rows = len(conn.execute(qd).get_as_df()) + ms = timeit(lambda: conn.execute(qd).get_as_df(), reps, "kuzu", warm) + rec = dict(system="kuzu", engine="kuzu", task="DEGALL", k=None, n=len(ndf), edges=len(edf), + warm_ms=ms, e_rows=rows, n_rows=rows, load_ms=load_ms) + print(f" kuzu {'':11} DEGALL{'':>13} {ms:10.3f}ms groups={rows:>10}") + if outf: outf.write(json.dumps(rec) + "\n"); outf.flush() + except Exception as ex: + print(f" kuzu DEGALL FAILED: {type(ex).__name__}: {ex}") + shutil.rmtree(dbp, ignore_errors=True) + + +def main(): + ndf, edf, nodes = load_graph() + print(f"===== graph: {len(ndf):,} nodes {len(edf):,} edges =====") + ks = [int(x) for x in os.environ.get("KS", "1000,10000,100000").split(",")] + engines = os.environ.get("ENGINES", "pandas,polars,cudf,polars-gpu").split(",") + systems = os.environ.get("SYSTEMS", "gfql,kuzu").split(",") + reps = int(os.environ.get("REPS", "10")); warm = int(os.environ.get("WARM", "2")) + seed = int(os.environ.get("SEED", "0")) + tmpdir = os.environ.get("TMPDIR_BENCH", "/tmp/bulkbench"); os.makedirs(tmpdir, exist_ok=True) + outf = open(os.environ["OUT"], "a") if os.environ.get("OUT") else None + if "gfql" in systems: + run_gfql(ndf, edf, nodes, ks, engines, reps, warm, outf, seed) + if "kuzu" in systems: + run_kuzu(ndf, edf, nodes, ks, reps, warm, outf, seed, tmpdir) + if outf: outf.close() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gfql/index_largegraph_bench.py b/benchmarks/gfql/index_largegraph_bench.py new file mode 100644 index 0000000000..ad36cad3f9 --- /dev/null +++ b/benchmarks/gfql/index_largegraph_bench.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Large REAL-graph CSR-index bench (Step 7). Power-law topology exposes what the +uniform deg-8 synthetic never did: the index is O(degree), so warm latency is flat +in N but scales with SEED DEGREE — a hub seed is the adversarial worst case. + +Same trust discipline as index_takeover_bench.py: every GFQL timing is GUARDED by +(index path actually taken via index_trace) AND (index result == scan result). A +cell failing either guard is reported INVALID, never as a speedup. + +Datasets (SNAP edge lists, gzipped `u v`, load once -> parquet cache): + com-Orkut 3.07M nodes / 117M edges https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz + com-LiveJournal 4.0M / 34.7M https://snap.stanford.edu/data/bigdata/communities/com-lj.ungraph.txt.gz + soc-LiveJournal1 4.8M / 69M (directed) https://snap.stanford.edu/data/soc-LiveJournal1.txt.gz + com-Friendster 65.6M / 1.8B (STRETCH) https://snap.stanford.edu/data/bigdata/communities/com-friendster.ungraph.txt.gz + twitter-2010 41.7M / 1.47B (STRETCH) +LDBC SNB sf10/sf100 via ~/Work/pyg-bench loader + the live snb-interactive-neo4j. + +Env: EDGELIST=/path/to/edges.txt.gz (or PARQUET=/path/edges.parquet) + DEG_PCTLS=50,90,99,100 MULTISEED=1,10,100,1000 ENGINES=pandas,polars,cudf,polars-gpu + REPS=15 OUT=/tmp/lg.jsonl MAXSCAN_REPS=3 (cap scan reps at large E) +""" +from __future__ import annotations +import gzip, json, os, statistics, time +import numpy as np +import pandas as pd +import graphistry +from graphistry.compute.gfql.index import index_trace + + +def load_graph(seed=0): + """Load a real edge list -> graphistry graph (int64 ids), parquet-cached.""" + pq = os.environ.get("PARQUET") + el = os.environ.get("EDGELIST") + if pq and os.path.exists(pq): + edf = pd.read_parquet(pq) + elif el: + cache = el + ".parquet" + if os.path.exists(cache): + edf = pd.read_parquet(cache) + else: + op = gzip.open if el.endswith(".gz") else open + with op(el, "rt") as f: + edf = pd.read_csv(f, sep=r"\s+", comment="#", header=None, + names=["src", "dst"], dtype=np.int64) + edf.to_parquet(cache) + print(f" cached parquet -> {cache}") + else: + # fallback synthetic power-law (Barabasi-ish via preferential attachment proxy) + rng = np.random.default_rng(seed) + n = int(os.environ.get("SYNTH_N", "1000000")); m = n * 8 + deg = rng.zipf(2.2, m) % n + edf = pd.DataFrame({"src": rng.integers(0, n, m, dtype=np.int64), "dst": deg.astype(np.int64)}) + nodes = np.unique(np.concatenate([edf["src"].values, edf["dst"].values])) + ndf = pd.DataFrame({"id": nodes}) + return graphistry.nodes(ndf, "id").edges(edf, "src", "dst"), ndf, edf + + +def degree_seeds(edf, pctls): + """Pick one seed id at each out-degree percentile (the O(degree) honesty sweep).""" + deg = edf.groupby("src").size() + out = {} + for p in pctls: + if p >= 100: + sid = int(deg.idxmax()); out["max"] = (int(deg.max()), sid) + else: + thr = np.percentile(deg.values, p) + cand = deg[deg >= thr] + sid = int(cand.index[0]); out[f"p{p}"] = (int(deg.loc[sid]), sid) + return out + + +def _sync(engine): + if engine in ("cudf", "polars-gpu"): + try: + import cupy as cp; cp.cuda.runtime.deviceSynchronize() + except Exception: + pass + + +def timeit(fn, reps, engine, warmup=2): + for _ in range(warmup): + fn(); _sync(engine) + ts = [] + for _ in range(reps): + t0 = time.perf_counter(); fn(); _sync(engine); ts.append((time.perf_counter() - t0) * 1e3) + ts.sort(); return statistics.median(ts) + + +def _sig(g): + n, e = g._nodes, g._edges + if "polars" in type(n).__module__ or "cudf" in type(n).__module__: n = n.to_pandas() + if "polars" in type(e).__module__ or "cudf" in type(e).__module__: e = e.to_pandas() + return (len(n), len(e), int(e["src"].sum()) + int(e["dst"].sum())) + + +def bench(g0, ndf, edf, engines, reps): + maxscan = int(os.environ.get("MAXSCAN_REPS", "3")) + E = len(edf) + pctls = [int(x) for x in os.environ.get("DEG_PCTLS", "50,90,99,100").split(",")] + multiseed = [int(x) for x in os.environ.get("MULTISEED", "1,10,100,1000").split(",")] + dseeds = degree_seeds(edf, pctls) + print(f" degree seeds: " + ", ".join(f"{k}=deg{d}" for k, (d, _) in dseeds.items())) + outf = open(os.environ["OUT"], "a") if os.environ.get("OUT") else None + for engine in engines: + try: + t0 = time.perf_counter(); gi = g0.gfql_index_all(engine=engine); _sync(engine) + build_ms = (time.perf_counter() - t0) * 1e3 + except Exception as ex: + print(f" {engine}: BUILD FAILED {type(ex).__name__}: {ex}"); continue + # T3: seed-degree sweep (1-hop), guarded + for tag, (deg, sid) in dseeds.items(): + seeds = pd.DataFrame({"id": [sid]}) + with index_trace() as steps: + gidx = gi.hop(nodes=seeds, engine=engine, hops=1, direction="forward") + took = any(s.get("path") == "index" for s in steps) + gscan = g0.hop(nodes=seeds, engine=engine, hops=1, direction="forward") + same = _sig(gidx) == _sig(gscan) + valid = took and same + wi = timeit(lambda: gi.hop(nodes=seeds, engine=engine, hops=1, direction="forward"), reps, engine) + ws = timeit(lambda: g0.hop(nodes=seeds, engine=engine, hops=1, direction="forward"), + min(reps, maxscan), engine) + rec = dict(system="gfql", engine=engine, task="degsweep", seed_deg=deg, n=len(ndf), edges=E, + valid=valid, warm_idx_ms=wi, warm_scan_ms=ws, speedup=ws / wi if wi else None, build_ms=build_ms) + print(f" {engine:11} deg={deg:>8} idx={wi:9.4f}ms scan={ws:10.3f}ms x{ws/wi:7.1f}{'' if valid else ' <scan) + rng = np.random.default_rng(0) + allids = ndf["id"].values + for k in multiseed: + seeds = pd.DataFrame({"id": rng.choice(allids, size=min(k, len(allids)), replace=False)}) + with index_trace() as steps: + gidx = gi.hop(nodes=seeds, engine=engine, hops=1, direction="forward") + took = any(s.get("path") == "index" for s in steps) + wi = timeit(lambda: gi.hop(nodes=seeds, engine=engine, hops=1, direction="forward"), reps, engine) + print(f" {engine:11} kseed={k:>6} idx={wi:9.4f}ms path={'index' if took else 'scan'}") + if outf: outf.write(json.dumps(dict(system="gfql", engine=engine, task="multiseed", kseed=k, + n=len(ndf), edges=E, took_index=took, warm_idx_ms=wi)) + "\n"); outf.flush() + if outf: outf.close() + + +def main(): + g0, ndf, edf = load_graph() + print(f"===== graph: {len(ndf):,} nodes {len(edf):,} edges =====") + engines = os.environ.get("ENGINES", "pandas,polars").split(",") + bench(g0, ndf, edf, engines, int(os.environ.get("REPS", "15"))) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gfql/index_vs_kuzu_prepared.py b/benchmarks/gfql/index_vs_kuzu_prepared.py new file mode 100644 index 0000000000..d19bf1c6b1 --- /dev/null +++ b/benchmarks/gfql/index_vs_kuzu_prepared.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Fairest GFQL-vs-kuzu seeded 1-hop: BOTH in-process, warm, kuzu using a PREPARED +statement (its fast path) + result fully materialized on both sides, same seed, +matched answer counts. Removes Cypher-parse-per-call from kuzu so the comparison is +engine-vs-engine, not engine-vs-(parse+engine). kuzu is embedded like GFQL (no bolt +network), so it's the cleanest peer. Env: PARQUET=/data/.parquet""" +import os, time, statistics, tempfile, shutil +import numpy as np, pandas as pd, graphistry, kuzu + + +def med(fn, reps=25, warm=4): + for _ in range(warm): fn() + ts = [] + for _ in range(reps): + t = time.perf_counter(); fn(); ts.append((time.perf_counter() - t) * 1e3) + ts.sort(); return statistics.median(ts) + + +def main(): + edf = pd.read_parquet(os.environ["PARQUET"]).astype({"src": np.int64, "dst": np.int64}) + nodes = np.unique(np.concatenate([edf["src"].values, edf["dst"].values])) + print(f"graph: {len(nodes):,} nodes / {len(edf):,} edges") + g = graphistry.nodes(pd.DataFrame({"id": nodes}), "id").edges(edf, "src", "dst") + gi = g.gfql_index_all(engine="pandas") + deg = edf.groupby("src").size() + # typical (median-degree) and hub seeds + for tag, sid in [("typical", int(deg[deg >= deg.median()].index[0])), ("hub", int(deg.idxmax()))]: + d = int(deg.loc[sid]); seeds = pd.DataFrame({"id": [sid]}) + gfql_rows = int(gi.hop(nodes=seeds, engine="pandas", hops=1)._edges.shape[0]) + gfql_ms = med(lambda: gi.hop(nodes=seeds, engine="pandas", hops=1)) + + dbp = tempfile.mkdtemp() + db = kuzu.Database(os.path.join(dbp, "kz")); conn = kuzu.Connection(db) + conn.execute("CREATE NODE TABLE N(id INT64, PRIMARY KEY(id))") + conn.execute("CREATE REL TABLE E(FROM N TO N)") + np_path = os.path.join(dbp, "n.parquet"); ep_path = os.path.join(dbp, "e.parquet") + pd.DataFrame({"id": nodes}).to_parquet(np_path) + edf.rename(columns={"src": "from", "dst": "to"}).to_parquet(ep_path) + conn.execute(f'COPY N FROM "{np_path}"'); conn.execute(f'COPY E FROM "{ep_path}"') + stmt = conn.prepare("MATCH (a:N {id:$sid})-[:E]->(b:N) RETURN b.id") + # Columnar materialization (kuzu's fast result path) == GFQL's DataFrame output. + def kq(): + conn.execute(stmt, {"sid": sid}).get_as_df() + kr = len(conn.execute(stmt, {"sid": sid}).get_as_df()) + kuzu_ms = med(kq) + ratio = kuzu_ms / gfql_ms if gfql_ms else float("nan") + print(f" {tag:8} deg={d:>7} GFQL-pandas {gfql_ms:8.4f}ms (rows={gfql_rows}) " + f"kuzu-prepared {kuzu_ms:8.4f}ms (rows={kr}) match={gfql_rows==kr} " + f"GFQL {'faster' if ratio>1 else 'SLOWER'} {ratio:.2f}x") + shutil.rmtree(dbp, ignore_errors=True) + + +if __name__ == "__main__": + main() From 944e5b16a8d376883fc316f200a6a6fd7d2a3284 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 1 Jul 2026 11:07:26 -0700 Subject: [PATCH 07/20] =?UTF-8?q?fix(gfql/index):=20engine-aware=20cost=20?= =?UTF-8?q?gate=20=E2=80=94=20resident=20index=20never=20loses=20to=20scan?= =?UTF-8?q?=20(F1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seeded-hop planner falls back to a full scan once the frontier covers too large a fraction of source keys (past the crossover, scanning all edges once beats many index probes). That crossover is engine-dependent: a vectorized-scan engine (polars/cuDF/GPU) scans far faster than pandas, so its crossover is much smaller. The single 0.5*n_keys threshold made a resident index under index_policy='use' run ~2x SLOWER than plain scan on polars for mid-size frontiers (measured ~frac 0.02-0.5; correct result, just slow). Now per-engine: _COST_GATE_FRAC={PANDAS:0.5}, default 0.02 (vectorized). GPU values provisional pending large-graph measurement on dgx. Small-frontier wins (the point of the index) unchanged. Data-driven (CPU, N=1e5 deg8): located polars crossover at ~frac 0.02 (pandas still winning 1.5x at 0.15). Post-fix sweep: polars mid-band flips to scan at ~1.0x (was 0.5x index); pandas unchanged; index==scan match everywhere. +1 engine-parametrized regression test (direct-hop path). Index suite 43 passed; index+chain+policy 120 passed CPU; ruff+mypy clean. Also logs benchmark receipts (RESULTS.md): CPU seeded-index proof (pandas 28x-2451x, polars 7x-145x, warm flat in N) and the explicit-vs-auto decision (default 'use'). Separate finding filed as #1670 (pandas Cypher RETURN is ~O(N)). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 +++ benchmarks/gfql/RESULTS.md | 3 +++ graphistry/compute/gfql/index/api.py | 12 +++++++++- .../tests/compute/gfql/index/test_index.py | 24 +++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dbdb35af9..c50068ae09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL `materialize_nodes()` on an edges-only graph under `engine='polars'`**: crashed with `AttributeError: 'Series' object has no attribute 'drop_duplicates'` — the node-id derivation used pandas-only `.drop_duplicates()` / `.reset_index()` / `.rename(columns=)` on a polars Series/DataFrame. Now polars-aware (`.unique(maintain_order=True)`, `.select().rename({...})`), matching the pandas oracle. Surfaced by any `let()` DAG on an edges-only graph under Polars (the DAG pre-materializes nodes). pandas/cuDF paths unchanged. - **CI: `compute_networkx` HITS test was scipy-version flaky**: `test_compute_networkx_hits_outputs_hubs_and_authorities` used a pure directed 3-cycle (`a→b→c→a`), whose adjacency is a permutation matrix (all singular values equal). networkx HITS computes scores via scipy `svds(k=1)`, which then returns an arbitrary vector from that fully-degenerate singular space; when its components sum to ~0, HITS's `h /= h.sum()` normalization blows up to `inf`/`nan`. Which vector `svds` returns is scipy/LAPACK-version dependent, so the test passed on the pinned scipy but failed on the `nx-upper-scipy` CI profile's newer scipy — a latent flake, not a code regression. Switched to a graph with a well-defined hub/authority structure (`a→b`, `a→c`, `b→c`), whose dominant singular vector is unique (Perron-Frobenius) and therefore version-stable. Test-only change.- **GFQL `materialize_nodes()` on an edges-only graph under `engine='polars'`**: crashed with `AttributeError: 'Series' object has no attribute 'drop_duplicates'` — the node-id derivation used pandas-only `.drop_duplicates()` / `.reset_index()` / `.rename(columns=)` on a polars Series/DataFrame. Now polars-aware (`.unique(maintain_order=True)`, `.select().rename({...})`), matching the pandas oracle. Surfaced by any `let()` DAG on an edges-only graph under Polars (the DAG pre-materializes nodes). pandas/cuDF paths unchanged.- **GFQL `engine='polars-gpu'` LazyFrame-input coercion now collects on the GPU executor**: `Engine.df_to_engine(lazyframe, POLARS_GPU)` materialized a `polars.LazyFrame` input with a bare `.collect()` on the CPU default executor — ignoring `gpu_executor()` and not distinguishing `POLARS` from `POLARS_GPU`. It now routes through the target-aware lazy collect, so a LazyFrame coerced under `polars-gpu` collects on the cudf-polars GPU executor (and `polars` honors `cpu_streaming()`). No change for already-materialized frame inputs (the common path). - **GFQL Polars engine `contains` honors `regex=`/`flags=`**: the native polars `filter_by_dict` lowering of the `Contains` predicate always used `str.contains(..., literal=False)`, so a **literal** request (`contains(pat, regex=False)`) was still regex-interpreted — a pattern with a metacharacter over-matched (e.g. `contains('a.c', regex=False)` matched `'abc'`), diverging from pandas/cuDF. It also dropped `flags=`. Now `regex=False` lowers to a literal match (`literal=True`; case-insensitive literal folds both sides, matching pandas' result), and regex mode maps `case=`/`flags=` (IGNORECASE/MULTILINE/DOTALL/VERBOSE) to a Rust-regex inline flag prefix (`(?ims…)`). +1 engine-parametrized differential-parity test. +- **GFQL adjacency-index cost gate is engine-aware (never slower than scan)**: the seeded-hop planner falls back to a full scan once the frontier covers too large a fraction of the source keys (past that, scanning all edges once beats many index probes). That crossover fraction is *engine-dependent* — a vectorized-scan engine (polars/cuDF/GPU) has a far faster scan, so its crossover is much smaller than pandas'. The gate previously used a single `0.5·n_keys` threshold, so on polars a **resident index under `index_policy='use'` ran ~2× slower than the plain scan** for mid-size frontiers (measured ~frac 0.02–0.5; correct result, just slow). The gate now uses a per-engine crossover (pandas ~0.5, vectorized engines ~0.02; GPU provisional pending large-graph measurement), so a resident index never loses to the un-indexed path on any engine. Small-frontier wins (the point of the index) are unchanged. +1 engine-parametrized regression test. +- **GFQL `ne()` / `<>` on NULL now follows openCypher/SQL 3-valued logic (pandas)**: `n({"col": ne(x)})` and cypher `WHERE n.col <> x` over a NULL/NA cell used to KEEP the null row on the pandas engine (`NaN != x` → True), diverging from cuDF and the polars engine (both drop it) — and even from pandas' own `WHERE NOT n.col = x` path. Per openCypher/SQL three-valued logic, `null <> x` is `null` (an unknown value cannot be proven unequal to `x`), so a null cell is **not** a match and the row is excluded — consistent with `eq`/`gt`/`lt`/`IN` (which already dropped nulls). Fixed the `NE` predicate to mask out nulls; this corrects both the `filter_dict` predicate path and the single-entity cypher `<>` WHERE path on pandas. cuDF/polars/polars-gpu were already conformant. Verified across all four engines (`ne`, `<>`, `NOT =`, `NOT IN` all drop the null). Note: this is a behavior change for `ne()` on nullable columns under the default pandas engine. (Broader openCypher null-semantics alignment + docs tracked in #1664.) +- **GFQL membership filter (`n({col: [..]})` / `IN`) on NULL follows openCypher 3VL (cuDF)**: a list/membership filter over a NULL cell — e.g. `n({"kind": ["x", "z", None]})` — used to KEEP the null row on the **cuDF** engine (cuDF `isin` matched a null cell against a `None` list element), diverging from pandas and polars (both exclude it). Per openCypher/SQL 3VL, `null IN [...]` is `null` → not a member → excluded. Fixed in `filter_by_dict` (`& notna()` on the membership branch); a no-op for pandas/polars, a fix for cuDF. (Part of the #1664 openCypher-conformance sweep.) - **GFQL `engine='polars-gpu'` silent CPU fallback removed (NO-CHEATING)**: the GPU collect used `pl.GPUEngine(raise_on_fail=False)`, so any plan node the cudf_polars backend can't execute would silently run **on CPU** and still be reported as a `polars-gpu` result — making `engine='polars-gpu'` indistinguishable from `engine='polars'` whenever the plan isn't fully GPU-capable (a benchmark showing near-identical `polars`/`polars-gpu` timings is exactly this tell). Flipped to `raise_on_fail=True` and translate the cudf_polars failure into a clear `NotImplementedError` pointing at `engine='polars'` for native CPU. `polars-gpu` is now **GPU-or-error**: any timing it produces is real on-device work, never CPU mislabeled as GPU. Verified on dgx-spark (LiveJournal 35M): the seeded-frontier `hop`/2-hop chain plan executes fully on GPU without raising (nvidia-smi 92% util during the loop), so existing GPU timings are unchanged — only the honesty guarantee is added. +1 regression test (the GPU-collect error path is translated, not swallowed). - **GFQL Polars chain dtype-mismatched join keys**: a chain (`MATCH (a)-[]->(b)…`) crashed under `engine='polars'` with `SchemaError: datatypes of join keys don't match` when an edge endpoint column's dtype differed from the node-id dtype across the int↔float boundary — e.g. a null in a `source`/`destination` column promotes it to `float64` while integer node ids stay `int64`. The hop already aligned join keys; the chain fast paths + combine did not. Now aligns endpoint join keys to the node-id dtype for the traversal and restores the original endpoint dtype on the output (matching pandas). No-op when dtypes already match. - **GFQL `chain()` OTel span placement**: the `gfql.chain` OpenTelemetry span (`@otel_traced`) had landed on the internal `_try_chain_fast_path` probe instead of the public `chain()` — so `chain()` lost its span and the span recorded the wrong function/attributes. Moved the decorator back onto `chain()`. diff --git a/benchmarks/gfql/RESULTS.md b/benchmarks/gfql/RESULTS.md index 624715682e..e13a2e305a 100644 --- a/benchmarks/gfql/RESULTS.md +++ b/benchmarks/gfql/RESULTS.md @@ -85,3 +85,6 @@ Summary-only log for notable benchmark runs. Raw per-scenario outputs live in | 2026-03-15 | DGX exploratory (feat/gfql-gpu-pagerank-benchmark) | `neo4j_pipeline_probe.py` Neo4j+GDS analog on SNAP GPlus | GPlus exact analog (`degree_q=0.995`, `pagerank_q=0.9995`) exceeded `3m07s` before the main transaction even finished closing, even after batching seed/core expansion to avoid transaction-memory OOM. | Raw output: `plans/gfql-gpu-pagerank-benchmark/results/neo4j_summary.md` | | 2026-03-15 | wip (`feat/gfql-gpu-pagerank-benchmark`) | `load_prepare_cpu_gpu.py` on SNAP Twitter (`degree_q=0.99`) | Cached prep median: CPU `0.2756s` vs GPU `0.1013s` (`2.72x`). Read `0.2156s` vs `0.0862s`; node prep `0.0620s` vs `0.0148s`; bind negligible. | Raw output: `plans/gfql-gpu-pagerank-benchmark/results/twitter_load_prepare_infer.json` | | 2026-03-15 | wip (`feat/gfql-gpu-pagerank-benchmark`) | `load_prepare_cpu_gpu.py` on SNAP GPlus (`degree_q=0.995`) | Cached prep median: CPU `8.7160s` vs GPU `3.9323s` (`2.22x`). Read `6.9096s` vs `3.0395s`; node prep `1.8097s` vs `0.8613s`; bind negligible. | Raw output: `plans/gfql-gpu-pagerank-benchmark/results/gplus_load_prepare_infer.json`; explicit integer-dtype probe rejected because Twitter regressed slightly and GPlus pandas overflowed. | +| 2026-07-01 | 1257dac9 (dev/gfql-seeded-traversal-index) | `index_perf.py` ENGINES=pandas,polars NS=1e4,1e5,1e6 DEG=8 REPS=10 SEEDS=1 (local CPU, small-run proof) | Seeded warm FLAT ~0.30–0.33ms across N=1e4→1e6 (edges 80k→8M); scan O(E): pandas 8.6→808ms (28×→2451×), polars 2.2→45.5ms (7×→145×). warm floor engine-independent (~0.3ms=numpy searchsorted). Parity via 41-passed test suite. | Raw: `plans/gfql-1658-seeded-index/receipts/p1-{pandas,polars}.jsonl` | +| 2026-07-01 | 1257dac9 (dev/gfql-seeded-traversal-index) | `explicit_vs_auto_sweep.py` (frontier sweep, pandas+polars, N=1e5 E=8e5, guarded match=True) | Index crossover engine-dependent: pandas wins to ~frac 0.5, polars scan ~18× faster → crossover ~frac 0.02. **DECISION: default `use`.** Finding F1(IMPORTANT): cost gate 0.5·n_keys miscalibrated for polars → resident index 0.4–0.5× (2× slower) in F≈1k–50k mid-band; fix=engine-aware gate. | Decision: `plans/gfql-1658-seeded-index/receipts/decision-explicit-vs-auto.md`; raw: `…/p3-explicit-vs-auto.jsonl` | +| 2026-07-01 | 1257dac9 (dev/gfql-seeded-traversal-index) | `lookup_probe.py` [D-index-wiring] point/range, N=500k M=2M id=1..N, pandas+polars CPU | Bug confirmed (node_id index ignored by filter path, 1.01× w/ index) BUT filter is sub-ms (pandas 0.79ms/polars 0.34ms); the point-lookup cost is the **Cypher RETURN row-pipeline**: RETURN a = pandas 94.7ms vs polars 1.2ms. DECISION: **REJECT node_id→filter wiring** (saves <1ms vs ~94ms RETURN); real lever = pandas RETURN path / use polars-cudf. | `plans/gfql-1658-seeded-index/receipts/lookup-finding.md` | diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 5976a0f0bf..2ea1abcf52 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -21,6 +21,15 @@ REGISTRY_ATTR = "_gfql_index_registry" +# Index-vs-scan crossover fraction (of distinct source keys): past this frontier a +# full scan is cheaper than per-seed index probes, so `use` falls back to scan and +# never loses to the un-indexed path. Engine-aware because vectorized-scan engines +# (polars/cudf/GPU) scan far faster than pandas, so their crossover is much smaller +# (F1). Measured N=1e5 deg8: pandas ~0.5, polars ~0.02. GPU values provisional +# (dgx-gated) — conservatively grouped with polars. +_COST_GATE_FRAC = {Engine.PANDAS: 0.5} +_COST_GATE_FRAC_DEFAULT = 0.02 + # --- lightweight, thread-local index decision trace (for gfql_explain) ------- import threading as _threading _TRACE = _threading.local() @@ -315,7 +324,8 @@ def maybe_index_hop( elif policy != "force": try: frontier_n = int(nodes.shape[0]) - if idx0.n_keys > 0 and frontier_n >= 0.5 * idx0.n_keys: + frac = _COST_GATE_FRAC.get(engine, _COST_GATE_FRAC_DEFAULT) + if idx0.n_keys > 0 and frontier_n >= frac * idx0.n_keys: return None except Exception: pass diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 3541b5e977..9240c6b23f 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -135,6 +135,30 @@ def test_index_policy_force_and_explain(graph, engine): assert _sig(r_scan) == _sig(r_force) +@pytest.mark.parametrize("engine", ENGINES) +def test_cost_gate_engine_aware_never_loses_to_scan(engine): + """F1: the index-vs-scan crossover depends on scan speed, so the cost gate + (maybe_index_hop) is engine-aware. A mid-frontier seeded hop (~frac 0.25 of source + keys) still beats the slow pandas scan (→ index path) but would lose to the fast + vectorized scan on polars/cudf/GPU (→ scan fallback), so a resident index never + runs slower than the un-indexed path. Result is identical on either path.""" + from graphistry.compute.gfql.index import index_trace + rng = np.random.default_rng(1) + N, deg = 4000, 8 + m = N * deg + edf = pd.DataFrame({"src": rng.integers(0, N, m), "dst": rng.integers(0, N, m)}) + ndf = pd.DataFrame({"id": np.arange(N)}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + seeds = pd.DataFrame({"id": np.arange(N // 4, dtype=np.int64)}) # frac ~0.25 of n_keys (~N) + gi = g.gfql_index_all(engine=engine) + with index_trace() as steps: + out = gi.hop(nodes=seeds, engine=engine, hops=1, direction="forward") + took_index = any(s.get("path") == "index" for s in steps) + assert took_index is (engine == "pandas"), (engine, steps) # vectorized engines → scan + scan_out = g.hop(nodes=seeds, engine=engine, hops=1, direction="forward") # no resident index + assert _sig(out) == _sig(scan_out), engine # correctness path-independent + + def test_column_mismatch_raises_not_silent(graph): # A custom column that doesn't match the binding must raise, not silently no-op. with pytest.raises(NotImplementedError): From 42897637b254f21e821863caf47d1e24d1955b62 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Thu, 2 Jul 2026 10:03:33 -0700 Subject: [PATCH 08/20] =?UTF-8?q?garden(gfql/index):=20team-polish=20?= =?UTF-8?q?=E2=80=94=20dead=20code,=20IF=20EXISTS,=20docstrings,=20bench?= =?UTF-8?q?=20honesty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review-wave findings on the index PR: - Delete dead engine_arrays.to_backend/cp_to_numpy (never called) + unused get_registry import (explain.py); remove index_ddl_smoke.py 'if False' scaffolding + unused variable. - DropIndex.missing_ok was serialized but never honored. Now real IF EXISTS semantics: 'DROP GFQL INDEX [IF EXISTS] ' — plain DROP of a missing index raises (SQL-style, matching the existing custom-name test), IF EXISTS / missing_ok=true is a no-op; wire default flipped to false (field is new in this PR — no consumers). +1 test covering DDL both forms, wire JSON both ways, and resident-drop. - Replace internal review-wave codes (F1/B1/B2/B3/I4/I5/I6/P0-1) in production comments with plain prose or in-repo test-name pointers (test-file section labels stay — they title the regression tests). - ComputeMixin: real docstrings on create_index/drop_index/show_indexes/ gfql_index_edges/gfql_index_all/gfql_explain (were one-liner-pointer or missing); gfql() docstring now documents index_policy (was invisible to help() despite being the flagship knob). - index_bulk_olap_bench.py: polars-gpu cell used raise_on_fail=False — the exact silent-CPU-fallback pattern the CHANGELOG bans; now GPU-or-error. - Label index_smoke/index_ddl_smoke as container-runnable mirrors of the canonical pytest suite; add benchmarks/gfql/README.md catalog for the 7 index_* harnesses (which pair is canonical for published claims). Validated: index suite 44 pass CPU-lane (cudf/polars-gpu lanes = local no-GPU artifact, dgx-verified separately); ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmarks/gfql/README.md | 27 +++++++++++++++ benchmarks/gfql/index_bulk_olap_bench.py | 4 ++- benchmarks/gfql/index_ddl_smoke.py | 10 +++--- benchmarks/gfql/index_smoke.py | 3 ++ graphistry/compute/ComputeMixin.py | 33 +++++++++++++++++-- graphistry/compute/gfql/index/api.py | 4 +-- graphistry/compute/gfql/index/build.py | 2 +- graphistry/compute/gfql/index/cypher_ddl.py | 13 ++++---- .../compute/gfql/index/engine_arrays.py | 21 ------------ graphistry/compute/gfql/index/explain.py | 4 +-- graphistry/compute/gfql/index/lookup.py | 4 +-- graphistry/compute/gfql/index/registry.py | 6 ++-- graphistry/compute/gfql/index/traverse.py | 8 ++--- graphistry/compute/gfql/index/wire.py | 10 ++++-- .../tests/compute/gfql/index/test_index.py | 28 ++++++++++++++++ 15 files changed, 126 insertions(+), 51 deletions(-) diff --git a/benchmarks/gfql/README.md b/benchmarks/gfql/README.md index 22de07ed86..458fe1c4e9 100644 --- a/benchmarks/gfql/README.md +++ b/benchmarks/gfql/README.md @@ -382,3 +382,30 @@ So the honest current comparison is: - Neo4j is workable for the smaller Twitter analog, but already materially slower than both Graphistry CPU and GPU on the exact same shape. - On the selected GPlus benchmark shape, Neo4j is already dramatically slower than Graphistry CPU (`83.61s`) and Graphistry GPU (`3.41s`) before teardown/cleanup is even done. - Raw notes: `plans/gfql-gpu-pagerank-benchmark/results/neo4j_summary.md` + +## GFQL physical-index harnesses (`index_*.py`) + +Catalog for the seeded-traversal adjacency-index work (PR #1658; see +`docs/source/gfql/index_adjacency.rst` for the user guide + published numbers): + +- `index_smoke.py` / `index_ddl_smoke.py` — correctness smokes: differential + parity (index path == scan path) and DDL/wire/`index_policy`/`gfql_explain` + round-trips. Container-runnable MIRRORS of the canonical pytest suite + (`graphistry/tests/compute/gfql/index/test_index.py`). +- `index_perf.py` — microbenchmark: seeded 1-hop/2-hop index-vs-scan across + engines and frontier sizes; the cost-gate calibration numbers come from here. +- `index_takeover_bench.py` + `index_vs_kuzu_prepared.py` — **canonical + competitor pair** behind the published "9–36x vs Kuzu" seeded-traversal + claims: guarded timings (path-taken assertions + result==scan oracle), + prepared-statement fairness on the Kuzu side. +- `index_vs_dbs.py` — broader 3-way sweep (GFQL / Kuzu / Neo4j) over the same + seeded shapes; superset of the pair above, heavier setup (dockerized Neo4j). +- `index_bulk_olap_bench.py` — bulk group-by/degree OLAP shapes across the 4 + engines (answers "does the index help bulk scans?" — it does not; scans win). +- `index_largegraph_bench.py` — scale probe on large real graphs + (flat-in-N behavior of the seeded path up to ~80M edges). + +Methodology for all: warm medians, one engine per process for large runs, +NO-CHEATING guards (a timing is void unless the intended path was taken AND +the result matches the scan oracle). Run on an idle box; GPU rows need +`--gpus all` and RAPIDS. diff --git a/benchmarks/gfql/index_bulk_olap_bench.py b/benchmarks/gfql/index_bulk_olap_bench.py index 257a3b9c35..5bad776d98 100644 --- a/benchmarks/gfql/index_bulk_olap_bench.py +++ b/benchmarks/gfql/index_bulk_olap_bench.py @@ -114,7 +114,9 @@ def degall(edf, engine, reps, warm): import polars as pl df = pl.from_pandas(edf) if engine == "polars-gpu": - eng = pl.GPUEngine(executor="in-memory", raise_on_fail=False) + # raise_on_fail=True: GPU-or-error — a silent CPU fallback would report + # CPU time as a polars-gpu result (banned; see the polars-gpu CHANGELOG fix) + eng = pl.GPUEngine(executor="in-memory", raise_on_fail=True) fn = lambda: df.lazy().group_by("src").len().collect(engine=eng) else: fn = lambda: df.group_by("src").len() diff --git a/benchmarks/gfql/index_ddl_smoke.py b/benchmarks/gfql/index_ddl_smoke.py index 46e32b7b2f..9f11ee0c78 100644 --- a/benchmarks/gfql/index_ddl_smoke.py +++ b/benchmarks/gfql/index_ddl_smoke.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Smoke test: Cypher DDL + JSON wire + index_policy auto/force + gfql_explain.""" +"""Smoke test: Cypher DDL + JSON wire + index_policy auto/force + gfql_explain. + +Container-runnable MIRROR of graphistry/tests/compute/gfql/index/test_index.py +(the pytest suite is canonical); kept for quick in-container smoke without pytest. +""" from __future__ import annotations import sys @@ -63,9 +67,7 @@ def main(): # --- index_policy auto/force build-on-demand + explain --- gp = make_graph() # fresh, no resident indexes - exp_use = gp.gfql_explain([], index_policy="use") if False else None # explain needs a real query - # explain with a seeded 1-hop chain expressed in cypher - q = "MATCH (a)-[e]->(b) RETURN b" # not seeded -> not coverable; use python chain instead + # explain with a seeded 1-hop chain (explain needs a real, seeded query) from graphistry.compute.ast import n, e_forward seeded_chain = [n({"id": 0}), e_forward(hops=1)] rep_off = gp.gfql_explain(seeded_chain, index_policy="off") diff --git a/benchmarks/gfql/index_smoke.py b/benchmarks/gfql/index_smoke.py index f0633cbcff..5d90e8a02f 100644 --- a/benchmarks/gfql/index_smoke.py +++ b/benchmarks/gfql/index_smoke.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 """Differential parity smoke test for GFQL physical indexes. +Container-runnable MIRROR of graphistry/tests/compute/gfql/index/test_index.py +(the pytest suite is canonical); kept for quick in-container smoke without pytest. + Oracle: the index fast path MUST return the same subgraph (node-id set + edge multiset) as the scan/join path, across engines / directions / hops / wavefront. A fast wrong answer is not a win. diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index f8b73036b9..8063bbf0a4 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -583,23 +583,42 @@ def hop(self, *args, **kwargs): # ---- GFQL physical indexes (pay-as-you-go seeded-traversal acceleration) ---- def create_index(self, kind, *, column=None, name=None, engine='auto'): + """Build a GFQL physical index for O(degree) seeded traversal. + + :param kind: 'edge_out_adj' (forward hops), 'edge_in_adj' (reverse hops), or 'node_id' (node lookup) + :param column: column to index (defaults to the binding for the kind, e.g. the edge source column) + :param name: optional custom index name (defaults to 'kind:column') + :param engine: 'auto' | 'pandas' | 'cudf' | 'polars' — array backend for the index + :returns: new Plottable with the index resident (original is untouched) + + **Example** + :: + + g2 = g.create_index('edge_out_adj') + g2.gfql([n({'id': 0}), e_forward(hops=2)], index_policy='use') + + See :doc:`gfql/index_adjacency` for policies, cost gating, and benchmarks. + """ from graphistry.compute.gfql.index import create_index as _ci return _ci(self, kind, column=column, name=name, engine=engine) - create_index.__doc__ = "Build a GFQL physical index (edge_out_adj|edge_in_adj|node_id). See graphistry/compute/gfql/index/api.py." def drop_index(self, kind=None): + """Drop one resident GFQL index (by kind) or all (kind=None). Idempotent; returns a new Plottable.""" from graphistry.compute.gfql.index import drop_index as _di return _di(self, kind) def show_indexes(self): + """Return a pandas DataFrame describing resident GFQL indexes (name, kind, column, valid). Empty if none; ``valid=False`` marks a stale index after a frame rebind.""" from graphistry.compute.gfql.index import show_indexes as _si return _si(self) def gfql_index_edges(self, direction='both', engine='auto'): + """Convenience: build the edge adjacency index(es) — 'forward', 'reverse', or 'both'. Returns a new Plottable.""" from graphistry.compute.gfql.index import gfql_index_edges as _gie return _gie(self, direction, engine=engine) def gfql_index_all(self, engine='auto'): + """Convenience: build all GFQL physical indexes (both edge adjacencies + node_id). Returns a new Plottable.""" from graphistry.compute.gfql.index import gfql_index_all as _gia return _gia(self, engine=engine) @@ -650,9 +669,19 @@ def gfql(self, *args, **kwargs): g = _copy.copy(self) g._gfql_index_policy = policy return gfql_base(g, *args, **kwargs) - gfql.__doc__ = gfql_base.__doc__ + gfql.__doc__ = (gfql_base.__doc__ or "") + """ + + **GFQL physical indexes** + + :param index_policy: 'off' (never use indexes), 'use' (use resident, + cost-gated; default planner behavior), 'auto' (build on demand), or + 'force' (always probe the index). Also accepts index DDL strings + (``CREATE GFQL INDEX ...``) / wire ops as the query — routed to the + index registry. See :meth:`create_index` and :doc:`gfql/index_adjacency`. + """ def gfql_explain(self, query, *, index_policy='use', engine='auto'): + """Explain how the GFQL planner would run ``query``: per-hop index-vs-scan choice, cost-gate numbers, and resident-index validity. Read-only (no execution). Returns a report object; print it for a human-readable plan.""" from graphistry.compute.gfql.index.explain import gfql_explain as _ge return _ge(self, query, index_policy=index_policy, engine=engine) diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 2ea1abcf52..efaf806193 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -25,7 +25,7 @@ # full scan is cheaper than per-seed index probes, so `use` falls back to scan and # never loses to the un-indexed path. Engine-aware because vectorized-scan engines # (polars/cudf/GPU) scan far faster than pandas, so their crossover is much smaller -# (F1). Measured N=1e5 deg8: pandas ~0.5, polars ~0.02. GPU values provisional +# (see test_index_cost_gate_engine_aware). Measured N=1e5 deg8: pandas ~0.5, polars ~0.02. GPU values provisional # (dgx-gated) — conservatively grouped with polars. _COST_GATE_FRAC = {Engine.PANDAS: 0.5} _COST_GATE_FRAC_DEFAULT = 0.02 @@ -332,7 +332,7 @@ def maybe_index_hop( # Honor max_hops: the scan resolves the hop count as ``max_hops or hops`` # (compute/hop.py); the index must run the SAME number of accumulating hops. - # (B1: max_hops was passed through *rest and silently ignored — the index ran + # (regression: max_hops was passed through *rest and silently ignored — the index ran # `hops` (default 1) while the scan ran max_hops → wrong answer.) _max_hops = rest.get("max_hops") eff_hops = _max_hops if _max_hops is not None else hops diff --git a/graphistry/compute/gfql/index/build.py b/graphistry/compute/gfql/index/build.py index 88e3129ab7..9eaa7c5556 100644 --- a/graphistry/compute/gfql/index/build.py +++ b/graphistry/compute/gfql/index/build.py @@ -78,7 +78,7 @@ def build_node_id_index( row position per unique key (``row_positions[group_offsets[:-1]]``, length U, aligned with ``unique_keys``) and (b) REFUSE (return None) when ids aren't unique: a unique-key CSR can't reproduce the scan's "all rows per id" semantics, so the - caller falls back to the correct ``select_by_ids`` isin path. (B2: a non-unique + caller falls back to the correct ``select_by_ids`` isin path. (Regression guard: a non-unique node-id index dropped reached nodes / emitted unrelated rows.)""" xp, backend = array_namespace(engine) keys = col_to_array(nodes, node_col, engine) diff --git a/graphistry/compute/gfql/index/cypher_ddl.py b/graphistry/compute/gfql/index/cypher_ddl.py index 0f95b50090..2655bc9500 100644 --- a/graphistry/compute/gfql/index/cypher_ddl.py +++ b/graphistry/compute/gfql/index/cypher_ddl.py @@ -1,8 +1,8 @@ """Targeted recognizer for GFQL index DDL Cypher statements. CREATE GFQL INDEX [] FOR [ON ] - DROP GFQL INDEX - DROP GFQL INDEX FOR [ON ] + DROP GFQL INDEX [IF EXISTS] + DROP GFQL INDEX [IF EXISTS] FOR [ON ] SHOW GFQL INDEXES The mandatory ``GFQL`` token disambiguates from standard property ``CREATE INDEX`` @@ -25,12 +25,12 @@ re.IGNORECASE, ) _DROP_FOR_RE = re.compile( - r"^\s*DROP\s+GFQL\s+INDEX\s+FOR\s+" + _KIND + r"^\s*DROP\s+GFQL\s+INDEX\s+(?PIF\s+EXISTS\s+)?FOR\s+" + _KIND + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$", re.IGNORECASE, ) _DROP_NAME_RE = re.compile( - r"^\s*DROP\s+GFQL\s+INDEX\s+(?P[A-Za-z_][\w:]*)\s*;?\s*$", + r"^\s*DROP\s+GFQL\s+INDEX\s+(?PIF\s+EXISTS\s+)?(?P[A-Za-z_][\w:]*)\s*;?\s*$", re.IGNORECASE, ) _SHOW_RE = re.compile(r"^\s*SHOW\s+GFQL\s+INDEXES\s*;?\s*$", re.IGNORECASE) @@ -54,10 +54,11 @@ def parse_index_ddl(query: str) -> Optional[Any]: name=m.group("name")) m = _DROP_FOR_RE.match(query) if m: - return DropIndex(kind=m.group("kind").lower(), column=m.group("col")) + return DropIndex(kind=m.group("kind").lower(), column=m.group("col"), + missing_ok=bool(m.group("ifexists"))) m = _DROP_NAME_RE.match(query) if m: - return DropIndex(name=m.group("name")) + return DropIndex(name=m.group("name"), missing_ok=bool(m.group("ifexists"))) if looks_like_index_ddl(query): raise ValueError( f"Malformed GFQL INDEX DDL: {query!r}. Expected e.g. " diff --git a/graphistry/compute/gfql/index/engine_arrays.py b/graphistry/compute/gfql/index/engine_arrays.py index 8417f6102d..eb3d2cdc58 100644 --- a/graphistry/compute/gfql/index/engine_arrays.py +++ b/graphistry/compute/gfql/index/engine_arrays.py @@ -51,27 +51,6 @@ def ids_to_array(ids: Any, col: str, engine: Engine) -> Any: return col_to_array(ids, col, engine) -def to_backend(arr: Any, backend: str) -> Any: - """Move a host/device array to the index backend (numpy<->cupy).""" - import numpy as np - - if backend == "cupy": - import cupy as cp # type: ignore - - return cp.asarray(arr) - if "cupy" in type(arr).__module__: - return cp_to_numpy(arr) - return np.asarray(arr) - - -def cp_to_numpy(arr: Any) -> Any: - try: - return arr.get() - except Exception: - import numpy as np - - return np.asarray(arr) - def take_rows(df: Any, positions: Any, engine: Engine) -> Any: """Positionally gather rows of ``df`` by an integer array ``positions``. diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index f12b705624..939c62774e 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -3,14 +3,14 @@ Seeded queries are cheap once indexed, so explain *executes* the query under a decision trace and reports the real per-hop index-vs-scan path (plus resident indexes). This makes "it silently scanned" detectable and assertable rather than -a mystery — the top human-factors need from the design review (P0-1). +a mystery — the top human-factors need from the design review. """ from __future__ import annotations from typing import Any, Dict, cast from graphistry.Engine import resolve_engine -from .api import index_trace, get_registry, show_indexes +from .api import index_trace, show_indexes def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = "auto") -> Dict[str, Any]: diff --git a/graphistry/compute/gfql/index/lookup.py b/graphistry/compute/gfql/index/lookup.py index b2441da067..bb198c3ff5 100644 --- a/graphistry/compute/gfql/index/lookup.py +++ b/graphistry/compute/gfql/index/lookup.py @@ -33,7 +33,7 @@ def lookup_edge_rows(index: AdjacencyIndex, frontier: Any, xp: Any): f = frontier if f.dtype != keys.dtype: - # I6: promote BOTH sides to a common dtype — never narrow the query to the key + # Promote BOTH sides to a common dtype — never narrow the query to the key # dtype (an int64 id cast to int32 keys wraps and false-matches). Widening a # sorted int array preserves order, so searchsorted stays valid. common = xp.promote_types(f.dtype, keys.dtype) @@ -85,7 +85,7 @@ def lookup_node_rows(index: NodeIdIndex, ids: Any, xp: Any) -> Any: return index.row_positions[:0] f = ids if f.dtype != keys.dtype: - common = xp.promote_types(f.dtype, keys.dtype) # I6: promote, never narrow + common = xp.promote_types(f.dtype, keys.dtype) # promote, never narrow f = f.astype(common) keys = keys.astype(common) pos = xp.searchsorted(keys, f) diff --git a/graphistry/compute/gfql/index/registry.py b/graphistry/compute/gfql/index/registry.py index 3bac0559f8..cb02f33993 100644 --- a/graphistry/compute/gfql/index/registry.py +++ b/graphistry/compute/gfql/index/registry.py @@ -29,7 +29,7 @@ def frame_fingerprint(df: Any, cols: Tuple[str, ...], engine: Engine) -> Tuple: + bound columns + engine. This is a SECONDARY guard; the primary validity check is object IDENTITY (``source_ref is df``, see ``get_valid``). We deliberately do NOT use ``id(df)`` here — a GC'd frame's id can be recycled by a new same-shape - frame, which `id`-equality would accept (stale index → wrong answer, I5). Holding + frame, which `id`-equality would accept (stale index → wrong answer). Holding a strong ref + identity is recycle-proof. (Pure-functional rebind via ``.edges()``/``.nodes()`` yields a new object → identity miss → safe scan.)""" try: @@ -57,7 +57,7 @@ class AdjacencyIndex: backend: str # 'numpy' | 'cupy' engine: Engine fingerprint: Tuple = field(compare=False, default=()) - source_ref: Any = field(compare=False, default=None) # the indexed frame (identity guard, I5) + source_ref: Any = field(compare=False, default=None) # the indexed frame (identity guard) n_edges: int = 0 n_keys: int = 0 name: Optional[str] = None @@ -112,7 +112,7 @@ def get_valid(self, kind: str, df: Any, cols: Tuple[str, ...], engine: Engine) - return None if idx.engine != engine: return None - # Primary: object IDENTITY (I5) — recycle-proof, since the index holds a strong + # Primary: object IDENTITY — recycle-proof, since the index holds a strong # ref so the frame's id can't be reused while indexed. `is` on a rebound frame # is False → safe miss. (source_ref None only for legacy/hand-built indexes.) if idx.source_ref is not None and idx.source_ref is not df: diff --git a/graphistry/compute/gfql/index/traverse.py b/graphistry/compute/gfql/index/traverse.py index 459093e95f..5b2a5ddb99 100644 --- a/graphistry/compute/gfql/index/traverse.py +++ b/graphistry/compute/gfql/index/traverse.py @@ -73,7 +73,7 @@ def index_seeded_hop( xp, _backend = array_namespace(engine) - # I6: do NOT narrow the seed to the index key dtype (a node-id int64 seed cast to + # Do NOT narrow the seed to the index key dtype (a node-id int64 seed cast to # an int32 edge-endpoint key wraps large ids → false match). lookup promotes both # sides to a common dtype; numpy/cupy set ops promote on concat. So we keep ids at # their natural width throughout and only ever widen. @@ -130,7 +130,7 @@ def index_seeded_hop( if materialize_endpoints and int(all_rows.shape[0]) > 0: src_vals = col_to_array(out_edges, src, engine) dst_vals = col_to_array(out_edges, dst, engine) - endpoints = xp.unique(xp.concatenate([src_vals, dst_vals])) # natural dtype (I6) + endpoints = xp.unique(xp.concatenate([src_vals, dst_vals])) # natural dtype, never narrowed needed = union1d(needed, endpoints, xp) # Materialize node rows. Prefer the node_id index (O(result·log N) searchsorted @@ -138,14 +138,14 @@ def index_seeded_hop( node_idx = registry.get_valid(NODE_ID, g._nodes, (node_col,), engine) if node_idx is not None: node_rows = lookup_node_rows(node_idx, needed, xp) - # I4: lookup returns rows in id-hit order; sort ascending so out_nodes keep + # lookup returns rows in id-hit order; sort ascending so out_nodes keep # the original .nodes table order (the index must never reorder .nodes). node_rows = xp.sort(node_rows) out_nodes = take_rows(g._nodes, node_rows, engine) else: out_nodes = select_by_ids(g._nodes, node_col, needed, engine) - # B3: the scan synthesizes a node row for EVERY edge endpoint, including ids + # The scan synthesizes a node row for EVERY edge endpoint, including ids # absent from the node table (compute/hop.py "Ensure all edge endpoints are # present in nodes"); the index only materializes existing rows. If any needed id # is missing from the materialized nodes, fall back to scan (return None) rather diff --git a/graphistry/compute/gfql/index/wire.py b/graphistry/compute/gfql/index/wire.py index e37fec4738..7075efdb47 100644 --- a/graphistry/compute/gfql/index/wire.py +++ b/graphistry/compute/gfql/index/wire.py @@ -5,7 +5,7 @@ ``to_json``/``from_json`` and are dispatched by ``index_op_from_json``. {"type": "CreateIndex", "kind": "edge_out_adj", "column": null, "name": null, "replace": false} - {"type": "DropIndex", "name": "edge_out_adj:src", "missing_ok": true} + {"type": "DropIndex", "name": "edge_out_adj:src", "missing_ok": true} # true = IF EXISTS (no-op when missing; default false = raise) {"type": "DropIndex", "kind": "edge_in_adj", "column": "dst", "missing_ok": true} {"type": "ShowIndexes"} @@ -45,7 +45,7 @@ class DropIndex: name: Optional[str] = None kind: Optional[str] = None column: Optional[str] = None - missing_ok: bool = True + missing_ok: bool = False # IF EXISTS semantics: True = dropping a missing index is a no-op def to_json(self) -> Dict[str, Any]: return {"type": "DropIndex", "name": self.name, "kind": self.kind, @@ -54,7 +54,7 @@ def to_json(self) -> Dict[str, Any]: @staticmethod def from_json(d: Dict[str, Any]) -> "DropIndex": return DropIndex(name=d.get("name"), kind=d.get("kind"), column=d.get("column"), - missing_ok=bool(d.get("missing_ok", True))) + missing_ok=bool(d.get("missing_ok", False))) @dataclass(frozen=True) @@ -112,10 +112,14 @@ def apply_index_op(g: Any, op: Any, *, engine: Any = "auto") -> Any: kind = next((k for k, ix in reg.indexes.items() if getattr(ix, "name", None) == op.name), None) if kind is None: + if op.missing_ok: + return g # IF EXISTS semantics: dropping a missing index is a no-op raise ValueError( f"DROP GFQL INDEX: no resident index named {op.name!r} " f"(resident: {sorted(getattr(ix, 'name', k) for k, ix in reg.indexes.items())})" ) + if kind is not None and not op.missing_ok and not get_registry(g).has(kind): + raise ValueError(f"DROP GFQL INDEX: no resident index of kind {kind!r}") return drop_index(g, kind) if isinstance(op, ShowIndexes): return show_indexes(g) diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 9240c6b23f..4caec78451 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -310,3 +310,31 @@ def test_drop_index_by_custom_name(): # unresolvable name raises (not silent) with pytest.raises(ValueError): g.gfql("DROP GFQL INDEX nonexistent_name") + + +def test_drop_index_if_exists_semantics(): + # missing_ok / IF EXISTS: plain DROP of a missing index raises (SQL-style); + # IF EXISTS makes it a no-op. Applies to both name- and kind-form. + g = graphistry.edges(pd.DataFrame({"src": [0, 1], "dst": [1, 2]}), "src", "dst").materialize_nodes() + + # no-op forms succeed unchanged + g2 = g.gfql("DROP GFQL INDEX IF EXISTS nonexistent_name") + assert g2.show_indexes().shape[0] == 0 + g3 = g.gfql("DROP GFQL INDEX IF EXISTS FOR edge_out_adj") + assert g3.show_indexes().shape[0] == 0 + + # plain forms raise when missing + with pytest.raises(ValueError): + g.gfql("DROP GFQL INDEX nonexistent_name") + with pytest.raises(ValueError): + g.gfql("DROP GFQL INDEX FOR edge_out_adj") + + # wire JSON honors missing_ok both ways + g4 = g.gfql({"type": "DropIndex", "name": "nope", "missing_ok": True}) + assert g4.show_indexes().shape[0] == 0 + with pytest.raises(ValueError): + g.gfql({"type": "DropIndex", "name": "nope", "missing_ok": False}) + + # and a resident index still drops through the plain form + gi = g.create_index("edge_out_adj") + assert gi.gfql("DROP GFQL INDEX FOR edge_out_adj").show_indexes().shape[0] == 0 From eddc424880e007f72703ee900c11c2530cd5ec7f Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 1 Jul 2026 13:54:05 -0700 Subject: [PATCH 09/20] feat(gfql): enrich gfql_explain with planner cost diagnostics (LP1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gfql_explain / index_trace now surface *why* the seeded-hop planner chose the index or the scan, not just used_index. Per-hop steps carry frontier_n, n_keys, a free Σ-seed-degree fanout estimate (seed_deg_sum/est_result_rows, read from the CSR group_offsets already on the adjacency index), the engine cost-gate threshold, and a human-readable decision_reason recorded at every bail point. The report lifts est_seed_cardinality/est_result_rows/ chosen_direction/decision_reason to the top level. Purely additive: the fanout estimate is computed only inside an index_trace()/gfql_explain context, so the hot traversal path pays nothing and behavior is unchanged. Stacked on the #1658 CSR index. +1 engine-parametrized test; ruff + mypy clean; 45 CPU index tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + graphistry/compute/gfql/index/api.py | 103 ++++++++++++++++-- graphistry/compute/gfql/index/explain.py | 9 ++ .../tests/compute/gfql/index/test_index.py | 26 +++++ 4 files changed, 129 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c50068ae09..670eee7168 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL native Polars engine — variable-length `min_hops>1` traversals (`engine='polars'`/`'polars-gpu'`)**: Forward/reverse lower-bound traversals (`e(min_hops=N, max_hops=M)`) now run natively on the Polars engine — no pandas bridge. The eager Polars hop runs pandas' min_hops algorithm vectorized: a NON-anti-joined BFS (the wavefront carries revisits so a cycle keeps bumping the reach depth until `max_hops`), a 3-case termination gate (`max_reached1` (needs connected-components + 2-core seed retention) and a *direct* `hop(min_hops>1)` (which would need pandas' separate un-labeled direct-hop node-output plus the `target_wave_front` threading the chain supplies — without them it silently diverges) both raise `NotImplementedError` for `engine='polars'` (use `chain()`/`gfql()`, or `engine='pandas'`). Validated by differential parity vs the pandas oracle: the 500-seed randomized chain fuzzer (`test_polars_chain_fuzz_parity`, hardened to compare null-aware node **attributes** and edge **multiplicity**, not just id/endpoint sets) is **500/500**, a min_hops+attribute-filter amplified fuzz and metamorphic invariants pass, and `engine='polars-gpu'` (cudf_polars) runs the full 500-seed fuzz **500/500** on-device. - **GFQL native Polars engine — cypher row pipeline (`engine='polars'`)**: Extended the Polars engine to the Cypher `MATCH … RETURN` row surface, natively vectorized. **NO CHEATING:** the polars engine never silently falls back to the pandas engine — every query runs natively on polars or raises an honest `NotImplementedError` pointing at `engine='pandas'` (falling back to pandas would misrepresent pandas performance as polars; only a human may consent to a bridge). `chain_polars` splits boundary `call()` ops (mirroring the pandas `_handle_boundary_calls`) and runs each trailing row op per-op native or raises. **Native polars** (no pandas round-trip): frame ops (`rows`/`limit`/`skip`/`distinct`/`drop_cols`), `select`/`with_`/`return_` projection (a conservative cypher-expr-AST → `pl.Expr` lowering covering property access, arithmetic, comparison, boolean, literals), `order_by` (`.sort`), `group_by` (`count`/`sum`/`avg`/`min`/`max`), `unwind` (literal-list cross-join), the result projection for property/expr columns, and entity-text `RETURN n` rendering for int/string/bool nodes (`pl.concat_str`). **Honestly deferred** (raise `NotImplementedError`, no pandas fallback): multi-entity `rows(binding_ops=…)`, cross-entity same-path `WHERE` (`DFSamePathExecutor`), float/temporal/nested entity-text, and exotic expressions (CASE/list/map/temporal, `collect` aggregates) — these are the forward native-engineering targets. Validated by differential parity vs pandas including a TCK-style conformance lane (`test_engine_polars_cypher_conformance.py`: native-only curated corpus + seeded fuzzer + NULL/3-valued-logic graph + entity-text escaping, plus a `DEFERRED` list asserting deferred queries raise rather than silently bridge) and benchmarked (`benchmarks/gfql/cypher_row_pipeline.py`). **Perf (interleaved, 1M nodes, each engine on its native-frame graph, all fully native):** polars wins **5.6–38×** across the surface — `RETURN n` ~38×, `ORDER BY` ~17×, `WHERE`+`ORDER BY`+`LIMIT` ~14×, traversals 6–7.5×, projections/aggregations/`DISTINCT` 5.6–6.9×. cuDF/pandas paths untouched. - **GFQL lazy Polars engine + GPU target (`engine='polars-gpu'`, cudf_polars)**: The Polars traversal engine now builds a single deferred `pl.LazyFrame` plan per single-hop and materializes `out_edges`+`out_nodes` in ONE `collect_all` on a chosen **execution target** (CPU or GPU). `engine='polars-gpu'` (`Engine.POLARS_GPU`, explicit opt-in only — AUTO never selects it) runs that same lazy plan on the RAPIDS cudf_polars backend (`pl.GPUEngine(raise_on_fail=True)` — NO-CHEATING: a GPU-incapable plan node **raises** rather than silently running on CPU and being reported as a GPU result; see Fixed). The collect-once design is what makes GPU pay off: a benchmark showed per-op eager GPU collect was a *regression* (repeated H2D), while collect-once is a **2.84× single-hop GPU win @1M** with CPU parity. Frames stay `pl.DataFrame` (handled like `POLARS` everywhere); the target is carried by a context var set at the chain/hop dispatch boundary, so `engine='polars'` (CPU) is byte-for-byte unchanged. Validated by differential parity `engine='polars-gpu' == engine='polars'` across the cypher conformance corpus + traversals (`test_engine_polars_gpu.py`, skips when no cudf_polars/GPU). Multi-hop and the chain forward/backward fusion (where the GPU win currently dilutes) are follow-up optimizations. +- **GFQL `gfql_explain` planner diagnostics**: `gfql_explain(...)` now surfaces *why* the seeded-hop planner chose the index or the scan, not just a `used_index` yes/no. Each per-hop step carries the frontier seed count (`frontier_n`), the number of distinct source keys (`n_keys`), a **free Σ-seed-degree fanout estimate** (`seed_deg_sum`/`est_result_rows`, read directly from the CSR `group_offsets` already on the adjacency index — no extra scan), the engine's cost-gate crossover fraction (`threshold_frac`), and a human-readable `decision_reason` (e.g. `"frontier below cost gate -> index"`, `"frontier N >= 0.02*n_keys (…) -> scan cheaper"`, `"query not index-coverable"`, `"policy=off"`). The report also lifts `est_seed_cardinality`, `est_result_rows`, `chosen_direction`, and the top-level `decision_reason` for quick inspection. Purely additive and diagnostic — the fanout estimate is computed **only** inside an `index_trace()`/`gfql_explain` context, so the hot traversal path pays nothing and traversal behavior is unchanged. + +### Added - **GFQL native Polars engine — more cypher row coverage (`toFloat`, `collect`/`collect(DISTINCT)`, `WHERE … IN`)**: three surfaces that previously raised `NotImplementedError` on `engine='polars'` now run natively, parity-validated vs the pandas oracle across all four engines (and honest-NIE where pandas can't be matched). **`toFloat(x)`** lowers int/uint/bool/float → `Float64` (NaN preserved — float64 has no separate null sentinel, unlike `toInteger`); a non-numeric String declines (NIE) because pandas `astype(float)` *raises* rather than null-on-failure. **`collect(x)` / `collect(DISTINCT x)`** aggregations complete the native `group_by` surface (every other agg was already native): drop nulls, preserve within-group first-occurrence order (`collect` keeps dups; `DISTINCT` dedups keep-first), all-null group → `[]`. **`where_rows`/`WHERE … IN [list]`** membership lowers to `is_in` (a null cell is excluded per openCypher 3VL). No change to any already-native path. - **GFQL Polars-CPU streaming collect (opt-in, large traversals)**: `GFQL_POLARS_CPU_STREAMING=1` runs the polars-CPU lazy collects (`hop`/`chain`) on the polars **streaming** executor instead of the default in-memory collect. Benchmarked ~1.04–1.11× faster on big multi-hop traversals (10M nodes / 80M edges: 20.0→18.0 s) and parity-identical, but ~0.86× (slower) on small/interactive sizes (streaming overhead) — so it is **opt-in, default off** (no change to default behavior). Use for large batch traversals where CPU is the target. - **GFQL Cypher `count(*)` short-circuit — O(1) instead of O(N) materialize**: a lone `RETURN count(*)` over a single node or edge pattern (`MATCH (n) RETURN count(*)`, `MATCH ()-[r]->() RETURN count(*)`) previously materialized the entire matched frame and ran a constant-key `group_by` just to count its rows. The lowering now emits a new `count_table` row op that reads the scanned table's height directly (or, when the pattern applies a filter, sums the boolean alias-mask column) in a single reduction — no full-frame copy, no group_by. Applies only to the provably-equivalent shapes: exactly one non-DISTINCT `count(*)`, no group keys / post-aggregate exprs / row-level `WHERE` / `UNWIND` / paging / multi-relationship binding, and either a pure node scan (`relationship_count == 0`) or a single relationship counted on its edge alias — every other shape (node-alias-over-relationship, multi-hop paths, `count(DISTINCT …)`, grouped counts) falls through to the general aggregate path unchanged. Engine-polymorphic across pandas/cuDF/polars/polars-gpu; differential parity verified on all four engines (`count_all_nodes`/`count_all_edges` cypher conformance cases + a `count_table` row-op subject case, chain and DAG surfaces). No change to any result value — only the execution path. diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index efaf806193..ca9fd84995 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -54,6 +54,46 @@ def _record(decision: dict) -> None: steps.append(decision) +def _trace_active() -> bool: + """True only inside an ``index_trace()`` / ``gfql_explain`` context. Diagnostic + enrichment (LP1) is computed only when this is True → zero hot-path cost.""" + return getattr(_TRACE, "steps", None) is not None + + +def _seed_id_array(nodes, node_col): + """Seed ids of the frontier as a host numpy array (device→host copy is fine — + only called under an explain trace). None on any failure.""" + try: + col = nodes[node_col] + if hasattr(col, "to_numpy"): + return col.to_numpy() + arr = getattr(col, "values", col) + return arr.get() if hasattr(arr, "get") else arr + except Exception: + return None + + +def _seed_deg_sum(idx, seed_ids) -> Optional[int]: + """Σ degree of the resident seeds — the true index expansion cost, FREE from the + CSR ``group_offsets`` already on the AdjacencyIndex. Diagnostic only (LP1); the + report's planner needs this fanout estimate exposed via EXPLAIN.""" + try: + import numpy as np + ks = idx.keys_sorted + ks = ks.get() if hasattr(ks, "get") else np.asarray(ks) + off = idx.group_offsets + off = off.get() if hasattr(off, "get") else np.asarray(off) + seed_ids = np.asarray(seed_ids) + pos = np.searchsorted(ks, seed_ids) + valid = pos < ks.size + pos_v = pos[valid] + seed_v = seed_ids[valid] + hit = pos_v[ks[pos_v] == seed_v] # keep only real key hits (robust to seed order) + return int((off[hit + 1] - off[hit]).sum()) + except Exception: + return None + + def get_registry(g: Plottable) -> GfqlIndexRegistry: return getattr(g, REGISTRY_ATTR, EMPTY_REGISTRY) @@ -279,11 +319,34 @@ def maybe_index_hop( (or buildable under auto/force), (b) the query is covered, (c) the frontier is not so large that a full scan is cheaper. Correctness is identical either way. """ - if policy == "off": + # Diagnostic trace (LP1) is populated only inside an explain context — build the + # base record + a `_bail` helper that logs *why* we fell back to scan. All of this + # is skipped entirely when not tracing, so the hot path pays nothing. + trace = _trace_active() + diag: dict = {} + if trace: + diag = { + "op": "hop", "direction": direction, "hops": hops, + "policy": policy, "engine": engine.value, + } + try: + diag["frontier_n"] = int(nodes.shape[0]) + except Exception: + pass + + def _bail(reason: str, extra: Optional[dict] = None) -> Optional[Plottable]: + if trace: + rec = {**diag, "path": "scan", "decision_reason": reason} + if extra: + rec.update(extra) + _record(rec) return None + + if policy == "off": + return _bail("policy=off") registry = get_registry(g) if registry.is_empty() and policy not in ("auto", "force"): - return None + return _bail("no resident index (policy=use)") if not _hop_is_index_coverable( nodes=nodes, to_fixed_point=to_fixed_point, hops=hops, min_hops=rest.get("min_hops"), max_hops=rest.get("max_hops"), @@ -301,32 +364,44 @@ def maybe_index_hop( include_zero_hop_seed=rest.get("include_zero_hop_seed", False), target_wave_front=rest.get("target_wave_front"), ): - return None + return _bail("query not index-coverable") node_col = g._node src, dst = g._source, g._destination if node_col is None or src is None or dst is None or g._edges is None or g._nodes is None: - return None + return _bail("graph missing node/edge columns") if policy in ("auto", "force"): registry = _ensure_indexes(g, registry, direction, engine, policy, nodes, src, dst, node_col) if registry.is_empty(): - return None + return _bail("no index available (build declined)") # Cost gate: if the frontier covers a large fraction of distinct sources, the # scan path is competitive — fall back (avoids index overhead on bulk-ish hops). idx0 = registry.get_valid( EDGE_OUT_ADJ if direction != "reverse" else EDGE_IN_ADJ, g._edges, (src, dst), engine ) + frac = _COST_GATE_FRAC.get(engine, _COST_GATE_FRAC_DEFAULT) + if trace and idx0 is not None: + # Free fanout estimate (Σ seed degree) from the CSR offsets — the planner + # signal the report wants EXPLAIN to surface (not just used-index yes/no). + seed_ids = _seed_id_array(nodes, node_col) + deg_sum = _seed_deg_sum(idx0, seed_ids) if seed_ids is not None else None + diag["n_keys"] = int(idx0.n_keys) + diag["seed_deg_sum"] = deg_sum + diag["est_result_rows"] = deg_sum + diag["threshold_frac"] = frac if idx0 is None: # required direction not resident (undirected needs both); let driver decide pass elif policy != "force": try: frontier_n = int(nodes.shape[0]) - frac = _COST_GATE_FRAC.get(engine, _COST_GATE_FRAC_DEFAULT) if idx0.n_keys > 0 and frontier_n >= frac * idx0.n_keys: - return None + return _bail( + f"frontier {frontier_n} >= {frac}*n_keys " + f"({frac * idx0.n_keys:.0f}) -> scan cheaper" + ) except Exception: pass @@ -341,9 +416,13 @@ def maybe_index_hop( hops=eff_hops, to_fixed_point=to_fixed_point, direction=direction, return_as_wave_front=return_as_wave_front, ) - _record({ - "op": "hop", "direction": direction, "hops": eff_hops, - "path": "index" if result is not None else "scan", - "policy": policy, "engine": engine.value, - }) + if trace: + _record({ + **diag, "hops": eff_hops, + "path": "index" if result is not None else "scan", + "decision_reason": ( + "frontier below cost gate -> index" if result is not None + else "index path not applicable -> scan" + ), + }) return result diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index 939c62774e..b90b7fb77f 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -23,11 +23,20 @@ def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = except Exception as ex: # report, don't raise — explain is diagnostic error = f"{type(ex).__name__}: {ex}" used_index = any(s.get("path") == "index" for s in steps) + # Surface the planner's cost signal at the top level (LP1): prefer the step that + # actually took the index, else the last decision. `est_seed_cardinality` = number + # of seeds; `est_result_rows` = estimated fanout (Σ seed degree, free from CSR). + ref = [s for s in steps if s.get("path") == "index"] or list(steps) + last = ref[-1] if ref else {} return { "engine": eng.value, "index_policy": index_policy, "resident_indexes": resident["name"].tolist() if len(resident) else [], "steps": list(steps), "used_index": used_index, + "est_seed_cardinality": last.get("frontier_n"), + "est_result_rows": last.get("est_result_rows"), + "chosen_direction": last.get("direction"), + "decision_reason": last.get("decision_reason"), "error": error, } diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 4caec78451..062169a5b3 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -135,6 +135,32 @@ def test_index_policy_force_and_explain(graph, engine): assert _sig(r_scan) == _sig(r_force) +@pytest.mark.parametrize("engine", ENGINES) +def test_explain_exposes_planner_diagnostics(graph, engine): + """LP1: gfql_explain surfaces the planner's cost signal — seed cardinality, the + free Σ-degree fanout estimate (from CSR group_offsets), chosen direction, the + cost-gate threshold, and a human-readable decision_reason — not just a used-index + yes/no. This is the EXPLAIN enrichment the indexing roadmap calls for.""" + chain = [n({"id": 0}), e_forward(hops=1)] + # force → index path taken → per-step + top-level diagnostics populated + rep = graph.gfql_explain(chain, index_policy="force", engine=engine) + assert rep["used_index"] is True, (engine, rep) + assert rep["chosen_direction"] == "forward" + assert isinstance(rep["est_result_rows"], int) and rep["est_result_rows"] >= 0 + assert "index" in (rep["decision_reason"] or ""), rep["decision_reason"] + step = next(s for s in rep["steps"] if s.get("path") == "index") + for k in ("frontier_n", "n_keys", "seed_deg_sum", "est_result_rows", "threshold_frac"): + assert k in step, (k, step) + assert step["est_result_rows"] == step["seed_deg_sum"] # est == free Σ-degree + assert step["seed_deg_sum"] >= 0 + + # off (index resident) → the planner records *why* it scanned, not just that it did + gi = graph.gfql_index_all(engine=engine) + rep_off = gi.gfql_explain(chain, index_policy="off", engine=engine) + assert rep_off["used_index"] is False + assert rep_off["decision_reason"] == "policy=off", rep_off + + @pytest.mark.parametrize("engine", ENGINES) def test_cost_gate_engine_aware_never_loses_to_scan(engine): """F1: the index-vs-scan crossover depends on scan speed, so the cost gate From af6631103d511d71b50e141aec44278916118569 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 1 Jul 2026 14:36:01 -0700 Subject: [PATCH 10/20] test(gfql): cover LP1 explain diagnostics fall-back branches Raise changed-line coverage on the gfql_explain diagnostics (LP1) from 76.7% to 91.4% (CI gate 80%): add tests for the cost-gate and not-index-coverable scan-fallback decision reasons, and a direct robustness test for the _seed_id_array / _seed_deg_sum helpers (degrade to None, never crash, since they run under the explain trace). Also drop _bail's unused `extra` parameter (dead code). Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/gfql/index/api.py | 7 +-- .../tests/compute/gfql/index/test_index.py | 52 +++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index ca9fd84995..c136476de7 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -334,12 +334,9 @@ def maybe_index_hop( except Exception: pass - def _bail(reason: str, extra: Optional[dict] = None) -> Optional[Plottable]: + def _bail(reason: str) -> Optional[Plottable]: if trace: - rec = {**diag, "path": "scan", "decision_reason": reason} - if extra: - rec.update(extra) - _record(rec) + _record({**diag, "path": "scan", "decision_reason": reason}) return None if policy == "off": diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 062169a5b3..4ff5e701fb 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -161,6 +161,58 @@ def test_explain_exposes_planner_diagnostics(graph, engine): assert rep_off["decision_reason"] == "policy=off", rep_off +def test_seed_diagnostic_helpers_are_robust(): + """LP1 helpers degrade to None instead of crashing on odd inputs (they run under + the explain trace and must never take down a real query).""" + from graphistry.compute.gfql.index.api import _seed_id_array, _seed_deg_sum + + class _Col: # a seed column with .values but no .to_numpy() (fallback branch) + values = np.array([1, 2, 3]) + + class _Frame: + def __getitem__(self, k): + return _Col() + + assert list(_seed_id_array(_Frame(), "id")) == [1, 2, 3] + assert _seed_id_array(None, "id") is None # nodes[None] raises → None, not a crash + + class _BadIdx: # missing keys_sorted/group_offsets → None, not AttributeError + pass + + assert _seed_deg_sum(_BadIdx(), np.array([0, 1])) is None + + +@pytest.mark.parametrize("engine", ENGINES) +def test_explain_decision_reasons_for_scan_fallbacks(engine): + """LP1: when the planner declines the index it records *why*, so a silent scan is + diagnosable. Covers the two fall-back branches: (a) a frontier past the cost gate, + (b) a query the index doesn't cover (min_hops>1).""" + from graphistry.compute.gfql.index import index_trace + rng = np.random.default_rng(2) + N, deg = 1000, 6 + edf = pd.DataFrame({"src": rng.integers(0, N, N * deg), "dst": rng.integers(0, N, N * deg)}) + ndf = pd.DataFrame({"id": np.arange(N)}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + gi = g.gfql_index_all(engine=engine) + + # (a) cost-gate fallback: frontier = all keys >> frac*n_keys → scan, with a reason + allseeds = pd.DataFrame({"id": np.arange(N, dtype=np.int64)}) + with index_trace() as steps: + gi.hop(nodes=allseeds, engine=engine, hops=1, direction="forward") + assert any("scan cheaper" in (s.get("decision_reason") or "") for s in steps), (engine, steps) + assert not any(s.get("path") == "index" for s in steps), (engine, steps) + + # (b) not-coverable fallback: a feature outside the index fast path (zero-hop seed). + # pandas-only: the Phase-1 polars hop rejects these features at its own engine layer + # before the index planner is consulted, so the index "not-coverable" bail is + # reachable via the pandas hop path here. + if engine == "pandas": + few = pd.DataFrame({"id": np.arange(4, dtype=np.int64)}) + with index_trace() as steps2: + gi.hop(nodes=few, engine=engine, hops=1, direction="forward", include_zero_hop_seed=True) + assert any(s.get("decision_reason") == "query not index-coverable" for s in steps2), (engine, steps2) + + @pytest.mark.parametrize("engine", ENGINES) def test_cost_gate_engine_aware_never_loses_to_scan(engine): """F1: the index-vs-scan crossover depends on scan speed, so the cost gate From ec0c2489e1264fca91a5344dc2d4ce9e35981f87 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 4 Jul 2026 15:57:14 -0700 Subject: [PATCH 11/20] test(gfql/index): run the index tests in the polars lane (covers the hop-entry hook) The polars lane's per-file coverage audit gates hop.py; after #1660's hop unification the file is a thin entry dominated by this PR's index hook, which the lane didn't exercise (57% < 87% floor). Adding the engine-parametrized index tests to POLARS_TEST_FILES exercises the hook for real: dgx-measured 92.86% (>87 floor), 1724 lane tests pass. Real coverage, not floor surgery. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/test-polars.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 55f6b1e7c1..ae429df75d 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -20,6 +20,9 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/test_engine_polars_cypher_conformance.py graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py graphistry/tests/compute/gfql/test_conformance_ledger.py + # index tests exercise the seeded-index hook in the polars hop entry (hop.py) — without + # them the hook dominates the now-thin file and trips its per-file coverage floor + graphistry/tests/compute/gfql/index/test_index.py ) COV_ARGS=() From 228338b4489d8cb4a92c2718a67fbf7bd967702c Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 09:29:00 -0700 Subject: [PATCH 12/20] fix(gfql/index): harden DDL refresh and policy validation --- CHANGELOG.md | 2 +- graphistry/compute/ComputeMixin.py | 3 +- graphistry/compute/gfql/index/__init__.py | 4 +- graphistry/compute/gfql/index/api.py | 102 +++++++++--------- graphistry/compute/gfql/index/build.py | 19 +++- graphistry/compute/gfql/index/cost.py | 56 ++++++++++ graphistry/compute/gfql/index/explain.py | 2 + graphistry/compute/gfql/index/policy.py | 20 ++++ graphistry/compute/gfql/index/wire.py | 6 +- .../tests/compute/gfql/index/test_index.py | 32 ++++++ 10 files changed, 183 insertions(+), 63 deletions(-) create mode 100644 graphistry/compute/gfql/index/cost.py create mode 100644 graphistry/compute/gfql/index/policy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 670eee7168..7645928ce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). - **GFQL Cypher numeric functions + `toLower`/`toUpper`/`lower`/`upper` (openCypher/neo4j/GQL-standard)**: added the standard scalar functions `floor`, `ceil` (alias `ceiling`, per Cypher 25 GQL conformance and the GQL grammar's `CEIL|CEILING`), `round(x)` / `round(x, precision)`, and `toLower` / `toUpper` plus their GQL-conformance aliases `lower` / `upper` (ISO GQL §20.24 character-string functions; neo4j accepts both spellings) — the idiomatic case-insensitive compare `WHERE toLower(n.name) = 'bob'` — across the Cypher `WHERE`/`RETURN` expression surface. Evaluated natively on pandas/cuDF and polars (differential-parity tested). **`round` follows neo4j's documented tie-breaking** (standards-vetted against the neo4j manual): precision 0 (and the 1-arg form) rounds ties toward **positive infinity** (`round(-1.5)` → `-1.0`, `round(2.5)` → `3.0`), and precision > 0 rounds ties **away from zero** (HALF_UP: `round(-1.55, 1)` → `-1.6`) — not the numpy/polars half-to-even defaults, which give wrong answers on ties. The `round(x, precision, mode)` 3-arg form is not yet supported. Complements the already-supported `abs`/`sqrt`/`sign` and chained comparisons (`WHERE 1 < n.age < 65`). The `^` exponentiation operator is deferred: standards vetting settled it as **left-associative** (the openCypher TCK pins left, and neo4j's shipped Cypher 5/25 parser left-folds `Pow` — the manual's "right to left" row is a docs bug), but the current conformance corpus marks it reject-expected, so re-adding it is a coordinated corpus change. `LIKE`/`ILIKE`/`BETWEEN` remain intentionally unimplemented (verified absent from both Cypher and ISO GQL — GQL's only `LIKE` is the unrelated `CREATE GRAPH TYPE … LIKE g` DDL). - **GFQL Cypher `=~` regex-match operator (openCypher/neo4j-standard)**: added the standard `=~` string predicate to the Cypher `WHERE`/expression grammar — `MATCH (n) WHERE n.name =~ '(?i)al.*' RETURN n`. Semantics match openCypher/neo4j: **Java-regex dialect, full-string/anchored match** (`n.name =~ 'AB'` matches only `'AB'`, not `'ABCDEF'`; use `.*`/`^..$` for partial), with inline flags (`(?i)`/`(?m)`/`(?s)`) honored; lowers to the existing `fullmatch` predicate. Works in the simple `WHERE prop =~ '...'` form (all engines via `filter_by_dict`) and composes through `AND`/`OR`/`NOT`/`RETURN` expressions via the shared expression engine (pandas/cuDF; polars supports the simple-WHERE form and declines the complex `OR`/`NOT` row-filter form with an honest `NotImplementedError` — the pre-existing polars `where_rows` limitation, not `=~`-specific). Also adds native polars `Match`/`Fullmatch` predicate lowering (previously `NotImplementedError`), so `=~` and the Python `match()`/`fullmatch()` predicates run natively on polars. Differential-parity tested against the pandas oracle. `LIKE`/`ILIKE` remain intentionally unimplemented (not in any graph standard — use `=~`/`CONTAINS`/`STARTS WITH`). -- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront, hardened by an adversarial review that found and fixed several index-vs-scan divergences the original scenarios missed (`max_hops` honored; duplicate node ids, edge endpoints absent from the node table, int32/int64 seed-vs-key dtype mismatches, and stale `.edges()` rebinds all now match scan or fall back — never a wrong answer), with object-identity fingerprinting (recycle-proof) and 6 added differential regression tests. **Perf (dgx-spark, deg-8, warm median; every measured cell guarded so the index path was actually taken AND the index result equals the scan result):** seeded latency is **flat in graph size** — GFQL-pandas 1-hop 0.124/0.122 ms at 0.8M/8M nodes (6.4M/64M edges) while the O(E) scan grows 105 → 1045 ms — and **beats kuzu (CSR) and neo4j by 9–28×** on CPU at 0.8M nodes (1-hop: GFQL-pandas 0.123 ms vs kuzu 1.15 ms vs neo4j 1.45 ms; 1–2-hop: 0.150 ms vs 4.25 ms vs 2.54 ms, matched answer counts). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity. cuDF/polars-GPU are flat but floored by ~3 ms (cuDF) GPU kernel-launch overhead — selective traversal is an indexing problem, not a compute one (CPU wins it). No change to default (un-indexed) behavior. +- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront, hardened by an adversarial review that found and fixed several index-vs-scan divergences the original scenarios missed (`max_hops` honored; duplicate node ids, edge endpoints absent from the node table, int32/int64 seed-vs-key dtype mismatches, and stale `.edges()` rebinds all now match scan or fall back — never a wrong answer), with object-identity fingerprinting (recycle-proof) and 6 added differential regression tests. **Perf (dgx-spark, deg-8, warm median; every measured cell guarded so the index path was actually taken AND the index result equals the scan result):** seeded latency is **flat in graph size** — GFQL-pandas 1-hop 0.124/0.122 ms at 0.8M/8M nodes (6.4M/64M edges) while the O(E) scan grows 105 → 1045 ms — and **beats kuzu (CSR) and neo4j by 9–28×** on CPU at 0.8M nodes (1-hop: GFQL-pandas 0.123 ms vs kuzu 1.15 ms vs neo4j 1.45 ms; 1–2-hop: 0.150 ms vs 4.25 ms vs 2.54 ms, matched answer counts). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity, invalid policy strings now fail fast, and repeated `CREATE GFQL INDEX ...` rebuilds stale resident indexes after frame rebinds. cuDF/polars-GPU are flat but floored by ~3 ms (cuDF) GPU kernel-launch overhead — selective traversal is an indexing problem, not a compute one (CPU wins it). No change to default (un-indexed) behavior. - **GFQL native Polars engine — traversals (`engine='polars'`)**: Added a native, vectorized Polars execution engine for the core GFQL traversals `hop()` and `chain()`, dispatched at the engine boundary so the production pandas/cuDF paths are untouched. `Engine.POLARS` is opt-in (explicit `engine='polars'`); `engine='auto'` with Polars input still coerces to pandas as before. Covers forward/reverse/undirected single-hop traversal, directed multi-hop chains, node/edge filter dicts and predicates (lowered to Polars expressions), `edge_match`/`source_node_match`/`destination_node_match`, `target_wave_front`, and alias names; the BFS advances via semi/anti joins (no per-row Python work). Validated by differential parity against the pandas engine (hop + chain test suites plus a randomized fuzzer) and benchmarked vs pandas (`benchmarks/gfql/pandas_vs_polars.py`) — Polars wins at scale (up to ~2.5x on multi-edge chains at millions of edges; crossover ~50–100k rows). Variable-length/multi-hop edges, undirected edges in multi-edge chains, hop labels, and node `query=` raise `NotImplementedError` for now (use `engine='pandas'`). - **GFQL native Polars engine — variable-length `min_hops>1` traversals (`engine='polars'`/`'polars-gpu'`)**: Forward/reverse lower-bound traversals (`e(min_hops=N, max_hops=M)`) now run natively on the Polars engine — no pandas bridge. The eager Polars hop runs pandas' min_hops algorithm vectorized: a NON-anti-joined BFS (the wavefront carries revisits so a cycle keeps bumping the reach depth until `max_hops`), a 3-case termination gate (`max_reached1` (needs connected-components + 2-core seed retention) and a *direct* `hop(min_hops>1)` (which would need pandas' separate un-labeled direct-hop node-output plus the `target_wave_front` threading the chain supplies — without them it silently diverges) both raise `NotImplementedError` for `engine='polars'` (use `chain()`/`gfql()`, or `engine='pandas'`). Validated by differential parity vs the pandas oracle: the 500-seed randomized chain fuzzer (`test_polars_chain_fuzz_parity`, hardened to compare null-aware node **attributes** and edge **multiplicity**, not just id/endpoint sets) is **500/500**, a min_hops+attribute-filter amplified fuzz and metamorphic invariants pass, and `engine='polars-gpu'` (cudf_polars) runs the full 500-seed fuzz **500/500** on-device. - **GFQL native Polars engine — cypher row pipeline (`engine='polars'`)**: Extended the Polars engine to the Cypher `MATCH … RETURN` row surface, natively vectorized. **NO CHEATING:** the polars engine never silently falls back to the pandas engine — every query runs natively on polars or raises an honest `NotImplementedError` pointing at `engine='pandas'` (falling back to pandas would misrepresent pandas performance as polars; only a human may consent to a bridge). `chain_polars` splits boundary `call()` ops (mirroring the pandas `_handle_boundary_calls`) and runs each trailing row op per-op native or raises. **Native polars** (no pandas round-trip): frame ops (`rows`/`limit`/`skip`/`distinct`/`drop_cols`), `select`/`with_`/`return_` projection (a conservative cypher-expr-AST → `pl.Expr` lowering covering property access, arithmetic, comparison, boolean, literals), `order_by` (`.sort`), `group_by` (`count`/`sum`/`avg`/`min`/`max`), `unwind` (literal-list cross-join), the result projection for property/expr columns, and entity-text `RETURN n` rendering for int/string/bool nodes (`pl.concat_str`). **Honestly deferred** (raise `NotImplementedError`, no pandas fallback): multi-entity `rows(binding_ops=…)`, cross-entity same-path `WHERE` (`DFSamePathExecutor`), float/temporal/nested entity-text, and exotic expressions (CASE/list/map/temporal, `collect` aggregates) — these are the forward native-engineering targets. Validated by differential parity vs pandas including a TCK-style conformance lane (`test_engine_polars_cypher_conformance.py`: native-only curated corpus + seeded fuzzer + NULL/3-valued-logic graph + entity-text escaping, plus a `DEFERRED` list asserting deferred queries raise rather than silently bridge) and benchmarked (`benchmarks/gfql/cypher_row_pipeline.py`). **Perf (interleaved, 1M nodes, each engine on its native-frame graph, all fully native):** polars wins **5.6–38×** across the surface — `RETURN n` ~38×, `ORDER BY` ~17×, `WHERE`+`ORDER BY`+`LIMIT` ~14×, traversals 6–7.5×, projections/aggregations/`DISTINCT` 5.6–6.9×. cuDF/pandas paths untouched. diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 8063bbf0a4..1da53b6153 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -665,9 +665,10 @@ def gfql(self, *args, **kwargs): g = self if policy is not None: + from graphistry.compute.gfql.index.policy import validate_index_policy import copy as _copy g = _copy.copy(self) - g._gfql_index_policy = policy + g._gfql_index_policy = validate_index_policy(policy) return gfql_base(g, *args, **kwargs) gfql.__doc__ = (gfql_base.__doc__ or "") + """ diff --git a/graphistry/compute/gfql/index/__init__.py b/graphistry/compute/gfql/index/__init__.py index 3166d28c6b..3f6fe81bfc 100644 --- a/graphistry/compute/gfql/index/__init__.py +++ b/graphistry/compute/gfql/index/__init__.py @@ -12,7 +12,7 @@ ) from .api import ( create_index, drop_index, show_indexes, gfql_index_edges, gfql_index_all, - get_registry, maybe_index_hop, index_name, REGISTRY_ATTR, index_trace, + get_registry, maybe_index_hop, index_name, index_trace, ) from .wire import ( CreateIndex, DropIndex, ShowIndexes, apply_index_op, index_op_from_json, @@ -26,7 +26,7 @@ "AdjacencyIndex", "NodeIdIndex", "create_index", "drop_index", "show_indexes", "gfql_index_edges", "gfql_index_all", "get_registry", "maybe_index_hop", "index_name", - "REGISTRY_ATTR", "index_trace", + "index_trace", "CreateIndex", "DropIndex", "ShowIndexes", "apply_index_op", "index_op_from_json", "is_index_op", "is_index_op_json", "parse_index_ddl", "looks_like_index_ddl", diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index c136476de7..c6b8084115 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -8,7 +8,7 @@ from __future__ import annotations import copy -from typing import Any, Optional, Union, cast +from typing import Any, Dict, List, Literal, Optional, Union, cast from graphistry.Engine import EngineAbstract, Engine, resolve_engine from graphistry.Plottable import Plottable @@ -18,18 +18,12 @@ ) from .build import build_adjacency_index, build_node_id_index from .traverse import index_seeded_hop +from .cost import cost_gate_frac, seed_deg_sum, seed_id_array +from .policy import IndexPolicy, validate_index_policy +# Private Plottable attachment key. Keep access behind get_registry()/show_indexes(). REGISTRY_ATTR = "_gfql_index_registry" -# Index-vs-scan crossover fraction (of distinct source keys): past this frontier a -# full scan is cheaper than per-seed index probes, so `use` falls back to scan and -# never loses to the un-indexed path. Engine-aware because vectorized-scan engines -# (polars/cudf/GPU) scan far faster than pandas, so their crossover is much smaller -# (see test_index_cost_gate_engine_aware). Measured N=1e5 deg8: pandas ~0.5, polars ~0.02. GPU values provisional -# (dgx-gated) — conservatively grouped with polars. -_COST_GATE_FRAC = {Engine.PANDAS: 0.5} -_COST_GATE_FRAC_DEFAULT = 0.02 - # --- lightweight, thread-local index decision trace (for gfql_explain) ------- import threading as _threading _TRACE = _threading.local() @@ -37,18 +31,18 @@ class index_trace: """Context manager: capture the per-hop index-vs-scan decisions made inside.""" - def __enter__(self): - self.steps = [] + def __enter__(self) -> List[Dict[str, Any]]: + self.steps: List[Dict[str, Any]] = [] self.prev = getattr(_TRACE, "steps", None) _TRACE.steps = self.steps return self.steps - def __exit__(self, *exc): + def __exit__(self, *exc: Any) -> Literal[False]: _TRACE.steps = self.prev return False -def _record(decision: dict) -> None: +def _record(decision: Dict[str, Any]) -> None: steps = getattr(_TRACE, "steps", None) if steps is not None: steps.append(decision) @@ -60,39 +54,10 @@ def _trace_active() -> bool: return getattr(_TRACE, "steps", None) is not None -def _seed_id_array(nodes, node_col): - """Seed ids of the frontier as a host numpy array (device→host copy is fine — - only called under an explain trace). None on any failure.""" - try: - col = nodes[node_col] - if hasattr(col, "to_numpy"): - return col.to_numpy() - arr = getattr(col, "values", col) - return arr.get() if hasattr(arr, "get") else arr - except Exception: - return None - - -def _seed_deg_sum(idx, seed_ids) -> Optional[int]: - """Σ degree of the resident seeds — the true index expansion cost, FREE from the - CSR ``group_offsets`` already on the AdjacencyIndex. Diagnostic only (LP1); the - report's planner needs this fanout estimate exposed via EXPLAIN.""" - try: - import numpy as np - ks = idx.keys_sorted - ks = ks.get() if hasattr(ks, "get") else np.asarray(ks) - off = idx.group_offsets - off = off.get() if hasattr(off, "get") else np.asarray(off) - seed_ids = np.asarray(seed_ids) - pos = np.searchsorted(ks, seed_ids) - valid = pos < ks.size - pos_v = pos[valid] - seed_v = seed_ids[valid] - hit = pos_v[ks[pos_v] == seed_v] # keep only real key hits (robust to seed order) - return int((off[hit + 1] - off[hit]).sum()) - except Exception: - return None +# Back-compat for existing private tests while helpers live in cost.py. +_seed_id_array = seed_id_array +_seed_deg_sum = seed_deg_sum def get_registry(g: Plottable) -> GfqlIndexRegistry: return getattr(g, REGISTRY_ATTR, EMPTY_REGISTRY) @@ -119,6 +84,27 @@ def _check_column(column: Optional[str], expected: str, kind: str) -> None: ) +def _is_resident_index_valid( + g: Plottable, + kind: str, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, +) -> bool: + """True when a resident index still matches the current graph frames.""" + eng = resolve_engine(cast(Any, engine), g) + registry = get_registry(g) + if kind in ADJ_KINDS: + src, dst = g._source, g._destination + if src is None or dst is None or g._edges is None: + return False + return registry.get_valid(kind, g._edges, (src, dst), eng) is not None + if kind == NODE_ID: + node_col = g._node + if node_col is None or g._nodes is None: + return False + return registry.get_valid(NODE_ID, g._nodes, (node_col,), eng) is not None + return False + + def create_index( g: Plottable, kind: str, @@ -275,7 +261,17 @@ def _hop_is_index_coverable( return True -def _ensure_indexes(g, registry, direction, engine, policy, nodes, src, dst, node_col): +def _ensure_indexes( + g: Plottable, + registry: GfqlIndexRegistry, + direction: str, + engine: Engine, + policy: IndexPolicy, + nodes: Any, + src: str, + dst: str, + node_col: str, +) -> GfqlIndexRegistry: """auto/force: build the indexes this seeded hop needs (opt-in pay-as-you-go). force => always build missing; auto => build only when the query looks @@ -310,7 +306,7 @@ def _ensure_indexes(g, registry, direction, engine, policy, nodes, src, dst, nod def maybe_index_hop( g: Plottable, engine: Engine, *, nodes, hops, direction, return_as_wave_front, - to_fixed_point=False, policy: str = "use", **rest, + to_fixed_point: bool = False, policy: str = "use", **rest: Any, ) -> Optional[Plottable]: """Planner entry called from hop(). Returns an index-built subgraph, or None to fall back to the scan/join path. @@ -319,11 +315,13 @@ def maybe_index_hop( (or buildable under auto/force), (b) the query is covered, (c) the frontier is not so large that a full scan is cheaper. Correctness is identical either way. """ + policy = validate_index_policy(policy) or "use" + # Diagnostic trace (LP1) is populated only inside an explain context — build the # base record + a `_bail` helper that logs *why* we fell back to scan. All of this # is skipped entirely when not tracing, so the hot path pays nothing. trace = _trace_active() - diag: dict = {} + diag: Dict[str, Any] = {} if trace: diag = { "op": "hop", "direction": direction, "hops": hops, @@ -378,12 +376,12 @@ def _bail(reason: str) -> Optional[Plottable]: idx0 = registry.get_valid( EDGE_OUT_ADJ if direction != "reverse" else EDGE_IN_ADJ, g._edges, (src, dst), engine ) - frac = _COST_GATE_FRAC.get(engine, _COST_GATE_FRAC_DEFAULT) + frac = cost_gate_frac(engine) if trace and idx0 is not None: # Free fanout estimate (Σ seed degree) from the CSR offsets — the planner # signal the report wants EXPLAIN to surface (not just used-index yes/no). - seed_ids = _seed_id_array(nodes, node_col) - deg_sum = _seed_deg_sum(idx0, seed_ids) if seed_ids is not None else None + seed_ids = seed_id_array(nodes, node_col) + deg_sum = seed_deg_sum(idx0, seed_ids) if seed_ids is not None else None diag["n_keys"] = int(idx0.n_keys) diag["seed_deg_sum"] = deg_sum diag["est_result_rows"] = deg_sum diff --git a/graphistry/compute/gfql/index/build.py b/graphistry/compute/gfql/index/build.py index 9eaa7c5556..b9edf1e440 100644 --- a/graphistry/compute/gfql/index/build.py +++ b/graphistry/compute/gfql/index/build.py @@ -6,14 +6,25 @@ """ from __future__ import annotations -from typing import Any, Optional, Tuple +from typing import Any, Optional, Protocol, Tuple from graphistry.Engine import Engine from .engine_arrays import array_namespace, col_to_array from .registry import AdjacencyIndex, NodeIdIndex, frame_fingerprint -def _csr_from_keys(keys: Any, xp: Any) -> Tuple[Any, Any, Any]: +class FrameLike(Protocol): + shape: Any + + def __getitem__(self, key: str) -> Any: + ... + + +ArrayLike = Any +ArrayNamespace = Any + + +def _csr_from_keys(keys: ArrayLike, xp: ArrayNamespace) -> Tuple[ArrayLike, ArrayLike, ArrayLike]: """(keys array over E rows) -> (unique_keys, group_offsets[U+1], row_positions[E]). row_positions = the original row indices grouped (contiguously) by key value. @@ -35,7 +46,7 @@ def _csr_from_keys(keys: Any, xp: Any) -> Tuple[Any, Any, Any]: def build_adjacency_index( - edges: Any, + edges: FrameLike, kind: str, key_col: str, other_col: str, @@ -66,7 +77,7 @@ def build_adjacency_index( def build_node_id_index( - nodes: Any, + nodes: FrameLike, node_col: str, engine: Engine, ) -> Optional[NodeIdIndex]: diff --git a/graphistry/compute/gfql/index/cost.py b/graphistry/compute/gfql/index/cost.py new file mode 100644 index 0000000000..09219281fe --- /dev/null +++ b/graphistry/compute/gfql/index/cost.py @@ -0,0 +1,56 @@ +"""Planner cost helpers for GFQL physical indexes.""" +from __future__ import annotations + +from typing import Any, Optional + +from graphistry.Engine import Engine + +# Index-vs-scan crossover fraction (of distinct source keys): past this frontier a +# full scan is cheaper than per-seed index probes, so `use` falls back to scan and +# never loses to the un-indexed path. Engine-aware because vectorized-scan engines +# (polars/cudf/GPU) scan far faster than pandas, so their crossover is much smaller +# (see test_index_cost_gate_engine_aware). Measured N=1e5 deg8: pandas ~0.5, +# polars ~0.02. GPU values provisional (dgx-gated) and conservatively grouped with +# polars. +_COST_GATE_FRAC = {Engine.PANDAS: 0.5} +_COST_GATE_FRAC_DEFAULT = 0.02 + + +def cost_gate_frac(engine: Engine) -> float: + """Return the index-vs-scan crossover fraction for an engine.""" + return _COST_GATE_FRAC.get(engine, _COST_GATE_FRAC_DEFAULT) + + +def seed_id_array(nodes: Any, node_col: str) -> Optional[Any]: + """Seed ids of the frontier as a host numpy array. + + Device-to-host copy is fine here because this is only used for diagnostic + planner-cost reporting under ``gfql_explain``. Returns ``None`` on any failure. + """ + try: + col = nodes[node_col] + if hasattr(col, "to_numpy"): + return col.to_numpy() + arr = getattr(col, "values", col) + return arr.get() if hasattr(arr, "get") else arr + except Exception: + return None + + +def seed_deg_sum(idx: Any, seed_ids: Any) -> Optional[int]: + """Sum resident seed degree from CSR offsets for explain diagnostics.""" + try: + import numpy as np + ks = idx.keys_sorted + ks = ks.get() if hasattr(ks, "get") else np.asarray(ks) + off = idx.group_offsets + off = off.get() if hasattr(off, "get") else np.asarray(off) + seed_ids = np.asarray(seed_ids) + pos = np.searchsorted(ks, seed_ids) + valid = pos < ks.size + pos_v = pos[valid] + seed_v = seed_ids[valid] + hit = pos_v[ks[pos_v] == seed_v] + return int((off[hit + 1] - off[hit]).sum()) + except Exception: + return None diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index b90b7fb77f..5adaef3970 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -11,9 +11,11 @@ from graphistry.Engine import resolve_engine from .api import index_trace, show_indexes +from .policy import validate_index_policy def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = "auto") -> Dict[str, Any]: + index_policy = validate_index_policy(index_policy) or "use" eng = resolve_engine(cast(Any, engine), g) resident = show_indexes(g) with index_trace() as steps: diff --git a/graphistry/compute/gfql/index/policy.py b/graphistry/compute/gfql/index/policy.py new file mode 100644 index 0000000000..91d546cc85 --- /dev/null +++ b/graphistry/compute/gfql/index/policy.py @@ -0,0 +1,20 @@ +"""GFQL index policy validation.""" +from __future__ import annotations + +from typing import Literal, Optional, Tuple, cast + +IndexPolicy = Literal["off", "use", "auto", "force"] +VALID_INDEX_POLICIES: Tuple[IndexPolicy, ...] = ("off", "use", "auto", "force") + + +def validate_index_policy(policy: Optional[str]) -> Optional[IndexPolicy]: + """Validate a public ``index_policy`` value. + + ``None`` means the caller did not override the default planner behavior. + """ + if policy is None: + return None + if policy not in VALID_INDEX_POLICIES: + allowed = ", ".join(repr(p) for p in VALID_INDEX_POLICIES) + raise ValueError(f"index_policy must be one of {allowed}; got {policy!r}") + return cast(IndexPolicy, policy) diff --git a/graphistry/compute/gfql/index/wire.py b/graphistry/compute/gfql/index/wire.py index 7075efdb47..7546936053 100644 --- a/graphistry/compute/gfql/index/wire.py +++ b/graphistry/compute/gfql/index/wire.py @@ -94,13 +94,13 @@ def apply_index_op(g: Any, op: Any, *, engine: Any = "auto") -> Any: CreateIndex/DropIndex -> new Plottable; ShowIndexes -> pandas DataFrame. """ - from .api import create_index, drop_index, show_indexes, index_name, get_registry + from .api import create_index, drop_index, show_indexes, get_registry, _is_resident_index_valid if isinstance(op, CreateIndex): if not op.replace: reg = get_registry(g) - if reg.has(op.kind): - return g # resident reuse (fingerprint validity checked at use time) + if reg.has(op.kind) and _is_resident_index_valid(g, op.kind, engine): + return g # valid resident index reuse return create_index(g, op.kind, column=op.column, name=op.name, engine=engine) if isinstance(op, DropIndex): kind = op.kind diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 4ff5e701fb..1dd1dafc36 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -121,6 +121,38 @@ def test_wire_roundtrip(graph): show = g.gfql({"type": "ShowIndexes"}) assert show.shape[0] == 1 +def test_create_rebuilds_stale_resident_index(): + g = graphistry.edges(pd.DataFrame({"src": [0, 1], "dst": [1, 2]}), "src", "dst").materialize_nodes() + gi = g.gfql("CREATE GFQL INDEX FOR edge_out_adj") + assert bool(gi.show_indexes().iloc[0]["valid"]) is True + + g2 = gi.edges(pd.DataFrame({"src": [2, 3, 4], "dst": [3, 4, 5]}), "src", "dst") + assert bool(g2.show_indexes().iloc[0]["valid"]) is False + + g3 = g2.gfql("CREATE GFQL INDEX FOR edge_out_adj") + row = g3.show_indexes().iloc[0] + assert bool(row["valid"]) is True + assert int(row["n_rows"]) == 3 + + g4 = g2.gfql({"type": "CreateIndex", "kind": "edge_out_adj"}) + row = g4.show_indexes().iloc[0] + assert bool(row["valid"]) is True + assert int(row["n_rows"]) == 3 + + +def test_invalid_index_policy_raises(graph): + chain = [n({"id": 0}), e_forward(hops=1)] + with pytest.raises(ValueError, match="index_policy"): + graph.gfql(chain, index_policy="bogus") + with pytest.raises(ValueError, match="index_policy"): + graph.gfql(chain, index_policy="") + with pytest.raises(ValueError, match="index_policy"): + graph.gfql_explain(chain, index_policy="bogus") + + # Documented values remain accepted. + graph.gfql(chain, index_policy="off") + graph.gfql_explain(chain, index_policy="use") + @pytest.mark.parametrize("engine", ENGINES) def test_index_policy_force_and_explain(graph, engine): From de5f6f35efb61b81bfba5fe22febc139eb3d23e2 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 09:32:10 -0700 Subject: [PATCH 13/20] ci(gfql): refresh cypher surface guard baseline --- bin/ci_cypher_surface_guard_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ci_cypher_surface_guard_baseline.json b/bin/ci_cypher_surface_guard_baseline.json index 704d9018f8..75ad90165a 100644 --- a/bin/ci_cypher_surface_guard_baseline.json +++ b/bin/ci_cypher_surface_guard_baseline.json @@ -13,5 +13,5 @@ "max_properties": 0 } }, - "lowering_py_max_lines": 8469 + "lowering_py_max_lines": 8478 } From 99c01754247a79c84aa9a2483263164915255673 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 10:07:32 -0700 Subject: [PATCH 14/20] fix(gfql/index): tighten index typing and cost tuning --- CHANGELOG.md | 2 +- graphistry/compute/ComputeMixin.py | 14 ++- graphistry/compute/gfql/index/__init__.py | 14 ++- graphistry/compute/gfql/index/api.py | 33 ++--- graphistry/compute/gfql/index/build.py | 25 ++-- graphistry/compute/gfql/index/cost.py | 58 ++++++++- graphistry/compute/gfql/index/cypher_ddl.py | 15 ++- .../compute/gfql/index/engine_arrays.py | 40 ++++--- graphistry/compute/gfql/index/explain.py | 47 ++++++-- graphistry/compute/gfql/index/lookup.py | 9 +- graphistry/compute/gfql/index/registry.py | 62 +++++----- graphistry/compute/gfql/index/traverse.py | 10 +- graphistry/compute/gfql/index/types.py | 113 ++++++++++++++++++ graphistry/compute/gfql/index/wire.py | 14 ++- graphistry/compute/hop.py | 3 +- .../tests/compute/gfql/index/test_index.py | 31 +++++ 16 files changed, 368 insertions(+), 122 deletions(-) create mode 100644 graphistry/compute/gfql/index/types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7645928ce9..856d156e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). - **GFQL Cypher numeric functions + `toLower`/`toUpper`/`lower`/`upper` (openCypher/neo4j/GQL-standard)**: added the standard scalar functions `floor`, `ceil` (alias `ceiling`, per Cypher 25 GQL conformance and the GQL grammar's `CEIL|CEILING`), `round(x)` / `round(x, precision)`, and `toLower` / `toUpper` plus their GQL-conformance aliases `lower` / `upper` (ISO GQL §20.24 character-string functions; neo4j accepts both spellings) — the idiomatic case-insensitive compare `WHERE toLower(n.name) = 'bob'` — across the Cypher `WHERE`/`RETURN` expression surface. Evaluated natively on pandas/cuDF and polars (differential-parity tested). **`round` follows neo4j's documented tie-breaking** (standards-vetted against the neo4j manual): precision 0 (and the 1-arg form) rounds ties toward **positive infinity** (`round(-1.5)` → `-1.0`, `round(2.5)` → `3.0`), and precision > 0 rounds ties **away from zero** (HALF_UP: `round(-1.55, 1)` → `-1.6`) — not the numpy/polars half-to-even defaults, which give wrong answers on ties. The `round(x, precision, mode)` 3-arg form is not yet supported. Complements the already-supported `abs`/`sqrt`/`sign` and chained comparisons (`WHERE 1 < n.age < 65`). The `^` exponentiation operator is deferred: standards vetting settled it as **left-associative** (the openCypher TCK pins left, and neo4j's shipped Cypher 5/25 parser left-folds `Pow` — the manual's "right to left" row is a docs bug), but the current conformance corpus marks it reject-expected, so re-adding it is a coordinated corpus change. `LIKE`/`ILIKE`/`BETWEEN` remain intentionally unimplemented (verified absent from both Cypher and ISO GQL — GQL's only `LIKE` is the unrelated `CREATE GRAPH TYPE … LIKE g` DDL). - **GFQL Cypher `=~` regex-match operator (openCypher/neo4j-standard)**: added the standard `=~` string predicate to the Cypher `WHERE`/expression grammar — `MATCH (n) WHERE n.name =~ '(?i)al.*' RETURN n`. Semantics match openCypher/neo4j: **Java-regex dialect, full-string/anchored match** (`n.name =~ 'AB'` matches only `'AB'`, not `'ABCDEF'`; use `.*`/`^..$` for partial), with inline flags (`(?i)`/`(?m)`/`(?s)`) honored; lowers to the existing `fullmatch` predicate. Works in the simple `WHERE prop =~ '...'` form (all engines via `filter_by_dict`) and composes through `AND`/`OR`/`NOT`/`RETURN` expressions via the shared expression engine (pandas/cuDF; polars supports the simple-WHERE form and declines the complex `OR`/`NOT` row-filter form with an honest `NotImplementedError` — the pre-existing polars `where_rows` limitation, not `=~`-specific). Also adds native polars `Match`/`Fullmatch` predicate lowering (previously `NotImplementedError`), so `=~` and the Python `match()`/`fullmatch()` predicates run natively on polars. Differential-parity tested against the pandas oracle. `LIKE`/`ILIKE` remain intentionally unimplemented (not in any graph standard — use `=~`/`CONTAINS`/`STARTS WITH`). -- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront, hardened by an adversarial review that found and fixed several index-vs-scan divergences the original scenarios missed (`max_hops` honored; duplicate node ids, edge endpoints absent from the node table, int32/int64 seed-vs-key dtype mismatches, and stale `.edges()` rebinds all now match scan or fall back — never a wrong answer), with object-identity fingerprinting (recycle-proof) and 6 added differential regression tests. **Perf (dgx-spark, deg-8, warm median; every measured cell guarded so the index path was actually taken AND the index result equals the scan result):** seeded latency is **flat in graph size** — GFQL-pandas 1-hop 0.124/0.122 ms at 0.8M/8M nodes (6.4M/64M edges) while the O(E) scan grows 105 → 1045 ms — and **beats kuzu (CSR) and neo4j by 9–28×** on CPU at 0.8M nodes (1-hop: GFQL-pandas 0.123 ms vs kuzu 1.15 ms vs neo4j 1.45 ms; 1–2-hop: 0.150 ms vs 4.25 ms vs 2.54 ms, matched answer counts). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity, invalid policy strings now fail fast, and repeated `CREATE GFQL INDEX ...` rebuilds stale resident indexes after frame rebinds. cuDF/polars-GPU are flat but floored by ~3 ms (cuDF) GPU kernel-launch overhead — selective traversal is an indexing problem, not a compute one (CPU wins it). No change to default (un-indexed) behavior. +- **GFQL physical adjacency indexes — pay-as-you-go seeded-traversal acceleration**: Opt-in CSR adjacency / node-id indexes that turn seeded `hop()`/`gfql()` traversal from an O(E) full edge scan into an O(degree) `searchsorted` gather. Sidecar over edge **row positions** (never reorders `.edges`/`.nodes`), fingerprint-validated so a `.edges()` rebind safely invalidates a stale index (treated as absent — never a wrong answer). Three uniform surfaces driving one registry: **Python** (`g.create_index('edge_out_adj'|'edge_in_adj'|'node_id')`, `drop_index`, `show_indexes`, `gfql_index_edges`/`gfql_index_all`, `gfql_explain`), **Cypher DDL** (`CREATE/DROP GFQL INDEX FOR `, `SHOW GFQL INDEXES` — the mandatory `GFQL` token disambiguates from standard property `CREATE INDEX`), and the **JSON wire protocol** (`{"type":"CreateIndex",...}` ops + `index_policy` in the request envelope). Optimizer policy `gfql(..., index_policy='use'|'auto'|'force'|'off')` (default `use` = use-resident-never-build). Engine-polymorphic (numpy host arrays for pandas/polars, **cupy on-device for cudf**); the seeded fast path is hooked at every scan site (`compute/hop.py`, the lazy polars hop, and the polars single-hop chain fast path) and falls back to the scan/join path for any uncovered feature (edge/source/dest match, `target_wave_front`, `min_hops>1`, labeling). Differential-parity verified (indexed subgraph == scan subgraph) across pandas/cudf/polars/polars-gpu × forward/reverse/undirected × 1–3 hop × wavefront, hardened by an adversarial review that found and fixed several index-vs-scan divergences the original scenarios missed (`max_hops` honored; duplicate node ids, edge endpoints absent from the node table, int32/int64 seed-vs-key dtype mismatches, and stale `.edges()` rebinds all now match scan or fall back — never a wrong answer), with object-identity fingerprinting (recycle-proof) and 6 added differential regression tests. **Perf (dgx-spark, deg-8, warm median; every measured cell guarded so the index path was actually taken AND the index result equals the scan result):** seeded latency is **flat in graph size** — GFQL-pandas 1-hop 0.124/0.122 ms at 0.8M/8M nodes (6.4M/64M edges) while the O(E) scan grows 105 → 1045 ms — and **beats kuzu (CSR) and neo4j by 9–28×** on CPU at 0.8M nodes (1-hop: GFQL-pandas 0.123 ms vs kuzu 1.15 ms vs neo4j 1.45 ms; 1–2-hop: 0.150 ms vs 4.25 ms vs 2.54 ms, matched answer counts). Build cost (one O(E log E) sort) is amortized over queries; `index_policy='auto'` builds only when the planner predicts selectivity, invalid policy strings now fail fast, repeated `CREATE GFQL INDEX ...` rebuilds stale resident indexes after frame rebinds, `gfql_explain()` has a typed report contract, and the planner cost gate is tunable via Python helpers or `GFQL_INDEX_COST_GATE_FRAC*` environment variables. cuDF/polars-GPU are flat but floored by ~3 ms (cuDF) GPU kernel-launch overhead — selective traversal is an indexing problem, not a compute one (CPU wins it). No change to default (un-indexed) behavior. - **GFQL native Polars engine — traversals (`engine='polars'`)**: Added a native, vectorized Polars execution engine for the core GFQL traversals `hop()` and `chain()`, dispatched at the engine boundary so the production pandas/cuDF paths are untouched. `Engine.POLARS` is opt-in (explicit `engine='polars'`); `engine='auto'` with Polars input still coerces to pandas as before. Covers forward/reverse/undirected single-hop traversal, directed multi-hop chains, node/edge filter dicts and predicates (lowered to Polars expressions), `edge_match`/`source_node_match`/`destination_node_match`, `target_wave_front`, and alias names; the BFS advances via semi/anti joins (no per-row Python work). Validated by differential parity against the pandas engine (hop + chain test suites plus a randomized fuzzer) and benchmarked vs pandas (`benchmarks/gfql/pandas_vs_polars.py`) — Polars wins at scale (up to ~2.5x on multi-edge chains at millions of edges; crossover ~50–100k rows). Variable-length/multi-hop edges, undirected edges in multi-edge chains, hop labels, and node `query=` raise `NotImplementedError` for now (use `engine='pandas'`). - **GFQL native Polars engine — variable-length `min_hops>1` traversals (`engine='polars'`/`'polars-gpu'`)**: Forward/reverse lower-bound traversals (`e(min_hops=N, max_hops=M)`) now run natively on the Polars engine — no pandas bridge. The eager Polars hop runs pandas' min_hops algorithm vectorized: a NON-anti-joined BFS (the wavefront carries revisits so a cycle keeps bumping the reach depth until `max_hops`), a 3-case termination gate (`max_reached1` (needs connected-components + 2-core seed retention) and a *direct* `hop(min_hops>1)` (which would need pandas' separate un-labeled direct-hop node-output plus the `target_wave_front` threading the chain supplies — without them it silently diverges) both raise `NotImplementedError` for `engine='polars'` (use `chain()`/`gfql()`, or `engine='pandas'`). Validated by differential parity vs the pandas oracle: the 500-seed randomized chain fuzzer (`test_polars_chain_fuzz_parity`, hardened to compare null-aware node **attributes** and edge **multiplicity**, not just id/endpoint sets) is **500/500**, a min_hops+attribute-filter amplified fuzz and metamorphic invariants pass, and `engine='polars-gpu'` (cudf_polars) runs the full 500-seed fuzz **500/500** on-device. - **GFQL native Polars engine — cypher row pipeline (`engine='polars'`)**: Extended the Polars engine to the Cypher `MATCH … RETURN` row surface, natively vectorized. **NO CHEATING:** the polars engine never silently falls back to the pandas engine — every query runs natively on polars or raises an honest `NotImplementedError` pointing at `engine='pandas'` (falling back to pandas would misrepresent pandas performance as polars; only a human may consent to a bridge). `chain_polars` splits boundary `call()` ops (mirroring the pandas `_handle_boundary_calls`) and runs each trailing row op per-op native or raises. **Native polars** (no pandas round-trip): frame ops (`rows`/`limit`/`skip`/`distinct`/`drop_cols`), `select`/`with_`/`return_` projection (a conservative cypher-expr-AST → `pl.Expr` lowering covering property access, arithmetic, comparison, boolean, literals), `order_by` (`.sort`), `group_by` (`count`/`sum`/`avg`/`min`/`max`), `unwind` (literal-list cross-join), the result projection for property/expr columns, and entity-text `RETURN n` rendering for int/string/bool nodes (`pl.concat_str`). **Honestly deferred** (raise `NotImplementedError`, no pandas fallback): multi-entity `rows(binding_ops=…)`, cross-entity same-path `WHERE` (`DFSamePathExecutor`), float/temporal/nested entity-text, and exotic expressions (CASE/list/map/temporal, `collect` aggregates) — these are the forward native-engineering targets. Validated by differential parity vs pandas including a TCK-style conformance lane (`test_engine_polars_cypher_conformance.py`: native-only curated corpus + seeded fuzzer + NULL/3-valued-logic graph + entity-text escaping, plus a `DEFERRED` list asserting deferred queries raise rather than silently bridge) and benchmarked (`benchmarks/gfql/cypher_row_pipeline.py`). **Perf (interleaved, 1M nodes, each engine on its native-frame graph, all fully native):** polars wins **5.6–38×** across the surface — `RETURN n` ~38×, `ORDER BY` ~17×, `WHERE`+`ORDER BY`+`LIMIT` ~14×, traversals 6–7.5×, projections/aggregations/`DISTINCT` 5.6–6.9×. cuDF/pandas paths untouched. diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 1da53b6153..cd8b8d07c7 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -1,6 +1,6 @@ import numpy as np import pandas as pd -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from typing_extensions import Literal from graphistry.Engine import Engine, EngineAbstract, EngineAbstractType, POLARS_ENGINES, resolve_engine, df_to_engine, df_concat, safe_merge from graphistry.Plottable import Plottable @@ -28,6 +28,10 @@ filter_nodes_by_dict as filter_nodes_by_dict_base ) +if TYPE_CHECKING: + from graphistry.compute.gfql.index.explain import GfqlExplainReport + + logger = setup_logger(__name__) @@ -681,7 +685,13 @@ def gfql(self, *args, **kwargs): index registry. See :meth:`create_index` and :doc:`gfql/index_adjacency`. """ - def gfql_explain(self, query, *, index_policy='use', engine='auto'): + def gfql_explain( + self, + query: object, + *, + index_policy: str = 'use', + engine: EngineAbstractType = 'auto', + ) -> 'GfqlExplainReport': """Explain how the GFQL planner would run ``query``: per-hop index-vs-scan choice, cost-gate numbers, and resident-index validity. Read-only (no execution). Returns a report object; print it for a human-readable plan.""" from graphistry.compute.gfql.index.explain import gfql_explain as _ge return _ge(self, query, index_policy=index_policy, engine=engine) diff --git a/graphistry/compute/gfql/index/__init__.py b/graphistry/compute/gfql/index/__init__.py index 3f6fe81bfc..8ab8796d4f 100644 --- a/graphistry/compute/gfql/index/__init__.py +++ b/graphistry/compute/gfql/index/__init__.py @@ -5,6 +5,10 @@ ``gfql_index_edges``, ``gfql_index_all``, and the planner entry ``maybe_index_hop``. These are wired onto Plottable via ComputeMixin. """ +from .types import ( + AdjacencyIndexKind, ArrayLike, ArrayNamespace, EdgeIndexDirection, + HopDirection, IndexBackend, IndexKind, IndexTraceStep, +) from .registry import ( GfqlIndexRegistry, EMPTY_REGISTRY, EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, ADJ_KINDS, ALL_KINDS, @@ -15,19 +19,25 @@ get_registry, maybe_index_hop, index_name, index_trace, ) from .wire import ( - CreateIndex, DropIndex, ShowIndexes, apply_index_op, index_op_from_json, + CreateIndex, DropIndex, ShowIndexes, IndexOp, apply_index_op, index_op_from_json, is_index_op, is_index_op_json, ) from .cypher_ddl import parse_index_ddl, looks_like_index_ddl +from .cost import cost_gate_frac, reset_cost_gate_frac, set_cost_gate_frac +from .explain import GfqlExplainReport __all__ = [ + "AdjacencyIndexKind", "ArrayLike", "ArrayNamespace", "EdgeIndexDirection", + "HopDirection", "IndexBackend", "IndexKind", "IndexTraceStep", "GfqlIndexRegistry", "EMPTY_REGISTRY", "EDGE_OUT_ADJ", "EDGE_IN_ADJ", "NODE_ID", "ADJ_KINDS", "ALL_KINDS", "AdjacencyIndex", "NodeIdIndex", "create_index", "drop_index", "show_indexes", "gfql_index_edges", "gfql_index_all", "get_registry", "maybe_index_hop", "index_name", "index_trace", - "CreateIndex", "DropIndex", "ShowIndexes", "apply_index_op", + "CreateIndex", "DropIndex", "ShowIndexes", "IndexOp", "apply_index_op", "index_op_from_json", "is_index_op", "is_index_op_json", "parse_index_ddl", "looks_like_index_ddl", + "cost_gate_frac", "reset_cost_gate_frac", "set_cost_gate_frac", + "GfqlExplainReport", ] diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index c6b8084115..60921d7e73 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -13,13 +13,14 @@ from graphistry.Engine import EngineAbstract, Engine, resolve_engine from graphistry.Plottable import Plottable from .registry import ( - GfqlIndexRegistry, EMPTY_REGISTRY, + AdjacencyIndex, GfqlIndexRegistry, EMPTY_REGISTRY, NodeIdIndex, EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, ADJ_KINDS, ALL_KINDS, ) from .build import build_adjacency_index, build_node_id_index from .traverse import index_seeded_hop from .cost import cost_gate_frac, seed_deg_sum, seed_id_array from .policy import IndexPolicy, validate_index_policy +from .types import EdgeIndexDirection, HopDirection, IndexKind # Private Plottable attachment key. Keep access behind get_registry()/show_indexes(). REGISTRY_ATTR = "_gfql_index_registry" @@ -69,11 +70,11 @@ def _attach(g: Plottable, registry: GfqlIndexRegistry) -> Plottable: return res -def index_name(kind: str, column: Optional[str]) -> str: +def index_name(kind: IndexKind, column: Optional[str]) -> str: return f"{kind}:{column}" if column else kind -def _check_column(column: Optional[str], expected: str, kind: str) -> None: +def _check_column(column: Optional[str], expected: str, kind: IndexKind) -> None: """A user-supplied ``column`` must match the binding the index keys on; a different column would be a silent no-op (registry is one-index-per-kind in v1), so reject it honestly rather than ignore it.""" @@ -86,7 +87,7 @@ def _check_column(column: Optional[str], expected: str, kind: str) -> None: def _is_resident_index_valid( g: Plottable, - kind: str, + kind: IndexKind, engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, ) -> bool: """True when a resident index still matches the current graph frames.""" @@ -107,7 +108,7 @@ def _is_resident_index_valid( def create_index( g: Plottable, - kind: str, + kind: IndexKind, *, column: Optional[str] = None, name: Optional[str] = None, @@ -161,7 +162,7 @@ def create_index( raise ValueError(f"Unknown GFQL index kind: {kind!r}. Expected one of {ALL_KINDS}.") -def drop_index(g: Plottable, kind: Optional[str] = None) -> Plottable: +def drop_index(g: Plottable, kind: Optional[IndexKind] = None) -> Plottable: """Drop one index (by kind) or all indexes (kind=None). Idempotent.""" registry = get_registry(g) if kind is None: @@ -187,12 +188,14 @@ def show_indexes(g: Plottable) -> Any: assert idx is not None # iterating registry.kinds() -> present if kind in ADJ_KINDS: assert g._source is not None and g._destination is not None - valid = registry.get_valid(kind, g._edges, (g._source, g._destination), idx.engine) is not None - n_keys, n_rows = idx.n_keys, idx.n_edges + adj = cast(AdjacencyIndex, idx) + valid = registry.get_valid(kind, g._edges, (g._source, g._destination), adj.engine) is not None + n_keys, n_rows = adj.n_keys, adj.n_edges else: # NODE_ID + node_idx = cast(NodeIdIndex, idx) valid = g._nodes is not None and registry.get_valid( - NODE_ID, g._nodes, (idx.key_col,), idx.engine) is not None - n_keys, n_rows = idx.n_nodes, 0 + NODE_ID, g._nodes, (node_idx.key_col,), node_idx.engine) is not None + n_keys, n_rows = node_idx.n_nodes, 0 rows.append({ "name": idx.name or index_name(kind, idx.key_col), "kind": kind, @@ -208,7 +211,7 @@ def show_indexes(g: Plottable) -> Any: return pd.DataFrame(rows, columns=cols) -def gfql_index_edges(g: Plottable, direction: str = "both", +def gfql_index_edges(g: Plottable, direction: EdgeIndexDirection = "both", engine: Union[EngineAbstract, str] = EngineAbstract.AUTO) -> Plottable: """Convenience: build edge adjacency index(es). direction: 'forward'|'reverse'|'both'.""" if direction in ("forward", "both"): @@ -264,7 +267,7 @@ def _hop_is_index_coverable( def _ensure_indexes( g: Plottable, registry: GfqlIndexRegistry, - direction: str, + direction: HopDirection, engine: Engine, policy: IndexPolicy, nodes: Any, @@ -305,7 +308,7 @@ def _ensure_indexes( def maybe_index_hop( - g: Plottable, engine: Engine, *, nodes, hops, direction, return_as_wave_front, + g: Plottable, engine: Engine, *, nodes: Any, hops: Optional[int], direction: HopDirection, return_as_wave_front: bool, to_fixed_point: bool = False, policy: str = "use", **rest: Any, ) -> Optional[Plottable]: """Planner entry called from hop(). Returns an index-built subgraph, or None to @@ -373,9 +376,9 @@ def _bail(reason: str) -> Optional[Plottable]: # Cost gate: if the frontier covers a large fraction of distinct sources, the # scan path is competitive — fall back (avoids index overhead on bulk-ish hops). - idx0 = registry.get_valid( + idx0 = cast(Optional[AdjacencyIndex], registry.get_valid( EDGE_OUT_ADJ if direction != "reverse" else EDGE_IN_ADJ, g._edges, (src, dst), engine - ) + )) frac = cost_gate_frac(engine) if trace and idx0 is not None: # Free fanout estimate (Σ seed degree) from the CSR offsets — the planner diff --git a/graphistry/compute/gfql/index/build.py b/graphistry/compute/gfql/index/build.py index b9edf1e440..344c4c9df4 100644 --- a/graphistry/compute/gfql/index/build.py +++ b/graphistry/compute/gfql/index/build.py @@ -6,22 +6,13 @@ """ from __future__ import annotations -from typing import Any, Optional, Protocol, Tuple +from typing import Optional, Tuple, cast from graphistry.Engine import Engine +from graphistry.compute.typing import DataFrameT from .engine_arrays import array_namespace, col_to_array from .registry import AdjacencyIndex, NodeIdIndex, frame_fingerprint - - -class FrameLike(Protocol): - shape: Any - - def __getitem__(self, key: str) -> Any: - ... - - -ArrayLike = Any -ArrayNamespace = Any +from .types import AdjacencyIndexKind, ArrayLike, ArrayNamespace def _csr_from_keys(keys: ArrayLike, xp: ArrayNamespace) -> Tuple[ArrayLike, ArrayLike, ArrayLike]: @@ -46,8 +37,8 @@ def _csr_from_keys(keys: ArrayLike, xp: ArrayNamespace) -> Tuple[ArrayLike, Arra def build_adjacency_index( - edges: FrameLike, - kind: str, + edges: DataFrameT, + kind: AdjacencyIndexKind, key_col: str, other_col: str, edge_id_col: Optional[str], @@ -70,14 +61,14 @@ def build_adjacency_index( backend=backend, engine=engine, fingerprint=frame_fingerprint(edges, fingerprint_cols, engine), - source_ref=edges, + source_ref=cast(DataFrameT, edges), n_edges=int(keys.shape[0]), n_keys=int(unique_keys.shape[0]), ) def build_node_id_index( - nodes: FrameLike, + nodes: DataFrameT, node_col: str, engine: Engine, ) -> Optional[NodeIdIndex]: @@ -105,6 +96,6 @@ def build_node_id_index( backend=backend, engine=engine, fingerprint=frame_fingerprint(nodes, (node_col,), engine), - source_ref=nodes, + source_ref=cast(DataFrameT, nodes), n_nodes=n_keys, ) diff --git a/graphistry/compute/gfql/index/cost.py b/graphistry/compute/gfql/index/cost.py index 09219281fe..754db74482 100644 --- a/graphistry/compute/gfql/index/cost.py +++ b/graphistry/compute/gfql/index/cost.py @@ -1,7 +1,8 @@ """Planner cost helpers for GFQL physical indexes.""" from __future__ import annotations -from typing import Any, Optional +import os +from typing import Any, Dict, Optional from graphistry.Engine import Engine @@ -14,10 +15,63 @@ # polars. _COST_GATE_FRAC = {Engine.PANDAS: 0.5} _COST_GATE_FRAC_DEFAULT = 0.02 +_COST_GATE_FRAC_OVERRIDES: Dict[Engine, float] = {} + + +def _validate_cost_gate_frac(frac: float) -> float: + if not 0.0 < frac <= 1.0: + raise ValueError(f"GFQL index cost gate fraction must be in (0, 1], got {frac!r}") + return frac + + +def set_cost_gate_frac(engine: Engine, frac: Optional[float]) -> None: + """Override the index-vs-scan crossover for one process. + + Passing ``None`` clears the code-level override for ``engine``. This is meant + for benchmark/diagnostic tuning; defaults remain stable and documented. + """ + if frac is None: + _COST_GATE_FRAC_OVERRIDES.pop(engine, None) + return + _COST_GATE_FRAC_OVERRIDES[engine] = _validate_cost_gate_frac(float(frac)) + + +def reset_cost_gate_frac(engine: Optional[Engine] = None) -> None: + """Clear code-level cost-gate overrides.""" + if engine is None: + _COST_GATE_FRAC_OVERRIDES.clear() + else: + _COST_GATE_FRAC_OVERRIDES.pop(engine, None) + + +def _env_cost_gate_frac(engine: Engine) -> Optional[float]: + names = ( + f"GFQL_INDEX_COST_GATE_FRAC_{engine.value.upper().replace('-', '_')}", + "GFQL_INDEX_COST_GATE_FRAC", + ) + for name in names: + raw = os.environ.get(name) + if raw is None or raw == "": + continue + try: + return _validate_cost_gate_frac(float(raw)) + except ValueError as ex: + raise ValueError(f"Invalid {name}={raw!r}: {ex}") from ex + return None def cost_gate_frac(engine: Engine) -> float: - """Return the index-vs-scan crossover fraction for an engine.""" + """Return the index-vs-scan crossover fraction for an engine. + + Precedence: code override via ``set_cost_gate_frac`` > engine/global env var + > baked default. Env vars are ``GFQL_INDEX_COST_GATE_FRAC`` and per-engine + ``GFQL_INDEX_COST_GATE_FRAC_PANDAS`` / ``..._POLARS_GPU`` style names. + """ + if engine in _COST_GATE_FRAC_OVERRIDES: + return _COST_GATE_FRAC_OVERRIDES[engine] + env = _env_cost_gate_frac(engine) + if env is not None: + return env return _COST_GATE_FRAC.get(engine, _COST_GATE_FRAC_DEFAULT) diff --git a/graphistry/compute/gfql/index/cypher_ddl.py b/graphistry/compute/gfql/index/cypher_ddl.py index 2655bc9500..e8bd02f405 100644 --- a/graphistry/compute/gfql/index/cypher_ddl.py +++ b/graphistry/compute/gfql/index/cypher_ddl.py @@ -13,9 +13,10 @@ from __future__ import annotations import re -from typing import Any, Optional +from typing import Optional, cast -from .wire import CreateIndex, DropIndex, ShowIndexes +from .types import IndexKind +from .wire import CreateIndex, DropIndex, ShowIndexes, IndexOp _KIND = r"(?Pedge_out_adj|edge_in_adj|node_id)" @@ -35,6 +36,8 @@ ) _SHOW_RE = re.compile(r"^\s*SHOW\s+GFQL\s+INDEXES\s*;?\s*$", re.IGNORECASE) +# Keep these regexes module-level: the DDL grammar is tiny, hot-path parse calls +# should not pay lazy-init branching, and GFQL temporal/row parsers follow the same pattern. _DDL_PREFIX = re.compile(r"^\s*(CREATE|DROP|SHOW)\s+GFQL\s+INDEX", re.IGNORECASE) @@ -42,19 +45,19 @@ def looks_like_index_ddl(query: str) -> bool: return bool(isinstance(query, str) and _DDL_PREFIX.match(query)) -def parse_index_ddl(query: str) -> Optional[Any]: - """Return a wire op (CreateIndex/DropIndex/ShowIndexes) or None.""" +def parse_index_ddl(query: str) -> Optional[IndexOp]: + """Return a typed wire op (CreateIndex/DropIndex/ShowIndexes) or None.""" if not isinstance(query, str): return None if _SHOW_RE.match(query): return ShowIndexes() m = _CREATE_RE.match(query) if m: - return CreateIndex(kind=m.group("kind").lower(), column=m.group("col"), + return CreateIndex(kind=cast(IndexKind, m.group("kind").lower()), column=m.group("col"), name=m.group("name")) m = _DROP_FOR_RE.match(query) if m: - return DropIndex(kind=m.group("kind").lower(), column=m.group("col"), + return DropIndex(kind=cast(IndexKind, m.group("kind").lower()), column=m.group("col"), missing_ok=bool(m.group("ifexists"))) m = _DROP_NAME_RE.match(query) if m: diff --git a/graphistry/compute/gfql/index/engine_arrays.py b/graphistry/compute/gfql/index/engine_arrays.py index eb3d2cdc58..b60fe024c6 100644 --- a/graphistry/compute/gfql/index/engine_arrays.py +++ b/graphistry/compute/gfql/index/engine_arrays.py @@ -12,12 +12,14 @@ """ from __future__ import annotations -from typing import Any, Tuple +from typing import Any, Tuple, cast from graphistry.Engine import Engine +from graphistry.compute.typing import DataFrameT +from .types import ArrayLike, ArrayNamespace, IndexBackend -def array_namespace(engine: Engine) -> Tuple[Any, str]: +def array_namespace(engine: Engine) -> Tuple[ArrayNamespace, IndexBackend]: """Return (array module, backend tag) for an engine. cudf indexes keep their arrays on-device (cupy); everything else uses numpy @@ -30,29 +32,29 @@ def array_namespace(engine: Engine) -> Tuple[Any, str]: try: import cupy as cp # type: ignore - return cp, "cupy" + return cast(ArrayNamespace, cp), "cupy" except Exception: # pragma: no cover - cupy always present with cudf - return np, "numpy" - return np, "numpy" + return cast(ArrayNamespace, np), "numpy" + return cast(ArrayNamespace, np), "numpy" -def col_to_array(df: Any, col: str, engine: Engine) -> Any: +def col_to_array(df: DataFrameT, col: str, engine: Engine) -> ArrayLike: """Extract a column as a backend-native 1-D array (numpy or cupy).""" if engine in (Engine.POLARS, Engine.POLARS_GPU): - return df.get_column(col).to_numpy() + return cast(ArrayLike, df.get_column(col).to_numpy()) if engine == Engine.CUDF: # cudf Series -> cupy array (stays on device) - return df[col].values - return df[col].to_numpy() + return cast(ArrayLike, df[col].values) + return cast(ArrayLike, df[col].to_numpy()) -def ids_to_array(ids: Any, col: str, engine: Engine) -> Any: +def ids_to_array(ids: DataFrameT, col: str, engine: Engine) -> ArrayLike: """Frontier ids (a frame/Series) -> backend array, matching index backend.""" return col_to_array(ids, col, engine) -def take_rows(df: Any, positions: Any, engine: Engine) -> Any: +def take_rows(df: DataFrameT, positions: ArrayLike, engine: Engine) -> DataFrameT: """Positionally gather rows of ``df`` by an integer array ``positions``. ``positions`` is a backend array (numpy for pandas/polars, cupy for cudf). @@ -62,12 +64,12 @@ def take_rows(df: Any, positions: Any, engine: Engine) -> Any: import numpy as np idx = np.asarray(positions) - return df[idx] + return cast(DataFrameT, df[idx]) # pandas / cudf: iloc accepts numpy (pandas) or cupy (cudf) int arrays - return df.iloc[positions] + return cast(DataFrameT, df.iloc[positions]) -def select_by_ids(df: Any, col: str, ids: Any, engine: Engine) -> Any: +def select_by_ids(df: DataFrameT, col: str, ids: ArrayLike, engine: Engine) -> DataFrameT: """Return rows of ``df`` whose ``col`` is in the id array ``ids`` (set semantics, preserves df row order). Engine-polymorphic, vectorized.""" if engine in (Engine.POLARS, Engine.POLARS_GPU): @@ -78,17 +80,17 @@ def select_by_ids(df: Any, col: str, ids: Any, engine: Engine) -> Any: # pola-rs/polars#22149) — vectorized AND preserves the left (df) row order, which # the node materialization relies on (table-order parity with the scan). ids_df = pl.DataFrame({col: np.asarray(ids)}).cast({col: df.schema[col]}) - return df.join(ids_df.unique(), on=col, how="semi") + return cast(DataFrameT, df.join(ids_df.unique(), on=col, how="semi")) if engine == Engine.CUDF: import cudf # type: ignore - return df[df[col].isin(cudf.Series(ids))] + return cast(DataFrameT, df[df[col].isin(cudf.Series(ids))]) import numpy as np - return df[df[col].isin(np.asarray(ids))] + return cast(DataFrameT, df[df[col].isin(np.asarray(ids))]) -def set_difference(cand: Any, visited: Any, xp: Any) -> Any: +def set_difference(cand: ArrayLike, visited: ArrayLike, xp: ArrayNamespace) -> ArrayLike: """cand minus visited (both backend arrays), vectorized via sorted membership.""" if int(visited.shape[0]) == 0: return cand @@ -101,7 +103,7 @@ def set_difference(cand: Any, visited: Any, xp: Any) -> Any: return cand[~ismember] -def union1d(a: Any, b: Any, xp: Any) -> Any: +def union1d(a: ArrayLike, b: ArrayLike, xp: ArrayNamespace) -> ArrayLike: if int(a.shape[0]) == 0: return xp.unique(b) if int(b.shape[0]) == 0: diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index 5adaef3970..d642afe950 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -7,20 +7,40 @@ """ from __future__ import annotations -from typing import Any, Dict, cast +from typing import Any, List, Optional, TypedDict, cast -from graphistry.Engine import resolve_engine +from graphistry.Engine import EngineAbstractType, resolve_engine from .api import index_trace, show_indexes -from .policy import validate_index_policy +from .policy import IndexPolicy, validate_index_policy +from .types import IndexTraceStep -def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = "auto") -> Dict[str, Any]: - index_policy = validate_index_policy(index_policy) or "use" - eng = resolve_engine(cast(Any, engine), g) +class GfqlExplainReport(TypedDict): + engine: str + index_policy: IndexPolicy + resident_indexes: List[str] + steps: List[IndexTraceStep] + used_index: bool + est_seed_cardinality: Optional[int] + est_result_rows: Optional[int] + chosen_direction: Optional[str] + decision_reason: Optional[str] + error: Optional[str] + + +def gfql_explain( + g: Any, + query: object, + *, + index_policy: str = "use", + engine: EngineAbstractType = "auto", +) -> GfqlExplainReport: + resolved_policy: IndexPolicy = validate_index_policy(index_policy) or "use" + eng = resolve_engine(engine, g) resident = show_indexes(g) with index_trace() as steps: try: - g.gfql(query, engine=engine, index_policy=index_policy) + g.gfql(query, engine=engine, index_policy=resolved_policy) error = None except Exception as ex: # report, don't raise — explain is diagnostic error = f"{type(ex).__name__}: {ex}" @@ -30,15 +50,16 @@ def gfql_explain(g: Any, query: Any, *, index_policy: str = "use", engine: str = # of seeds; `est_result_rows` = estimated fanout (Σ seed degree, free from CSR). ref = [s for s in steps if s.get("path") == "index"] or list(steps) last = ref[-1] if ref else {} + resident_names = cast(List[str], resident["name"].tolist() if len(resident) else []) return { "engine": eng.value, - "index_policy": index_policy, - "resident_indexes": resident["name"].tolist() if len(resident) else [], + "index_policy": resolved_policy, + "resident_indexes": resident_names, "steps": list(steps), "used_index": used_index, - "est_seed_cardinality": last.get("frontier_n"), - "est_result_rows": last.get("est_result_rows"), - "chosen_direction": last.get("direction"), - "decision_reason": last.get("decision_reason"), + "est_seed_cardinality": cast(Optional[int], last.get("frontier_n")), + "est_result_rows": cast(Optional[int], last.get("est_result_rows")), + "chosen_direction": cast(Optional[str], last.get("direction")), + "decision_reason": cast(Optional[str], last.get("decision_reason")), "error": error, } diff --git a/graphistry/compute/gfql/index/lookup.py b/graphistry/compute/gfql/index/lookup.py index bb198c3ff5..14fa469021 100644 --- a/graphistry/compute/gfql/index/lookup.py +++ b/graphistry/compute/gfql/index/lookup.py @@ -6,12 +6,13 @@ """ from __future__ import annotations -from typing import Any +from typing import Tuple from .registry import AdjacencyIndex, NodeIdIndex +from .types import ArrayLike, ArrayNamespace -def lookup_edge_rows(index: AdjacencyIndex, frontier: Any, xp: Any): +def lookup_edge_rows(index: AdjacencyIndex, frontier: ArrayLike, xp: ArrayNamespace) -> Tuple[ArrayLike, ArrayLike]: """frontier (backend array of seed ids, deduped) -> (edge_rows, matched_ids). ``edge_rows`` = row positions of all edges incident to the frontier. @@ -59,7 +60,7 @@ def lookup_edge_rows(index: AdjacencyIndex, frontier: Any, xp: Any): return index.row_positions[flat], matched_ids -def _expand_ranges(start: Any, counts: Any, total: int, xp: Any) -> Any: +def _expand_ranges(start: ArrayLike, counts: ArrayLike, total: int, xp: ArrayNamespace) -> ArrayLike: """Vectorized [start, start+count) range concat WITHOUT np.repeat (cupy's ``repeat`` rejects array ``repeats``). Builds a per-output segment id via a boundary-marker cumsum, then gathers start/offset by segment. @@ -76,7 +77,7 @@ def _expand_ranges(start: Any, counts: Any, total: int, xp: Any) -> Any: return start[seg] + pos_in -def lookup_node_rows(index: NodeIdIndex, ids: Any, xp: Any) -> Any: +def lookup_node_rows(index: NodeIdIndex, ids: ArrayLike, xp: ArrayNamespace) -> ArrayLike: """ids (backend array) -> node row positions for those that exist (in id order of the index hits). Used to materialize node rows for a result id set.""" keys = index.keys_sorted diff --git a/graphistry/compute/gfql/index/registry.py b/graphistry/compute/gfql/index/registry.py index cb02f33993..6024ee0172 100644 --- a/graphistry/compute/gfql/index/registry.py +++ b/graphistry/compute/gfql/index/registry.py @@ -11,20 +11,24 @@ from __future__ import annotations from dataclasses import dataclass, field, replace -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, Union, cast from graphistry.Engine import Engine +from graphistry.compute.typing import DataFrameT +from .types import AdjacencyIndexKind, ArrayLike, IndexBackend, IndexKind # Index kinds (v1). Property/label/type indexes share this registry shape later. -EDGE_OUT_ADJ = "edge_out_adj" -EDGE_IN_ADJ = "edge_in_adj" -NODE_ID = "node_id" +EDGE_OUT_ADJ: AdjacencyIndexKind = "edge_out_adj" +EDGE_IN_ADJ: AdjacencyIndexKind = "edge_in_adj" +NODE_ID: IndexKind = "node_id" -ADJ_KINDS = (EDGE_OUT_ADJ, EDGE_IN_ADJ) -ALL_KINDS = (EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID) +ADJ_KINDS: Tuple[AdjacencyIndexKind, ...] = (EDGE_OUT_ADJ, EDGE_IN_ADJ) +ALL_KINDS: Tuple[IndexKind, ...] = (EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID) +FrameFingerprint = Tuple[int, Tuple[str, ...], str] -def frame_fingerprint(df: Any, cols: Tuple[str, ...], engine: Engine) -> Tuple: + +def frame_fingerprint(df: DataFrameT, cols: Tuple[str, ...], engine: Engine) -> FrameFingerprint: """Cheap O(1) structural fingerprint of the frame an index is built over: length + bound columns + engine. This is a SECONDARY guard; the primary validity check is object IDENTITY (``source_ref is df``, see ``get_valid``). We deliberately do @@ -46,18 +50,18 @@ class AdjacencyIndex: Lookup of a frontier of ids is O(F log U + result) via searchsorted + vectorized range expansion — sublinear in E, never a full edge scan. """ - kind: str # EDGE_OUT_ADJ | EDGE_IN_ADJ + kind: AdjacencyIndexKind # EDGE_OUT_ADJ | EDGE_IN_ADJ key_col: str # endpoint we key on (src for out, dst for in) other_col: str # opposite endpoint (the neighbor we emit) edge_id_col: Optional[str] # edge-id binding if present (else row pos == id) - keys_sorted: Any # distinct key ids, ascending (len U) [array] - group_offsets: Any # CSR offsets into row_positions (len U+1) [array] - row_positions: Any # edge row indices grouped by key (len E) [array] - other_values: Any # neighbor id per edge row, ORIGINAL order (len E) [array] - backend: str # 'numpy' | 'cupy' + keys_sorted: ArrayLike # distinct key ids, ascending (len U) [array] + group_offsets: ArrayLike # CSR offsets into row_positions (len U+1) [array] + row_positions: ArrayLike # edge row indices grouped by key (len E) [array] + other_values: ArrayLike # neighbor id per edge row, ORIGINAL order (len E) [array] + backend: IndexBackend # 'numpy' | 'cupy' engine: Engine - fingerprint: Tuple = field(compare=False, default=()) - source_ref: Any = field(compare=False, default=None) # the indexed frame (identity guard) + fingerprint: FrameFingerprint = field(compare=False, default=(-1, (), "")) + source_ref: Optional[DataFrameT] = field(compare=False, default=None) # the indexed frame (identity guard) n_edges: int = 0 n_keys: int = 0 name: Optional[str] = None @@ -67,12 +71,12 @@ class AdjacencyIndex: class NodeIdIndex: """Sorted node-id -> node row position (find seed/endpoint rows fast).""" key_col: str - keys_sorted: Any - row_positions: Any - backend: str + keys_sorted: ArrayLike + row_positions: ArrayLike + backend: IndexBackend engine: Engine - fingerprint: Tuple = field(compare=False, default=()) - source_ref: Any = field(compare=False, default=None) # the indexed frame (identity guard, I5) + fingerprint: FrameFingerprint = field(compare=False, default=(-1, (), "")) + source_ref: Optional[DataFrameT] = field(compare=False, default=None) # the indexed frame (identity guard, I5) n_nodes: int = 0 name: Optional[str] = None @@ -80,31 +84,31 @@ class NodeIdIndex: @dataclass(frozen=True) class GfqlIndexRegistry: """Immutable kind -> index map. ``with_index`` / ``without`` return copies.""" - indexes: Dict[str, Any] = field(default_factory=dict) + indexes: Dict[IndexKind, Union[AdjacencyIndex, NodeIdIndex]] = field(default_factory=dict) - def with_index(self, kind: str, index: Any) -> "GfqlIndexRegistry": + def with_index(self, kind: IndexKind, index: Union[AdjacencyIndex, NodeIdIndex]) -> "GfqlIndexRegistry": new = dict(self.indexes) new[kind] = index return GfqlIndexRegistry(new) - def without(self, kind: str) -> "GfqlIndexRegistry": + def without(self, kind: IndexKind) -> "GfqlIndexRegistry": new = dict(self.indexes) new.pop(kind, None) return GfqlIndexRegistry(new) - def get(self, kind: str) -> Optional[Any]: + def get(self, kind: IndexKind) -> Optional[Union[AdjacencyIndex, NodeIdIndex]]: return self.indexes.get(kind) - def has(self, kind: str) -> bool: + def has(self, kind: IndexKind) -> bool: return kind in self.indexes - def kinds(self) -> Tuple[str, ...]: - return tuple(sorted(self.indexes.keys())) + def kinds(self) -> Tuple[IndexKind, ...]: + return cast(Tuple[IndexKind, ...], tuple(sorted(self.indexes.keys()))) def is_empty(self) -> bool: return not self.indexes - def get_valid(self, kind: str, df: Any, cols: Tuple[str, ...], engine: Engine) -> Optional[Any]: + def get_valid(self, kind: IndexKind, df: DataFrameT, cols: Tuple[str, ...], engine: Engine) -> Optional[Union[AdjacencyIndex, NodeIdIndex]]: """Return the index for ``kind`` only if its fingerprint still matches the live frame + engine; else None (treat as absent).""" idx = self.indexes.get(kind) @@ -122,7 +126,7 @@ def get_valid(self, kind: str, df: Any, cols: Tuple[str, ...], engine: Engine) - return idx -def index_nbytes(idx: Any) -> int: +def index_nbytes(idx: Union[AdjacencyIndex, NodeIdIndex]) -> int: """Approximate resident memory of an index's sidecar arrays (bytes).""" total = 0 for attr in ("keys_sorted", "group_offsets", "row_positions", "other_values"): diff --git a/graphistry/compute/gfql/index/traverse.py b/graphistry/compute/gfql/index/traverse.py index 5b2a5ddb99..ab7cc6bad4 100644 --- a/graphistry/compute/gfql/index/traverse.py +++ b/graphistry/compute/gfql/index/traverse.py @@ -12,7 +12,7 @@ """ from __future__ import annotations -from typing import Any, List, Optional +from typing import Any, List, Optional, cast from graphistry.Engine import Engine from graphistry.Plottable import Plottable @@ -21,14 +21,14 @@ set_difference, union1d, ) from .lookup import lookup_edge_rows, lookup_node_rows -from .registry import EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, GfqlIndexRegistry +from .registry import EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, AdjacencyIndex, GfqlIndexRegistry, NodeIdIndex def _indices_for_direction( registry: GfqlIndexRegistry, direction: str, edges: Any, cols, engine: Engine ) -> Optional[List[Any]]: - out_idx = registry.get_valid(EDGE_OUT_ADJ, edges, cols, engine) - in_idx = registry.get_valid(EDGE_IN_ADJ, edges, cols, engine) + out_idx = cast(Optional[AdjacencyIndex], registry.get_valid(EDGE_OUT_ADJ, edges, cols, engine)) + in_idx = cast(Optional[AdjacencyIndex], registry.get_valid(EDGE_IN_ADJ, edges, cols, engine)) if direction == "forward": chosen = [out_idx] elif direction == "reverse": @@ -135,7 +135,7 @@ def index_seeded_hop( # Materialize node rows. Prefer the node_id index (O(result·log N) searchsorted # gather) over an O(N) isin scan — this keeps warm seeded latency flat in N. - node_idx = registry.get_valid(NODE_ID, g._nodes, (node_col,), engine) + node_idx = cast(Optional[NodeIdIndex], registry.get_valid(NODE_ID, g._nodes, (node_col,), engine)) if node_idx is not None: node_rows = lookup_node_rows(node_idx, needed, xp) # lookup returns rows in id-hit order; sort ascending so out_nodes keep diff --git a/graphistry/compute/gfql/index/types.py b/graphistry/compute/gfql/index/types.py new file mode 100644 index 0000000000..c17514d161 --- /dev/null +++ b/graphistry/compute/gfql/index/types.py @@ -0,0 +1,113 @@ +"""Shared internal types for GFQL physical indexes.""" +from __future__ import annotations + +from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union + +from graphistry.compute.typing import DataFrameT + +IndexKind = Literal["edge_out_adj", "edge_in_adj", "node_id"] +AdjacencyIndexKind = Literal["edge_out_adj", "edge_in_adj"] +IndexBackend = Literal["numpy", "cupy"] +HopDirection = Literal["forward", "reverse", "undirected"] +EdgeIndexDirection = Literal["forward", "reverse", "both"] + +FrameLike = DataFrameT + + +class ArrayLike(Protocol): + """Small numpy/cupy-like 1-D array surface used by CSR indexes.""" + + shape: Tuple[int, ...] + dtype: Any + nbytes: int + + def __getitem__(self, key: Any) -> "ArrayLike": + ... + + def __setitem__(self, key: Any, value: Any) -> None: + ... + + def __ne__(self, other: Any) -> "ArrayLike": # type: ignore[override] + ... + + def __eq__(self, other: Any) -> "ArrayLike": # type: ignore[override] + ... + + def __lt__(self, other: Any) -> "ArrayLike": + ... + + def __gt__(self, other: Any) -> "ArrayLike": + ... + + def __invert__(self) -> "ArrayLike": + ... + + def __add__(self, other: Any) -> "ArrayLike": + ... + + def __radd__(self, other: Any) -> "ArrayLike": + ... + + def __sub__(self, other: Any) -> "ArrayLike": + ... + + def __rsub__(self, other: Any) -> "ArrayLike": + ... + + def astype(self, dtype: Any) -> "ArrayLike": + ... + + def sum(self) -> Any: + ... + + +class ArrayNamespace(Protocol): + """Tiny numpy/cupy namespace surface used by CSR build/lookup.""" + + int64: Any + + def zeros(self, shape: Any, dtype: Any = ...) -> ArrayLike: + ... + + def ones(self, shape: Any, dtype: Any = ...) -> ArrayLike: + ... + + def argsort(self, a: ArrayLike) -> ArrayLike: + ... + + def nonzero(self, a: ArrayLike) -> Tuple[ArrayLike, ...]: + ... + + def concatenate(self, arrays: Any) -> ArrayLike: + ... + + def asarray(self, a: Any, dtype: Any = ...) -> ArrayLike: + ... + + def cumsum(self, a: ArrayLike) -> ArrayLike: + ... + + def arange(self, *args: Any, **kwargs: Any) -> ArrayLike: + ... + + def searchsorted(self, a: ArrayLike, v: ArrayLike) -> ArrayLike: + ... + + def where(self, condition: ArrayLike, x: Any, y: Any) -> ArrayLike: + ... + + def sort(self, a: ArrayLike) -> ArrayLike: + ... + + def unique(self, a: ArrayLike) -> ArrayLike: + ... + + def promote_types(self, type1: Any, type2: Any) -> Any: + ... + + +IndexTraceStep = Dict[str, Any] +IndexTrace = List[IndexTraceStep] + +GfqlIndexNameList = List[str] +OptionalInt = Optional[int] diff --git a/graphistry/compute/gfql/index/wire.py b/graphistry/compute/gfql/index/wire.py index 7546936053..12322e3edf 100644 --- a/graphistry/compute/gfql/index/wire.py +++ b/graphistry/compute/gfql/index/wire.py @@ -15,14 +15,15 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union, cast from .registry import ALL_KINDS +from .types import IndexKind @dataclass(frozen=True) class CreateIndex: - kind: str + kind: IndexKind column: Optional[str] = None name: Optional[str] = None replace: bool = False @@ -36,14 +37,14 @@ def from_json(d: Dict[str, Any]) -> "CreateIndex": kind = d.get("kind") if kind not in ALL_KINDS: raise ValueError(f"CreateIndex.kind must be one of {ALL_KINDS}, got {kind!r}") - return CreateIndex(kind=kind, column=d.get("column"), name=d.get("name"), + return CreateIndex(kind=cast(IndexKind, kind), column=d.get("column"), name=d.get("name"), replace=bool(d.get("replace", False))) @dataclass(frozen=True) class DropIndex: name: Optional[str] = None - kind: Optional[str] = None + kind: Optional[IndexKind] = None column: Optional[str] = None missing_ok: bool = False # IF EXISTS semantics: True = dropping a missing index is a no-op @@ -68,6 +69,7 @@ def from_json(d: Dict[str, Any]) -> "ShowIndexes": INDEX_OP_TYPES = ("CreateIndex", "DropIndex", "ShowIndexes") +IndexOp = Union[CreateIndex, DropIndex, ShowIndexes] def is_index_op(obj: Any) -> bool: @@ -78,7 +80,7 @@ def is_index_op_json(d: Any) -> bool: return isinstance(d, dict) and d.get("type") in INDEX_OP_TYPES -def index_op_from_json(d: Dict[str, Any]) -> Any: +def index_op_from_json(d: Dict[str, Any]) -> IndexOp: t = d.get("type") if t == "CreateIndex": return CreateIndex.from_json(d) @@ -89,7 +91,7 @@ def index_op_from_json(d: Dict[str, Any]) -> Any: raise ValueError(f"Not a GFQL index op: type={t!r}") -def apply_index_op(g: Any, op: Any, *, engine: Any = "auto") -> Any: +def apply_index_op(g: Any, op: IndexOp, *, engine: Any = "auto") -> Any: """Execute a DDL op against a Plottable's index registry. CreateIndex/DropIndex -> new Plottable; ShowIndexes -> pandas DataFrame. diff --git a/graphistry/compute/hop.py b/graphistry/compute/hop.py index 97a19190fd..853965c523 100644 --- a/graphistry/compute/hop.py +++ b/graphistry/compute/hop.py @@ -115,11 +115,12 @@ def hop(self: Plottable, # returns None to fall back. Coercion above is a no-op when already in-engine, # so the index fingerprint (keyed on the live edge frame) still matches. from graphistry.compute.gfql.index import get_registry, maybe_index_hop + from graphistry.compute.gfql.index.types import HopDirection _idx_policy = getattr(self, "_gfql_index_policy", "use") if (not get_registry(self).is_empty()) or _idx_policy in ("auto", "force"): _idx_nodes = df_to_engine(nodes, engine_concrete) if nodes is not None else None _indexed = maybe_index_hop( - self, engine_concrete, nodes=_idx_nodes, hops=hops, direction=direction, + self, engine_concrete, nodes=_idx_nodes, hops=hops, direction=cast(HopDirection, direction), return_as_wave_front=return_as_wave_front, to_fixed_point=to_fixed_point, policy=_idx_policy, min_hops=min_hops, max_hops=max_hops, diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 1dd1dafc36..13435ceba1 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -269,6 +269,37 @@ def test_cost_gate_engine_aware_never_loses_to_scan(engine): assert _sig(out) == _sig(scan_out), engine # correctness path-independent +def test_cost_gate_frac_tuning(monkeypatch): + from graphistry.Engine import Engine + from graphistry.compute.gfql.index import ( + cost_gate_frac, reset_cost_gate_frac, set_cost_gate_frac, + ) + + reset_cost_gate_frac() + monkeypatch.delenv("GFQL_INDEX_COST_GATE_FRAC", raising=False) + monkeypatch.delenv("GFQL_INDEX_COST_GATE_FRAC_PANDAS", raising=False) + assert cost_gate_frac(Engine.PANDAS) == 0.5 + + monkeypatch.setenv("GFQL_INDEX_COST_GATE_FRAC", "0.11") + assert cost_gate_frac(Engine.PANDAS) == 0.11 + + monkeypatch.setenv("GFQL_INDEX_COST_GATE_FRAC_PANDAS", "0.22") + assert cost_gate_frac(Engine.PANDAS) == 0.22 + + set_cost_gate_frac(Engine.PANDAS, 0.33) + assert cost_gate_frac(Engine.PANDAS) == 0.33 + + set_cost_gate_frac(Engine.PANDAS, None) + assert cost_gate_frac(Engine.PANDAS) == 0.22 + + with pytest.raises(ValueError, match="cost gate fraction"): + set_cost_gate_frac(Engine.PANDAS, 0) + monkeypatch.setenv("GFQL_INDEX_COST_GATE_FRAC_PANDAS", "2") + with pytest.raises(ValueError, match="GFQL_INDEX_COST_GATE_FRAC_PANDAS"): + cost_gate_frac(Engine.PANDAS) + reset_cost_gate_frac() + + def test_column_mismatch_raises_not_silent(graph): # A custom column that doesn't match the binding must raise, not silently no-op. with pytest.raises(NotImplementedError): From 305228b5e0f96df96b655b9190245e683c1f3acf Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 10:11:34 -0700 Subject: [PATCH 15/20] fix(gfql/index): support older mypy narrowing --- graphistry/compute/gfql/index/api.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 60921d7e73..e26c31c9df 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -20,7 +20,7 @@ from .traverse import index_seeded_hop from .cost import cost_gate_frac, seed_deg_sum, seed_id_array from .policy import IndexPolicy, validate_index_policy -from .types import EdgeIndexDirection, HopDirection, IndexKind +from .types import AdjacencyIndexKind, EdgeIndexDirection, HopDirection, IndexKind # Private Plottable attachment key. Keep access behind get_registry()/show_indexes(). REGISTRY_ATTR = "_gfql_index_registry" @@ -135,12 +135,13 @@ def create_index( raise ValueError( "edge adjacency index requires bound edges with source/destination columns" ) - key_col = src if kind == EDGE_OUT_ADJ else dst - _check_column(column, key_col, kind) - other = dst if kind == EDGE_OUT_ADJ else src - idx = build_adjacency_index(g._edges, kind, key_col, other, g._edge, eng, (src, dst)) - idx = replace(idx, name=name or index_name(kind, key_col)) - registry = registry.with_index(kind, idx) + adj_kind = cast(AdjacencyIndexKind, kind) + key_col = src if adj_kind == EDGE_OUT_ADJ else dst + _check_column(column, key_col, adj_kind) + other = dst if adj_kind == EDGE_OUT_ADJ else src + idx = build_adjacency_index(g._edges, adj_kind, key_col, other, g._edge, eng, (src, dst)) + idx = replace(idx, name=name or index_name(adj_kind, key_col)) + registry = registry.with_index(adj_kind, idx) return _attach(g, registry) if kind == NODE_ID: From 948c8661fc5f8d35cb84f947fbf45b9f49830a4a Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 10:18:09 -0700 Subject: [PATCH 16/20] fix(gfql/index): type index api tracing --- graphistry/compute/gfql/index/api.py | 78 +++++++++++++++++--------- graphistry/compute/gfql/index/types.py | 24 ++++++-- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index e26c31c9df..0151e386c2 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -10,7 +10,10 @@ import copy from typing import Any, Dict, List, Literal, Optional, Union, cast +import pandas as pd + from graphistry.Engine import EngineAbstract, Engine, resolve_engine +from graphistry.compute.typing import DataFrameT from graphistry.Plottable import Plottable from .registry import ( AdjacencyIndex, GfqlIndexRegistry, EMPTY_REGISTRY, NodeIdIndex, @@ -20,7 +23,10 @@ from .traverse import index_seeded_hop from .cost import cost_gate_frac, seed_deg_sum, seed_id_array from .policy import IndexPolicy, validate_index_policy -from .types import AdjacencyIndexKind, EdgeIndexDirection, HopDirection, IndexKind +from .types import ( + AdjacencyIndexKind, EdgeIndexDirection, HopDirection, IndexKind, + IndexTrace, IndexTraceStep, +) # Private Plottable attachment key. Keep access behind get_registry()/show_indexes(). REGISTRY_ATTR = "_gfql_index_registry" @@ -32,28 +38,35 @@ class index_trace: """Context manager: capture the per-hop index-vs-scan decisions made inside.""" - def __enter__(self) -> List[Dict[str, Any]]: - self.steps: List[Dict[str, Any]] = [] - self.prev = getattr(_TRACE, "steps", None) - _TRACE.steps = self.steps + def __enter__(self) -> IndexTrace: + self.steps: IndexTrace = [] + self.prev = _get_trace_steps() + _set_trace_steps(self.steps) return self.steps def __exit__(self, *exc: Any) -> Literal[False]: - _TRACE.steps = self.prev + _set_trace_steps(self.prev) return False -def _record(decision: Dict[str, Any]) -> None: - steps = getattr(_TRACE, "steps", None) +def _get_trace_steps() -> Optional[IndexTrace]: + return cast(Optional[IndexTrace], getattr(_TRACE, "steps", None)) + + +def _set_trace_steps(steps: Optional[IndexTrace]) -> None: + _TRACE.steps = steps + + +def _record(decision: IndexTraceStep) -> None: + steps = _get_trace_steps() if steps is not None: steps.append(decision) def _trace_active() -> bool: """True only inside an ``index_trace()`` / ``gfql_explain`` context. Diagnostic - enrichment (LP1) is computed only when this is True → zero hot-path cost.""" - return getattr(_TRACE, "steps", None) is not None - + enrichment (LP1) is computed only when this is True -> zero hot-path cost.""" + return _get_trace_steps() is not None # Back-compat for existing private tests while helpers live in cost.py. @@ -61,7 +74,7 @@ def _trace_active() -> bool: _seed_deg_sum = seed_deg_sum def get_registry(g: Plottable) -> GfqlIndexRegistry: - return getattr(g, REGISTRY_ATTR, EMPTY_REGISTRY) + return cast(GfqlIndexRegistry, getattr(g, REGISTRY_ATTR, EMPTY_REGISTRY)) def _attach(g: Plottable, registry: GfqlIndexRegistry) -> Plottable: @@ -171,7 +184,7 @@ def drop_index(g: Plottable, kind: Optional[IndexKind] = None) -> Plottable: return _attach(g, registry.without(kind)) -def show_indexes(g: Plottable) -> Any: +def show_indexes(g: Plottable) -> pd.DataFrame: """Return a pandas DataFrame describing resident indexes (empty if none). ``valid`` reflects live fingerprint validity against the current frames — a @@ -179,11 +192,10 @@ def show_indexes(g: Plottable) -> Any: is auto-skipped (scan fallback) until rebuilt. ``nbytes`` is the resident sidecar-array footprint (the pay-as-you-go memory signal). """ - import pandas as pd from .registry import index_nbytes registry = get_registry(g) - rows = [] + rows: List[Dict[str, object]] = [] for kind in registry.kinds(): idx = registry.get(kind) assert idx is not None # iterating registry.kinds() -> present @@ -243,11 +255,25 @@ def gfql_index_all(g: Plottable, # Coverage: features the index fast path does NOT yet handle -> caller scans. def _hop_is_index_coverable( - *, nodes, to_fixed_point, hops, min_hops, max_hops, - output_min_hops, output_max_hops, label_node_hops, label_edge_hops, - label_seeds, edge_match, source_node_match, destination_node_match, - source_node_query, destination_node_query, edge_query, - include_zero_hop_seed, target_wave_front, + *, + nodes: Optional[DataFrameT], + to_fixed_point: bool, + hops: Optional[int], + min_hops: Optional[int], + max_hops: Optional[int], + output_min_hops: Optional[int], + output_max_hops: Optional[int], + label_node_hops: Optional[str], + label_edge_hops: Optional[str], + label_seeds: bool, + edge_match: Optional[object], + source_node_match: Optional[object], + destination_node_match: Optional[object], + source_node_query: Optional[str], + destination_node_query: Optional[str], + edge_query: Optional[str], + include_zero_hop_seed: bool, + target_wave_front: Optional[DataFrameT], ) -> bool: if nodes is None: return False @@ -271,7 +297,7 @@ def _ensure_indexes( direction: HopDirection, engine: Engine, policy: IndexPolicy, - nodes: Any, + nodes: DataFrameT, src: str, dst: str, node_col: str, @@ -281,7 +307,7 @@ def _ensure_indexes( force => always build missing; auto => build only when the query looks selective (frontier small vs E), else leave registry as-is (scan). """ - needed = [] + needed: List[AdjacencyIndexKind] = [] if direction in ("forward", "undirected"): needed.append(EDGE_OUT_ADJ) if direction in ("reverse", "undirected"): @@ -325,7 +351,7 @@ def maybe_index_hop( # base record + a `_bail` helper that logs *why* we fell back to scan. All of this # is skipped entirely when not tracing, so the hot path pays nothing. trace = _trace_active() - diag: Dict[str, Any] = {} + diag: IndexTraceStep = {} if trace: diag = { "op": "hop", "direction": direction, "hops": hops, @@ -338,7 +364,7 @@ def maybe_index_hop( def _bail(reason: str) -> Optional[Plottable]: if trace: - _record({**diag, "path": "scan", "decision_reason": reason}) + _record(cast(IndexTraceStep, {**diag, "path": "scan", "decision_reason": reason})) return None if policy == "off": @@ -416,12 +442,12 @@ def _bail(reason: str) -> Optional[Plottable]: return_as_wave_front=return_as_wave_front, ) if trace: - _record({ + _record(cast(IndexTraceStep, { **diag, "hops": eff_hops, "path": "index" if result is not None else "scan", "decision_reason": ( "frontier below cost gate -> index" if result is not None else "index path not applicable -> scan" ), - }) + })) return result diff --git a/graphistry/compute/gfql/index/types.py b/graphistry/compute/gfql/index/types.py index c17514d161..c69e4050fb 100644 --- a/graphistry/compute/gfql/index/types.py +++ b/graphistry/compute/gfql/index/types.py @@ -1,7 +1,7 @@ """Shared internal types for GFQL physical indexes.""" from __future__ import annotations -from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union +from typing import Any, List, Literal, Optional, Protocol, Tuple, TypedDict, Union from graphistry.compute.typing import DataFrameT @@ -106,8 +106,22 @@ def promote_types(self, type1: Any, type2: Any) -> Any: ... -IndexTraceStep = Dict[str, Any] -IndexTrace = List[IndexTraceStep] +IndexPath = Literal["scan", "index"] + + +class IndexTraceStep(TypedDict, total=False): + op: str + direction: HopDirection + hops: Optional[int] + policy: str + engine: str + frontier_n: int + path: IndexPath + decision_reason: str + n_keys: int + seed_deg_sum: Optional[int] + est_result_rows: Optional[int] + threshold_frac: float -GfqlIndexNameList = List[str] -OptionalInt = Optional[int] + +IndexTrace = List[IndexTraceStep] From c0cf065cccb4ae52de8bb1318016f05f1cdd601a Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 13:28:12 -0700 Subject: [PATCH 17/20] fix(gfql/index): type seeded traversal internals --- graphistry/compute/gfql/index/traverse.py | 34 +++++++++++++---------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/graphistry/compute/gfql/index/traverse.py b/graphistry/compute/gfql/index/traverse.py index ab7cc6bad4..80dc73c810 100644 --- a/graphistry/compute/gfql/index/traverse.py +++ b/graphistry/compute/gfql/index/traverse.py @@ -12,9 +12,10 @@ """ from __future__ import annotations -from typing import Any, List, Optional, cast +from typing import List, Optional, Tuple, cast from graphistry.Engine import Engine +from graphistry.compute.typing import DataFrameT from graphistry.Plottable import Plottable from .engine_arrays import ( array_namespace, col_to_array, ids_to_array, take_rows, select_by_ids, @@ -22,36 +23,39 @@ ) from .lookup import lookup_edge_rows, lookup_node_rows from .registry import EDGE_OUT_ADJ, EDGE_IN_ADJ, NODE_ID, AdjacencyIndex, GfqlIndexRegistry, NodeIdIndex +from .types import ArrayLike, HopDirection def _indices_for_direction( - registry: GfqlIndexRegistry, direction: str, edges: Any, cols, engine: Engine -) -> Optional[List[Any]]: + registry: GfqlIndexRegistry, + direction: HopDirection, + edges: DataFrameT, + cols: Tuple[str, str], + engine: Engine, +) -> Optional[List[AdjacencyIndex]]: out_idx = cast(Optional[AdjacencyIndex], registry.get_valid(EDGE_OUT_ADJ, edges, cols, engine)) in_idx = cast(Optional[AdjacencyIndex], registry.get_valid(EDGE_IN_ADJ, edges, cols, engine)) if direction == "forward": - chosen = [out_idx] - elif direction == "reverse": - chosen = [in_idx] - else: # undirected - chosen = [out_idx, in_idx] - if any(ix is None for ix in chosen): + return None if out_idx is None else [out_idx] + if direction == "reverse": + return None if in_idx is None else [in_idx] + if out_idx is None or in_idx is None: return None - return chosen + return [out_idx, in_idx] def index_seeded_hop( g: Plottable, registry: GfqlIndexRegistry, *, - nodes: Any, + nodes: DataFrameT, node_col: str, src: str, dst: str, engine: Engine, hops: Optional[int], to_fixed_point: bool, - direction: str, + direction: HopDirection, return_as_wave_front: bool, ) -> Optional[Plottable]: if nodes is None or g._edges is None or g._nodes is None: @@ -81,7 +85,7 @@ def index_seeded_hop( frontier = seed visited = seed[:0] - edge_rows_parts: List[Any] = [] + edge_rows_parts: List[ArrayLike] = [] first = True hop_count = 0 @@ -92,8 +96,8 @@ def index_seeded_hop( break hop_count += 1 - matched_parts: List[Any] = [] - neigh_parts: List[Any] = [] + matched_parts: List[ArrayLike] = [] + neigh_parts: List[ArrayLike] = [] for ix in indices: rows, matched = lookup_edge_rows(ix, frontier, xp) edge_rows_parts.append(rows) From f3e6d33ad9db4e7590e17570d5bca6d12680e5e6 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 16:33:38 -0700 Subject: [PATCH 18/20] fix(gfql/index): lazily compile index DDL regexes --- graphistry/compute/gfql/index/cypher_ddl.py | 49 ++++++++++++------- .../tests/compute/gfql/index/test_index.py | 24 +++++++++ 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/graphistry/compute/gfql/index/cypher_ddl.py b/graphistry/compute/gfql/index/cypher_ddl.py index e8bd02f405..e289a93e49 100644 --- a/graphistry/compute/gfql/index/cypher_ddl.py +++ b/graphistry/compute/gfql/index/cypher_ddl.py @@ -12,54 +12,65 @@ """ from __future__ import annotations +from functools import lru_cache import re -from typing import Optional, cast +from typing import Optional, Pattern, Tuple, cast from .types import IndexKind from .wire import CreateIndex, DropIndex, ShowIndexes, IndexOp _KIND = r"(?Pedge_out_adj|edge_in_adj|node_id)" -_CREATE_RE = re.compile( +_CREATE_PATTERN = ( r"^\s*CREATE\s+GFQL\s+INDEX\s+(?:(?P[A-Za-z_]\w*)\s+)?FOR\s+" + _KIND - + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$", - re.IGNORECASE, + + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$" ) -_DROP_FOR_RE = re.compile( +_DROP_FOR_PATTERN = ( r"^\s*DROP\s+GFQL\s+INDEX\s+(?PIF\s+EXISTS\s+)?FOR\s+" + _KIND - + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$", - re.IGNORECASE, + + r"(?:\s+ON\s+(?P[A-Za-z_]\w*))?\s*;?\s*$" ) -_DROP_NAME_RE = re.compile( - r"^\s*DROP\s+GFQL\s+INDEX\s+(?PIF\s+EXISTS\s+)?(?P[A-Za-z_][\w:]*)\s*;?\s*$", - re.IGNORECASE, +_DROP_NAME_PATTERN = ( + r"^\s*DROP\s+GFQL\s+INDEX\s+(?PIF\s+EXISTS\s+)?(?P[A-Za-z_][\w:]*)\s*;?\s*$" ) -_SHOW_RE = re.compile(r"^\s*SHOW\s+GFQL\s+INDEXES\s*;?\s*$", re.IGNORECASE) +_SHOW_PATTERN = r"^\s*SHOW\s+GFQL\s+INDEXES\s*;?\s*$" +_DDL_PREFIX_PATTERN = r"^\s*(CREATE|DROP|SHOW)\s+GFQL\s+INDEX" -# Keep these regexes module-level: the DDL grammar is tiny, hot-path parse calls -# should not pay lazy-init branching, and GFQL temporal/row parsers follow the same pattern. -_DDL_PREFIX = re.compile(r"^\s*(CREATE|DROP|SHOW)\s+GFQL\s+INDEX", re.IGNORECASE) + +@lru_cache(maxsize=1) +def _ddl_prefix_re() -> Pattern[str]: + return re.compile(_DDL_PREFIX_PATTERN, re.IGNORECASE) + + +@lru_cache(maxsize=1) +def _ddl_res() -> Tuple[Pattern[str], Pattern[str], Pattern[str], Pattern[str]]: + return ( + re.compile(_SHOW_PATTERN, re.IGNORECASE), + re.compile(_CREATE_PATTERN, re.IGNORECASE), + re.compile(_DROP_FOR_PATTERN, re.IGNORECASE), + re.compile(_DROP_NAME_PATTERN, re.IGNORECASE), + ) def looks_like_index_ddl(query: str) -> bool: - return bool(isinstance(query, str) and _DDL_PREFIX.match(query)) + return bool(isinstance(query, str) and _ddl_prefix_re().match(query)) def parse_index_ddl(query: str) -> Optional[IndexOp]: """Return a typed wire op (CreateIndex/DropIndex/ShowIndexes) or None.""" if not isinstance(query, str): return None - if _SHOW_RE.match(query): + show_re, create_re, drop_for_re, drop_name_re = _ddl_res() + if show_re.match(query): return ShowIndexes() - m = _CREATE_RE.match(query) + m = create_re.match(query) if m: return CreateIndex(kind=cast(IndexKind, m.group("kind").lower()), column=m.group("col"), name=m.group("name")) - m = _DROP_FOR_RE.match(query) + m = drop_for_re.match(query) if m: return DropIndex(kind=cast(IndexKind, m.group("kind").lower()), column=m.group("col"), missing_ok=bool(m.group("ifexists"))) - m = _DROP_NAME_RE.match(query) + m = drop_name_re.match(query) if m: return DropIndex(name=m.group("name"), missing_ok=bool(m.group("ifexists"))) if looks_like_index_ddl(query): diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 13435ceba1..2d3c17913f 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -4,6 +4,8 @@ the scan/join path. Engine-parametrized (pandas/cudf/polars/polars-gpu) with importorskip so GPU lanes run only where available. """ +import importlib + import numpy as np import pandas as pd import pytest @@ -104,6 +106,28 @@ def test_cypher_ddl_recognizer(): parse_index_ddl("CREATE GFQL INDEX FOR bogus_kind") +def test_cypher_ddl_regexes_are_lazy(monkeypatch): + import graphistry.compute.gfql.index.cypher_ddl as ddl + + real_compile = ddl.re.compile + calls = [] + + def tracking_compile(*args, **kwargs): + calls.append(args[0]) + return real_compile(*args, **kwargs) + + monkeypatch.setattr(ddl.re, "compile", tracking_compile) + ddl = importlib.reload(ddl) + + assert calls == [] + assert ddl.looks_like_index_ddl("CREATE GFQL INDEX FOR edge_out_adj") + assert len(calls) == 1 + assert ddl.parse_index_ddl("SHOW GFQL INDEXES") == ShowIndexes() + assert len(calls) == 5 + assert isinstance(ddl.parse_index_ddl("DROP GFQL INDEX FOR edge_in_adj"), DropIndex) + assert len(calls) == 5 + + def test_cypher_ddl_via_gfql(graph): g = graph.gfql("CREATE GFQL INDEX FOR edge_out_adj") assert get_registry(g).has("edge_out_adj") From 59fdfe8e7a144be7d07d24ce50d7ae8f0c14e9b8 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 16:49:16 -0700 Subject: [PATCH 19/20] fix(gfql/index): share array protocol typing --- graphistry/compute/ComputeMixin.py | 3 +- graphistry/compute/__init__.py | 4 +- graphistry/compute/gfql/index/explain.py | 2 +- graphistry/compute/gfql/index/types.py | 96 +----------------------- graphistry/compute/typing.py | 93 ++++++++++++++++++++++- 5 files changed, 99 insertions(+), 99 deletions(-) diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index cd8b8d07c7..8016c1eff4 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -30,6 +30,7 @@ if TYPE_CHECKING: from graphistry.compute.gfql.index.explain import GfqlExplainReport + from graphistry.compute.gfql.index.policy import IndexPolicy logger = setup_logger(__name__) @@ -689,7 +690,7 @@ def gfql_explain( self, query: object, *, - index_policy: str = 'use', + index_policy: 'IndexPolicy' = 'use', engine: EngineAbstractType = 'auto', ) -> 'GfqlExplainReport': """Explain how the GFQL planner would run ``query``: per-hop index-vs-scan choice, cost-gate numbers, and resident-index validity. Read-only (no execution). Returns a report object; print it for a human-readable plan.""" diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 839669ff6a..28bca7047b 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -59,7 +59,7 @@ notnull, NotNull, ) from .gfql.same_path_types import col, compare -from .typing import DataFrameT +from .typing import ArrayLike, ArrayNamespace, DataFrameT __all__ = [ # Core classes @@ -100,5 +100,5 @@ # WHERE helpers 'col', 'compare', # Types - 'DataFrameT' + 'ArrayLike', 'ArrayNamespace', 'DataFrameT' ] diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index d642afe950..3880d544e0 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -32,7 +32,7 @@ def gfql_explain( g: Any, query: object, *, - index_policy: str = "use", + index_policy: IndexPolicy = "use", engine: EngineAbstractType = "auto", ) -> GfqlExplainReport: resolved_policy: IndexPolicy = validate_index_policy(index_policy) or "use" diff --git a/graphistry/compute/gfql/index/types.py b/graphistry/compute/gfql/index/types.py index c69e4050fb..af3aa800f6 100644 --- a/graphistry/compute/gfql/index/types.py +++ b/graphistry/compute/gfql/index/types.py @@ -1,9 +1,9 @@ """Shared internal types for GFQL physical indexes.""" from __future__ import annotations -from typing import Any, List, Literal, Optional, Protocol, Tuple, TypedDict, Union +from typing import List, Literal, Optional, TypedDict, Union -from graphistry.compute.typing import DataFrameT +from graphistry.compute.typing import ArrayLike, ArrayNamespace, DataFrameT IndexKind = Literal["edge_out_adj", "edge_in_adj", "node_id"] AdjacencyIndexKind = Literal["edge_out_adj", "edge_in_adj"] @@ -14,98 +14,6 @@ FrameLike = DataFrameT -class ArrayLike(Protocol): - """Small numpy/cupy-like 1-D array surface used by CSR indexes.""" - - shape: Tuple[int, ...] - dtype: Any - nbytes: int - - def __getitem__(self, key: Any) -> "ArrayLike": - ... - - def __setitem__(self, key: Any, value: Any) -> None: - ... - - def __ne__(self, other: Any) -> "ArrayLike": # type: ignore[override] - ... - - def __eq__(self, other: Any) -> "ArrayLike": # type: ignore[override] - ... - - def __lt__(self, other: Any) -> "ArrayLike": - ... - - def __gt__(self, other: Any) -> "ArrayLike": - ... - - def __invert__(self) -> "ArrayLike": - ... - - def __add__(self, other: Any) -> "ArrayLike": - ... - - def __radd__(self, other: Any) -> "ArrayLike": - ... - - def __sub__(self, other: Any) -> "ArrayLike": - ... - - def __rsub__(self, other: Any) -> "ArrayLike": - ... - - def astype(self, dtype: Any) -> "ArrayLike": - ... - - def sum(self) -> Any: - ... - - -class ArrayNamespace(Protocol): - """Tiny numpy/cupy namespace surface used by CSR build/lookup.""" - - int64: Any - - def zeros(self, shape: Any, dtype: Any = ...) -> ArrayLike: - ... - - def ones(self, shape: Any, dtype: Any = ...) -> ArrayLike: - ... - - def argsort(self, a: ArrayLike) -> ArrayLike: - ... - - def nonzero(self, a: ArrayLike) -> Tuple[ArrayLike, ...]: - ... - - def concatenate(self, arrays: Any) -> ArrayLike: - ... - - def asarray(self, a: Any, dtype: Any = ...) -> ArrayLike: - ... - - def cumsum(self, a: ArrayLike) -> ArrayLike: - ... - - def arange(self, *args: Any, **kwargs: Any) -> ArrayLike: - ... - - def searchsorted(self, a: ArrayLike, v: ArrayLike) -> ArrayLike: - ... - - def where(self, condition: ArrayLike, x: Any, y: Any) -> ArrayLike: - ... - - def sort(self, a: ArrayLike) -> ArrayLike: - ... - - def unique(self, a: ArrayLike) -> ArrayLike: - ... - - def promote_types(self, type1: Any, type2: Any) -> Any: - ... - - IndexPath = Literal["scan", "index"] diff --git a/graphistry/compute/typing.py b/graphistry/compute/typing.py index 00125c8d9c..d17805fc0b 100644 --- a/graphistry/compute/typing.py +++ b/graphistry/compute/typing.py @@ -1,5 +1,5 @@ import pandas as pd -from typing import Any, TYPE_CHECKING, TypeVar, Union +from typing import Any, Protocol, TYPE_CHECKING, Tuple, TypeVar, Union # TODO stubs for Union[cudf.DataFrame, dask.DataFrame, ..] at checking time if TYPE_CHECKING: @@ -15,3 +15,94 @@ # Type variable for return type preservation in predicates T = TypeVar('T') + +class ArrayLike(Protocol): + """Small numpy/cupy-like 1-D array surface used by compute kernels.""" + + shape: Tuple[int, ...] + dtype: Any + nbytes: int + + def __getitem__(self, key: Any) -> "ArrayLike": + ... + + def __setitem__(self, key: Any, value: Any) -> None: + ... + + def __ne__(self, other: Any) -> "ArrayLike": # type: ignore[override] + ... + + def __eq__(self, other: Any) -> "ArrayLike": # type: ignore[override] + ... + + def __lt__(self, other: Any) -> "ArrayLike": + ... + + def __gt__(self, other: Any) -> "ArrayLike": + ... + + def __invert__(self) -> "ArrayLike": + ... + + def __add__(self, other: Any) -> "ArrayLike": + ... + + def __radd__(self, other: Any) -> "ArrayLike": + ... + + def __sub__(self, other: Any) -> "ArrayLike": + ... + + def __rsub__(self, other: Any) -> "ArrayLike": + ... + + def astype(self, dtype: Any) -> "ArrayLike": + ... + + def sum(self) -> Any: + ... + + +class ArrayNamespace(Protocol): + """Small numpy/cupy namespace surface used by compute kernels.""" + + int64: Any + + def zeros(self, shape: Any, dtype: Any = ...) -> ArrayLike: + ... + + def ones(self, shape: Any, dtype: Any = ...) -> ArrayLike: + ... + + def argsort(self, a: ArrayLike) -> ArrayLike: + ... + + def nonzero(self, a: ArrayLike) -> Tuple[ArrayLike, ...]: + ... + + def concatenate(self, arrays: Any) -> ArrayLike: + ... + + def asarray(self, a: Any, dtype: Any = ...) -> ArrayLike: + ... + + def cumsum(self, a: ArrayLike) -> ArrayLike: + ... + + def arange(self, *args: Any, **kwargs: Any) -> ArrayLike: + ... + + def searchsorted(self, a: ArrayLike, v: ArrayLike) -> ArrayLike: + ... + + def where(self, condition: ArrayLike, x: Any, y: Any) -> ArrayLike: + ... + + def sort(self, a: ArrayLike) -> ArrayLike: + ... + + def unique(self, a: ArrayLike) -> ArrayLike: + ... + + def promote_types(self, type1: Any, type2: Any) -> Any: + ... From c1b4bf101e173af142b2b28cd65b436c89c31716 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 8 Jul 2026 17:01:17 -0700 Subject: [PATCH 20/20] fix(gfql): type explain query inputs --- graphistry/compute/ComputeMixin.py | 3 ++- graphistry/compute/__init__.py | 3 ++- graphistry/compute/gfql/index/explain.py | 10 +++++++--- graphistry/compute/gfql/query_types.py | 11 +++++++++++ graphistry/compute/gfql_unified.py | 5 +++-- graphistry/compute/gfql_validate.py | 3 ++- 6 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 graphistry/compute/gfql/query_types.py diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 8016c1eff4..4edda70b6a 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: from graphistry.compute.gfql.index.explain import GfqlExplainReport from graphistry.compute.gfql.index.policy import IndexPolicy + from graphistry.compute.gfql.query_types import GFQLQuery logger = setup_logger(__name__) @@ -688,7 +689,7 @@ def gfql(self, *args, **kwargs): def gfql_explain( self, - query: object, + query: 'GFQLQuery', *, index_policy: 'IndexPolicy' = 'use', engine: EngineAbstractType = 'auto', diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 28bca7047b..bbdb9a0f37 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -59,6 +59,7 @@ notnull, NotNull, ) from .gfql.same_path_types import col, compare +from .gfql.query_types import GFQLQuery from .typing import ArrayLike, ArrayNamespace, DataFrameT __all__ = [ @@ -100,5 +101,5 @@ # WHERE helpers 'col', 'compare', # Types - 'ArrayLike', 'ArrayNamespace', 'DataFrameT' + 'ArrayLike', 'ArrayNamespace', 'DataFrameT', 'GFQLQuery' ] diff --git a/graphistry/compute/gfql/index/explain.py b/graphistry/compute/gfql/index/explain.py index 3880d544e0..d4b7891827 100644 --- a/graphistry/compute/gfql/index/explain.py +++ b/graphistry/compute/gfql/index/explain.py @@ -7,13 +7,17 @@ """ from __future__ import annotations -from typing import Any, List, Optional, TypedDict, cast +from typing import TYPE_CHECKING, List, Optional, TypedDict, cast from graphistry.Engine import EngineAbstractType, resolve_engine +from graphistry.compute.gfql.query_types import GFQLQuery from .api import index_trace, show_indexes from .policy import IndexPolicy, validate_index_policy from .types import IndexTraceStep +if TYPE_CHECKING: + from graphistry.compute.ComputeMixin import ComputeMixin + class GfqlExplainReport(TypedDict): engine: str @@ -29,8 +33,8 @@ class GfqlExplainReport(TypedDict): def gfql_explain( - g: Any, - query: object, + g: ComputeMixin, + query: GFQLQuery, *, index_policy: IndexPolicy = "use", engine: EngineAbstractType = "auto", diff --git a/graphistry/compute/gfql/query_types.py b/graphistry/compute/gfql/query_types.py new file mode 100644 index 0000000000..5da75eb40c --- /dev/null +++ b/graphistry/compute/gfql/query_types.py @@ -0,0 +1,11 @@ +"""Shared public GFQL query type aliases.""" +from __future__ import annotations + +from typing import Any, Dict, List, Union + +from graphistry.compute.ast import ASTLet, ASTObject +from graphistry.compute.chain import Chain + + +GFQLQuery = Union[ASTObject, List[ASTObject], ASTLet, Chain, Dict[str, Any], str] +"""Accepted local GFQL query inputs: AST objects/chains/DAGs, JSON dicts, or strings.""" diff --git a/graphistry/compute/gfql_unified.py b/graphistry/compute/gfql_unified.py index bb0e7ac841..be088c674c 100644 --- a/graphistry/compute/gfql_unified.py +++ b/graphistry/compute/gfql_unified.py @@ -9,6 +9,7 @@ from graphistry.util import setup_logger from .ast import ASTObject, ASTLet, ASTNode, ASTEdge, ASTCall from .chain import Chain, chain as chain_impl +from .gfql.query_types import GFQLQuery from .chain_let import chain_let as chain_let_impl from .execution_context import ExecutionContext from .gfql.policy import ( @@ -1276,7 +1277,7 @@ def _materialize_split_alias_columns( def _gfql_otel_attrs( self: Plottable, - query: Union[ASTObject, List[ASTObject], ASTLet, Chain, dict, str], + query: GFQLQuery, engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, output: Optional[str] = None, policy: Optional[Dict[str, PolicyFunction]] = None, @@ -1507,7 +1508,7 @@ def _fire_postcompile_policy( @otel_traced("gfql.run", attrs_fn=_gfql_otel_attrs) def gfql(self: Plottable, - query: Union[ASTObject, List[ASTObject], ASTLet, Chain, dict, str], + query: GFQLQuery, engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, output: Optional[str] = None, policy: Optional[Dict[str, PolicyFunction]] = None, diff --git a/graphistry/compute/gfql_validate.py b/graphistry/compute/gfql_validate.py index bce8125b56..76b94bd60e 100644 --- a/graphistry/compute/gfql_validate.py +++ b/graphistry/compute/gfql_validate.py @@ -18,6 +18,7 @@ from graphistry.compute.gfql.cypher.parser import parse_cypher from graphistry.compute.gfql.frontends.cypher.binder import FrontendBinder from graphistry.compute.gfql.ir.compilation import GraphSchemaCatalog, PlanContext +from graphistry.compute.gfql.query_types import GFQLQuery from graphistry.compute.gfql.same_path_types import ( WhereComparison, normalize_where_entries, @@ -26,7 +27,7 @@ from graphistry.compute.validate.validate_schema import validate_chain_schema -GFQLValidationQuery = Union[ASTObject, List[ASTObject], ASTLet, Chain, dict, str] +GFQLValidationQuery = GFQLQuery def _serialize_error(exc: Exception, *, stage: str) -> Dict[str, Any]: if hasattr(exc, "to_dict") and callable(getattr(exc, "to_dict")):