diff --git a/CHANGELOG.md b/CHANGELOG.md index 267a72c416..ec46960a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Performance +- **GFQL seeded fast paths use resident indexes for PROPERTY-seeded lookups (#1658 x #1755)**: the resident-index acceleration previously required the seed filter to hit the node BINDING column, so the common LDBC/user pattern — graph bound on a synthetic key while `MATCH (m {id: $x})` filters the `id` property — never engaged it and paid O(N)/O(E) scans. The gates are now decoupled: the seed row falls back to the property scan when needed, but the CSR adjacency frontier gather and node-id candidate gathers engage whenever valid indexes are resident (binding-column values are the index key domain regardless of how the seed rows were found). Same decline contract otherwise (stale/absent index, non-numeric ids). Pinned: `test_property_seeded_engages_adjacency_index` (rebound-key graph, engagement + parity, pandas+polars). - **GFQL seeded fast path covers single-alias property RETURNs (LDBC IS5 shape) (#1755)**: `MATCH (m {id})-[:T]->(p) RETURN p.a AS x, p.b` — which lowers to `rows(source=p)+select(items)` rather than a whole-row projection — now takes the seeded typed-hop fast path: the deduped destination rows are renamed/selected directly (value-identical to the rows-pivot + select pipeline; row order may differ). Dtype parity with the full pandas path is exact: non-id properties ride the rows-pivot there, which upcasts int/float to float64 and bool to object, so the lean tail applies the same casts (the id property keeps its dtype); dtype classes not verified against the pivot (datetimes, pandas extension dtypes, categoricals) decline. `res._edges` is the same empty edges frame the full path yields (never None). Conservative declines keep full-path semantics for everything else: cross-alias refs (`RETURN m.x, p.y`), mixed whole-row+property, DISTINCT/ORDER BY/LIMIT (extra lowered ops), expr items, properties absent from the node frame (the full path's null/error semantics must apply), and requested-vs-actual engine mismatches (the full path converts to the requested engine). Differential fast-vs-full sweep across all of the above on pandas + polars: exact parity incl. dtypes and edges, engagement pinned both ways. Measured on LDBC SNB SF1 (dgx, harness row-validated): `message-creator` (IS5) 116.3→38.3 ms on polars (the residue is the seed scan, targeted separately by resident-index coverage). - **GFQL seeded typed-hop fast paths use resident indexes (#1658 x #1755)**: with `gfql_index_all()` resident, the seeded fast-path helpers (`_seeded_typed_hop_pandas_cudf`, `_seeded_typed_return_dst_pandas_cudf`, `_seeded_typed_return_dst_polars`) replace their three full-frame scans — O(N) seed-row lookup, O(E) frontier `isin`, O(N) endpoint gather — with positional index lookups (node-id searchsorted + CSR adjacency gather), so a seeded Cypher lookup stops paying graph-size costs entirely. Decline-gated to fall back to the identical scan body: absent/stale index (fingerprint+identity via `get_valid`), non-numeric id families (object/str ids keep the scan path's null semantics), seed not id-filtered. Results identical either way (row order may differ; node-id index implies unique ids). dgx 50k/200k covered-shape: pandas 1.71→0.86ms, polars 2.02→0.69ms (vs Kuzu's same-box 1.06ms); SF1-scale 3.2M/17M polars 47.2→2.25ms — polars numbers measured with a polars-engine index resident, which today requires `gfql_index_all(engine='polars')` on polars frames (until #1767 lands, AUTO index builds swap polars frames to pandas; pandas/cuDF flows are unaffected and independent). Pinned in `TestResidentIndexSeededFastPath` (parity+engagement pandas/polars, string-id decline, stale-index decline, uint64/int64 promote decline, reverse-direction serving). - **GFQL seeded typed-hop fast path (#1755)**: a seeded typed 1-hop — native chain `[n({id}), e_forward(), n({type})]` or Cypher `MATCH (m {id})-[:T]->(p) RETURN p` — previously paid the full two-pass chain machinery (~20-40ms: whole-frame combines, full-column type filters, rows-pivot projection). A new seed-first fast path recognizes exactly this shape and reduces the graph to the seed's 1-hop neighborhood before any of that work: native chain 32.6→0.93ms (35×), Cypher RETURN 39.2→1.9ms (20×) on pandas, with cuDF covered via the shared DataFrame API (30.8→4.7ms). Value-identical to the full path by construction (same rows/columns/dtypes; row order and index may differ) — the helpers return `None` and fall through for anything outside the exact shape or carrying full-path side-channels (multi-hop, variable-length, undirected, predicate filters, reverse patterns, missing bindings, list-`labels` columns, policy hooks, same-path WHERE, OPTIONAL MATCH null rows, and WITH..MATCH carried seeds all decline). Null ids/endpoints never link (membership sets are null-dropped, matching the full pipeline's joins). Verified with an independent oracle (results checked against the creator set hand-computed from the raw frames, not merely fast-vs-slow agreement) plus a differential fast-vs-full sweep across shapes × engines in `test_seeded_typed_hop_fastpath.py`. diff --git a/graphistry/compute/chain_fast_paths.py b/graphistry/compute/chain_fast_paths.py index 3cd1b42727..70066f6460 100644 --- a/graphistry/compute/chain_fast_paths.py +++ b/graphistry/compute/chain_fast_paths.py @@ -184,16 +184,28 @@ def _seeded_typed_hop_pandas_cudf( # O(E) frontier isin, and the O(N) candidate gather become positional lookups. # Any decline (no index, stale fingerprint, unsafe id cast) falls back to the # scan body below — identical results either way, only speed differs. - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if (n0f and node in n0f) else None + # Decoupled index use: the seed row lookup uses the node-id index ONLY when + # the seed filter includes the binding column, but the frontier edge gather + # (CSR adjacency) and candidate gathers (node-id index) engage regardless of + # HOW the seed rows were found — their inputs are binding-column values, which + # are the index key domain. (LDBC/user pattern: seed on the `id` PROPERTY + # while the graph binds a different key column — previously disqualified the + # whole index path.) + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if n0f else None seed_nodes = edges = cand = None if ctx is not None: nid, adj, xp, idx_engine = ctx - seed_nodes = _index_node_rows(nid, [n0f[node]], xp, idx_engine, nodes_df) + if node in n0f: + seed_nodes = _index_node_rows(nid, [n0f[node]], xp, idx_engine, nodes_df) if seed_nodes is not None: for k, v in n0f.items(): if k != node: seed_nodes = seed_nodes[seed_nodes[k] == v] - edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) + else: + seed_nodes = nodes_df + for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1): + seed_nodes = seed_nodes[seed_nodes[k] == v] + edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) if edges is not None: if ef: for k, v in ef.items(): @@ -266,16 +278,24 @@ def _seeded_typed_return_dst_pandas_cudf( # frame, never materializing an object column over the whole node table. # Membership sets are dropna()'d: pandas .isin matches NaN<->NaN, but the full # pipeline's joins never join on null keys, so a null id/endpoint must not link. - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if node in n0f else None + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) seed_nodes = edges = dstn = None if ctx is not None: nid, adj, xp, idx_engine = ctx - seed_nodes = _index_node_rows(nid, [n0f[node]], xp, idx_engine, nodes_df) + if node in n0f: + seed_nodes = _index_node_rows(nid, [n0f[node]], xp, idx_engine, nodes_df) if seed_nodes is not None: for k, v in n0f.items(): if k != node: seed_nodes = seed_nodes[seed_nodes[k] == v] - edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) + else: + # property-seeded (binding col not in the filter): scan the seed row, + # then the CSR/node-index gathers below still engage — binding-column + # values are the index key domain no matter how the seed was found. + seed_nodes = nodes_df + for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1): + seed_nodes = seed_nodes[seed_nodes[k] == v] + edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) if edges is not None: if ef: for k, v in ef.items(): @@ -331,16 +351,23 @@ def _seeded_typed_return_dst_polars( # Membership sets are drop_nulls()'d (null ids/endpoints never link, matching # the full pipeline's joins) and passed via .implode() (Series-arg is_in is # deprecated in polars 1.42, see polars#22149). - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if node in n0f else None + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) seed_nodes = edges = dstn = None if ctx is not None: nid, adj, xp, idx_engine = ctx - seed_nodes = _index_node_rows(nid, [n0f[node]], xp, idx_engine, nodes_df) + if node in n0f: + seed_nodes = _index_node_rows(nid, [n0f[node]], xp, idx_engine, nodes_df) if seed_nodes is not None: for k, v in n0f.items(): if k != node: seed_nodes = seed_nodes.filter(pl.col(k) == v) - edges = _index_edge_rows(adj, seed_nodes.get_column(node), xp, idx_engine, edges_df) + else: + # property-seeded: scan the seed row; CSR/node-index gathers below + # still engage (binding-column values = index key domain). + seed_nodes = nodes_df + for k, v in n0f.items(): + seed_nodes = seed_nodes.filter(pl.col(k) == v) + edges = _index_edge_rows(adj, seed_nodes.get_column(node), xp, idx_engine, edges_df) if edges is not None: for k, v in ef.items(): edges = edges.filter(pl.col(k) == v) diff --git a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py index 75c596eb0a..1b880590d3 100644 --- a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py @@ -962,3 +962,41 @@ def med(g): scan = med(graphistry.nodes(ndf, "id").edges(edf, "src", "dst")) indexed = med(graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql_index_all()) assert indexed * 1.5 < scan, f"indexed {indexed*1e3:.2f}ms not >=1.5x faster than scan {scan*1e3:.2f}ms" + + @pytest.mark.parametrize("engine", ["pandas", "polars"]) + def test_property_seeded_engages_adjacency_index(self, engine, monkeypatch): + """Decoupled-index pin (the SF1-harness/LDBC pattern): the graph binds a + synthetic key column while the seed filter hits the `id` PROPERTY. The + node-id index can't serve the seed row, but the CSR adjacency + node-id + gathers must still engage — binding-column values are the key domain + regardless of how the seed rows were found.""" + if engine == "polars": + pytest.importorskip("polars") + import graphistry.compute.chain_fast_paths as cfp + ndf, edf = self._frames() + # rebind: __key__ is the node/edge key domain; `id` becomes a plain property + ndf = ndf.rename(columns={"id": "prop_id"}).assign(__key__=lambda d: d.index.to_numpy()) + key_of = dict(zip(ndf["prop_id"], ndf["__key__"])) + edf = edf.dropna(subset=["src", "dst"]) + edf = edf[edf["src"].isin(key_of) & edf["dst"].isin(key_of)] + edf = edf.assign(src=edf["src"].map(key_of), dst=edf["dst"].map(key_of)) + ndf = ndf.rename(columns={"prop_id": "id"}) + if engine == "polars": + import polars as pl + mk = lambda: graphistry.nodes(pl.from_pandas(ndf), "__key__").edges(pl.from_pandas(edf), "src", "dst") # noqa: E731 + else: + mk = lambda: graphistry.nodes(ndf, "__key__").edges(edf, "src", "dst") # noqa: E731 + q = "MATCH (m {id: 33.0})-[:KNOWS]->(p) RETURN p" + plain = mk().gfql(q, engine=engine) + serves = {"e": 0} + oe = cfp._index_edge_rows + + def spy(*a, **k): + out = oe(*a, **k) + serves["e"] += out is not None + return out + monkeypatch.setattr(cfp, "_index_edge_rows", spy) + indexed = mk().gfql_index_all(engine=engine).gfql(q, engine=engine) + monkeypatch.setattr(cfp, "_index_edge_rows", oe) + assert serves["e"] > 0, "adjacency index did not serve the property-seeded lookup" + pd.testing.assert_frame_equal(self._canon(indexed), self._canon(plain))