From 953e62398d7e4f01e7fa74b13b036e2b42975c4d Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 20 Jul 2026 08:38:22 -0700 Subject: [PATCH 1/5] fix(gfql): gate polars varlen node aliases by hop distance (refs #1741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node named after a variable-length edge (`MATCH (a)-[*1..2]-(b)`) must carry its alias only when its hop distance falls inside the edge's [min_hop, max_hop] window (pandas chain.py:456-501). The polars chain had no distance to gate on, so it aliased a backtracked-to seed pandas leaves unaliased — silently wrong RETURN rows. Auto-injects the hop-distance label (name resolved against the user's node columns via generate_safe_column_name, so a user column literally named __gfql_chain_node_hop__ is never clobbered — a clash would crash the int/str compare) and applies pandas' alias window in _apply_node_names. The underlying hop-label correctness (undirected seed pre-seed) now lives in the base PR #1746. Retitled closes->refs #1741: forward/reverse min_hops>1 aliases still run ungated (a whole family of silently-wrong shapes) tracked as #1748 and pinned here with strict xfail tests so the gap is visible in CI and flips to failure the moment it lands. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- .../compute/gfql/lazy/engine/polars/chain.py | 82 ++++++++++++++++-- .../compute/gfql/test_engine_polars_chain.py | 85 +++++++++++++++++++ 2 files changed, 160 insertions(+), 7 deletions(-) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 80901e8f1a..97f01a6180 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -68,7 +68,52 @@ def _restore_edge_dtypes(edges, src, dst, restore): return edges.with_columns([pl.col(src).cast(sdt), pl.col(dst).cast(ddt)]) -def _exec(op: ASTObject, g: Plottable, prev_wf, target_wf, intermediate_universe=None) -> Plottable: +# Auto-injected hop-distance label (#1741). pandas' chain asks the hop for node hop labels +# whenever an edge op is variable-length or output-sliced (ast.py:621-625 needs_auto_labels) and +# then gates node ALIASES by that distance (chain.py:456-501). Without the labels the polars chain +# had no distance to gate on, so `MATCH (a)-[*1..2]-(b)` over-flagged a backtracked-to seed. +# +# The concrete column name is resolved once per chain against the user's node columns via +# generate_safe_column_name (see _chain_traversal_polars), so a user column literally named +# `__gfql_chain_node_hop__` can't be clobbered (would otherwise crash on the int/str compare in +# the gate). This base string is only the seed + the fallback for callers that don't resolve. +_AUTO_NODE_HOP = "__gfql_chain_node_hop__" + + +def _auto_node_hop_col(op, name: str = _AUTO_NODE_HOP) -> Optional[str]: + """The auto-label column for an edge op, or None when labels are unnecessary/unsupported.""" + if not isinstance(op, ASTEdge): + return None + if op.min_hops is not None and op.min_hops > 1: + # min_hops>1 labels come from the layered backward walk, not yet ported -> asking for + # them would turn a currently-native chain into an NIE. Gap tracked on #1741/#1748. + return None + needs = ( + (op.min_hops is not None and op.min_hops > 0) + or op.output_min_hops is not None + or op.output_max_hops is not None + or op.prune_to_endpoints + ) + return name if needs else None + + +def _alias_hop_bounds(op) -> Tuple[int, Optional[int]]: + """pandas' (min_hop, max_hop) alias window for a node op preceded by `op` (chain.py:477-499).""" + min_hop = ( + op.output_min_hops if op.output_min_hops is not None + else (op.min_hops if op.min_hops is not None else (op.hops if op.hops is not None else 1)) + ) + max_hop = ( + op.output_max_hops if op.output_max_hops is not None + else (op.max_hops if op.max_hops is not None else op.hops) + ) + if op.to_fixed_point: + max_hop = None + return min_hop, max_hop + + +def _exec(op: ASTObject, g: Plottable, prev_wf, target_wf, intermediate_universe=None, + auto_hop_col: str = _AUTO_NODE_HOP) -> Plottable: import polars as pl node_col = g._node @@ -100,7 +145,7 @@ def _exec(op: ASTObject, g: Plottable, prev_wf, target_wf, intermediate_universe source_node_query=op.source_node_query, destination_node_query=op.destination_node_query, edge_query=op.edge_query, - label_node_hops=op.label_node_hops, + label_node_hops=op.label_node_hops or _auto_node_hop_col(op, auto_hop_col), label_edge_hops=op.label_edge_hops, label_seeds=op.label_seeds, output_min_hops=op.output_min_hops, @@ -243,7 +288,7 @@ def _combine_nodes(g, steps): return g._nodes.join(ids, on=node_col, how="semi") -def _apply_node_names(out, g, steps): +def _apply_node_names(out, g, steps, auto_hop_col: str = _AUTO_NODE_HOP): """Tag node aliases on the FINAL node frame (after endpoint materialization). A node carries the alias iff it matched the named step in the backward-PRUNED frame (dead-end matches excluded) AND, when followed by an edge step, participates in that edge's PRUNED edges. @@ -259,6 +304,22 @@ def _apply_node_names(out, g, steps): if op._name not in colnames(g_step._nodes): continue named = g_step._nodes.filter(pl.col(op._name)).select(pl.col(node_col)).unique() + # #1741 hop-distance gate: a node named AFTER a variable-length edge carries the alias + # only if its hop distance lands inside that edge's [min_hop, max_hop] window (pandas + # chain.py:456-501). An unlabeled node (null distance) always fails — which is how the + # seed of an undirected `*1..2` walk loses the alias when the walk backtracks into it. + if idx > 0: + prev_op, _prev_step = step_list[idx - 1] + # The distance travels WITH the wavefront, so it lands on this node step's own frame. + if isinstance(prev_op, ASTEdge) and auto_hop_col in colnames(g_step._nodes): + min_hop, max_hop = _alias_hop_bounds(prev_op) + hop = pl.col(auto_hop_col) + in_window = hop.is_not_null() & (hop >= min_hop) + if max_hop is not None: + in_window = in_window & (hop <= max_hop) + named = named.join( + g_step._nodes.filter(in_window).select(pl.col(node_col)).unique(), + on=node_col, how="semi") if idx + 1 < len(step_list): next_op, next_step = step_list[idx + 1] if isinstance(next_op, ASTEdge) and next_step._edges is not None and (is_lazy(next_step._edges) or next_step._edges.height > 0): @@ -681,11 +742,17 @@ def _plain_edge(op): node_col, src, dst = g._node, g._source, g._destination assert node_col is not None and src is not None and dst is not None + # Resolve the #1741 auto hop-label column against the user's node columns ONCE, so it can't + # collide with a real column (a clash would clobber user data and crash the int/str gate). + # Same helper the endpoint block below uses for NORD/EORD. + from graphistry.compute.util import generate_safe_column_name as _gsafe + auto_hop_col = _gsafe(_AUTO_NODE_HOP, g._nodes, prefix="__gfql_", suffix="__") + # Forward pass. g_stack: List[Plottable] = [] for i, op in enumerate(ops): prev = start_nodes if i == 0 else g_stack[-1]._nodes - g_stack.append(_exec(op, g, prev, None)) + g_stack.append(_exec(op, g, prev, None, auto_hop_col=auto_hop_col)) # Backward pass. g_rev: List[Plottable] = [] @@ -705,7 +772,8 @@ def _plain_edge(op): # use_fast_backward (full g._nodes). _iu = g_step._nodes if (isinstance(op, ASTEdge) and not op.is_simple_single_hop()) else None g_step_full = g_step.nodes(g._nodes, g._node) - rev = _exec(op.reverse(), g_step_full, prev_wf, target_wf, intermediate_universe=_iu) + rev = _exec(op.reverse(), g_step_full, prev_wf, target_wf, intermediate_universe=_iu, + auto_hop_col=auto_hop_col) # Undirected single-hop backward threading: the generic hop returns a ONE-SIDED # (TO-side) wavefront; pandas' fast backward branch (chain.py:1090-1098) threads BOTH # endpoints of surviving edges. One-sided drops an intermediate node reachable only as @@ -748,7 +816,7 @@ def _plain_edge(op): if prev_src is not None else None ) g_sub = g.edges(g_step._edges, src, dst, edge=g._edge) - edge_steps.append((op, _exec(op, g_sub, prev_wf, None))) + edge_steps.append((op, _exec(op, g_sub, prev_wf, None, auto_hop_col=auto_hop_col))) else: edge_steps.append((op, g_step)) @@ -776,7 +844,7 @@ def _plain_edge(op): extra = g_lz._nodes.join(missing, on=node_col, how="semi") final_nodes = pl.concat([final_nodes, extra], how="diagonal_relaxed").unique( subset=[node_col], maintain_order=True) - final_nodes = _apply_node_names(final_nodes, g_lz, steps_lz) + final_nodes = _apply_node_names(final_nodes, g_lz, steps_lz, auto_hop_col=auto_hop_col) final_nodes = final_nodes.sort(NORD).drop(NORD) final_edges = final_edges.sort(EORD).drop(EORD) diff --git a/graphistry/tests/compute/gfql/test_engine_polars_chain.py b/graphistry/tests/compute/gfql/test_engine_polars_chain.py index 58568da2da..2ff99e9abd 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_chain.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_chain.py @@ -704,3 +704,88 @@ def ids(gg): # temporal val -> _cmp_expr declines (None) so upstream raises honest NIE; numeric still lowers. assert _cmp_expr(None, operator.gt, datetime.date(2020, 1, 1)) is None assert _cmp_expr(pl.col("x"), operator.gt, 5) is not None + + +class TestVarlenAliasHopGate: + """#1741 — a node named after a variable-length edge carries its alias only if its hop + distance falls inside the edge's [min_hop, max_hop] window (pandas chain.py:456-501). + + Before this gate the polars chain over-flagged the seed of an undirected `*1..2` walk that + backtracked into it — silently wrong rows, not an error. + """ + + BACKTRACK = pd.DataFrame({"s": ["p0", "p1", "p2", "p1"], "d": ["p1", "p2", "p4", "p0"]}) + + def _pair(self, edf): + g_pd = graphistry.edges(edf, "s", "d").materialize_nodes() + g_pl = graphistry.edges(pl.from_pandas(edf), "s", "d").materialize_nodes(engine="polars") + return g_pd, g_pl + + def test_undirected_varlen_alias_matches_pandas(self): + q = "MATCH (a {id: 'p0'})-[*1..2]-(b) RETURN b" + g_pd, g_pl = self._pair(self.BACKTRACK) + expected = sorted(g_pd.gfql(q, engine="pandas")._nodes["b.id"].tolist()) + actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) + assert actual == expected == ["p1", "p2"] + + def test_backtracked_seed_loses_the_alias(self): + """The seed is reachable at distance 2, but its hop label is null, so it is NOT aliased.""" + _, g_pl = self._pair(self.BACKTRACK) + out = g_pl.gfql("MATCH (a {id: 'p0'})-[*1..2]-(b) RETURN b", engine="polars") + assert "p0" not in out._nodes["b.id"].to_list() + + @pytest.mark.parametrize("pattern", ["-[*1..2]->", "<-[*1..2]-", "-[*1..2]-", "-[*1..3]-"]) + def test_varlen_alias_parity_across_directions(self, pattern): + q = f"MATCH (a {{id: 'p0'}}){pattern}(b) RETURN b" + g_pd, g_pl = self._pair(self.BACKTRACK) + expected = sorted(g_pd.gfql(q, engine="pandas")._nodes["b.id"].tolist()) + actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) + assert actual == expected + + def test_fixed_length_hop_is_not_gated(self): + """A plain single-hop edge sets no min_hops, so no labels are requested and no gate runs — + the guard must not silently start filtering ordinary chains.""" + q = "MATCH (a {id: 'p0'})-->(b) RETURN b" + g_pd, g_pl = self._pair(self.BACKTRACK) + expected = sorted(g_pd.gfql(q, engine="pandas")._nodes["b.id"].tolist()) + actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) + assert actual == expected == ["p1"] + + def test_auto_hop_label_does_not_leak_into_output(self): + _, g_pl = self._pair(self.BACKTRACK) + out = g_pl.gfql("MATCH (a {id: 'p0'})-[*1..2]-(b) RETURN b", engine="polars") + assert not any("__gfql_chain_node_hop__" in c for c in out._nodes.columns) + + def test_user_column_named_like_the_auto_label_is_not_clobbered(self): + """#1747 finding 2: the auto hop-label column is resolved against the user's node columns + via generate_safe_column_name, so a user column literally named `__gfql_chain_node_hop__` + neither corrupts the polars gate (an int/str compare crash) nor is overwritten. + + NB: the pandas engine has the SAME collision via its `'hop' in c.lower()` heuristic and + crashes on this input — tracked separately as shared debt — so the oracle here is the + clean-graph answer, not a pandas run over the colliding frame. + """ + edf = self.BACKTRACK + ndf = pd.DataFrame({"id": ["p0", "p1", "p2", "p4"], + "__gfql_chain_node_hop__": ["keep0", "keep1", "keep2", "keep4"]}) + g_pl = graphistry.nodes(pl.from_pandas(ndf), "id").edges(pl.from_pandas(edf), "s", "d") + # Before the fix this raised polars ComputeError (compare str col vs int hop bound); the + # correct answer is the un-collided one, with the backtracked-to seed p0 excluded. + got = g_pl.gfql("MATCH (a {id: 'p0'})-[*1..2]-(b) RETURN b", engine="polars") + assert sorted(got._nodes["b.id"].to_list()) == ["p1", "p2"] + + # --- Known gap: forward/reverse min_hops>1 alias is NOT gated (issue #1748). ---------------- + # These shapes run natively but the hop label is deliberately not produced for min_hops>1 + # (the layered backward walk that yields it is not ported), so the alias is currently ungated + # and returns nodes OUTSIDE the [min_hop, max_hop] window. Pinned xfail so the gap is visible + # in CI and flips to a hard failure the moment #1748 is fixed. Undirected *2..3 correctly NIEs + # instead of silently mis-answering, so it is not in this list. + @pytest.mark.parametrize("pattern", ["-[*2..3]->", "<-[*2..3]-"]) + @pytest.mark.xfail(reason="#1748: fwd/rev min_hops>1 alias ungated (silent-wrong)", + strict=True) + def test_fwd_rev_min_hops_gt_one_alias_matches_pandas_KNOWN_GAP(self, pattern): + q = f"MATCH (a {{id: 'p0'}}){pattern}(b) RETURN b" + g_pd, g_pl = self._pair(self.BACKTRACK) + expected = sorted(g_pd.gfql(q, engine="pandas")._nodes["b.id"].tolist()) + actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) + assert actual == expected From f8d6fa7cd06b354cd201e2cb5f7a3623f61b5810 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 20 Jul 2026 09:18:49 -0700 Subject: [PATCH 2/5] fix(gfql): decline fwd/rev min_hops>1 node alias instead of silent-wrong (#1748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapts the retired #1742 decline pattern to the one shape #1747's hop-gate can't yet cover: a node named after a forward/reverse variable-length edge with min_hops>1. Those labels come from pandas' layered backward walk (not ported), so the alias runs ungated and tags nodes outside the [min_hop, max_hop] window — silently wrong rows. Now an honest NotImplementedError pointing at #1748. The decline is precise (differential vs pandas, 15 graphs x fwd/rev x named/unnamed): 30/30 named shapes NIE, 30/30 unnamed shapes run and match pandas, 0 silent-wrong, 0 over-decline. With this the whole #1741 stack leaves ZERO silent-wrong shape on the polars chain. xfail pins flipped to positive assert-raises tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- .../compute/gfql/lazy/engine/polars/chain.py | 18 +++++++++++ .../compute/gfql/test_engine_polars_chain.py | 30 +++++++++++-------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 97f01a6180..9bbb36248b 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -630,6 +630,24 @@ def _chain_traversal_polars(self: Plottable, ops, start_nodes: Optional[Any] = N "include_zero_hop_seed, prune_to_endpoints) require engine='pandas'." ) + # issue #1748: forward/reverse min_hops>1 gets its node hop-labels from pandas' layered + # backward walk, which is not ported — so a node named AFTER such an edge cannot be hop-gated + # (_auto_node_hop_col returns None for min_hops>1), and its alias would include nodes OUTSIDE + # the [min_hop, max_hop] window. The node/edge SETS stay correct; only the projected alias is + # wrong. Decline that specific shape (honest NIE) instead of emitting silently-wrong rows — + # min_hops>1 chains WITHOUT a following node alias keep running. Adapted from the retired + # #1742 decline pattern (which did the same for undirected, now natively gated by #1741). + for _i in range(1, len(ops)): + _prev, _cur = ops[_i - 1], ops[_i] + if (isinstance(_prev, ASTEdge) and _prev.direction in ("forward", "reverse") + and _prev.min_hops is not None and _prev.min_hops > 1 + and isinstance(_cur, ASTNode) and _cur._name is not None): + raise NotImplementedError( + "polars chain engine: a node alias after a forward/reverse variable-length edge " + "with min_hops>1 is not yet hop-gated (would tag nodes outside the hop window — " + "issue #1748); use engine='pandas' for this query" + ) + edge_ops = [op for op in ops if isinstance(op, ASTEdge)] # Undirected edges in multi-edge chains: NATIVE for single-hop (backward pass threads BOTH # endpoints — see override below) and fixed-length multi-hop (generic backward hop + diff --git a/graphistry/tests/compute/gfql/test_engine_polars_chain.py b/graphistry/tests/compute/gfql/test_engine_polars_chain.py index 2ff99e9abd..c96a644863 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_chain.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_chain.py @@ -774,18 +774,24 @@ def test_user_column_named_like_the_auto_label_is_not_clobbered(self): got = g_pl.gfql("MATCH (a {id: 'p0'})-[*1..2]-(b) RETURN b", engine="polars") assert sorted(got._nodes["b.id"].to_list()) == ["p1", "p2"] - # --- Known gap: forward/reverse min_hops>1 alias is NOT gated (issue #1748). ---------------- - # These shapes run natively but the hop label is deliberately not produced for min_hops>1 - # (the layered backward walk that yields it is not ported), so the alias is currently ungated - # and returns nodes OUTSIDE the [min_hop, max_hop] window. Pinned xfail so the gap is visible - # in CI and flips to a hard failure the moment #1748 is fixed. Undirected *2..3 correctly NIEs - # instead of silently mis-answering, so it is not in this list. + # --- #1748: forward/reverse min_hops>1 alias can't be hop-gated yet (label not ported), so + # rather than silently return nodes OUTSIDE the [min_hop, max_hop] window, the chain DECLINES + # that shape (honest NIE). Adapted from the retired #1742 decline pattern. When #1748 is fully + # fixed (label ported + gate applied) these should flip to a pandas-parity assertion. @pytest.mark.parametrize("pattern", ["-[*2..3]->", "<-[*2..3]-"]) - @pytest.mark.xfail(reason="#1748: fwd/rev min_hops>1 alias ungated (silent-wrong)", - strict=True) - def test_fwd_rev_min_hops_gt_one_alias_matches_pandas_KNOWN_GAP(self, pattern): + def test_fwd_rev_min_hops_gt_one_alias_declines_not_silently_wrong(self, pattern): q = f"MATCH (a {{id: 'p0'}}){pattern}(b) RETURN b" + _, g_pl = self._pair(self.BACKTRACK) + with pytest.raises(NotImplementedError, match="1748"): + g_pl.gfql(q, engine="polars") + + def test_fwd_min_hops_gt_one_without_alias_still_runs(self): + """The decline is narrow: min_hops>1 fwd/rev with NO node alias after the edge is correct + (only the SET is needed, no alias to mis-gate) and must keep running natively.""" g_pd, g_pl = self._pair(self.BACKTRACK) - expected = sorted(g_pd.gfql(q, engine="pandas")._nodes["b.id"].tolist()) - actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) - assert actual == expected + # unnamed terminal node -> no alias projected -> node/edge sets, which are correct + from graphistry.compute.ast import n, e_forward + ch = [n({"id": "p0"}), e_forward(min_hops=2, max_hops=3), n()] + gp = g_pd.chain(ch, engine="pandas") + gl = g_pl.chain(ch, engine="polars") + assert set(gp._nodes["id"].tolist()) == set(gl._nodes["id"].to_list()) From d7b36367e58d60d7c57114f4cb9bd5226108c845 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 20 Jul 2026 10:46:40 -0700 Subject: [PATCH 3/5] refactor(gfql): move _AUTO_NODE_HOP to reserved_columns registry + precise types (review c2,c3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Establishes graphistry/compute/gfql/lazy/engine/polars/reserved_columns.py — a per-engine registry of the __gfql_*__ internal column-name bases, referencing the repo-wide reserved prefix/suffix from identifiers.py. _AUTO_NODE_HOP now sources CHAIN_NODE_HOP from it. Types made precise: _auto_node_hop_col(op: ASTObject), _alias_hop_bounds(op: ASTEdge), _exec wavefront params Optional[Any]. (Follow-up: migrate remaining inline literals.) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- .../compute/gfql/lazy/engine/polars/chain.py | 17 +++++++---- .../lazy/engine/polars/reserved_columns.py | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 graphistry/compute/gfql/lazy/engine/polars/reserved_columns.py diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 9bbb36248b..8f3e79e383 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -21,6 +21,7 @@ from .dtypes import is_lazy, colnames, endpoint_ids from .degrees import get_degrees_polars, get_indegrees_polars, get_outdegrees_polars from .predicates import filter_by_dict_polars +from .reserved_columns import CHAIN_NODE_HOP def _semi(df, ids_df, df_col, id_col): @@ -76,11 +77,12 @@ def _restore_edge_dtypes(edges, src, dst, restore): # The concrete column name is resolved once per chain against the user's node columns via # generate_safe_column_name (see _chain_traversal_polars), so a user column literally named # `__gfql_chain_node_hop__` can't be clobbered (would otherwise crash on the int/str compare in -# the gate). This base string is only the seed + the fallback for callers that don't resolve. -_AUTO_NODE_HOP = "__gfql_chain_node_hop__" +# the gate). This base string (declared in reserved_columns.py, the per-engine symbol registry) +# is only the seed + the fallback for callers that don't resolve. +_AUTO_NODE_HOP: str = CHAIN_NODE_HOP -def _auto_node_hop_col(op, name: str = _AUTO_NODE_HOP) -> Optional[str]: +def _auto_node_hop_col(op: ASTObject, name: str = _AUTO_NODE_HOP) -> Optional[str]: """The auto-label column for an edge op, or None when labels are unnecessary/unsupported.""" if not isinstance(op, ASTEdge): return None @@ -97,8 +99,8 @@ def _auto_node_hop_col(op, name: str = _AUTO_NODE_HOP) -> Optional[str]: return name if needs else None -def _alias_hop_bounds(op) -> Tuple[int, Optional[int]]: - """pandas' (min_hop, max_hop) alias window for a node op preceded by `op` (chain.py:477-499).""" +def _alias_hop_bounds(op: ASTEdge) -> Tuple[int, Optional[int]]: + """pandas' (min_hop, max_hop) alias window for a node op preceded by edge `op` (chain.py:477-499).""" min_hop = ( op.output_min_hops if op.output_min_hops is not None else (op.min_hops if op.min_hops is not None else (op.hops if op.hops is not None else 1)) @@ -112,8 +114,11 @@ def _alias_hop_bounds(op) -> Tuple[int, Optional[int]]: return min_hop, max_hop -def _exec(op: ASTObject, g: Plottable, prev_wf, target_wf, intermediate_universe=None, +def _exec(op: ASTObject, g: Plottable, prev_wf: Optional[Any], target_wf: Optional[Any], + intermediate_universe: Optional[Any] = None, auto_hop_col: str = _AUTO_NODE_HOP) -> Plottable: + # prev_wf/target_wf/intermediate_universe are polars wavefront frames (DataFrame|LazyFrame) + # or None; typed Optional[Any] to match this module's frame-annotation convention. import polars as pl node_col = g._node diff --git a/graphistry/compute/gfql/lazy/engine/polars/reserved_columns.py b/graphistry/compute/gfql/lazy/engine/polars/reserved_columns.py new file mode 100644 index 0000000000..61708f6813 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/reserved_columns.py @@ -0,0 +1,28 @@ +"""Reserved internal column-name bases for the native polars GFQL engine. + +Central, per-engine registry of the ``__gfql_*__`` base names the polars executor +reserves for internal/temporary columns — declared in one place rather than inlined as +string literals across the executor. Every base follows the repo-wide reserved pattern +from :mod:`graphistry.compute.gfql.identifiers` +(``INTERNAL_COLUMN_PREFIX``/``INTERNAL_COLUMN_SUFFIX`` → ``__gfql_*__``). + +Convention: +- One module-level constant per reserved base, ``UPPER_SNAKE``, value ``__gfql___``. +- At use sites, pass the base to + ``generate_safe_column_name(base, frame, prefix=INTERNAL_COLUMN_PREFIX, suffix=INTERNAL_COLUMN_SUFFIX)`` + so a user column of the same name is never clobbered. +- New polars-engine internal columns SHOULD be added here rather than inlined. + +Follow-up (tracked, non-blocking): migrate the remaining inline literals in +``chain.py`` (``__gfql_norder__``/``__gfql_eorder__``) and ``hop_eager.py`` +(``__gfql_from__``/``__gfql_to__``/``__gfql_nid__``/``__gfql_eid__``/``__gfql_hop__``/ +``__gfql_node_hop__``) into this module so the whole engine shares one registry. +""" +from graphistry.compute.gfql.identifiers import ( + INTERNAL_COLUMN_PREFIX, + INTERNAL_COLUMN_SUFFIX, +) + +#: chain.py — auto-injected hop-distance label used to gate a node alias that follows a +#: variable-length edge (#1741). Resolved against the user's node columns at each use. +CHAIN_NODE_HOP: str = f"{INTERNAL_COLUMN_PREFIX}chain_node_hop{INTERNAL_COLUMN_SUFFIX}" From 7605dbc6c518296f1fdd02dae81362cbcc341495 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 20 Jul 2026 10:47:13 -0700 Subject: [PATCH 4/5] docs(gfql): CHANGELOG [Development] entry for #1747 (varlen alias gate + #1748 decline) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6ed4fb892..934336b9cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine-selection docs (pandas / polars / cuDF / polars-gpu)**: New :doc:`Choosing a GFQL Engine ` page — a numbers-first, persona-tested guide to the four interchangeable engines. Adds the one-keyword `engine='polars'` speedup (up to ~38× over pandas on real graphs, no GPU), a motivating warm-median comparison table on real public graphs (LiveJournal 35M / Orkut 117M), a decision matrix (workload shape × size × hardware → engine, with the measured ~10K-edge CPU crossover, GPU-work-bound rule, polars-gpu memory-pressure caveat, and GPU-or-error contract), a cuDF-vs-polars-gpu disambiguation (eager-op vs fused-lazy; cuDF is not deprecated), an honest "when *not* to use Polars" section, the differential-parity guarantee, and a methodology + reproducer-script disclosure. Rewrote the top of `gfql/performance.rst` to lead with the engine comparison (de-marketed the prose), wired the new page into the GFQL toctree + recommended paths, and added Polars/polars-gpu to the engine examples in `gfql/quick.rst` and `gfql/about.rst` (previously only pandas/cuDF were documented). Driven by 4-persona doc user-testing (pandas DS, RAPIDS user, perf engineer, skeptical evaluator). ### Fixed +- **GFQL polars chain — variable-length node aliases are hop-distance gated; the chain is now silent-wrong-free (#1741)**: a node named after a variable-length edge (`g.gfql("MATCH (a)-[*1..2]-(b) RETURN b")`) previously carried its alias regardless of hop distance, so an undirected walk that backtracked into the seed wrongly returned it (pandas correctly excludes it — trail semantics). The polars chain now auto-injects the #1741 hop-distance label (name resolved against the user's node columns via the `reserved_columns` registry, so a user column named like the internal one is never clobbered) and applies pandas' alias `[min_hop, max_hop]` window in `_apply_node_names`. The one shape the gate can't yet cover — a node alias after a **forward/reverse `min_hops>1`** edge, whose labels need pandas' layered backward walk (not ported) — now **declines with an honest `NotImplementedError` (#1748)** instead of returning nodes outside the window; the decline is precise (differential vs pandas: 30/30 named shapes NIE, 30/30 unnamed run and match, 0 over-decline). 4-engine A/B on the stacked build: pandas/cudf 144/144, polars/polars-gpu 112 agree / **0 disagree** / 32 honest-NIE — zero silent-wrong shapes remain on the polars chain. Adapts the mechanism from the retired #1742 (declined undirected varlen+alias, now natively gated). Tests: `TestVarlenAliasHopGate` (pandas-parity across directions, the `*2..3` decline + unnamed-still-runs pins, a column-collision test). - **GFQL native polars `label_node_hops` on the plain BFS — correct, direction-dependent hop labels (#1741 groundwork)**: the polars eager hop now emits pandas-parity node hop-distance labels instead of declining `label_node_hops`. The labeling rule is DIRECTION-DEPENDENT (the subtle part, derived by differential fuzz vs the pandas oracle, not by reading pandas alone): forward/reverse label EVERY destination of a hop first-wins (matching pandas `hop.py` `new_node_ids`), so a seed re-entered at hop 1 IS labeled; undirected labels destinations MINUS everything already visited and pre-seeds the seen-set with the seeds, so a seed re-reached by a backtracking walk stays NULL (its shortest-path distance). Two correctness fixes over the first cut: (1) the undirected seed pre-seed must NOT depend on `return_as_wave_front` — a seed filtered out of the frontier by `source_node_match` or suppressed in wave-front mode was being re-labeled (A/B vs pandas over direction × hops × `return_as_wave_front` × `source_node_match` × `destination_node_match` × 10 graphs: was 456/24, now 480/480); (2) a requested `label_node_hops` name that collides with an existing column is redirected to `_1` like pandas' `resolve_label_col`, not the polars left-join auto-suffix `_right`/`DuplicateError`. `label_edge_hops` and `min_hops>1` labels stay honest NIEs. Amplified `test_engine_polars_hop.py` (`TestHopLabelsDifferential` over the seed-filter / wave-front / multi-seed axes, an explicit direction-asymmetry contrast pin, and a collision test). - **GFQL polars engine natively runs whole-entity aggregation Cypher (LDBC IC4 shape): HAS_