From 31dc8b4a90e2db4d5c0e4fc46e295ced3b797a68 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 4 Jul 2026 22:58:34 -0700 Subject: [PATCH 01/10] =?UTF-8?q?feat(gfql):=20search=5Fany=20cross-column?= =?UTF-8?q?=20search=20op=20=E2=80=94=20native=204=20engines=20(viz-filter?= =?UTF-8?q?=20L2-a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel (gfql/search_any.py): OR-across-columns, case-insensitive substring default, regex opt-in, dtype gate AS SEMANTICS (string cols always; int cols iff numeric- literal term per the inspector /^[0-9.-]+$/ gate; floats/dates/bools via explicit columns=); per-column mask = the parity-hardened Contains predicate (all cuDF workarounds + honest declines inherited). Row op w/ alias-prefix scoping + safelist + builder; polars native (schema dtype gate; lower()-fold literal default — the common viz call never touches regex; rust-guard on regex path; any_horizontal); ComputeMixin twins search_nodes/search_edges (pandas/cuDF; polars frames honest-NIE v1). Matrix: 5-case oracle pins + 4-engine parity-or-NIE + twins test; ledger calls-axis waiver (count_table pattern) + rowops exercised. dgx: 1996 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/ComputeMixin.py | 47 ++++++++++++ graphistry/compute/ast.py | 23 ++++++ graphistry/compute/gfql/call/validation.py | 15 ++++ .../compute/gfql/lazy/engine/polars/chain.py | 13 +++- .../compute/gfql/lazy/engine/polars/search.py | 72 ++++++++++++++++++ graphistry/compute/gfql/row/pipeline.py | 48 ++++++++++++ graphistry/compute/gfql/search_any.py | 73 +++++++++++++++++++ .../compute/gfql/test_conformance_ledger.py | 1 + .../test_engine_polars_conformance_matrix.py | 58 ++++++++++++++- 9 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 graphistry/compute/gfql/lazy/engine/polars/search.py create mode 100644 graphistry/compute/gfql/search_any.py diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 624a0ffc93..6ac7365dc4 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -487,6 +487,53 @@ def get_topological_levels( out_df = safe_merge(g2_base._nodes, levels_df, on=g2_base._node, how='left') return self.nodes(out_df) + def search_nodes(self, term, columns=None, case_sensitive=False, regex=False): + """Keep nodes where ANY column matches ``term`` (viz-filter L2 inspector + semantics: OR across columns; case-insensitive substring default; regex + opt-in; string columns always, integer columns iff the term is a numeric + literal — floats/dates via explicit ``columns=``). pandas/cuDF native; + polars frames raise NotImplementedError (use the cypher ``search_any`` op). + """ + from graphistry.compute.gfql.search_any import search_any_mask + from graphistry.compute.exceptions import ErrorCode, GFQLValidationError + df = self._nodes + if df is None: + return self + if "polars" in type(df).__module__: + raise NotImplementedError( + "search_nodes is not yet native on polars frames; use the cypher " + "search_any op or engine='pandas'") + mask = search_any_mask( + df, term, case_sensitive=case_sensitive, regex=regex, columns=columns) + if mask is None: + raise GFQLValidationError( + ErrorCode.E108, + "search_nodes columns= includes a column absent from the nodes table", + field="columns", value=columns, + suggestion="List only columns present on the nodes table.") + return self.nodes(df[mask]) + + def search_edges(self, term, columns=None, case_sensitive=False, regex=False): + """Keep edges where ANY column matches ``term`` — see :meth:`search_nodes`.""" + from graphistry.compute.gfql.search_any import search_any_mask + from graphistry.compute.exceptions import ErrorCode, GFQLValidationError + df = self._edges + if df is None: + return self + if "polars" in type(df).__module__: + raise NotImplementedError( + "search_edges is not yet native on polars frames; use the cypher " + "search_any op or engine='pandas'") + mask = search_any_mask( + df, term, case_sensitive=case_sensitive, regex=regex, columns=columns) + if mask is None: + raise GFQLValidationError( + ErrorCode.E108, + "search_edges columns= includes a column absent from the edges table", + field="columns", value=columns, + suggestion="List only columns present on the edges table.") + return self.edges(df[mask]) + def prune_self_edges(self): return self.edges(self._edges[ self._edges[self._source] != self._edges[self._destination] ]) diff --git a/graphistry/compute/ast.py b/graphistry/compute/ast.py index df858e4b3b..1ca8ef3ae6 100644 --- a/graphistry/compute/ast.py +++ b/graphistry/compute/ast.py @@ -1680,6 +1680,29 @@ def anti_semi_apply( return ASTCall("anti_semi_apply", params) +def search_any( + *, + alias: str, + term: str, + out_col: str, + case_sensitive: bool = False, + regex: bool = False, + columns: Optional[Sequence[str]] = None, +) -> ASTCall: + """Annotate active rows with a cross-column search marker (viz-filter L2): + ``out_col`` True where ANY of ``alias``'s columns matches ``term`` — OR across + columns, case-insensitive substring default, regex opt-in, dtype-gated (string + columns always; integer columns iff numeric-literal term).""" + params: Dict[str, Any] = {"alias": alias, "term": term, "out_col": out_col} + if case_sensitive: + params["case_sensitive"] = True + if regex: + params["regex"] = True + if columns is not None: + params["columns"] = list(columns) + return ASTCall("search_any", params) + + def semi_apply_mark( *, binding_ops: List[Dict[str, Any]], diff --git a/graphistry/compute/gfql/call/validation.py b/graphistry/compute/gfql/call/validation.py index 93d571c922..095a9d74da 100644 --- a/graphistry/compute/gfql/call/validation.py +++ b/graphistry/compute/gfql/call/validation.py @@ -406,6 +406,21 @@ def _semi_apply_mark_added_node_cols(params: Dict[str, object]) -> Set[str]: description='Filter active rows by anti-semi joining against correlated binding rows', ), + 'search_any': _safelist_entry( + {'alias', 'term', 'out_col', 'case_sensitive', 'regex', 'columns'}, + required_params={'alias', 'term', 'out_col'}, + param_validators={ + 'alias': is_non_empty_string, + 'term': is_string, + 'out_col': is_non_empty_string, + 'case_sensitive': is_bool, + 'regex': is_bool, + 'columns': is_non_empty_list_of_strings, + }, + description='Annotate active rows with a cross-column search marker (OR across columns)', + schema_effects=_schema_effects(adds_node_cols=_semi_apply_mark_added_node_cols), + ), + 'semi_apply_mark': _safelist_entry( {'binding_ops', 'join_aliases', 'out_col', 'neq'}, required_params={'binding_ops', 'join_aliases', 'out_col'}, diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index ed38b68c39..01d81077a2 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -353,10 +353,8 @@ def _try_native_row_op(g_cur, op): from graphistry.Engine import Engine from .row_pipeline import ( select_polars, with_columns_polars, order_by_polars, group_by_polars, - unwind_polars, where_rows_polars, - ) - from .pattern_apply import ( - rows_binding_ops_polars, semi_apply_mark_polars, anti_semi_apply_polars, + unwind_polars, where_rows_polars, rows_binding_ops_polars, search_any_polars, + semi_apply_mark_polars, anti_semi_apply_polars, ) fn = getattr(op, "function", None) @@ -379,6 +377,13 @@ def _try_native_row_op(g_cur, op): g_cur, op.params["binding_ops"], op.params["join_aliases"], op.params["out_col"], neq=op.params.get("neq"), ) + if fn == "search_any": + return search_any_polars( + g_cur, op.params["alias"], op.params["term"], op.params["out_col"], + case_sensitive=op.params.get("case_sensitive", False), + regex=op.params.get("regex", False), + columns=op.params.get("columns"), + ) if fn == "anti_semi_apply": return anti_semi_apply_polars( g_cur, op.params["binding_ops"], op.params["join_aliases"], diff --git a/graphistry/compute/gfql/lazy/engine/polars/search.py b/graphistry/compute/gfql/lazy/engine/polars/search.py new file mode 100644 index 0000000000..8f720eac87 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/search.py @@ -0,0 +1,72 @@ +"""Native polars lowering for the ``search_any`` cross-column search row op +(viz-filter L2; semantics in gfql/search_any.py). Module-per-family, split from +row_pipeline.py (the expression/projection lowering core) — degrees.py precedent.""" +from __future__ import annotations + +from typing import Optional, Sequence + +from graphistry.Plottable import Plottable + +from .row_pipeline import _active_table, _rewrap + + +def search_any_polars( + g: Plottable, + alias: str, + term: str, + out_col: str, + case_sensitive: bool = False, + regex: bool = False, + columns: Optional[Sequence[str]] = None, +) -> Optional[Plottable]: + """Native polars ``search_any`` (viz-filter L2): OR-across-columns marker, same + dtype gate as the pandas kernel (string cols always; int cols iff numeric-literal + term; float/date/bool auto-gated out). Regex path applies the same Rust-regex + decline gate as Contains; literal default folds via lowercase (never regex). + None declines (honest NIE).""" + import polars as pl + from graphistry.compute.gfql.search_any import is_numeric_term + from .predicates import _regex_rust_incompatible + left = _active_table(g) + if left is None: + return None + prefix = f"{alias}." + prefixed = [c for c in left.columns if c.startswith(prefix)] + if prefixed: + pool = {c[len(prefix):]: c for c in prefixed} + else: + pool = {c: c for c in left.columns + if not c.startswith("__gfql_") and c != alias} + schema = dict(left.schema) + if columns is not None: + if any(c not in pool for c in columns): + return None # missing explicit column: decline -> pandas raises the clear error + chosen = [pool[c] for c in columns] + else: + numeric_ok = is_numeric_term(term) + chosen = [] + for real in pool.values(): + dt = schema[real] + if dt == pl.String: + chosen.append(real) + elif numeric_ok and dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64, + pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64): + chosen.append(real) + if len(left) == 0 or not chosen: + marked = left.with_columns( + pl.lit(False).alias(out_col) if len(left) else pl.lit(None).cast(pl.Boolean).alias(out_col)) + return _rewrap(g, marked) + if regex and _regex_rust_incompatible(term): + return None + exprs = [] + for real in chosen: + base = pl.col(real) if schema[real] == pl.String else pl.col(real).cast(pl.String) + if regex: + pat = term if case_sensitive else f"(?i){term}" + exprs.append(base.str.contains(pat, literal=False)) + elif case_sensitive: + exprs.append(base.str.contains(term, literal=True)) + else: + exprs.append(base.str.to_lowercase().str.contains(term.lower(), literal=True)) + marked = left.with_columns(pl.any_horizontal(exprs).fill_null(False).alias(out_col)) + return _rewrap(g, marked) diff --git a/graphistry/compute/gfql/row/pipeline.py b/graphistry/compute/gfql/row/pipeline.py index 9f80da31b5..294d068744 100644 --- a/graphistry/compute/gfql/row/pipeline.py +++ b/graphistry/compute/gfql/row/pipeline.py @@ -168,6 +168,7 @@ def _gfql_cudf_list_sort_series_requires_host_bridge(series: Any) -> bool: "rows", "semi_apply_mark", "anti_semi_apply", + "search_any", "join_apply", "where_rows", "select", @@ -4171,6 +4172,52 @@ def _gfql_apply_neq_filter(right_df: Any, neq: Optional[List[str]], node_id_col: cols.append(cand) return right_df.loc[right_df[cols[0]] != right_df[cols[1]]] + def search_any( + self, + alias: str, + term: str, + out_col: str, + case_sensitive: bool = False, + regex: bool = False, + columns: Optional[List[str]] = None, + ) -> "Plottable": + """Cross-column search marker (viz-filter L2, panel-algebra D2): ``out_col`` is + True where ANY of the alias's columns matches ``term`` — OR across columns, + case-insensitive substring by default, regex opt-in, dtype-gated (string cols + always; int cols iff numeric-literal term; kernel in gfql/search_any.py).""" + from graphistry.compute.gfql.search_any import search_any_mask + left_df = self._gfql_get_active_table() + if left_df is None: + empty = self._gfql_empty_frame() + empty[out_col] = pd.Series(dtype="bool") + return self._gfql_row_table(empty) + if len(left_df) == 0: + out_df = left_df.copy() + out_df[out_col] = pd.Series(dtype="bool") + return self._gfql_row_table(out_df) + prefix = f"{alias}." + prefixed = [c for c in left_df.columns if c.startswith(prefix)] + if prefixed: + sub = left_df[prefixed].rename(columns={c: c[len(prefix):] for c in prefixed}) + else: + # single-entity row table: bare columns (internals + the alias marker excluded) + sub = left_df[[c for c in left_df.columns + if not c.startswith("__gfql_") and c != alias]] + mask = search_any_mask( + sub, term, case_sensitive=case_sensitive, regex=regex, columns=columns) + if mask is None: + raise GFQLValidationError( + ErrorCode.E108, + "searchAny columns= includes a column absent from the searched table", + field="columns", + value=columns, + suggestion="List only columns present on the searched entity.", + language="cypher", + ) + out_df = left_df.copy() + out_df[out_col] = mask + return self._gfql_row_table(out_df) + def anti_semi_apply( self, binding_ops: List[Dict[str, Any]], @@ -4814,6 +4861,7 @@ def bind(self) -> "Plottable": _ROW_PIPELINE_DISPATCH: Dict[str, Callable[..., "Plottable"]] = { "rows": RowPipelineMixin.rows, + "search_any": RowPipelineMixin.search_any, "semi_apply_mark": RowPipelineMixin.semi_apply_mark, "anti_semi_apply": RowPipelineMixin.anti_semi_apply, "join_apply": RowPipelineMixin.join_apply, diff --git a/graphistry/compute/gfql/search_any.py b/graphistry/compute/gfql/search_any.py new file mode 100644 index 0000000000..969447063d --- /dev/null +++ b/graphistry/compute/gfql/search_any.py @@ -0,0 +1,73 @@ +"""Cross-column search kernel (viz-filter L2, panel-algebra D2): OR-across-columns +substring/regex match, dtype-gated AS SEMANTICS — string columns always; integer +columns iff the term is a numeric literal (inspector gate); float/date/bool are +auto-gated OUT (float stringification is engine-divergent — reach them via explicit +``columns=``). Per-column matching delegates to the parity-hardened ``Contains`` +predicate, so every pandas/cuDF quirk and honest decline gate carries over.""" +import re +from typing import Any, List, Optional + +# inspector's numeric-term gate (streamgl-viz sortAndFilterRowsByQuery.js) +_NUMERIC_TERM_RE = re.compile(r"^[0-9.\-]+$") + + +def is_numeric_term(term: str) -> bool: + return bool(_NUMERIC_TERM_RE.match(term)) + + +def _is_searchable_string_dtype(dtype: Any) -> bool: + import pandas.api.types as pat # cuDF mirrors the pandas dtype API + return bool(pat.is_string_dtype(dtype)) or bool(pat.is_object_dtype(dtype)) + + +def _is_int_dtype(dtype: Any) -> bool: + import pandas.api.types as pat + return bool(pat.is_integer_dtype(dtype)) + + +def search_candidate_columns( + df: Any, term: str, columns: Optional[List[str]] +) -> Optional[List[str]]: + """Columns to search: the explicit list (None if any is missing — caller declines + loudly) or the auto dtype gate.""" + if columns is not None: + return list(columns) if all(c in df.columns for c in columns) else None + numeric_ok = is_numeric_term(term) + out: List[str] = [] + for c in df.columns: + dt = df[c].dtype + if _is_searchable_string_dtype(dt): + out.append(c) + elif numeric_ok and _is_int_dtype(dt): + out.append(c) + return out + + +def search_any_mask( + df: Any, + term: str, + *, + case_sensitive: bool = False, + regex: bool = False, + columns: Optional[List[str]] = None, +) -> Optional[Any]: + """Boolean row mask over ``df`` (pandas or cuDF), or None to decline (an explicit + column is missing). Null cells never match; no candidate columns -> all-False.""" + from graphistry.compute.predicates.str import Contains + cols = search_candidate_columns(df, term, columns) + if cols is None: + return None + if not cols or len(df) == 0: + if len(df.columns) == 0: + return None + return df[df.columns[0]].isna() & False # engine-safe all-False + pred = Contains(term, case=case_sensitive, regex=regex, na=False) + mask: Optional[Any] = None + for c in cols: + s = df[c] + if not _is_searchable_string_dtype(s.dtype): + s = s.astype(str) # canonical toString for int / explicit columns + m = pred(s) + mask = m if mask is None else (mask | m) + assert mask is not None # cols is non-empty here + return mask.fillna(False) diff --git a/graphistry/tests/compute/gfql/test_conformance_ledger.py b/graphistry/tests/compute/gfql/test_conformance_ledger.py index 6e0706f176..22aae6fca0 100644 --- a/graphistry/tests/compute/gfql/test_conformance_ledger.py +++ b/graphistry/tests/compute/gfql/test_conformance_ledger.py @@ -137,6 +137,7 @@ def _exercised_function_names() -> set: "order_by": "row sort; native on polars chain, NIE via call()/DAG executor; not asserted via call(). TODO.", "unwind": "list explode; native on polars chain, NIE via call()/DAG executor; not asserted via call(). TODO.", "group_by": "grouped aggregation; native on polars chain, NIE via call()/DAG executor; not asserted via call(). TODO.", + "search_any": "cross-column search marker (viz-filter L2); exercised as a labeled subject via test_search_any_op_all_engines (chain call surface, oracle pins + 4-engine parity-or-NIE), not via a direct call() consistency label.", "count_table": "count(*) short-circuit fast path (table height / source-mask sum); native frame op emitted by the cypher lowering, exercised as a labeled subject via _ROW_OP_CASES + the count_all_nodes/edges cypher cases, not via a direct call() consistency label. TODO.", "semi_apply_mark": "correlated EXISTS-mark; NATIVE on polars (viz-filter L1), exercised implicitly by the matrix EXISTS cypher cases; no direct labeled subject.", "anti_semi_apply": "anti-semi correlated filter; NATIVE on polars (viz-filter L1), exercised implicitly by the matrix NOT-EXISTS cypher case; no direct labeled subject.", diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index 40a34c817f..d3132af1cd 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -389,6 +389,62 @@ def test_scalar_fns_null_cells_parity(): _assert_invariant(g, q, f"nullcells {q}") +def test_search_any_op_all_engines(): + """viz-filter L2 `search_any` row op (panel-algebra D2 inspector semantics): + OR-across-columns; case-insensitive substring DEFAULT; regex opt-in; dtype gate = + string cols always + int cols iff numeric-literal term (floats gated OUT of auto); + explicit columns= override; nulls never match. Oracle-pinned + 4-engine + parity-or-NIE via the chain call surface.""" + import pandas as pd + from graphistry.compute.ast import search_any as search_any_op + from graphistry.compute.exceptions import GFQLValidationError + nd = pd.DataFrame({ + "id": [0, 1, 2, 3], + "name": ["Alpha", "beta", None, "gamma7"], + "num": pd.Series([7, 77, 3, 4], dtype="int64"), + "f": [0.5, 1.5, 2.5, 3.5], + }) + ed = pd.DataFrame({"s": [0, 1], "d": [1, 2], "eid": [0, 1]}) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + + def q(**kw): + return [n(name="a"), search_any_op(alias="a", out_col="__hit__", **kw)] + + oracle = { + "default_ci": (dict(term="ALPHA"), [True, False, False, False]), + "case_sensitive": (dict(term="ALPHA", case_sensitive=True), [False, False, False, False]), + "numeric_int_gate": (dict(term="7"), [True, True, False, True]), # num=7, num=77, name gamma7; f EXCLUDED + "explicit_cols": (dict(term="7", columns=["name"]), [False, False, False, True]), + "regex": (dict(term="a.pha", regex=True), [True, False, False, False]), # (?i) default + } + for label, (kw, expected) in oracle.items(): + out = g.gfql(q(**kw), engine="pandas") + got = _to_pd(out._nodes).sort_values("id")["__hit__"].tolist() + assert got == expected, f"search_any oracle {label}: {got} != {expected}" + _assert_invariant(g, q(**kw), f"search_any {label}") + with pytest.raises(GFQLValidationError): + g.gfql(q(term="x", columns=["nope"]), engine="pandas") + + +def test_search_nodes_edges_twins(): + """g.search_nodes/search_edges (persona-pass python twins): same kernel, own table + only, Plottable out; polars frames decline honestly.""" + import pandas as pd + from graphistry.compute.exceptions import GFQLValidationError + nd = pd.DataFrame({"id": [0, 1, 2], "name": ["Alpha", "beta", None]}) + ed = pd.DataFrame({"s": [0, 1], "d": [1, 2], "etype": ["Knows", "likes"], "eid": [0, 1]}) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + assert sorted(g.search_nodes("ALPHA")._nodes["id"].tolist()) == [0] + assert sorted(g.search_nodes("alpha", case_sensitive=True)._nodes["id"].tolist()) == [] + assert sorted(g.search_edges("knows")._edges["eid"].tolist()) == [0] + with pytest.raises(GFQLValidationError): + g.search_nodes("x", columns=["nope"]) + import polars as pl + gpl = graphistry.nodes(pl.from_pandas(nd), "id").edges(pl.from_pandas(ed), "s", "d").bind(edge="eid") + with pytest.raises(NotImplementedError): + gpl.search_nodes("x") + + def test_exists_polars_internal_decline_branches(): """Coverage pins for the honest-decline branches of the polars semi-apply family (each returns None -> the boundary lane raises NIE): non-single/unnamed/query=/ @@ -592,7 +648,7 @@ def _rowop_exercised(): "with_", "unwind", "rows", "skip", "limit", "distinct", "drop_cols", "order_by", "select", "return_", "where_rows", "group_by", - "count_table", + "count_table", "search_any", } From 5169ebbf8243b97394d64e609555262e094325de Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 4 Jul 2026 23:06:40 -0700 Subject: [PATCH 02/10] feat(gfql/cypher): searchAny WHERE predicate + docs (viz-filter L2-b/c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cypher surface: searchAny(alias, 'term'[, {caseSensitive, regex, columns}]) lifts at lowering assembly into a search_any pre-filter + fresh marker column (the pattern-predicate marker mechanism) — composes through AND/OR/NOT; strict opts validation (unknown keys/bad bools/bad lists/unbound alias -> clear E108); oracle-pinned cypher-surface cases + 4-engine parity-or-NIE incl. NOT/AND composition; docs + CHANGELOG. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + docs/source/gfql/cypher.rst | 12 ++ graphistry/compute/gfql/cypher/lowering.py | 108 ++++++++++++++++++ .../compute/gfql/lazy/engine/polars/chain.py | 7 +- .../compute/gfql/cypher/test_lowering.py | 33 ++++++ .../test_engine_polars_conformance_matrix.py | 25 ++++ 6 files changed, 184 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b386c5869..04dff3db04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added +- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (folded via lowercase, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). - **GFQL Cypher `EXISTS { }` pattern-existence subqueries (openCypher-standard), native on all four engines**: `WHERE EXISTS { (n)-[:R]->() }` and `WHERE NOT EXISTS { (n)--() }` now parse and run — the declarative prune-isolated building blocks for the streamgl-viz filter pipeline. An `EXISTS` body reuses the existing pattern-predicate lowering wholesale (`semi_apply_mark` / `anti_semi_apply` row ops), so pandas/cuDF worked immediately; the polars engine gains NATIVE lowerings for the semi-apply family (correlated key sets computed by the polars chain executor's named-flag columns; order-preserving `is_in` joins) plus `rows(binding_ops=...)` for the single-entity row table — previously all honest-NIE. Aliases introduced inside the braces are existentially scoped (`EXISTS { (n)--(m) }` allowed with `m` unbound outside — bare pattern predicates keep the conservative guard), inline property maps work, and the one supported inner `WHERE` form is endpoint inequality — `EXISTS { (n)--(m) WHERE m <> n }`, the drop-self-loop prune-isolated flavor (pandas/cuDF filter the correlated bindings; polars excludes self-loop edges, which is exactly the `m <> n` witness). Both prune flavors are oracle-pinned in the conformance matrix on a self-loop discriminator graph, 4-engine parity-or-NIE. Honest declines with clear errors: `EXISTS` in RETURN/WITH projections, general inner `WHERE`, multi-pattern bodies, full `MATCH..RETURN` subquery bodies, multi-alias correlation on polars. - **GFQL polars execution config is Python-settable and live**: `set_cpu_streaming(bool)` and `set_gpu_executor('in-memory'|'streaming')` in `graphistry.compute.gfql.lazy` (plus the public `GPU_EXECUTORS` options and `GpuExecutor` type) set the CPU-streaming / GPU-executor knobs from Python. They resolve **Python override > environment variable > default**, read **live** per collect — previously these were env-only (`GFQL_POLARS_CPU_STREAMING` / `GFQL_POLARS_GPU_EXECUTOR`) and frozen at import, so neither a Python setting nor a post-import env change took effect. `None` resets a setter to env/default. - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). diff --git a/docs/source/gfql/cypher.rst b/docs/source/gfql/cypher.rst index 761bc89df4..384caf1d31 100644 --- a/docs/source/gfql/cypher.rst +++ b/docs/source/gfql/cypher.rst @@ -314,6 +314,18 @@ and ``RETURN`` expressions: ``size``, and conversions ``toInteger`` / ``toFloat`` / ``toString`` / ``toBoolean`` and ``coalesce``. - Regex ``=~`` (see WHERE Forms above). +- ``searchAny(entity, term[, opts])`` — cross-column search predicate (WHERE + position; GFQL extension for the viz filter pipeline): True where ANY of the + entity's columns matches ``term``. Inspector semantics: OR across columns, + case-insensitive substring by default, regex opt-in; dtype-gated — string + columns always, integer columns iff the term is a numeric literal + (``/^[0-9.-]+$/``); floats/dates/booleans only via the explicit list. Options + map: ``{caseSensitive: true, regex: true, columns: ['name', ...]}`` (unknown + keys error, listing the valid ones). Composes with other WHERE predicates + through AND/OR/NOT; nodes and edges independently searchable with different + terms. Runs natively on all four engines; the regex path obeys the same + per-engine decline rules as ``=~``. Python twins: + :meth:`ComputeMixin.search_nodes` / :meth:`ComputeMixin.search_edges`. ``LIKE`` / ``ILIKE`` and ``BETWEEN`` are intentionally not provided — they are not part of Cypher or GQL; use ``=~`` / ``CONTAINS`` / ``STARTS WITH`` and diff --git a/graphistry/compute/gfql/cypher/lowering.py b/graphistry/compute/gfql/cypher/lowering.py index c326eb7002..fae4693b4d 100644 --- a/graphistry/compute/gfql/cypher/lowering.py +++ b/graphistry/compute/gfql/cypher/lowering.py @@ -24,6 +24,7 @@ order_by, return_, rows, + search_any, semi_apply_mark, serialize_binding_ops, select, @@ -3843,6 +3844,103 @@ def _append_page_ops( params=params, ) +_SEARCH_ANY_CALL_RE = re.compile( + r"\bsearchAny\s*\(\s*([A-Za-z_]\w*)\s*,\s*" + r"('(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" + r"\s*(?:,\s*\{([^{}]*)\})?\s*\)", + re.IGNORECASE, +) +_SEARCH_ANY_OPT_KEYS = {"casesensitive": "case_sensitive", "regex": "regex", "columns": "columns"} +_SEARCH_ANY_COLUMNS_RE = re.compile( + r"^\[\s*(?:'[^']*'|\"[^\"]*\")(?:\s*,\s*(?:'[^']*'|\"[^\"]*\"))*\s*\]$") +_SEARCH_ANY_COL_ITEM_RE = re.compile(r"'([^']*)'|\"([^\"]*)\"") + + +def _parse_search_any_opts(opts_text: str, *, line: int, column: int) -> Dict[str, Any]: + """Parse searchAny's option map literal — strict: unknown keys error listing the + valid ones (persona pass: predictable options beat silent typo-tolerance).""" + out: Dict[str, Any] = {} + if not opts_text.strip(): + return out + depth = 0 + parts: List[str] = [] + cur = "" + for ch in opts_text: + if ch in "[(": + depth += 1 + elif ch in "])": + depth -= 1 + if ch == "," and depth == 0: + parts.append(cur) + cur = "" + else: + cur += ch + parts.append(cur) + for part in parts: + m = re.match(r"\s*([A-Za-z_]\w*)\s*:\s*(.+?)\s*$", part) + key = m.group(1) if m else part.strip() + canon = _SEARCH_ANY_OPT_KEYS.get(key.lower()) if m else None + if m is None or canon is None: + raise _unsupported( + f"searchAny got an unsupported option {key!r}", + field="where", + value=opts_text, + line=line, + column=column, + ) + val_text = m.group(2) + if canon in ("case_sensitive", "regex"): + low = val_text.lower() + if low not in ("true", "false"): + raise _unsupported( + f"searchAny option {key!r} must be true or false", + field="where", value=val_text, line=line, column=column, + ) + out[canon] = low == "true" + else: + if not _SEARCH_ANY_COLUMNS_RE.match(val_text): + raise _unsupported( + "searchAny option 'columns' must be a list of string literals", + field="where", value=val_text, line=line, column=column, + ) + out[canon] = [a or b for a, b in _SEARCH_ANY_COL_ITEM_RE.findall(val_text)] + return out + + +def _lift_search_any_from_row_where( + expr: ExpressionText, + *, + alias_targets: Mapping[str, ASTObject], + existing_cols: AbstractSet[str], +) -> Tuple[str, List[ASTCall]]: + """viz-filter L2-b: rewrite each ``searchAny(alias, 'term'[, {opts}])`` in the + WHERE row-expression into a fresh boolean MARKER column + a ``search_any`` row + pre-filter (exactly the pattern-predicate marker mechanism), so the remaining + boolean expression composes through AND/OR/NOT unchanged.""" + calls: List[ASTCall] = [] + used: set = set(existing_cols) + + def _sub(m: "re.Match[str]") -> str: + alias, term_lit, opts_text = m.group(1), m.group(2), m.group(3) or "" + if alias not in alias_targets: + raise _unsupported( + f"searchAny references alias {alias!r} not bound in the active MATCH scope", + field="where", value=alias, + line=expr.span.line, column=expr.span.column, + ) + term = term_lit[1:-1].replace("\\'", "'").replace('\\"', '"') + opts = _parse_search_any_opts( + opts_text, line=expr.span.line, column=expr.span.column) + base = f"__gfql_search_any_{expr.span.line}_{expr.span.column}_{len(calls)}__" + out_col = _fresh_temp_name(used, base) + used.add(out_col) + calls.append(search_any(alias=alias, term=term, out_col=out_col, **opts)) + return out_col + + new_text = _SEARCH_ANY_CALL_RE.sub(_sub, expr.text) + return new_text, calls + + def _append_match_row_where( row_steps: List[ASTObject], *, @@ -6113,6 +6211,16 @@ def lower_match_query( span=query.where.span if query.where is not None else merged_match.span, ) + if row_where is not None: + # viz-filter L2-b: lift searchAny(...) calls into search_any pre-filters + + # marker columns HERE (assembly level, like the pattern-predicate markers), + # so every LoweredCypherMatch consumer sees the lifted form. + lifted_text, search_calls = _lift_search_any_from_row_where( + row_where, alias_targets=alias_targets, existing_cols=frozenset()) + if search_calls: + row_pre_filters.extend(search_calls) + row_where = ExpressionText(text=lifted_text, span=row_where.span) + return LoweredCypherMatch( query=ops, where=where_out, diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 01d81077a2..3f4f0a9202 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -353,9 +353,12 @@ def _try_native_row_op(g_cur, op): from graphistry.Engine import Engine from .row_pipeline import ( select_polars, with_columns_polars, order_by_polars, group_by_polars, - unwind_polars, where_rows_polars, rows_binding_ops_polars, search_any_polars, - semi_apply_mark_polars, anti_semi_apply_polars, + unwind_polars, where_rows_polars, ) + from .pattern_apply import ( + rows_binding_ops_polars, semi_apply_mark_polars, anti_semi_apply_polars, + ) + from .search import search_any_polars fn = getattr(op, "function", None) if _call_native_on_polars(op): diff --git a/graphistry/tests/compute/gfql/cypher/test_lowering.py b/graphistry/tests/compute/gfql/cypher/test_lowering.py index 79936509f4..e1f8e75561 100644 --- a/graphistry/tests/compute/gfql/cypher/test_lowering.py +++ b/graphistry/tests/compute/gfql/cypher/test_lowering.py @@ -2022,6 +2022,39 @@ def test_lower_exists_drop_self_flavor_emits_neq_semi_apply() -> None: assert sorted(anti.params.get("neq") or []) == ["m", "n"] +def test_lower_search_any_where_emits_marker_prefilter() -> None: + """viz-filter L2-b: WHERE searchAny(a, 'x'[, {opts}]) lifts to a search_any row + pre-filter + a fresh marker column in the residual boolean expression (the + pattern-predicate marker mechanism), composing through AND/OR/NOT.""" + lowered = lower_match_query(_parse_query( + "MATCH (a) WHERE searchAny(a, 'foo') AND a.x > 1 RETURN a.id AS id")) + pre = [op for op in lowered.row_pre_filters + if isinstance(op, ASTCall) and op.function == "search_any"] + assert len(pre) == 1 + assert pre[0].params["alias"] == "a" + assert pre[0].params["term"] == "foo" + marker = pre[0].params["out_col"] + assert marker.startswith("__gfql_search_any_") + assert lowered.row_where is not None and marker in lowered.row_where.text + assert "searchAny" not in lowered.row_where.text + + lowered2 = lower_match_query(_parse_query( + "MATCH (a) WHERE searchAny(a, '7', {caseSensitive: true, regex: false, columns: ['name']}) RETURN a.id AS id")) + op2 = [op for op in lowered2.row_pre_filters + if isinstance(op, ASTCall) and op.function == "search_any"][0] + assert op2.params.get("case_sensitive") is True + assert op2.params.get("columns") == ["name"] + + for q, phrase in [ + ("MATCH (a) WHERE searchAny(a, 'x', {nope: true}) RETURN a", "unsupported option"), + ("MATCH (a) WHERE searchAny(zzz, 'x') RETURN a", "not bound"), + ("MATCH (a) WHERE searchAny(a, 'x', {caseSensitive: 'yes'}) RETURN a", "true or false"), + ]: + with pytest.raises(GFQLValidationError) as exc_info: + lower_match_query(_parse_query(q)) + assert phrase in exc_info.value.message, (q, exc_info.value.message) + + def test_exists_subquery_unsupported_bodies_decline_clearly() -> None: """viz-filter L1 v1 boundaries: inner WHERE, multi-pattern bodies, and full MATCH..RETURN subquery bodies decline with a clear message (never a wrong answer).""" diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index d3132af1cd..0bd0649ac5 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -426,6 +426,31 @@ def q(**kw): g.gfql(q(term="x", columns=["nope"]), engine="pandas") +def test_search_any_cypher_surface_all_engines(): + """viz-filter L2-b: the cypher WHERE searchAny(...) surface — marker lift + + composition with other predicates; oracle-pinned + 4-engine parity-or-NIE.""" + import pandas as pd + nd = pd.DataFrame({ + "id": [0, 1, 2, 3], + "name": ["Alpha", "beta", None, "gamma7"], + "num": pd.Series([7, 77, 3, 4], dtype="int64"), + }) + ed = pd.DataFrame({"s": [0, 1], "d": [1, 2], "eid": [0, 1]}) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + oracle = { + "MATCH (a) WHERE searchAny(a, 'ALPHA') RETURN a.id AS id": [0], + "MATCH (a) WHERE searchAny(a, '7') RETURN a.id AS id": [0, 1, 3], + "MATCH (a) WHERE searchAny(a, '7', {columns: ['name']}) RETURN a.id AS id": [3], + "MATCH (a) WHERE searchAny(a, 'a.pha', {regex: true}) RETURN a.id AS id": [0], + "MATCH (a) WHERE searchAny(a, 'a') AND a.num > 10 RETURN a.id AS id": [1], + "MATCH (a) WHERE NOT searchAny(a, 'a') RETURN a.id AS id": [2], + } + for q, expected in oracle.items(): + pdf = _to_pd(g.gfql(q, engine="pandas")._nodes) + assert sorted(pdf["id"].tolist()) == expected, f"oracle drift: {q}" + _assert_invariant(g, q, f"searchAny-cypher {q}") + + def test_search_nodes_edges_twins(): """g.search_nodes/search_edges (persona-pass python twins): same kernel, own table only, Plottable out; polars frames decline honestly.""" From fdbc463174f9810988123a29b601999fc6d21326 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 08:58:59 -0700 Subject: [PATCH 03/10] =?UTF-8?q?fix(gfql/searchAny):=20wave-1=20B1/B2=20+?= =?UTF-8?q?=20I1/I2/I5=20=E2=80=94=20literal-masked=20lift,=20cuDF=20=3D~-?= =?UTF-8?q?parity=20regex=20guards,=20null-cell=20mask,=20polars=20E108,?= =?UTF-8?q?=20$param=20clear-decline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B1: _lift_search_any_from_row_where now finds calls against a literal/comment- masked copy of the WHERE text, so searchAny(...) inside a string literal is data (was: bare .sub rewrote literal contents — silent wrong answer / spurious unbound-alias error). I5: any residual searchAny( outside literals after the lift (e.g. $param term) raises a clear E108 instead of leaking downstream. B2: the cuDF kernel path applies the same decline rules as =~ (_cudf_regex_prep + _cudf_casefold_or_decline) before delegating to Contains — NIE on lookaround/ backrefs/inline-flags/unsound folds instead of libcudf crashes or \D->\d inversion. I1: null cells never match — explicit non-string columns mask isna() back out after astype(str) ('nan'/'' artifacts). I2: polars missing-explicit-column now raises the same E108 as pandas (the comment claimed a fallback that doesn't exist). Pins for each in the conformance matrix. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/gfql/cypher/lowering.py | 31 +++++- .../compute/gfql/lazy/engine/polars/search.py | 13 ++- graphistry/compute/gfql/search_any.py | 24 ++++- .../test_engine_polars_conformance_matrix.py | 94 +++++++++++++++++++ 4 files changed, 155 insertions(+), 7 deletions(-) diff --git a/graphistry/compute/gfql/cypher/lowering.py b/graphistry/compute/gfql/cypher/lowering.py index fae4693b4d..2c295585cb 100644 --- a/graphistry/compute/gfql/cypher/lowering.py +++ b/graphistry/compute/gfql/cypher/lowering.py @@ -57,6 +57,7 @@ from graphistry.compute.predicates.is_in import is_in from graphistry.compute.predicates.logical import all_of from graphistry.compute.predicates.str import contains as str_contains, endswith, fullmatch, never_match, startswith +from graphistry.compute.gfql.cypher.parser import _mask_quoted_backticked_and_commented_for_scan from graphistry.compute.gfql.language_defs import GFQL_AGGREGATION_FUNCTIONS from graphistry.compute.gfql.expr_parser import ( BinaryOp, @@ -3916,7 +3917,11 @@ def _lift_search_any_from_row_where( """viz-filter L2-b: rewrite each ``searchAny(alias, 'term'[, {opts}])`` in the WHERE row-expression into a fresh boolean MARKER column + a ``search_any`` row pre-filter (exactly the pattern-predicate marker mechanism), so the remaining - boolean expression composes through AND/OR/NOT unchanged.""" + boolean expression composes through AND/OR/NOT unchanged. + + Matches are found against a literal/comment-masked copy of the text so a + ``searchAny(...)`` occurring INSIDE a string literal is data, not a call + (wave-1 B1: the bare ``.sub`` rewrote literal contents — silent wrong answer).""" calls: List[ASTCall] = [] used: set = set(existing_cols) @@ -3937,7 +3942,29 @@ def _sub(m: "re.Match[str]") -> str: calls.append(search_any(alias=alias, term=term, out_col=out_col, **opts)) return out_col - new_text = _SEARCH_ANY_CALL_RE.sub(_sub, expr.text) + masked = _mask_quoted_backticked_and_commented_for_scan(expr.text) + parts: List[str] = [] + last = 0 + for m in _SEARCH_ANY_CALL_RE.finditer(expr.text): + if masked[m.start()] != expr.text[m.start()]: + continue # inside a string literal / comment: data, not a call + parts.append(expr.text[last:m.start()]) + parts.append(_sub(m)) + last = m.end() + parts.append(expr.text[last:]) + new_text = "".join(parts) + # Anything still spelled searchAny( outside literals is a form the lift can't + # parse (e.g. a $param term — the persona-pass signature is literal-only): fail + # loudly here instead of leaking an unknown function downstream (wave-1 I5). + masked_new = _mask_quoted_backticked_and_commented_for_scan(new_text) + if re.search(r"\bsearchAny\s*\(", masked_new, re.IGNORECASE): + raise _unsupported( + "searchAny requires a string-literal term and literal options, e.g. " + "searchAny(n, 'term', {caseSensitive: false}) — parameters ($p) and " + "expressions are not supported", + field="where", value=expr.text, + line=expr.span.line, column=expr.span.column, + ) return new_text, calls diff --git a/graphistry/compute/gfql/lazy/engine/polars/search.py b/graphistry/compute/gfql/lazy/engine/polars/search.py index 8f720eac87..f3c52d018b 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/search.py +++ b/graphistry/compute/gfql/lazy/engine/polars/search.py @@ -40,7 +40,18 @@ def search_any_polars( schema = dict(left.schema) if columns is not None: if any(c not in pool for c in columns): - return None # missing explicit column: decline -> pandas raises the clear error + # same validation error as the pandas row pipeline (there is no pandas + # fallback behind this dispatch — a generic NIE here would misreport a + # user input error as an engine gap; wave-1 I2) + from graphistry.compute.exceptions import ErrorCode, GFQLValidationError + raise GFQLValidationError( + ErrorCode.E108, + "searchAny columns= includes a column absent from the searched table", + field="columns", + value=list(columns), + suggestion="List only columns present on the searched entity.", + language="cypher", + ) chosen = [pool[c] for c in columns] else: numeric_ok = is_numeric_term(term) diff --git a/graphistry/compute/gfql/search_any.py b/graphistry/compute/gfql/search_any.py index 969447063d..94e0614ec7 100644 --- a/graphistry/compute/gfql/search_any.py +++ b/graphistry/compute/gfql/search_any.py @@ -53,7 +53,9 @@ def search_any_mask( ) -> Optional[Any]: """Boolean row mask over ``df`` (pandas or cuDF), or None to decline (an explicit column is missing). Null cells never match; no candidate columns -> all-False.""" - from graphistry.compute.predicates.str import Contains + from graphistry.compute.predicates.str import ( + Contains, _cudf_casefold_or_decline, _cudf_regex_prep, + ) cols = search_candidate_columns(df, term, columns) if cols is None: return None @@ -61,13 +63,27 @@ def search_any_mask( if len(df.columns) == 0: return None return df[df.columns[0]].isna() & False # engine-safe all-False - pred = Contains(term, case=case_sensitive, regex=regex, na=False) + pat, case = term, case_sensitive + if regex and "cudf" in type(df).__module__: + # Same decline rules as =~ (Match/Fullmatch): NIE on lookaround/backrefs/ + # inline flags instead of a libcudf crash, and refuse unsound casefolds + # instead of Contains' blind pat.lower() (\D -> \d INVERTS) — wave-1 B2. + pat, case = _cudf_regex_prep(pat, case) + if not case: + pat = _cudf_casefold_or_decline(pat) # pre-folded; Contains' .lower() is a no-op + pred = Contains(pat, case=case, regex=regex, na=False) mask: Optional[Any] = None for c in cols: s = df[c] + m: Any if not _is_searchable_string_dtype(s.dtype): - s = s.astype(str) # canonical toString for int / explicit columns - m = pred(s) + # canonical toString for int / explicit columns; pandas astype(str) + # stringifies nulls ("nan"/"") so mask them back out — null cells + # never match on any engine (wave-1 I1) + nulls = s.isna() + m = pred(s.astype(str)) & ~nulls + else: + m = pred(s) mask = m if mask is None else (mask | m) assert mask is not None # cols is non-empty here return mask.fillna(False) diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index 0bd0649ac5..60b0881308 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -470,6 +470,100 @@ def test_search_nodes_edges_twins(): gpl.search_nodes("x") +def test_search_any_literal_and_param_guards(): + """wave-1 B1/I5: `searchAny(...)` spelled INSIDE a string literal is data — no + lift, no spurious unbound-alias error, literal compares byte-identically; a + non-literal term ($param) fails loudly at lowering with a usable message + instead of leaking an unknown function downstream.""" + from graphistry.compute.exceptions import GFQLValidationError + lit = 'see searchAny(zzz, "x") docs' + nd = pd.DataFrame({"id": [0, 1], "note": [lit, "plain"]}) + ed = pd.DataFrame({"s": [0], "d": [1], "eid": [0]}) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + # zzz is unbound: if the lift saw inside the literal this would error/rewrite + q1 = "MATCH (n) WHERE n.note = 'see searchAny(zzz, \"x\") docs' RETURN n.id AS id" + assert _to_pd(g.gfql(q1, engine="pandas")._nodes)["id"].tolist() == [0] + _assert_invariant(g, q1, "searchAny-in-literal") + # real call AND a literal mention in the same WHERE: only the real call lifts + q2 = ("MATCH (n) WHERE searchAny(n, 'plain') AND n.note <> 'searchAny(zzz, \"y\")' " + "RETURN n.id AS id") + assert _to_pd(g.gfql(q2, engine="pandas")._nodes)["id"].tolist() == [1] + _assert_invariant(g, q2, "searchAny-real-plus-literal") + # $param term: the lift can't parse it — clear E108, not an obscure failure. + # Without params the generic missing-param validation fires first (also clear); + # WITH params supplied, the residual-searchAny guard is the one that speaks. + qp = "MATCH (n) WHERE searchAny(n, $q) RETURN n.id AS id" + with pytest.raises(GFQLValidationError, match="Missing Cypher parameter"): + g.gfql(qp, engine="pandas") + with pytest.raises(GFQLValidationError, match="string-literal term"): + g.gfql(qp, engine="pandas", params={"q": "plain"}) + + +def test_search_any_cudf_regex_guards(): + """wave-1 B2: the cuDF search path honors the same decline rules as =~ — + NIE on lookaround (libcudf kernel-compile crash otherwise) and on unsound + case-insensitive folds (blind pat.lower() turns \\D into \\d — INVERTS); + sound patterns still run natively.""" + cudf = pytest.importorskip("cudf") + from graphistry.compute.gfql.search_any import search_any_mask + df = cudf.DataFrame({"name": ["Deed", "1234"]}) + with pytest.raises(NotImplementedError): + search_any_mask(df, r"\D+", regex=True) # case-insensitive default: unsound fold + with pytest.raises(NotImplementedError): + search_any_mask(df, r"x(?=y)", regex=True, case_sensitive=True) # lookaround + got = search_any_mask(df, r"de{2}d", regex=True) # sound fold: native + assert got.to_pandas().tolist() == [True, False] + got_cs = search_any_mask(df, r"\d{4}", regex=True, case_sensitive=True) + assert got_cs.to_pandas().tolist() == [False, True] + + +def test_search_any_null_and_float_stringify(): + """wave-1 I1/I3: null cells NEVER match — including explicit non-string columns + where pandas astype(str) would stringify them ('nan'/''); and explicit + float-column stringification is parity-pinned (term '7' discriminates 7.5).""" + from graphistry.compute.ast import search_any as search_any_op + nd = pd.DataFrame({ + "id": [0, 1, 2], + "f": [7.5, None, 1.25], + "ni": pd.array([7, None, 3], dtype="Int64"), + }) + ed = pd.DataFrame({"s": [0], "d": [1], "eid": [0]}) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + + def q(**kw): + return [n(name="a"), search_any_op(alias="a", out_col="__hit__", **kw)] + + for term in ["nan", "NA", "None"]: # the null-stringification spellings + kw = dict(term=term, columns=["f", "ni"]) + got = _to_pd(g.gfql(q(**kw), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() + assert got == [False, False, False], f"null cells matched {term!r}: {got}" + _assert_invariant(g, q(**kw), f"search_any nulls {term}") + # floats stay OUT of the auto gate even when they'd match stringified ('7' in '7.5'); + # numeric term still probes int cols (id, ni) — none contain '7' except ni=7 + got = _to_pd(g.gfql(q(term="7"), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() + assert got == [True, False, False], f"auto gate drift: {got}" # ni=7 only, NOT f=7.5 + _assert_invariant(g, q(term="7"), "search_any float auto-gate") + # explicit float column: canonical toString, discriminating pin (I3) + kw = dict(term="7", columns=["f"]) + got = _to_pd(g.gfql(q(**kw), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() + assert got == [True, False, False], f"float stringify drift: {got}" + _assert_invariant(g, q(**kw), "search_any float explicit") + + +def test_search_any_edge_alias(): + """wave-1 I4: edge-alias searchAny — correct on pandas; polars declines with an + honest NIE (documented v1 gap), never a silent wrong answer.""" + nd = pd.DataFrame({"id": [0, 1, 2]}) + ed = pd.DataFrame({ + "s": [0, 1], "d": [1, 2], "eid": [0, 1], + "etype": ["Knows", "likes"], + }) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + q = "MATCH (a)-[r]->(b) WHERE searchAny(r, 'knows') RETURN r.eid AS eid" + assert _to_pd(g.gfql(q, engine="pandas")._nodes)["eid"].tolist() == [0] + _assert_invariant(g, q, "searchAny-edge-alias") + + def test_exists_polars_internal_decline_branches(): """Coverage pins for the honest-decline branches of the polars semi-apply family (each returns None -> the boundary lane raises NIE): non-single/unnamed/query=/ From d4053b171131b564240fcfbea9e4be8abfc4cc41 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 09:26:01 -0700 Subject: [PATCH 04/10] =?UTF-8?q?fix(gfql/searchAny):=20wave-2=20W2-1..4?= =?UTF-8?q?=20+=20S1/S4=20=E2=80=94=20polars=20bool=20canonicalize=20+=20e?= =?UTF-8?q?xplicit-dtype=20gate,=20discriminating=20float=20pin,=20polars?= =?UTF-8?q?=20E108=20pin,=20docs=20caveats,=20rescan-after-skip=20lift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W2-3: polars explicit-column search now canonicalizes Boolean rendering to pandas' 'True'/'False' (cast gave 'true' — a SILENT caseSensitive divergence) with nulls kept null, and declines explicit columns beyond string/int/float/ bool (temporal/categorical/nested stringification is engine-divergent) — NIE, never silent. W2-1: float auto-gate pin made discriminating (7.25 moved to a row with no int '7'). W2-2: polars-engine missing-explicit-column E108 pinned directly. W2-4: docs/CHANGELOG caveat the edge-alias polars NIE + dtype gate. S1: lift rescans from m.start()+1 after an in-literal skip so a quote-spanning pseudo-match can't swallow a real call. S4: two-searchAny composition pin. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- docs/source/gfql/cypher.rst | 6 ++- graphistry/compute/gfql/cypher/lowering.py | 13 ++++++- .../compute/gfql/lazy/engine/polars/search.py | 26 ++++++++++++- .../test_engine_polars_conformance_matrix.py | 38 ++++++++++++++++--- 5 files changed, 75 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04dff3db04..82c015897a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added -- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (folded via lowercase, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). +- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (folded via lowercase, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). Honest polars declines (NIE, use engine='pandas'): edge-alias searchAny, and explicit columns beyond string/int/float/bool dtypes. - **GFQL Cypher `EXISTS { }` pattern-existence subqueries (openCypher-standard), native on all four engines**: `WHERE EXISTS { (n)-[:R]->() }` and `WHERE NOT EXISTS { (n)--() }` now parse and run — the declarative prune-isolated building blocks for the streamgl-viz filter pipeline. An `EXISTS` body reuses the existing pattern-predicate lowering wholesale (`semi_apply_mark` / `anti_semi_apply` row ops), so pandas/cuDF worked immediately; the polars engine gains NATIVE lowerings for the semi-apply family (correlated key sets computed by the polars chain executor's named-flag columns; order-preserving `is_in` joins) plus `rows(binding_ops=...)` for the single-entity row table — previously all honest-NIE. Aliases introduced inside the braces are existentially scoped (`EXISTS { (n)--(m) }` allowed with `m` unbound outside — bare pattern predicates keep the conservative guard), inline property maps work, and the one supported inner `WHERE` form is endpoint inequality — `EXISTS { (n)--(m) WHERE m <> n }`, the drop-self-loop prune-isolated flavor (pandas/cuDF filter the correlated bindings; polars excludes self-loop edges, which is exactly the `m <> n` witness). Both prune flavors are oracle-pinned in the conformance matrix on a self-loop discriminator graph, 4-engine parity-or-NIE. Honest declines with clear errors: `EXISTS` in RETURN/WITH projections, general inner `WHERE`, multi-pattern bodies, full `MATCH..RETURN` subquery bodies, multi-alias correlation on polars. - **GFQL polars execution config is Python-settable and live**: `set_cpu_streaming(bool)` and `set_gpu_executor('in-memory'|'streaming')` in `graphistry.compute.gfql.lazy` (plus the public `GPU_EXECUTORS` options and `GpuExecutor` type) set the CPU-streaming / GPU-executor knobs from Python. They resolve **Python override > environment variable > default**, read **live** per collect — previously these were env-only (`GFQL_POLARS_CPU_STREAMING` / `GFQL_POLARS_GPU_EXECUTOR`) and frozen at import, so neither a Python setting nor a post-import env change took effect. `None` resets a setter to env/default. - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). diff --git a/docs/source/gfql/cypher.rst b/docs/source/gfql/cypher.rst index 384caf1d31..31f9059d6c 100644 --- a/docs/source/gfql/cypher.rst +++ b/docs/source/gfql/cypher.rst @@ -323,7 +323,11 @@ and ``RETURN`` expressions: map: ``{caseSensitive: true, regex: true, columns: ['name', ...]}`` (unknown keys error, listing the valid ones). Composes with other WHERE predicates through AND/OR/NOT; nodes and edges independently searchable with different - terms. Runs natively on all four engines; the regex path obeys the same + terms. Runs natively on all four engines for node aliases; an edge-alias + ``searchAny(r, ...)`` declines honestly on polars pending multi-entity + binding-row support (use ``engine='pandas'``), and explicit non-string + columns beyond ints/floats/bools likewise decline on polars rather than risk + divergent stringification. The regex path obeys the same per-engine decline rules as ``=~``. Python twins: :meth:`ComputeMixin.search_nodes` / :meth:`ComputeMixin.search_edges`. diff --git a/graphistry/compute/gfql/cypher/lowering.py b/graphistry/compute/gfql/cypher/lowering.py index 2c295585cb..3623e4d992 100644 --- a/graphistry/compute/gfql/cypher/lowering.py +++ b/graphistry/compute/gfql/cypher/lowering.py @@ -3945,12 +3945,21 @@ def _sub(m: "re.Match[str]") -> str: masked = _mask_quoted_backticked_and_commented_for_scan(expr.text) parts: List[str] = [] last = 0 - for m in _SEARCH_ANY_CALL_RE.finditer(expr.text): + pos = 0 + while True: + m = _SEARCH_ANY_CALL_RE.search(expr.text, pos) + if m is None: + break if masked[m.start()] != expr.text[m.start()]: - continue # inside a string literal / comment: data, not a call + # starts inside a string literal / comment: data, not a call — resume + # just past the START (not the end: an in-literal pseudo-match can span + # across quotes and swallow a real call inside its span; wave-2 S1) + pos = m.start() + 1 + continue parts.append(expr.text[last:m.start()]) parts.append(_sub(m)) last = m.end() + pos = m.end() parts.append(expr.text[last:]) new_text = "".join(parts) # Anything still spelled searchAny( outside literals is a form the lift can't diff --git a/graphistry/compute/gfql/lazy/engine/polars/search.py b/graphistry/compute/gfql/lazy/engine/polars/search.py index f3c52d018b..fdc270b183 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/search.py +++ b/graphistry/compute/gfql/lazy/engine/polars/search.py @@ -69,9 +69,33 @@ def search_any_polars( return _rewrap(g, marked) if regex and _regex_rust_incompatible(term): return None + # Explicit columns= reaches beyond the auto gate: only dtypes whose canonical + # toString provably matches the pandas kernel are searched natively — floats/ints + # render identically; Boolean is canonicalized below (polars 'true' vs pandas + # 'True' was a SILENT divergence under caseSensitive — wave-2 W2-3); everything + # else (temporal, categorical, nested) declines honestly. + _stringify_ok = { + pl.String, pl.Boolean, pl.Float32, pl.Float64, + pl.Int8, pl.Int16, pl.Int32, pl.Int64, + pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, + } + if any(schema[real] not in _stringify_ok for real in chosen): + return None exprs = [] for real in chosen: - base = pl.col(real) if schema[real] == pl.String else pl.col(real).cast(pl.String) + dt = schema[real] + if dt == pl.String: + base = pl.col(real) + elif dt == pl.Boolean: + # null cells must STAY null (never match) — bare when/otherwise would + # send null conditions to the 'False' branch + base = ( + pl.when(pl.col(real).is_null()).then(pl.lit(None, dtype=pl.String)) + .when(pl.col(real)).then(pl.lit("True")) + .otherwise(pl.lit("False")) + ) + else: + base = pl.col(real).cast(pl.String) if regex: pat = term if case_sensitive else f"(?i){term}" exprs.append(base.str.contains(pat, literal=False)) diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index 60b0881308..8ebe458017 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -444,6 +444,9 @@ def test_search_any_cypher_surface_all_engines(): "MATCH (a) WHERE searchAny(a, 'a.pha', {regex: true}) RETURN a.id AS id": [0], "MATCH (a) WHERE searchAny(a, 'a') AND a.num > 10 RETURN a.id AS id": [1], "MATCH (a) WHERE NOT searchAny(a, 'a') RETURN a.id AS id": [2], + # two lifts in one WHERE compose independently (wave-2 S4): + # 'alpha'~{0}; '7'~{0 (num=7), 1 (77), 3 (gamma7)}; AND => [0] + "MATCH (a) WHERE searchAny(a, 'alpha') AND searchAny(a, '7') RETURN a.id AS id": [0], } for q, expected in oracle.items(): pdf = _to_pd(g.gfql(q, engine="pandas")._nodes) @@ -522,10 +525,15 @@ def test_search_any_null_and_float_stringify(): where pandas astype(str) would stringify them ('nan'/''); and explicit float-column stringification is parity-pinned (term '7' discriminates 7.5).""" from graphistry.compute.ast import search_any as search_any_op + from graphistry.compute.exceptions import GFQLValidationError nd = pd.DataFrame({ "id": [0, 1, 2], - "f": [7.5, None, 1.25], + # 7.25 sits on a row with NO int-column '7' so a float wrongly entering the + # auto gate flips row 2 — the wave-1 fixture (7.5 on the ni=7 row) could not + # discriminate (wave-2 W2-1) + "f": [0.5, None, 7.25], "ni": pd.array([7, None, 3], dtype="Int64"), + "flag": pd.array([True, None, False], dtype="boolean"), }) ed = pd.DataFrame({"s": [0], "d": [1], "eid": [0]}) g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") @@ -538,16 +546,36 @@ def q(**kw): got = _to_pd(g.gfql(q(**kw), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() assert got == [False, False, False], f"null cells matched {term!r}: {got}" _assert_invariant(g, q(**kw), f"search_any nulls {term}") - # floats stay OUT of the auto gate even when they'd match stringified ('7' in '7.5'); - # numeric term still probes int cols (id, ni) — none contain '7' except ni=7 + # floats stay OUT of the auto gate even when they'd match stringified ('7' in + # '7.25'); numeric term still probes int cols — ni=7 hits row 0 ONLY got = _to_pd(g.gfql(q(term="7"), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() - assert got == [True, False, False], f"auto gate drift: {got}" # ni=7 only, NOT f=7.5 + assert got == [True, False, False], f"auto gate drift: {got}" # NOT [.., .., True] _assert_invariant(g, q(term="7"), "search_any float auto-gate") # explicit float column: canonical toString, discriminating pin (I3) kw = dict(term="7", columns=["f"]) got = _to_pd(g.gfql(q(**kw), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() - assert got == [True, False, False], f"float stringify drift: {got}" + assert got == [False, False, True], f"float stringify drift: {got}" _assert_invariant(g, q(**kw), "search_any float explicit") + # explicit BOOL column under caseSensitive: pandas renders 'True' — polars must + # match (canonicalized) or NIE, never silently miss (wave-2 W2-3) + kw = dict(term="True", columns=["flag"], case_sensitive=True) + got = _to_pd(g.gfql(q(**kw), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() + assert got == [True, False, False], f"bool stringify drift: {got}" + _assert_invariant(g, q(**kw), "search_any bool explicit cs") + # missing explicit column raises the SAME E108 natively on polars — a generic + # NIE here would misreport a user error as an engine gap (wave-2 W2-2) + import polars as pl + gpl = graphistry.nodes(pl.from_pandas(nd), "id").edges( + pl.from_pandas(ed), "s", "d").bind(edge="eid") + with pytest.raises(GFQLValidationError, match="absent"): + gpl.gfql(q(term="x", columns=["nope"]), engine="polars") + # explicit TEMPORAL column: stringification is engine-divergent — polars + # declines honestly (NIE) instead of risking a silent mismatch (wave-2 W2-3) + ndt = nd.assign(t=pd.to_datetime(["2020-01-02", "2021-03-04", "2022-05-06"])) + gplt = graphistry.nodes(pl.from_pandas(ndt), "id").edges( + pl.from_pandas(ed), "s", "d").bind(edge="eid") + with pytest.raises(NotImplementedError): + gplt.gfql(q(term="2020", columns=["t"]), engine="polars") def test_search_any_edge_alias(): From b5a38254af588a3ae46a5a86d8bfb00ee3e0f7ae Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 09:42:18 -0700 Subject: [PATCH 05/10] ci(gfql): polars coverage-baseline entry for search.py (L2 module) dgx-measured 86.96% on the CI-verbatim CPU polars lane; floor 79.0 per the baseline's ~8-below convention (also absorbs the wave-2 canonicalize/dtype- gate lines, whose decline/parity branches are pinned in the matrix). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../compute/gfql/coverage_baselines/ci-polars-py3.12.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json b/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json index 6ba869bf0e..9a4ebc4e77 100644 --- a/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json +++ b/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json @@ -1,5 +1,5 @@ { - "_note": "Per-file line-coverage floors for the native POLARS engine (profile gfql-polars, bin/test-polars.sh). COMPLETE list required (audit fails un-baselined target files). Floors set conservatively (~8% below dgx-measured; measured: chain 94.3 row_pipeline 87.9 predicates 91 projection 73.2 hop 95.1 hop_eager 98.3 degrees 90.7 dtypes 100 lazy/__init__ 98.6). pattern_apply 75.4 (2026-07-05, module split from row_pipeline). Raise as coverage grows.", + "_note": "Per-file line-coverage floors for the native POLARS engine (profile gfql-polars, bin/test-polars.sh). COMPLETE list required (audit fails un-baselined target files). Floors set conservatively (~8% below dgx-measured; measured: chain 94.3 row_pipeline 87.9 predicates 91 projection 73.2 hop 95.1 hop_eager 98.3 degrees 90.7 dtypes 100 lazy/__init__ 98.6). pattern_apply 75.4 (2026-07-05, module split from row_pipeline). search 87.0 (2026-07-05, L2). Raise as coverage grows.", "files": { "graphistry/compute/gfql/lazy/__init__.py": 90.0, "graphistry/compute/gfql/lazy/engine/__init__.py": 90.0, @@ -12,6 +12,7 @@ "graphistry/compute/gfql/lazy/engine/polars/pattern_apply.py": 67.0, "graphistry/compute/gfql/lazy/engine/polars/predicates.py": 83.0, "graphistry/compute/gfql/lazy/engine/polars/projection.py": 65.0, - "graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py": 79.0 + "graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py": 79.0, + "graphistry/compute/gfql/lazy/engine/polars/search.py": 79.0 } } From b36119a7422b816e6dbce388d8e87abe315628b0 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 09:56:57 -0700 Subject: [PATCH 06/10] =?UTF-8?q?fix(gfql/searchAny):=20wave-3=20W3-1=20?= =?UTF-8?q?=E2=80=94=20polars=20declines=20explicit=20float=20columns=20(r?= =?UTF-8?q?epr=20divergence),=20precedence=20pin,=20wording?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Floats leave the polars _stringify_ok gate: pandas str(1e16)='1e+16' vs the Rust formatter's '1e16' is a silent exponent-regime divergence — the SAME decision the polars toString lowering already made (row_pipeline.py declines Float 'repr diverges'). Pinned NIE for explicit float columns on polars; pandas oracle pin unchanged. Missing-column-beats-dtype-gate precedence pinned (E108, not NIE). Docs/CHANGELOG: float added to the polars decline list; 'folded via lowercase' -> 'case-folded' (pandas upper-folds internally). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- docs/source/gfql/cypher.rst | 4 ++-- graphistry/compute/gfql/lazy/engine/polars/search.py | 12 +++++++----- .../gfql/test_engine_polars_conformance_matrix.py | 12 +++++++++++- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82c015897a..22ce48a00a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added -- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (folded via lowercase, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). Honest polars declines (NIE, use engine='pandas'): edge-alias searchAny, and explicit columns beyond string/int/float/bool dtypes. +- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (case-folded, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). Honest polars declines (NIE, use engine='pandas'): edge-alias searchAny, and explicit columns beyond string/int/bool dtypes (incl. floats: repr diverges in the exponent regime). - **GFQL Cypher `EXISTS { }` pattern-existence subqueries (openCypher-standard), native on all four engines**: `WHERE EXISTS { (n)-[:R]->() }` and `WHERE NOT EXISTS { (n)--() }` now parse and run — the declarative prune-isolated building blocks for the streamgl-viz filter pipeline. An `EXISTS` body reuses the existing pattern-predicate lowering wholesale (`semi_apply_mark` / `anti_semi_apply` row ops), so pandas/cuDF worked immediately; the polars engine gains NATIVE lowerings for the semi-apply family (correlated key sets computed by the polars chain executor's named-flag columns; order-preserving `is_in` joins) plus `rows(binding_ops=...)` for the single-entity row table — previously all honest-NIE. Aliases introduced inside the braces are existentially scoped (`EXISTS { (n)--(m) }` allowed with `m` unbound outside — bare pattern predicates keep the conservative guard), inline property maps work, and the one supported inner `WHERE` form is endpoint inequality — `EXISTS { (n)--(m) WHERE m <> n }`, the drop-self-loop prune-isolated flavor (pandas/cuDF filter the correlated bindings; polars excludes self-loop edges, which is exactly the `m <> n` witness). Both prune flavors are oracle-pinned in the conformance matrix on a self-loop discriminator graph, 4-engine parity-or-NIE. Honest declines with clear errors: `EXISTS` in RETURN/WITH projections, general inner `WHERE`, multi-pattern bodies, full `MATCH..RETURN` subquery bodies, multi-alias correlation on polars. - **GFQL polars execution config is Python-settable and live**: `set_cpu_streaming(bool)` and `set_gpu_executor('in-memory'|'streaming')` in `graphistry.compute.gfql.lazy` (plus the public `GPU_EXECUTORS` options and `GpuExecutor` type) set the CPU-streaming / GPU-executor knobs from Python. They resolve **Python override > environment variable > default**, read **live** per collect — previously these were env-only (`GFQL_POLARS_CPU_STREAMING` / `GFQL_POLARS_GPU_EXECUTOR`) and frozen at import, so neither a Python setting nor a post-import env change took effect. `None` resets a setter to env/default. - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). diff --git a/docs/source/gfql/cypher.rst b/docs/source/gfql/cypher.rst index 31f9059d6c..235b450a6a 100644 --- a/docs/source/gfql/cypher.rst +++ b/docs/source/gfql/cypher.rst @@ -326,8 +326,8 @@ and ``RETURN`` expressions: terms. Runs natively on all four engines for node aliases; an edge-alias ``searchAny(r, ...)`` declines honestly on polars pending multi-entity binding-row support (use ``engine='pandas'``), and explicit non-string - columns beyond ints/floats/bools likewise decline on polars rather than risk - divergent stringification. The regex path obeys the same + columns beyond ints/bools likewise decline on polars rather than risk + divergent stringification (float repr differs in the exponent regime). The regex path obeys the same per-engine decline rules as ``=~``. Python twins: :meth:`ComputeMixin.search_nodes` / :meth:`ComputeMixin.search_edges`. diff --git a/graphistry/compute/gfql/lazy/engine/polars/search.py b/graphistry/compute/gfql/lazy/engine/polars/search.py index fdc270b183..f9424defee 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/search.py +++ b/graphistry/compute/gfql/lazy/engine/polars/search.py @@ -70,12 +70,14 @@ def search_any_polars( if regex and _regex_rust_incompatible(term): return None # Explicit columns= reaches beyond the auto gate: only dtypes whose canonical - # toString provably matches the pandas kernel are searched natively — floats/ints - # render identically; Boolean is canonicalized below (polars 'true' vs pandas - # 'True' was a SILENT divergence under caseSensitive — wave-2 W2-3); everything - # else (temporal, categorical, nested) declines honestly. + # toString provably matches the pandas kernel are searched natively — ints render + # identically; Boolean is canonicalized below (polars 'true' vs pandas 'True' was + # a SILENT divergence under caseSensitive — wave-2 W2-3). Float DECLINES like the + # polars toString lowering (row_pipeline.py): repr diverges in the exponent + # regime (pandas str(1e16)='1e+16' vs Rust-formatter '1e16' — wave-3 W3-1). + # Temporal/categorical/nested likewise decline honestly. _stringify_ok = { - pl.String, pl.Boolean, pl.Float32, pl.Float64, + pl.String, pl.Boolean, pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, } diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index 8ebe458017..57ee8ce604 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -551,7 +551,9 @@ def q(**kw): got = _to_pd(g.gfql(q(term="7"), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() assert got == [True, False, False], f"auto gate drift: {got}" # NOT [.., .., True] _assert_invariant(g, q(term="7"), "search_any float auto-gate") - # explicit float column: canonical toString, discriminating pin (I3) + # explicit float column: canonical toString on pandas, discriminating pin (I3); + # polars DECLINES floats — repr diverges in the exponent regime, same decision + # as the polars toString lowering (wave-3 W3-1); NIE tolerated by the invariant kw = dict(term="7", columns=["f"]) got = _to_pd(g.gfql(q(**kw), engine="pandas")._nodes).sort_values("id")["__hit__"].tolist() assert got == [False, False, True], f"float stringify drift: {got}" @@ -569,6 +571,10 @@ def q(**kw): pl.from_pandas(ed), "s", "d").bind(edge="eid") with pytest.raises(GFQLValidationError, match="absent"): gpl.gfql(q(term="x", columns=["nope"]), engine="polars") + # explicit FLOAT column on polars: pinned NIE (exponent-regime repr divergence, + # wave-3 W3-1 — same decision as the polars toString lowering) + with pytest.raises(NotImplementedError): + gpl.gfql(q(term="7", columns=["f"]), engine="polars") # explicit TEMPORAL column: stringification is engine-divergent — polars # declines honestly (NIE) instead of risking a silent mismatch (wave-2 W2-3) ndt = nd.assign(t=pd.to_datetime(["2020-01-02", "2021-03-04", "2022-05-06"])) @@ -576,6 +582,10 @@ def q(**kw): pl.from_pandas(ed), "s", "d").bind(edge="eid") with pytest.raises(NotImplementedError): gplt.gfql(q(term="2020", columns=["t"]), engine="polars") + # precedence: a MISSING column is a user error (E108) even when another listed + # column would trip the dtype gate — a refactor must not flip it to NIE (wave-3) + with pytest.raises(GFQLValidationError, match="absent"): + gplt.gfql(q(term="x", columns=["nope", "t"]), engine="polars") def test_search_any_edge_alias(): From 09b6528d4c19c3b13221121acd4f434fd794054f Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 10:15:29 -0700 Subject: [PATCH 07/10] =?UTF-8?q?fix(gfql/searchAny):=20cuDF=20explicit-co?= =?UTF-8?q?lumn=20dtype=20gate=20=E2=80=94=20float/temporal=20stringify=20?= =?UTF-8?q?diverges=20from=20pandas=20(dgx-probed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Probe (rapids 26.02): cudf.Series.astype(str) renders 0.1+0.2 as '0.3' (pandas '0.30000000000000004'), 1e16 as '1.0e+16' (pandas '1e+16'), and truncates long mantissas ('123456789.1') — the explicit-columns float path silently diverged from the pandas oracle. cuDF now declines explicit non-string/int/bool columns (NIE), symmetric with the polars gate; string/int/bool stay native (bool parity pinned). Kernel pin added; docs/CHANGELOG updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- docs/source/gfql/cypher.rst | 4 ++-- graphistry/compute/gfql/call/executor.py | 14 +++++++++----- graphistry/compute/gfql/search_any.py | 17 +++++++++++++++++ .../test_engine_polars_conformance_matrix.py | 7 +++++++ 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22ce48a00a..933c020029 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added -- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (case-folded, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). Honest polars declines (NIE, use engine='pandas'): edge-alias searchAny, and explicit columns beyond string/int/bool dtypes (incl. floats: repr diverges in the exponent regime). +- **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (case-folded, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). Honest declines (NIE, use engine='pandas'): edge-alias searchAny on polars, and explicit columns beyond string/int/bool dtypes on polars AND cuDF (incl. floats: repr diverges across engines — dgx-probed). - **GFQL Cypher `EXISTS { }` pattern-existence subqueries (openCypher-standard), native on all four engines**: `WHERE EXISTS { (n)-[:R]->() }` and `WHERE NOT EXISTS { (n)--() }` now parse and run — the declarative prune-isolated building blocks for the streamgl-viz filter pipeline. An `EXISTS` body reuses the existing pattern-predicate lowering wholesale (`semi_apply_mark` / `anti_semi_apply` row ops), so pandas/cuDF worked immediately; the polars engine gains NATIVE lowerings for the semi-apply family (correlated key sets computed by the polars chain executor's named-flag columns; order-preserving `is_in` joins) plus `rows(binding_ops=...)` for the single-entity row table — previously all honest-NIE. Aliases introduced inside the braces are existentially scoped (`EXISTS { (n)--(m) }` allowed with `m` unbound outside — bare pattern predicates keep the conservative guard), inline property maps work, and the one supported inner `WHERE` form is endpoint inequality — `EXISTS { (n)--(m) WHERE m <> n }`, the drop-self-loop prune-isolated flavor (pandas/cuDF filter the correlated bindings; polars excludes self-loop edges, which is exactly the `m <> n` witness). Both prune flavors are oracle-pinned in the conformance matrix on a self-loop discriminator graph, 4-engine parity-or-NIE. Honest declines with clear errors: `EXISTS` in RETURN/WITH projections, general inner `WHERE`, multi-pattern bodies, full `MATCH..RETURN` subquery bodies, multi-alias correlation on polars. - **GFQL polars execution config is Python-settable and live**: `set_cpu_streaming(bool)` and `set_gpu_executor('in-memory'|'streaming')` in `graphistry.compute.gfql.lazy` (plus the public `GPU_EXECUTORS` options and `GpuExecutor` type) set the CPU-streaming / GPU-executor knobs from Python. They resolve **Python override > environment variable > default**, read **live** per collect — previously these were env-only (`GFQL_POLARS_CPU_STREAMING` / `GFQL_POLARS_GPU_EXECUTOR`) and frozen at import, so neither a Python setting nor a post-import env change took effect. `None` resets a setter to env/default. - **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). diff --git a/docs/source/gfql/cypher.rst b/docs/source/gfql/cypher.rst index 235b450a6a..9bbd28229b 100644 --- a/docs/source/gfql/cypher.rst +++ b/docs/source/gfql/cypher.rst @@ -326,8 +326,8 @@ and ``RETURN`` expressions: terms. Runs natively on all four engines for node aliases; an edge-alias ``searchAny(r, ...)`` declines honestly on polars pending multi-entity binding-row support (use ``engine='pandas'``), and explicit non-string - columns beyond ints/bools likewise decline on polars rather than risk - divergent stringification (float repr differs in the exponent regime). The regex path obeys the same + columns beyond ints/bools likewise decline on polars and cuDF rather than + risk divergent stringification (float repr differs across engines). The regex path obeys the same per-engine decline rules as ``=~``. Python twins: :meth:`ComputeMixin.search_nodes` / :meth:`ComputeMixin.search_edges`. diff --git a/graphistry/compute/gfql/call/executor.py b/graphistry/compute/gfql/call/executor.py index eb6b0aa0bf..1b0b1a25c8 100644 --- a/graphistry/compute/gfql/call/executor.py +++ b/graphistry/compute/gfql/call/executor.py @@ -303,12 +303,16 @@ def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: En ) from error if isinstance(error, GFQLTypeError): raise error - if isinstance(error, NotImplementedError) and engine in (Engine.POLARS, Engine.POLARS_GPU): + if isinstance(error, NotImplementedError) and ( + engine in (Engine.POLARS, Engine.POLARS_GPU) or is_row_pipeline_call(function) + ): # Honest engine-capability decline — propagate as-is so the DAG surface matches the chain - # surface's NotImplementedError. Sources: call_mode='strict' off-engine decline, and the - # polars-gpu GPU-or-error when cuDF is unavailable (_bridge_graph_for_offengine_call). - # Gated to the polars engines: a pandas/cudf NIE (e.g. fa2_layout requiring a GPU) must - # still fall through to the GFQLTypeError(E303) wrapper below, not leak as a bare NIE. + # surface's NotImplementedError. Sources: call_mode='strict' off-engine decline, the + # polars-gpu GPU-or-error when cuDF is unavailable (_bridge_graph_for_offengine_call), + # and row-pipeline kernels' per-engine declines on ANY engine (searchAny's cuDF + # regex/dtype gates — the parity-or-NIE contract needs them to SURFACE as NIE). + # Non-row-pipeline pandas/cudf NIEs (e.g. fa2_layout requiring a GPU) still fall + # through to the GFQLTypeError(E303) wrapper below, not leak as a bare NIE. raise error if error is not None: raise GFQLTypeError( diff --git a/graphistry/compute/gfql/search_any.py b/graphistry/compute/gfql/search_any.py index 94e0614ec7..638044d4da 100644 --- a/graphistry/compute/gfql/search_any.py +++ b/graphistry/compute/gfql/search_any.py @@ -59,6 +59,23 @@ def search_any_mask( cols = search_candidate_columns(df, term, columns) if cols is None: return None + if columns is not None and "cudf" in type(df).__module__: + # Explicit columns= reaches beyond the auto gate; cuDF's astype(str) float + # rendering DIVERGES from pandas (dgx-probed: 0.1+0.2 -> '0.3' vs + # '0.30000000000000004'; 1e16 -> '1.0e+16' vs '1e+16'; long mantissas + # truncate) and temporal is unverified — decline honestly rather than + # silently mismatch the pandas oracle (wave-3 W3-1). string/int/bool render + # identically (bool parity is pinned) and stay native. + import pandas.api.types as pat + for c in cols: + dt = df[c].dtype + if not (_is_searchable_string_dtype(dt) or _is_int_dtype(dt) + or bool(pat.is_bool_dtype(dt))): + raise NotImplementedError( + "cuDF searchAny explicit columns support string/int/bool dtypes " + "only (float/temporal stringification diverges from pandas); " + "use engine='pandas'" + ) if not cols or len(df) == 0: if len(df.columns) == 0: return None diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index 57ee8ce604..65be30cd06 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -518,6 +518,13 @@ def test_search_any_cudf_regex_guards(): assert got.to_pandas().tolist() == [True, False] got_cs = search_any_mask(df, r"\d{4}", regex=True, case_sensitive=True) assert got_cs.to_pandas().tolist() == [False, True] + # explicit FLOAT column on cuDF: astype(str) rendering diverges from pandas + # (dgx-probed: 0.1+0.2 -> '0.3', 1e16 -> '1.0e+16', mantissa truncation) — + # honest NIE, never a silent oracle mismatch (wave-3 W3-1); string stays native + dff = cudf.DataFrame({"f": [0.1 + 0.2, 7.25], "s": ["x", "y"]}) + with pytest.raises(NotImplementedError): + search_any_mask(dff, "0.3", columns=["f"]) + assert search_any_mask(dff, "x", columns=["s"]).to_pandas().tolist() == [True, False] def test_search_any_null_and_float_stringify(): From b569fbf8b9cd92c660e85c7cdaf6f2e070bbb317 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 10:33:15 -0700 Subject: [PATCH 08/10] =?UTF-8?q?docs(gfql/searchAny):=20twin=20+=20kernel?= =?UTF-8?q?=20docstrings=20=E2=80=94=20cuDF=20explicit=20float/date=20decl?= =?UTF-8?q?ine=20(wave-4=20S-w4a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/ComputeMixin.py | 6 ++++-- graphistry/compute/gfql/search_any.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 6ac7365dc4..03b135f122 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -491,8 +491,10 @@ def search_nodes(self, term, columns=None, case_sensitive=False, regex=False): """Keep nodes where ANY column matches ``term`` (viz-filter L2 inspector semantics: OR across columns; case-insensitive substring default; regex opt-in; string columns always, integer columns iff the term is a numeric - literal — floats/dates via explicit ``columns=``). pandas/cuDF native; - polars frames raise NotImplementedError (use the cypher ``search_any`` op). + literal — floats/dates via explicit ``columns=`` on pandas ONLY: cuDF + declines them, its float/temporal stringification diverges from pandas). + pandas/cuDF native; polars frames raise NotImplementedError (use the + cypher ``search_any`` op). """ from graphistry.compute.gfql.search_any import search_any_mask from graphistry.compute.exceptions import ErrorCode, GFQLValidationError diff --git a/graphistry/compute/gfql/search_any.py b/graphistry/compute/gfql/search_any.py index 638044d4da..6ae8eccbb2 100644 --- a/graphistry/compute/gfql/search_any.py +++ b/graphistry/compute/gfql/search_any.py @@ -1,9 +1,12 @@ """Cross-column search kernel (viz-filter L2, panel-algebra D2): OR-across-columns substring/regex match, dtype-gated AS SEMANTICS — string columns always; integer columns iff the term is a numeric literal (inspector gate); float/date/bool are -auto-gated OUT (float stringification is engine-divergent — reach them via explicit -``columns=``). Per-column matching delegates to the parity-hardened ``Contains`` -predicate, so every pandas/cuDF quirk and honest decline gate carries over.""" +auto-gated OUT (float stringification is engine-divergent — explicit ``columns=`` +reaches bool on both engines, float/date on pandas only: cuDF declines them, its +astype(str) rendering diverges from pandas — dgx-probed). Per-column matching +delegates to the parity-hardened ``Contains`` predicate, so every pandas/cuDF +quirk and honest decline gate carries over; cuDF regex obeys the same decline +rules as ``=~``.""" import re from typing import Any, List, Optional From 898f112f82984b28e26a9b7b72039d42abeeb54e Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 15:38:22 -0700 Subject: [PATCH 09/10] =?UTF-8?q?type(gfql/searchAny):=20precise=20cross-e?= =?UTF-8?q?ngine=20types=20in=20the=20kernel=20=E2=80=94=20DataFrameT/Seri?= =?UTF-8?q?esT=20(no=20Any)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the codebase's cross-engine aliases (compute/typing.py, as predicates/str.py already does): frames are DataFrameT, masks/series are SeriesT; dtype params are "object" (stricter than Any — only passed to the pandas dtype API). Annotation- only; mypy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/gfql/search_any.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/graphistry/compute/gfql/search_any.py b/graphistry/compute/gfql/search_any.py index 6ae8eccbb2..045dc4a0c2 100644 --- a/graphistry/compute/gfql/search_any.py +++ b/graphistry/compute/gfql/search_any.py @@ -8,7 +8,9 @@ quirk and honest decline gate carries over; cuDF regex obeys the same decline rules as ``=~``.""" import re -from typing import Any, List, Optional +from typing import List, Optional + +from graphistry.compute.typing import DataFrameT, SeriesT # inspector's numeric-term gate (streamgl-viz sortAndFilterRowsByQuery.js) _NUMERIC_TERM_RE = re.compile(r"^[0-9.\-]+$") @@ -18,18 +20,18 @@ def is_numeric_term(term: str) -> bool: return bool(_NUMERIC_TERM_RE.match(term)) -def _is_searchable_string_dtype(dtype: Any) -> bool: +def _is_searchable_string_dtype(dtype: object) -> bool: import pandas.api.types as pat # cuDF mirrors the pandas dtype API return bool(pat.is_string_dtype(dtype)) or bool(pat.is_object_dtype(dtype)) -def _is_int_dtype(dtype: Any) -> bool: +def _is_int_dtype(dtype: object) -> bool: import pandas.api.types as pat return bool(pat.is_integer_dtype(dtype)) def search_candidate_columns( - df: Any, term: str, columns: Optional[List[str]] + df: DataFrameT, term: str, columns: Optional[List[str]] ) -> Optional[List[str]]: """Columns to search: the explicit list (None if any is missing — caller declines loudly) or the auto dtype gate.""" @@ -47,13 +49,13 @@ def search_candidate_columns( def search_any_mask( - df: Any, + df: DataFrameT, term: str, *, case_sensitive: bool = False, regex: bool = False, columns: Optional[List[str]] = None, -) -> Optional[Any]: +) -> Optional[SeriesT]: """Boolean row mask over ``df`` (pandas or cuDF), or None to decline (an explicit column is missing). Null cells never match; no candidate columns -> all-False.""" from graphistry.compute.predicates.str import ( @@ -92,10 +94,10 @@ def search_any_mask( if not case: pat = _cudf_casefold_or_decline(pat) # pre-folded; Contains' .lower() is a no-op pred = Contains(pat, case=case, regex=regex, na=False) - mask: Optional[Any] = None + mask: Optional[SeriesT] = None for c in cols: s = df[c] - m: Any + m: SeriesT if not _is_searchable_string_dtype(s.dtype): # canonical toString for int / explicit columns; pandas astype(str) # stringifies nulls ("nan"/"") so mask them back out — null cells From 9388be7f70740611d18ef44db86d50e871caa3e9 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 5 Jul 2026 15:58:43 -0700 Subject: [PATCH 10/10] fix(gfql/searchAny): skip nested/object-non-string columns in the auto gate (inspector 2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match the streamgl-viz inspector (sortAndFilterRowsByQuery.js shouldSearch): the auto gate only fires on string dtype. A pandas OBJECT column can hold lists/dicts; we included it as 'string' then Contains-over-lists silently never-matched (+ a fillna FutureWarning). Now _has_string_content inspects object columns via infer_dtype and skips non-string content — clean skip, not silent no-match. Real string / StringDtype / cuDF-str columns still searched; polars/cudf already used typed columns (no object ambiguity). Pandas-path fix; pin verifies pandas skip + polars parity. Reference-parity spec: research/searchany-inspector-parity.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphistry/compute/gfql/search_any.py | 26 ++++++++++++++++-- .../test_engine_polars_conformance_matrix.py | 27 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/graphistry/compute/gfql/search_any.py b/graphistry/compute/gfql/search_any.py index 045dc4a0c2..04977bfb46 100644 --- a/graphistry/compute/gfql/search_any.py +++ b/graphistry/compute/gfql/search_any.py @@ -30,18 +30,40 @@ def _is_int_dtype(dtype: object) -> bool: return bool(pat.is_integer_dtype(dtype)) +def _has_string_content(df: DataFrameT, c: object) -> bool: + """True iff column ``c`` holds STRINGS (a real string dtype, or an object column whose + values are actually strings). An object column of lists/dicts/mixed is NOT string + content — the streamgl-viz inspector skips such columns (``shouldSearch`` only fires on + ``dataType === 'string'``), so the auto gate skips them too rather than include-then- + silently-never-match on a ``Contains``-over-lists path (viz-filter searchAny 2a).""" + import pandas.api.types as pat + s = df[c] + is_cudf = "cudf" in type(s).__module__ + # The list/dict ambiguity is PANDAS-only (numpy `object` can hold arbitrary python + # objects); cuDF/polars use typed columns (a list is a typed List, not object), and + # infer_dtype does NOT accept a cuDF Series — so only inspect contents for numpy-object. + if not is_cudf and s.dtype == object: + try: + return pat.infer_dtype(s, skipna=True) in ("string", "empty") + except Exception: + return False + return bool(pat.is_string_dtype(s.dtype)) # StringDtype / cuDF str; cuDF List -> False + + def search_candidate_columns( df: DataFrameT, term: str, columns: Optional[List[str]] ) -> Optional[List[str]]: """Columns to search: the explicit list (None if any is missing — caller declines - loudly) or the auto dtype gate.""" + loudly) or the auto dtype gate (mirrors the streamgl-viz inspector's ``shouldSearch``: + string cols always; integer cols iff the term is a numeric literal; nested/bool/other + skipped — see research/searchany-inspector-parity.md).""" if columns is not None: return list(columns) if all(c in df.columns for c in columns) else None numeric_ok = is_numeric_term(term) out: List[str] = [] for c in df.columns: dt = df[c].dtype - if _is_searchable_string_dtype(dt): + if _has_string_content(df, c): out.append(c) elif numeric_ok and _is_int_dtype(dt): out.append(c) diff --git a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py index 65be30cd06..204c95e145 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py @@ -609,6 +609,33 @@ def test_search_any_edge_alias(): _assert_invariant(g, q, "searchAny-edge-alias") +def test_search_any_nested_object_column_skipped(): + """viz-filter searchAny 2a (inspector parity): a list/object-non-string column is + SKIPPED by the auto gate (the streamgl-viz inspector's shouldSearch only fires on string + dtype), not included-then-silently-never-matched. This is fundamentally a PANDAS-path fix + (object-dtype is ambiguous — it can hold lists); polars/cudf use typed List columns and + never had the bug. Verify the pandas skip + polars parity directly (a pandas list column + does not round-trip to cudf — an orthogonal representation limit, not searchAny's).""" + nd = pd.DataFrame({ + "id": [0, 1, 2], + "name": ["xx", "yy", "zz"], # real string col; carries NO tag token + "tags": [["a", "acme"], ["b"], ["acme"]], # object/list col — must be SKIPPED + }) + ed = pd.DataFrame({"s": [0], "d": [1], "eid": [0]}) + g = graphistry.nodes(nd, "id").edges(ed, "s", "d").bind(edge="eid") + # 'acme' exists ONLY inside the list column: correct = zero matches (tags skipped); a + # regression that searched tags would return ids 0,2. + q = "MATCH (a) WHERE searchAny(a, 'acme') RETURN a.id AS id" + assert _to_pd(g.gfql(q, engine="pandas")._nodes)["id"].tolist() == [], "list col must be skipped" + if "polars" in _NONPANDAS_ENGINES: # polars typed-List column: also skips → same empty result + assert _run(g, q, "polars") == _run(g, q, "pandas"), "polars nested-skip divergence" + # sanity: the real string column is still searched (list column present, unaffected) + q2 = "MATCH (a) WHERE searchAny(a, 'yy') RETURN a.id AS id" + assert _to_pd(g.gfql(q2, engine="pandas")._nodes)["id"].tolist() == [1] + if "polars" in _NONPANDAS_ENGINES: + assert _run(g, q2, "polars") == _run(g, q2, "pandas"), "polars string-search divergence" + + def test_exists_polars_internal_decline_branches(): """Coverage pins for the honest-decline branches of the polars semi-apply family (each returns None -> the boundary lane raises NIE): non-single/unnamed/query=/