From c2138b0d35826baa43b248553bff51e7c652032c Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Thu, 16 Jul 2026 05:51:43 -0700 Subject: [PATCH] feat(gfql): add aggregate fast paths and final hardening --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 4 +- bin/ci_cypher_surface_guard_baseline.json | 2 +- bin/coverage_audit.py | 17 +- bin/test-polars.sh | 15 +- graphistry/Engine.py | 16 +- graphistry/compute/gfql/cypher/lowering.py | 28 +- graphistry/compute/gfql/index/__init__.py | 7 +- graphistry/compute/gfql/index/api.py | 14 +- graphistry/compute/gfql/index/cost.py | 40 -- graphistry/compute/gfql/index/types.py | 1 - .../compute/gfql/lazy/engine/polars/chain.py | 17 +- .../gfql/lazy/engine/polars/row_pipeline.py | 84 ++- graphistry/compute/gfql/row/frame_ops.py | 3 +- graphistry/compute/gfql/row/prefilter.py | 20 +- graphistry/compute/gfql_unified.py | 619 +++++++++++++++++- .../gfql/cypher/test_compiler_policy.py | 72 +- .../compute/gfql/cypher/test_lowering.py | 456 ++++++++++++- .../compute/gfql/cypher/test_row_pushdown.py | 34 +- .../tests/compute/gfql/index/test_index.py | 91 +-- .../gfql/test_engine_polars_binding_rows.py | 75 +++ .../tests/compute/test_engine_coercion.py | 42 -- 22 files changed, 1318 insertions(+), 341 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b95768bc2..43a5e955b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1193,7 +1193,7 @@ jobs: cat build/gfql-coverage-audit/gfql-coverage-audit.md >> "$GITHUB_STEP_SUMMARY" - name: Upload GFQL coverage audit - if: ${{ matrix.python-version == '3.12' }} + if: ${{ always() && matrix.python-version == '3.12' }} uses: actions/upload-artifact@v4 with: name: gfql-coverage-audit-py3.12 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a7d6c1e3c..cc55e0e5a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine-selection docs (pandas / polars / cuDF / polars-gpu)**: New :doc:`Choosing a GFQL Engine ` page — a numbers-first, persona-tested guide to the four interchangeable engines. Adds the one-keyword `engine='polars'` speedup (up to ~38× over pandas on real graphs, no GPU), a motivating warm-median comparison table on real public graphs (LiveJournal 35M / Orkut 117M), a decision matrix (workload shape × size × hardware → engine, with the measured ~10K-edge CPU crossover, GPU-work-bound rule, polars-gpu memory-pressure caveat, and GPU-or-error contract), a cuDF-vs-polars-gpu disambiguation (eager-op vs fused-lazy; cuDF is not deprecated), an honest "when *not* to use Polars" section, the differential-parity guarantee, and a methodology + reproducer-script disclosure. Rewrote the top of `gfql/performance.rst` to lead with the engine comparison (de-marketed the prose), wired the new page into the GFQL toctree + recommended paths, and added Polars/polars-gpu to the engine examples in `gfql/quick.rst` and `gfql/about.rst` (previously only pandas/cuDF were documented). Driven by 4-persona doc user-testing (pandas DS, RAPIDS user, perf engineer, skeptical evaluator). ### Added -- **GFQL Cypher row-pipeline single-alias predicate pushdown**: Planner-generated `rows(alias_prefilters=...)` hints pre-filter eligible node and single-hop edge alias frames before binding joins on pandas/cuDF while retaining the original post-join filters for exact semantics and engine-safe fallback. - **GFQL viz-filter-pipeline acceptance suite + regression benchmark (viz-filter L3)**: `test_viz_pipeline_conformance.py` — curated full-panel pipelines (node+edge filters, exclusion-dominates composition, `(pred OR IS NULL)` keep-null leaves, both EXISTS prune-isolated flavors, searchAny composition, deterministic paging), graph-state prune shapes with exact node+edge pins, a 40-seed panel-state fuzzer with an independent plain-pandas second oracle, and a case/regex/unicode trick matrix (ß/İ full-case-mapping pins, metachar literal-vs-regex, null cells, Categorical) — all parity-or-NIE across pandas/cuDF/polars/polars-gpu. `benchmarks/gfql/viz_filter_pipeline.py` — six streamgl-viz panel scenarios (filters, keep-self GRAPH prune, EXISTS prune, node/edge search, combined) at 100K/1M/10M with native-frame-per-engine fairness, an NIE-tolerant matrix, and JSON receipts (first receipt: 100k × 4 engines, everything within the ~350ms interactive reference except pandas combined). Documented findings from the first runs: the same-path WHERE route dedupes parallel edges (diverges from the panel algebra's edge multiplicity — pinned + tracked), and edge-alias searchAny declines on polars (tracked). - **GFQL Cypher `searchAny(entity, term[, opts])` cross-column search predicate + `g.search_nodes()`/`g.search_edges()` (viz-filter L2, native on all four engines)**: True where ANY of the entity's columns matches the term — the streamgl-viz inspector's table-search semantics as a composable WHERE predicate: OR across columns; case-insensitive substring by DEFAULT (case-folded, never regex — the common call avoids every engine regex limit); regex opt-in obeying the same per-engine decline rules as `=~`; dtype gate AS SEMANTICS (string columns always; integer columns iff the term is a numeric literal, per the inspector's `/^[0-9.-]+$/` gate; floats/dates/booleans reachable via the explicit `columns:` list). Options map `{caseSensitive, regex, columns}` is strict-validated (unknown keys error listing the valid ones); unbound aliases and missing explicit columns error clearly; null cells never match. Lowered like the pattern-predicate markers (a `search_any` row op + fresh marker column), so it composes through AND/OR/NOT and different node/edge terms coexist in one pipeline. Per-column matching reuses the parity-hardened `Contains` predicate on pandas/cuDF and a lowercase-fold/any_horizontal lowering on polars; oracle-pinned + 4-engine parity-or-NIE conformance cases. Python twins `g.search_nodes(term, columns=, case_sensitive=, regex=)` / `g.search_edges(...)` filter their own table and return a Plottable (polars-frame twins decline honestly for now — use the cypher op). Honest declines (NIE, use engine='pandas'): edge-alias searchAny on polars, and explicit columns beyond string/int/bool dtypes on polars AND cuDF (incl. floats: repr diverges across engines — dgx-probed). - **GFQL Cypher `EXISTS { }` pattern-existence subqueries (openCypher-standard), native on all four engines**: `WHERE EXISTS { (n)-[:R]->() }` and `WHERE NOT EXISTS { (n)--() }` now parse and run — the declarative prune-isolated building blocks for the streamgl-viz filter pipeline. An `EXISTS` body reuses the existing pattern-predicate lowering wholesale (`semi_apply_mark` / `anti_semi_apply` row ops), so pandas/cuDF worked immediately; the polars engine gains NATIVE lowerings for the semi-apply family (correlated key sets computed by the polars chain executor's named-flag columns; order-preserving `is_in` joins) plus `rows(binding_ops=...)` for the single-entity row table — previously all honest-NIE. Aliases introduced inside the braces are existentially scoped (`EXISTS { (n)--(m) }` allowed with `m` unbound outside — bare pattern predicates keep the conservative guard), inline property maps work, and the one supported inner `WHERE` form is endpoint inequality — `EXISTS { (n)--(m) WHERE m <> n }`, the drop-self-loop prune-isolated flavor (pandas/cuDF filter the correlated bindings; polars excludes self-loop edges, which is exactly the `m <> n` witness). Both prune flavors are oracle-pinned in the conformance matrix on a self-loop discriminator graph, 4-engine parity-or-NIE. Honest declines with clear errors: `EXISTS` in RETURN/WITH projections, general inner `WHERE`, multi-pattern bodies, full `MATCH..RETURN` subquery bodies, multi-alias correlation on polars. @@ -41,9 +40,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL native Polars bindings-row tables (`rows(binding_ops)`) — traversal Cypher on polars (#1709)**: the Cypher multi-alias lowering's `rows(binding_ops=...)` op (one row per matched path) now runs natively on `engine='polars'` for **fixed-length connected patterns** — unblocking traversal-shaped Cypher that previously NIE'd: multi-alias property projections (`MATCH (a)-[e]->(b) RETURN a.x, e.w, b.y`), top-k in-degree (`RETURN b.id, count(a) ... ORDER BY ... LIMIT`, graph-benchmark q1/q2), and fixed multi-hop counts (`MATCH (a)-->(b)-->(c) RETURN count(*)`, q8/q9), with forward/reverse/undirected edges, node/edge filters, and edge-alias payload columns — **plus bounded directed variable-length segments** (`-[*i..k]->`, typed `-[:TYPE*i..k]->`, exactly-k; iterative pair joins with Cypher path multiplicity and zero-hop rows), covering the q3 shape (`MATCH (a:Person)-[:FOLLOWS*1..2]->(b:Person) ... RETURN avg(b.age)`). With this, **all graph-benchmark q1–q9 shapes run as real GFQL Cypher on pandas, cuDF, and polars**. Also adds native `with_(extend=True)` (emitted by the bindings-path aggregate lowering) and an honest decline for `group_by(key_prefixes=...)` (whole-row bindings grouping — was a latent silent-wrong-key trap). **Honestly deferred** (`NotImplementedError`, no pandas bridge): unbounded `[*]`, undirected/aliased variable-length, shortestPath scalar bindings, node/edge `query=`/endpoint-match params, cartesian (`MATCH (a),(b)`) mode, seeded re-entry contexts. Differential parity vs the pandas oracle (+20 tests incl. path multiplicity and undirected self-loops; conformance corpus extended). **Perf @500k nodes/2M edges (CPU):** q1 top-k in-degree **17×** vs pandas (2559→150ms), seeded 2-hop count 7.3×, seeded 1-hop projection 13×. polars-gpu runs the same code path (eager joins; GPU verification queued on the shared GPU box). ### Fixed -- **GFQL seeded indexes now engage for small Polars/cuDF frontiers**: NaN-free native Polars frames retain object identity during normalization, preserving resident CSR validity, while a tunable absolute floor prevents the fractional cost gate from routing tiny seeded hops over low-cardinality slices to a full scan. Scan/index result parity is unchanged; the default floor is a routing heuristic whose engine/data crossover remains subject to measurement. - **GFQL Cypher `GRAPH { }` residual predicates now fail safely or apply as graph masks**: `GRAPH { MATCH ... WHERE ... }` no longer silently drops predicates that the graph-state path cannot apply. Safe one-node/one-edge residual filters, including disjunctions and `searchAny(...)`, are applied before graph-state matching; unsupported pattern-predicate, multi-alias, and Polars graph-residual cases now raise clear validation errors instead of returning an over-broad subgraph. -- **GFQL Cypher `=~` / scalar-fn cross-engine hardening (review wave, dgx-verified)**: (1) composed `=~` (`WHERE … =~ … OR …`, `RETURN`-expression position) now works on `engine='cudf'` — the series evaluator used raw `.str.fullmatch`, which cuDF lacks, and the resulting `AttributeError` was masked as "unsupported predicate op" (it now routes through the `Fullmatch` predicate's engine workarounds, and honest `NotImplementedError` declines pass through instead of being re-labeled); (2) the cuDF fullmatch emulation anchors alternations as a whole (`^(ab|cd)$` — bare `^ab|cd$` silently matched `'abXXX'`); (3) the cuDF case-insensitive `(?i)` lowercase-folding workaround now declines the fold-unsound shapes — uppercase escape classes (`.lower()` turns `\D` into `\d`, silently inverting the predicate), case-crossing character ranges (`(?i)[A-z]` silently narrowed; `[X-b]` folded to an invalid range), and non-ASCII patterns — while lowercase escapes (`\d`, `\.`) keep folding as before; lookaround, backreferences, and named-group refs decline up front (libcudf rejects them at kernel-compile time); (4) polars `Match`/`Fullmatch` lowering applies the same Rust-regex guard as `Contains` (lookaround/backrefs decline instead of a non-NIE `ComputeError` at collect); (5) `toLower`/`toUpper` on a non-string column decline like neo4j's type error instead of broadcasting the stringified Series repr (pandas/cuDF) or raising a non-NIE `SchemaError` (polars-gpu); (6) polars `floor`/`ceil`/`round` cast to `Float64` so integer columns return Float like neo4j and the pandas engine; (7) invalid regex patterns on the composed path raise a clear "invalid regex pattern" error instead of "unsupported predicate op". +- **GFQL Cypher `=~` / scalar-fn cross-engine hardening (review wave, dgx-verified)**: (1) composed `=~` (`WHERE … =~ … OR …`, `RETURN`-expression position) now works on `engine='cudf'` — the series evaluator used raw `.str.fullmatch`, which cuDF lacks, and the resulting `AttributeError` was masked as "unsupported predicate op" (it now routes through the `Fullmatch` predicate's engine workarounds, and honest `NotImplementedError` declines pass through instead of being re-labeled); (2) the cuDF fullmatch emulation anchors alternations as a whole (`^(ab|cd)$` — bare `^ab|cd$` silently matched `'abXXX'`); (3) the cuDF case-insensitive `(?i)` lowercase-folding workaround now declines the fold-unsound shapes — uppercase escape classes (`.lower()` turns `\D` into `\d`, silently inverting the predicate), case-crossing character ranges (`(?i)[A-z]` silently narrowed; `[X-b]` folded to an invalid range), and non-ASCII patterns — while lowercase escapes (`\d`, `\.`) keep folding as before; lookaround, backreferences, and named-group refs decline up front (libcudf rejects them at kernel-compile time); (4) polars `Match`/`Fullmatch` lowering applies the same Rust-regex guard as `Contains` (lookaround/backrefs decline instead of a non-NIE `ComputeError` at collect); (5) `toLower`/`toUpper` on a non-string column decline with a Cypher-standard type error instead of broadcasting the stringified Series repr (pandas/cuDF) or raising a non-NIE `SchemaError` (polars-gpu); (6) polars `floor`/`ceil`/`round` cast to `Float64` so integer columns return Float per Cypher semantics, matching the pandas engine; (7) invalid regex patterns on the composed path raise a clear "invalid regex pattern" error instead of "unsupported predicate op". - **GFQL Cypher `round()` hardening (review wave, dgx-verified)**: (1) the `polars` extra's floor is now `polars>=1.29` — `Expr.round(mode=)` shipped in py-1.29.0 (pola-rs/polars#22248), not 1.5 as previously pinned, so 1.5–1.28 installs crashed with a raw `TypeError` on `round(x, p>0)` under `engine='polars'`; (2) `round(x, p>308)` is the identity on both engines (a float64 has no digits there) instead of pandas raising through an unclear decline while polars returned identity — parity restored, `10.0**p` overflow guarded; (3) polars `round(x, p>0)` normalizes `-0.0` to `+0.0` like the pandas kernel (`round(-0.04, 1)` was `0.0` on pandas vs `-0.0` on polars — invisible to value equality, pinned by a sign-bit test); (4) documented the precision>0 decimal-string deviation vs neo4j (`round(2.675, 2)` = `2.67` binary-double here vs `2.68` BigDecimal there) and added deterministic tie/hazard matrix cases so ties actually reach cuDF/polars-gpu (the fixture's normal-distribution floats never tied). - **GFQL `materialize_nodes()` on an edges-only graph under `engine='polars'`**: crashed with `AttributeError: 'Series' object has no attribute 'drop_duplicates'` — the node-id derivation used pandas-only `.drop_duplicates()` / `.reset_index()` / `.rename(columns=)` on a polars Series/DataFrame. Now polars-aware (`.unique(maintain_order=True)`, `.select().rename({...})`), matching the pandas oracle. Surfaced by any `let()` DAG on an edges-only graph under Polars (the DAG pre-materializes nodes). pandas/cuDF paths unchanged. - **CI: `compute_networkx` HITS test was scipy-version flaky**: `test_compute_networkx_hits_outputs_hubs_and_authorities` used a pure directed 3-cycle (`a→b→c→a`), whose adjacency is a permutation matrix (all singular values equal). networkx HITS computes scores via scipy `svds(k=1)`, which then returns an arbitrary vector from that fully-degenerate singular space; when its components sum to ~0, HITS's `h /= h.sum()` normalization blows up to `inf`/`nan`. Which vector `svds` returns is scipy/LAPACK-version dependent, so the test passed on the pinned scipy but failed on the `nx-upper-scipy` CI profile's newer scipy — a latent flake, not a code regression. Switched to a graph with a well-defined hub/authority structure (`a→b`, `a→c`, `b→c`), whose dominant singular vector is unique (Perron-Frobenius) and therefore version-stable. Test-only change.- **GFQL `materialize_nodes()` on an edges-only graph under `engine='polars'`**: crashed with `AttributeError: 'Series' object has no attribute 'drop_duplicates'` — the node-id derivation used pandas-only `.drop_duplicates()` / `.reset_index()` / `.rename(columns=)` on a polars Series/DataFrame. Now polars-aware (`.unique(maintain_order=True)`, `.select().rename({...})`), matching the pandas oracle. Surfaced by any `let()` DAG on an edges-only graph under Polars (the DAG pre-materializes nodes). pandas/cuDF paths unchanged.- **GFQL `engine='polars-gpu'` LazyFrame-input coercion now collects on the GPU executor**: `Engine.df_to_engine(lazyframe, POLARS_GPU)` materialized a `polars.LazyFrame` input with a bare `.collect()` on the CPU default executor — ignoring `gpu_executor()` and not distinguishing `POLARS` from `POLARS_GPU`. It now routes through the target-aware lazy collect, so a LazyFrame coerced under `polars-gpu` collects on the cudf-polars GPU executor (and `polars` honors `cpu_streaming()`). No change for already-materialized frame inputs (the common path). diff --git a/bin/ci_cypher_surface_guard_baseline.json b/bin/ci_cypher_surface_guard_baseline.json index bbf8dbb481..393b035ad3 100644 --- a/bin/ci_cypher_surface_guard_baseline.json +++ b/bin/ci_cypher_surface_guard_baseline.json @@ -13,5 +13,5 @@ "max_properties": 0 } }, - "lowering_py_max_lines": 8511 + "lowering_py_max_lines": 9125 } diff --git a/bin/coverage_audit.py b/bin/coverage_audit.py index 980e8fdd6f..8a5db836d2 100755 --- a/bin/coverage_audit.py +++ b/bin/coverage_audit.py @@ -614,8 +614,23 @@ def main(argv: Optional[Sequence[str]] = None) -> int: print(f"Wrote {json_path}") if exit_code: print(f"pytest failed with exit code {exit_code}", file=sys.stderr) - if any(check.status == "fail" for check in baseline_checks): + failing_baseline_checks = [check for check in baseline_checks if check.status == "fail"] + if failing_baseline_checks: print("per-file coverage baseline failed", file=sys.stderr) + for check in failing_baseline_checks: + actual = "missing" if check.actual_percent is None else f"{check.actual_percent:.2f}%" + delta = "n/a" if check.delta_percent is None else f"{check.delta_percent:+.2f}%" + print( + " {path}: actual={actual} floor={floor:.2f}% tolerance={tolerance:.2f}% delta={delta} reason={reason}".format( + path=check.path, + actual=actual, + floor=check.min_percent, + tolerance=check.tolerance_percent, + delta=delta, + reason=check.reason, + ), + file=sys.stderr, + ) return exit_code or 3 return exit_code diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 6d7965cee6..a6a6f5b936 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -3,7 +3,7 @@ set -ex # Run from project root # - Extra args are passed through to the pytest phase -# - Set POLARS_COV=1 to collect coverage over the graphistry package; the coverage +# - Set POLARS_COV=1 to collect coverage over graphistry/compute; the coverage # data file location is taken from $COVERAGE_FILE (as the CI py3.12 lane sets it) # - Non-zero exit code on fail @@ -17,25 +17,18 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/test_engine_polars_hop.py graphistry/tests/compute/gfql/test_engine_polars_chain.py graphistry/tests/compute/gfql/test_engine_polars_row_pipeline.py + graphistry/tests/compute/gfql/test_engine_polars_binding_rows.py graphistry/tests/compute/gfql/test_engine_polars_cypher_conformance.py graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py graphistry/tests/compute/gfql/test_conformance_ledger.py # index tests exercise the seeded-index hook in the polars hop entry (hop.py) — without # them the hook dominates the now-thin file and trips its per-file coverage floor graphistry/tests/compute/gfql/index/test_index.py - # Engine coercion tests: the polars paths (df_to_engine, _pl_nan_to_null identity) only - # run where polars is installed, and this lane is the only coverage-collecting lane - # with polars — the core py3.14 lane has no polars extra, so those tests skip there - graphistry/tests/compute/test_engine_coercion.py ) -# The whole graphistry package is measured here (not just graphistry/compute) because -# polars-only branches outside compute/ (e.g. Engine._pl_nan_to_null) are unreachable in -# the core coverage lane (no polars extra); coverage sources must be dirs/packages, and a -# dotted module source (graphistry.Engine) breaks numpy under pytest COV_ARGS=() if [ -n "${POLARS_COV:-}" ]; then - COV_ARGS=(--cov=graphistry --cov-report=) + COV_ARGS=(--cov=graphistry/compute --cov-report=) fi python -B -m pytest -vv "${COV_ARGS[@]}" "${POLARS_TEST_FILES[@]}" "$@" @@ -44,7 +37,7 @@ python -B -m pytest -vv "${COV_ARGS[@]}" "${POLARS_TEST_FILES[@]}" "$@" # appended into the same coverage data file when POLARS_COV=1 (CI audit reads it) COV_APPEND_ARGS=() if [ -n "${POLARS_COV:-}" ]; then - COV_APPEND_ARGS=(--cov=graphistry --cov-report= --cov-append) + COV_APPEND_ARGS=(--cov=graphistry/compute --cov-report= --cov-append) fi python -B -m pytest -vv "${COV_APPEND_ARGS[@]}" \ graphistry/tests/compute/gfql/cypher/test_lowering.py -k polars diff --git a/graphistry/Engine.py b/graphistry/Engine.py index a19c8f1449..a3d93c2939 100644 --- a/graphistry/Engine.py +++ b/graphistry/Engine.py @@ -214,25 +214,11 @@ def _pl_nan_to_null(df): polars / Arrow / cuDF input carrying genuine NaN is treated as MISSING like the pandas oracle (which skipna/dropna's NaN). Without this, ``engine='polars'`` on a frame with a real NaN keeps rows a filter/aggregation should drop (silent divergence from pandas). - No-op when there are no float columns. - - Identity-stable: returns the *same* frame object when no float column actually carries a - NaN. ``fill_nan(None)`` on a NaN-free column is a value-level no-op but still yields a - FRESH frame; that identity churn defeats frame-identity caches keyed on ``source_ref is - df`` (the #1658 CSR index) -- see #1726. So we probe for real NaN presence per float - column (``is_nan`` only applies to float dtypes, hence the schema gate) and rebuild only - when -- and only the columns where -- a NaN is genuinely present. Values are identical to - the old unconditional rewrite (``fill_nan(None)`` never touches non-NaN entries).""" + No-op when there are no float columns.""" import polars as pl float_cols = [c for c, dt in df.schema.items() if dt in (pl.Float32, pl.Float64)] if not float_cols: return df - if isinstance(df, pl.DataFrame): - nan_cols = [c for c in float_cols if df.get_column(c).is_nan().any()] - if not nan_cols: - return df - return df.with_columns([pl.col(c).fill_nan(None) for c in nan_cols]) - # LazyFrame (rare): no cheap eager NaN probe, keep the original unconditional rewrite. return df.with_columns([pl.col(c).fill_nan(None) for c in float_cols]) diff --git a/graphistry/compute/gfql/cypher/lowering.py b/graphistry/compute/gfql/cypher/lowering.py index e42a16db65..d7a67ba019 100644 --- a/graphistry/compute/gfql/cypher/lowering.py +++ b/graphistry/compute/gfql/cypher/lowering.py @@ -2265,7 +2265,7 @@ def _projection_ref_from_expr_safe( def _clause_has_mixed_aggregate_item( - query: Any, + query: CypherQuery, *, alias_targets: Mapping[str, ASTObject], params: Optional[Mapping[str, Any]], @@ -2275,12 +2275,12 @@ def _clause_has_mixed_aggregate_item( cross-source items have ambiguous multiplicity and must keep the conservative fail-fast; a clean split (``c.city`` and ``avg(p.age)`` as separate items) does not trip this (#1273 / rejects-unsound-multi-source-overlap contract).""" - return_clause = getattr(query, "return_", None) + return_clause = query.return_ exprs: List[Tuple[str, int, int]] = [] if return_clause is not None: for item in return_clause.items: exprs.append((item.expression.text, item.span.line, item.span.column)) - order_by = getattr(query, "order_by", None) # top-level ORDER BY (not on ReturnClause) + order_by = query.order_by # top-level ORDER BY (not on ReturnClause) if order_by is not None: for order_item in order_by.items: exprs.append((order_item.expression.text, order_item.span.line, order_item.span.column)) @@ -2324,7 +2324,7 @@ def _expr_has_aggregate(node: ExprNode) -> bool: def _binding_prop_alias_set( - query: Any, + query: CypherQuery, *, alias_targets: Mapping[str, ASTObject], params: Optional[Mapping[str, Any]], @@ -2342,18 +2342,18 @@ def _binding_prop_alias_set( itself is EXACT: ``_expr_match_alias_usage`` non-aggregate refs are precisely the property / whole-entity uses; aggregate-only refs (``count(a)``) are excluded. """ - if getattr(query, "with_stages", None): + if query.with_stages: return None - if getattr(query, "where", None) is not None: + if query.where is not None: return None - matches = cast(Sequence[MatchClause], getattr(query, "matches", ()) or ()) + matches = query.matches or () if len(matches) != 1: return None # multi-MATCH / cartesian — conservative match_clause = matches[0] - if getattr(match_clause, "where", None) is not None or getattr(match_clause, "optional", False): + if match_clause.where is not None or match_clause.optional: return None - return_clause = getattr(query, "return_", None) - if return_clause is None or getattr(return_clause, "where", None) is not None: + return_clause = query.return_ + if return_clause is None: return None # A repeated node alias (e.g. `MATCH (n)-[:LOOP]->(n)`) enforces n==n via hidden @@ -2365,7 +2365,7 @@ def _binding_prop_alias_set( for el in _match_pattern_elements(match_clause) if isinstance(el, NodePattern) and el.variable is not None ] - except Exception: + except (GFQLValidationError, RuntimeError): return None if len(node_vars) != len(set(node_vars)): return None @@ -2374,9 +2374,9 @@ def _binding_prop_alias_set( agg_specs = _collect_aggregate_specs_for_clause( return_clause, params=params, alias_targets=alias_targets ) - except Exception: + except (GFQLValidationError, RuntimeError): return None - if any(getattr(spec, "func", None) == "collect" for spec in agg_specs): + if any(spec.func == "collect" for spec in agg_specs): return None node_aliases = {a for a, t in alias_targets.items() if isinstance(t, ASTNode)} @@ -2386,7 +2386,7 @@ def _binding_prop_alias_set( exprs: List[Tuple[str, int, int]] = [] for item in return_clause.items: exprs.append((item.expression.text, item.span.line, item.span.column)) - order_by = getattr(query, "order_by", None) # top-level ORDER BY (not on ReturnClause) + order_by = query.order_by # top-level ORDER BY (not on ReturnClause) if order_by is not None: for order_item in order_by.items: exprs.append((order_item.expression.text, order_item.span.line, order_item.span.column)) diff --git a/graphistry/compute/gfql/index/__init__.py b/graphistry/compute/gfql/index/__init__.py index f58588bb79..7ee6cbf4f8 100644 --- a/graphistry/compute/gfql/index/__init__.py +++ b/graphistry/compute/gfql/index/__init__.py @@ -23,9 +23,7 @@ is_index_op, is_index_op_json, ) from .cypher_ddl import parse_index_ddl, looks_like_index_ddl -from .cost import ( - cost_gate_frac, cost_gate_min_frontier, reset_cost_gate_frac, set_cost_gate_frac, -) +from .cost import cost_gate_frac, reset_cost_gate_frac, set_cost_gate_frac from .explain import GfqlExplainReport __all__ = [ @@ -40,7 +38,6 @@ "CreateIndex", "DropIndex", "ShowIndexes", "IndexOp", "apply_index_op", "index_op_from_json", "is_index_op", "is_index_op_json", "parse_index_ddl", "looks_like_index_ddl", - "cost_gate_frac", "cost_gate_min_frontier", "reset_cost_gate_frac", - "set_cost_gate_frac", + "cost_gate_frac", "reset_cost_gate_frac", "set_cost_gate_frac", "GfqlExplainReport", ] diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index f5296bd911..bc6f751e5a 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -21,7 +21,7 @@ ) from .build import build_adjacency_index, build_node_id_index from .traverse import index_seeded_hop -from .cost import cost_gate_frac, cost_gate_min_frontier, seed_deg_sum, seed_id_array +from .cost import cost_gate_frac, seed_deg_sum, seed_id_array from .policy import IndexPolicy, validate_index_policy from .types import ( AdjacencyIndexKind, EdgeIndexDirection, HopDirection, IndexKind, @@ -424,15 +424,10 @@ def _bail(reason: str) -> Optional[Plottable]: # Cost gate: if the frontier covers a large fraction of distinct sources, the # scan path is competitive — fall back (avoids index overhead on bulk-ish hops). - # Small-frontier floor: a frontier of <= cost_gate_min_frontier() seeds always - # indexes — the frac gate scales with n_keys, so on small/low-cardinality slices - # it would otherwise send even a single-seed hop to the O(E) scan. Pure routing - # heuristic either way: index and scan return identical results. idx0 = cast(Optional[AdjacencyIndex], registry.get_valid( EDGE_OUT_ADJ if direction != "reverse" else EDGE_IN_ADJ, g._edges, (src, dst), engine )) frac = cost_gate_frac(engine) - min_frontier = cost_gate_min_frontier() if trace and idx0 is not None: # Free fanout estimate (Σ seed degree) from the CSR offsets — the planner # signal the report wants EXPLAIN to surface (not just used-index yes/no). @@ -442,19 +437,16 @@ def _bail(reason: str) -> Optional[Plottable]: diag["seed_deg_sum"] = deg_sum diag["est_result_rows"] = deg_sum diag["threshold_frac"] = frac - diag["min_frontier_floor"] = min_frontier if idx0 is None: # required direction not resident (undirected needs both); let driver decide pass elif resolved_policy != "force": try: frontier_n = int(nodes.shape[0]) - if (frontier_n > min_frontier - and idx0.n_keys > 0 and frontier_n >= frac * idx0.n_keys): + if idx0.n_keys > 0 and frontier_n >= frac * idx0.n_keys: return _bail( f"frontier {frontier_n} >= {frac}*n_keys " - f"({frac * idx0.n_keys:.0f}) and > floor ({min_frontier}) " - f"-> scan cheaper" + f"({frac * idx0.n_keys:.0f}) -> scan cheaper" ) except (AttributeError, TypeError, ValueError): pass diff --git a/graphistry/compute/gfql/index/cost.py b/graphistry/compute/gfql/index/cost.py index bd8ca53862..754db74482 100644 --- a/graphistry/compute/gfql/index/cost.py +++ b/graphistry/compute/gfql/index/cost.py @@ -17,46 +17,6 @@ _COST_GATE_FRAC_DEFAULT = 0.02 _COST_GATE_FRAC_OVERRIDES: Dict[Engine, float] = {} -# Absolute small-frontier floor: at or below this many seed rows the planner NEVER -# bails to scan on the frac gate. The frac gate scales with distinct-key cardinality, -# so on small / low-cardinality edge slices (e.g. per-edge-type homogeneous frames: -# n_keys <= 1/0.02 = 50 at the polars/cudf frac) even a single-node seed trips it and -# the hop scans O(E) despite a resident O(degree) index. A frontier of <= K seeds -# bounds CSR lookup work to K searchsorted probes plus the matching-row gather, -# avoiding that fractional-gate artifact. The actual index/scan crossover remains -# engine- and data-dependent: K is a conservative, environment-tunable default, -# not a measured universal threshold. Constant (not a function of n_keys) on -# purpose: scaling with n_keys is the frac gate's job; the floor bounds absolute -# per-hop probe work where frac*n_keys collapses below a handful of seeds. Uniform -# across engines (pandas's 0.5 frac only overlaps the floor when n_keys <= 2*K). -# Purely a routing heuristic: index and scan return identical results. -_COST_GATE_MIN_FRONTIER_DEFAULT = 16 - - -def cost_gate_min_frontier() -> int: - """Return the absolute small-frontier floor for the index-vs-scan cost gate. - - Frontiers of at most this many seeds always take a resident index, regardless - of the frac gate. ``0`` disables the floor (pure frac gating). Env-overridable - via ``GFQL_INDEX_COST_GATE_MIN_FRONTIER`` for benchmark/diagnostic tuning. - """ - raw = os.environ.get("GFQL_INDEX_COST_GATE_MIN_FRONTIER") - if raw is None or raw == "": - return _COST_GATE_MIN_FRONTIER_DEFAULT - try: - val = int(raw) - except ValueError as ex: - raise ValueError( - f"Invalid GFQL_INDEX_COST_GATE_MIN_FRONTIER={raw!r}: " - f"expected a non-negative integer" - ) from ex - if val < 0: - raise ValueError( - f"Invalid GFQL_INDEX_COST_GATE_MIN_FRONTIER={raw!r}: " - f"expected a non-negative integer" - ) - return val - def _validate_cost_gate_frac(frac: float) -> float: if not 0.0 < frac <= 1.0: diff --git a/graphistry/compute/gfql/index/types.py b/graphistry/compute/gfql/index/types.py index 47a088b73c..af3aa800f6 100644 --- a/graphistry/compute/gfql/index/types.py +++ b/graphistry/compute/gfql/index/types.py @@ -30,7 +30,6 @@ class IndexTraceStep(TypedDict, total=False): seed_deg_sum: Optional[int] est_result_rows: Optional[int] threshold_frac: float - min_frontier_floor: int IndexTrace = List[IndexTraceStep] diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 012a76e5ee..a546ef849a 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -362,6 +362,14 @@ def _try_native_row_op(g_cur, op): fn = getattr(op, "function", None) if fn == "rows" and op.params.get("binding_ops") is not None: + # Single-entity boundary rows emitted by MATCH (n) / EXISTS seeds are + # handled by the pattern-apply helper. Try that narrow shape before the + # connected multi-alias bindings table path, which intentionally declines + # node-only binding_ops. + if op.params.get("source") is None: + out = rows_binding_ops_polars(g_cur, op.params["binding_ops"]) + if out is not None: + return out # Multi-alias bindings table (#1709): native for fixed-length connected # patterns; binding_rows_polars declines (None → NIE) outside that subset. if op.params.get("alias_endpoints") is not None: @@ -372,15 +380,6 @@ def _try_native_row_op(g_cur, op): if _call_native_on_polars(op): # frame ops (rows/limit/skip/distinct/drop_cols) — engine-polymorphic return op.execute(g=g_cur, prev_node_wavefront=None, target_wave_front=None, engine=Engine.POLARS) - # correlated pattern-existence family (EXISTS { } / pattern predicates): native - # via chain_polars-computed key sets; unsupported shapes return None -> honest NIE. - if ( - fn == "rows" - and op.params.get("binding_ops") is not None - and op.params.get("alias_endpoints") is None - and op.params.get("source") is None - ): - return rows_binding_ops_polars(g_cur, op.params["binding_ops"]) if fn == "semi_apply_mark": # required params are safelist-validated — direct indexing (an or-default # here could only mask an unvalidated call); neq is the optional one. diff --git a/graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py b/graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py index 88853cb45a..17b071446b 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py +++ b/graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py @@ -23,6 +23,7 @@ from graphistry.compute.gfql.expr_parser import ExprNode, FunctionCall from graphistry.Plottable import Plottable +from graphistry.utils.json import JSONVal from .dtypes import is_float as _dtype_is_float, is_int as _dtype_is_int, is_numeric as _dtype_is_numeric, is_stringlike as _dtype_is_stringlike @@ -822,8 +823,8 @@ def select_extend_polars(g: Plottable, items: Sequence[Any]) -> Optional[Plottab def binding_rows_polars( g: Plottable, - binding_ops: Sequence[Any], - attach_prop_aliases: Optional[List[str]] = None, + binding_ops: Sequence[Dict[str, JSONVal]], + attach_prop_aliases: Optional[Sequence[str]] = None, ) -> Optional[Plottable]: """Native polars bindings-row table for FIXED-LENGTH connected patterns (#1709). @@ -843,13 +844,13 @@ def binding_rows_polars( never bridges to pandas. Parity gate: differential tests vs the pandas oracle. """ import polars as pl - from graphistry.compute.ast import ASTEdge, ASTNode, from_json as ast_from_json + from graphistry.compute.ast import ASTEdge, ASTNode, ASTObject, from_json as ast_from_json from graphistry.compute.gfql.lazy import collect as _lazy_collect from graphistry.compute.gfql.row.pipeline import RowPipelineMixin from graphistry.compute.gfql.same_path.edge_semantics import EdgeSemantics from .predicates import filter_by_dict_polars - def _names(lf: Any) -> List[str]: + def _names(lf: pl.LazyFrame) -> List[str]: # LazyFrame column names WITHOUT collecting data (schema-only resolve). return lf.collect_schema().names() @@ -864,7 +865,7 @@ def _names(lf: Any) -> List[str]: # Bounded re-entry seeds the first alias from carried rows — pandas-only. return None - ops = [ast_from_json(op_json, validate=False) for op_json in binding_ops] + ops: List[ASTObject] = [ast_from_json(op_json, validate=False) for op_json in binding_ops] # Shared validation (engine-agnostic): raises the canonical GFQLValidationError # for malformed op sequences / duplicate aliases — same error as pandas. RowPipelineMixin._gfql_validate_binding_ops(ops) @@ -875,7 +876,7 @@ def _names(lf: Any) -> List[str]: for idx, op in enumerate(ops): if idx % 2 == 0: - if not isinstance(op, ASTNode) or getattr(op, "query", None) is not None: + if not isinstance(op, ASTNode) or op.query is not None: return None else: if not isinstance(op, ASTEdge): @@ -891,22 +892,22 @@ def _names(lf: Any) -> List[str]: op.direction == "undirected" or bool(op.to_fixed_point) or (op.max_hops is None and op.hops is None) - or isinstance(getattr(op, "_name", None), str) + or isinstance(op._name, str) ): return None if op.direction not in ("forward", "reverse", "undirected"): return None if any( - getattr(op, attr, None) is not None - for attr in ( - "edge_query", "source_node_match", "destination_node_match", - "source_node_query", "destination_node_query", - "label_node_hops", "label_edge_hops", - "output_min_hops", "output_max_hops", + value is not None + for value in ( + op.edge_query, op.source_node_match, op.destination_node_match, + op.source_node_query, op.destination_node_query, + op.label_node_hops, op.label_edge_hops, + op.output_min_hops, op.output_max_hops, ) ): return None - if bool(getattr(op, "label_seeds", False)) or bool(getattr(op, "include_zero_hop_seed", False)): + if bool(op.label_seeds) or bool(op.include_zero_hop_seed): return None # Duplicate-id + HAS_