diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38a81f13b6..0dc7cb19dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1020,7 +1020,7 @@ jobs: # PR hygiene only: combine coverage collected by existing CPU test jobs and # gate executable package lines touched by this PR. Historical uncovered # lines outside the diff are intentionally ignored. - needs: [changes, test-core-python, test-gfql-core, generate-lockfiles] + needs: [changes, test-core-python, test-gfql-core, test-polars, generate-lockfiles] if: ${{ github.event_name == 'pull_request' && (needs.changes.outputs.python == 'true' || needs.changes.outputs.gfql == 'true' || needs.changes.outputs.infra == 'true') && !(needs.changes.outputs.docs_only_latest == 'true') }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -1063,6 +1063,12 @@ jobs: name: gfql-coverage-audit-py3.12 path: build/changed-line-coverage/artifacts/gfql + - name: Download Polars coverage data + uses: actions/download-artifact@v4 + with: + name: polars-coverage-py3.12 + path: build/changed-line-coverage/artifacts/polars + - name: Changed-line coverage gate env: BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -1073,7 +1079,8 @@ jobs: python -m coverage combine \ --data-file=build/changed-line-coverage/combined.coverage \ build/changed-line-coverage/artifacts/core/coverage.core-py3.14 \ - build/changed-line-coverage/artifacts/gfql/coverage.gfql-py3.12 + build/changed-line-coverage/artifacts/gfql/coverage.gfql-py3.12 \ + build/changed-line-coverage/artifacts/polars/coverage.polars-py3.12 python bin/changed_line_coverage.py \ --data-file build/changed-line-coverage/combined.coverage \ --base-ref "$BASE_SHA" \ @@ -1389,10 +1396,39 @@ jobs: uv pip install -e . --no-deps - name: Polars tests + if: ${{ matrix.python-version != '3.12' }} run: | source pygraphistry/bin/activate ./bin/test-polars.sh + - name: Polars tests with coverage + if: ${{ matrix.python-version == '3.12' }} + run: | + source pygraphistry/bin/activate + mkdir -p build/polars-coverage + COVERAGE_FILE=build/polars-coverage/coverage.polars-py3.12 POLARS_COV=1 ./bin/test-polars.sh + + - name: Polars GFQL coverage audit (per-file floors) + if: ${{ matrix.python-version == '3.12' }} + run: | + source pygraphistry/bin/activate + python bin/coverage_audit.py \ + --profile gfql-polars \ + --skip-tests \ + --engine-label ci-polars-py3.12 \ + --output-dir build/polars-coverage-audit \ + --data-file build/polars-coverage/coverage.polars-py3.12 \ + --baseline-file graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json + cat build/polars-coverage-audit/gfql-polars-coverage-audit.md | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload polars coverage + if: ${{ matrix.python-version == '3.12' }} + uses: actions/upload-artifact@v4 + with: + name: polars-coverage-py3.12 + path: build/polars-coverage/ + retention-days: 14 + test-core-umap: needs: [changes, test-minimal-python, test-gfql-core, generate-lockfiles] if: ${{ success() && (needs.changes.outputs.ai == 'true' || needs.changes.outputs.infra == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad2e1055b..d5d0407d6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,40 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Development] -### Fixed - +### Added +- **GFQL polars execution config is Python-settable and live**: `set_cpu_streaming(bool)` and `set_gpu_executor('in-memory'|'streaming')` in `graphistry.compute.gfql.lazy` (plus the public `GPU_EXECUTORS` options and `GpuExecutor` type) set the CPU-streaming / GPU-executor knobs from Python. They resolve **Python override > environment variable > default**, read **live** per collect — previously these were env-only (`GFQL_POLARS_CPU_STREAMING` / `GFQL_POLARS_GPU_EXECUTOR`) and frozen at import, so neither a Python setting nor a post-import env change took effect. `None` resets a setter to env/default. +- **GFQL engine conversion honors the `validate`/`warn` convention**: `Engine.df_to_engine(df, engine, *, validate=, warn=)` threads the repo-wide `validate` (`'strict'`/`'strict-fast'`/`'autofix'`; `True`→strict, `False`→autofix) + `warn` protocol into the pandas→polars and pandas→cuDF converters. On a mixed-type object column that Arrow/polars/cuDF cannot represent, `strict` raises (`NotImplementedError` for polars, `ArrowConversionError` for cuDF) and `autofix` coerces the column to string and warns — the same convention as `plot()`/`upload()`. Each engine keeps its established default (polars `strict` = parity-or-raise; cuDF `autofix` = its shipped best-effort coercion, now `warn`-suppressible). +- **GFQL native Polars engine — traversals (`engine='polars'`)**: Added a native, vectorized Polars execution engine for the core GFQL traversals `hop()` and `chain()`, dispatched at the engine boundary so the production pandas/cuDF paths are untouched. `Engine.POLARS` is opt-in (explicit `engine='polars'`); `engine='auto'` with Polars input still coerces to pandas as before. Covers forward/reverse/undirected single-hop traversal, directed multi-hop chains, node/edge filter dicts and predicates (lowered to Polars expressions), `edge_match`/`source_node_match`/`destination_node_match`, `target_wave_front`, and alias names; the BFS advances via semi/anti joins (no per-row Python work). Validated by differential parity against the pandas engine (hop + chain test suites plus a randomized fuzzer) and benchmarked vs pandas (`benchmarks/gfql/pandas_vs_polars.py`) — Polars wins at scale (up to ~2.5x on multi-edge chains at millions of edges; crossover ~50–100k rows). Variable-length/multi-hop edges, undirected edges in multi-edge chains, hop labels, and node `query=` raise `NotImplementedError` for now (use `engine='pandas'`). +- **GFQL native Polars engine — variable-length `min_hops>1` traversals (`engine='polars'`/`'polars-gpu'`)**: Forward/reverse lower-bound traversals (`e(min_hops=N, max_hops=M)`) now run natively on the Polars engine — no pandas bridge. The eager Polars hop runs pandas' min_hops algorithm vectorized: a NON-anti-joined BFS (the wavefront carries revisits so a cycle keeps bumping the reach depth until `max_hops`), a 3-case termination gate (`max_reached1` (needs connected-components + 2-core seed retention) and a *direct* `hop(min_hops>1)` (which would need pandas' separate un-labeled direct-hop node-output plus the `target_wave_front` threading the chain supplies — without them it silently diverges) both raise `NotImplementedError` for `engine='polars'` (use `chain()`/`gfql()`, or `engine='pandas'`). Validated by differential parity vs the pandas oracle: the 500-seed randomized chain fuzzer (`test_polars_chain_fuzz_parity`, hardened to compare null-aware node **attributes** and edge **multiplicity**, not just id/endpoint sets) is **500/500**, a min_hops+attribute-filter amplified fuzz and metamorphic invariants pass, and `engine='polars-gpu'` (cudf_polars) runs the full 500-seed fuzz **500/500** on-device. +- **GFQL native Polars engine — cypher row pipeline (`engine='polars'`)**: Extended the Polars engine to the Cypher `MATCH … RETURN` row surface, natively vectorized. **NO CHEATING:** the polars engine never silently falls back to the pandas engine — every query runs natively on polars or raises an honest `NotImplementedError` pointing at `engine='pandas'` (falling back to pandas would misrepresent pandas performance as polars; only a human may consent to a bridge). `chain_polars` splits boundary `call()` ops (mirroring the pandas `_handle_boundary_calls`) and runs each trailing row op per-op native or raises. **Native polars** (no pandas round-trip): frame ops (`rows`/`limit`/`skip`/`distinct`/`drop_cols`), `select`/`with_`/`return_` projection (a conservative cypher-expr-AST → `pl.Expr` lowering covering property access, arithmetic, comparison, boolean, literals), `order_by` (`.sort`), `group_by` (`count`/`sum`/`avg`/`min`/`max`), `unwind` (literal-list cross-join), the result projection for property/expr columns, and entity-text `RETURN n` rendering for int/string/bool nodes (`pl.concat_str`). **Honestly deferred** (raise `NotImplementedError`, no pandas fallback): multi-entity `rows(binding_ops=…)`, cross-entity same-path `WHERE` (`DFSamePathExecutor`), float/temporal/nested entity-text, and exotic expressions (CASE/list/map/temporal, `collect` aggregates) — these are the forward native-engineering targets. Validated by differential parity vs pandas including a TCK-style conformance lane (`test_engine_polars_cypher_conformance.py`: native-only curated corpus + seeded fuzzer + NULL/3-valued-logic graph + entity-text escaping, plus a `DEFERRED` list asserting deferred queries raise rather than silently bridge) and benchmarked (`benchmarks/gfql/cypher_row_pipeline.py`). **Perf (interleaved, 1M nodes, each engine on its native-frame graph, all fully native):** polars wins **5.6–38×** across the surface — `RETURN n` ~38×, `ORDER BY` ~17×, `WHERE`+`ORDER BY`+`LIMIT` ~14×, traversals 6–7.5×, projections/aggregations/`DISTINCT` 5.6–6.9×. cuDF/pandas paths untouched. +- **GFQL lazy Polars engine + GPU target (`engine='polars-gpu'`, cudf_polars)**: The Polars traversal engine now builds a single deferred `pl.LazyFrame` plan per single-hop and materializes `out_edges`+`out_nodes` in ONE `collect_all` on a chosen **execution target** (CPU or GPU). `engine='polars-gpu'` (`Engine.POLARS_GPU`, explicit opt-in only — AUTO never selects it) runs that same lazy plan on the RAPIDS cudf_polars backend (`pl.GPUEngine(raise_on_fail=True)` — NO-CHEATING: a GPU-incapable plan node **raises** rather than silently running on CPU and being reported as a GPU result; see Fixed). The collect-once design is what makes GPU pay off: a benchmark showed per-op eager GPU collect was a *regression* (repeated H2D), while collect-once is a **2.84× single-hop GPU win @1M** with CPU parity. Frames stay `pl.DataFrame` (handled like `POLARS` everywhere); the target is carried by a context var set at the chain/hop dispatch boundary, so `engine='polars'` (CPU) is byte-for-byte unchanged. Validated by differential parity `engine='polars-gpu' == engine='polars'` across the cypher conformance corpus + traversals (`test_engine_polars_gpu.py`, skips when no cudf_polars/GPU). Multi-hop and the chain forward/backward fusion (where the GPU win currently dilutes) are follow-up optimizations. +- **GFQL native Polars engine — more cypher row coverage (`toFloat`, `collect`/`collect(DISTINCT)`, `WHERE … IN`)**: three surfaces that previously raised `NotImplementedError` on `engine='polars'` now run natively, parity-validated vs the pandas oracle across all four engines (and honest-NIE where pandas can't be matched). **`toFloat(x)`** lowers int/uint/bool/float → `Float64` (NaN preserved — float64 has no separate null sentinel, unlike `toInteger`); a non-numeric String declines (NIE) because pandas `astype(float)` *raises* rather than null-on-failure. **`collect(x)` / `collect(DISTINCT x)`** aggregations complete the native `group_by` surface (every other agg was already native): drop nulls, preserve within-group first-occurrence order (`collect` keeps dups; `DISTINCT` dedups keep-first), all-null group → `[]`. **`where_rows`/`WHERE … IN [list]`** membership lowers to `is_in` (a null cell is excluded per openCypher 3VL). No change to any already-native path. +- **GFQL Polars-CPU streaming collect (opt-in, large traversals)**: `GFQL_POLARS_CPU_STREAMING=1` runs the polars-CPU lazy collects (`hop`/`chain`) on the polars **streaming** executor instead of the default in-memory collect. Benchmarked ~1.04–1.11× faster on big multi-hop traversals (10M nodes / 80M edges: 20.0→18.0 s) and parity-identical, but ~0.86× (slower) on small/interactive sizes (streaming overhead) — so it is **opt-in, default off** (no change to default behavior). Use for large batch traversals where CPU is the target. +- **GFQL Cypher `count(*)` short-circuit — O(1) instead of O(N) materialize**: a lone `RETURN count(*)` over a single node or edge pattern (`MATCH (n) RETURN count(*)`, `MATCH ()-[r]->() RETURN count(*)`) previously materialized the entire matched frame and ran a constant-key `group_by` just to count its rows. The lowering now emits a new `count_table` row op that reads the scanned table's height directly (or, when the pattern applies a filter, sums the boolean alias-mask column) in a single reduction — no full-frame copy, no group_by. Applies only to the provably-equivalent shapes: exactly one non-DISTINCT `count(*)`, no group keys / post-aggregate exprs / row-level `WHERE` / `UNWIND` / paging / multi-relationship binding, and either a pure node scan (`relationship_count == 0`) or a single relationship counted on its edge alias — every other shape (node-alias-over-relationship, multi-hop paths, `count(DISTINCT …)`, grouped counts) falls through to the general aggregate path unchanged. Engine-polymorphic across pandas/cuDF/polars/polars-gpu; differential parity verified on all four engines (`count_all_nodes`/`count_all_edges` cypher conformance cases + a `count_table` row-op subject case, chain and DAG surfaces). No change to any result value — only the execution path. +- **GFQL `engine='polars'`/`'polars-gpu'` can now run off-engine analytic `call()`s (umap/hypergraph/compute_cugraph/…) — `call_mode='auto'` (default)**: a GFQL `call()` that invokes a Plottable-method analytic with **no native polars implementation** (and never will — `umap`, `hypergraph`, `compute_cugraph`, `compute_igraph`, `layout_*`, `collapse`, `get_topological_levels`, …) previously raised `NotImplementedError` under a polars engine, forcing users off polars for the whole pipeline. It now runs as a **mode-gated, warned modality switch**: `call_mode='auto'` (default) bridges the graph off-engine (`polars`→pandas, `polars-gpu`→cuDF **on-device**), runs the analytic, and coerces the result back to polars **losslessly via Arrow**, warning once per (process, function). `call_mode='strict'` keeps the honest `NotImplementedError` decline (for benchmark integrity / a hard memory ceiling). `polars-gpu` is **GPU-or-error**: it bridges to cuDF and declines rather than silently dropping a GPU analytic to host pandas. This is **deliberately narrower than CHAIN traversal/filter/row ops**, which stay parity-or-NIE (a bridge there would hide a missing impl and cheat a benchmark) — the split is mechanical (`is_row_pipeline_call`), and the chain and DAG surfaces bridge consistently. Configurable via `graphistry.compute.gfql.lazy.set_call_mode('auto'|'strict')` (Python) or `GFQL_POLARS_CALL_MODE` (env), resolving Python override > env > default('auto'), read live. Verified on dgx: `compute_cugraph` PageRank is byte-parity with the pandas oracle under both `polars` and `polars-gpu`. + +### Fixed +- **GFQL `materialize_nodes()` on an edges-only graph under `engine='polars'`**: crashed with `AttributeError: 'Series' object has no attribute 'drop_duplicates'` — the node-id derivation used pandas-only `.drop_duplicates()` / `.reset_index()` / `.rename(columns=)` on a polars Series/DataFrame. Now polars-aware (`.unique(maintain_order=True)`, `.select().rename({...})`), matching the pandas oracle. Surfaced by any `let()` DAG on an edges-only graph under Polars (the DAG pre-materializes nodes). pandas/cuDF paths unchanged. +- **GFQL `engine='polars-gpu'` LazyFrame-input coercion now collects on the GPU executor**: `Engine.df_to_engine(lazyframe, POLARS_GPU)` materialized a `polars.LazyFrame` input with a bare `.collect()` on the CPU default executor — ignoring `gpu_executor()` and not distinguishing `POLARS` from `POLARS_GPU`. It now routes through the target-aware lazy collect, so a LazyFrame coerced under `polars-gpu` collects on the cudf-polars GPU executor (and `polars` honors `cpu_streaming()`). No change for already-materialized frame inputs (the common path). +- **GFQL Polars engine `contains` honors `regex=`/`flags=`**: the native polars `filter_by_dict` lowering of the `Contains` predicate always used `str.contains(..., literal=False)`, so a **literal** request (`contains(pat, regex=False)`) was still regex-interpreted — a pattern with a metacharacter over-matched (e.g. `contains('a.c', regex=False)` matched `'abc'`), diverging from pandas/cuDF. It also dropped `flags=`. Now `regex=False` lowers to a literal match (`literal=True`; case-insensitive literal folds both sides, matching pandas' result), and regex mode maps `case=`/`flags=` (IGNORECASE/MULTILINE/DOTALL/VERBOSE) to a Rust-regex inline flag prefix (`(?ims…)`). +1 engine-parametrized differential-parity test. +- **GFQL `engine='polars-gpu'` silent CPU fallback removed (NO-CHEATING)**: the GPU collect used `pl.GPUEngine(raise_on_fail=False)`, so any plan node the cudf_polars backend can't execute would silently run **on CPU** and still be reported as a `polars-gpu` result — making `engine='polars-gpu'` indistinguishable from `engine='polars'` whenever the plan isn't fully GPU-capable (a benchmark showing near-identical `polars`/`polars-gpu` timings is exactly this tell). Flipped to `raise_on_fail=True` and translate the cudf_polars failure into a clear `NotImplementedError` pointing at `engine='polars'` for native CPU. `polars-gpu` is now **GPU-or-error**: any timing it produces is real on-device work, never CPU mislabeled as GPU. Verified on dgx-spark (LiveJournal 35M): the seeded-frontier `hop`/2-hop chain plan executes fully on GPU without raising (nvidia-smi 92% util during the loop), so existing GPU timings are unchanged — only the honesty guarantee is added. +1 regression test (the GPU-collect error path is translated, not swallowed). +- **GFQL Polars chain dtype-mismatched join keys**: a chain (`MATCH (a)-[]->(b)…`) crashed under `engine='polars'` with `SchemaError: datatypes of join keys don't match` when an edge endpoint column's dtype differed from the node-id dtype across the int↔float boundary — e.g. a null in a `source`/`destination` column promotes it to `float64` while integer node ids stay `int64`. The hop already aligned join keys; the chain fast paths + combine did not. Now aligns endpoint join keys to the node-id dtype for the traversal and restores the original endpoint dtype on the output (matching pandas). No-op when dtypes already match. +- **GFQL `chain()` OTel span placement**: the `gfql.chain` OpenTelemetry span (`@otel_traced`) had landed on the internal `_try_chain_fast_path` probe instead of the public `chain()` — so `chain()` lost its span and the span recorded the wrong function/attributes. Moved the decorator back onto `chain()`. +- **GFQL Polars engine adversarial-review correctness fixes (#1648)**: a multi-dimension adversarial review of the native polars engine found three reachable divergences from the pandas oracle, now fixed (NO-CHEATING — match pandas or decline honestly): (1) **duplicate alias** — a chain reusing an alias (`[n(name='a'), e(), n(name='a')]`) returned a malformed colliding-join schema (`a`/`a_right`) instead of raising; now raises the same `GFQLValidationError` E201 as pandas. (2) **integer-literal division** — `5/2` lowered to polars true division (`2.5`) but Cypher folds it to integer division (`2`), a silent wrong order when embedded non-monotonically (`ORDER BY n.val % (10/4)`); now declines (`NotImplementedError`). Column `/` int (Float on both engines) is unaffected. (3) **chain seed dtype** — an internal `start_nodes` seed whose id-column dtype diverged from the node-id dtype (e.g. an empty crossfilter selection defaulting to float64 vs int64 node ids) crashed the combine join with `SchemaError`; now aligns the seed join key (mirroring the hop). Also removed stale "pandas bridge" docstrings (the bridge was removed earlier), DRY-consolidated the cross-type/NaN dtype classifiers into `engine_polars/dtypes.py`, and documented a narrow `filter_by_dict` genuine-NaN residual (unreachable on the `from_pandas` ingestion path). +3 regression tests. +- **GFQL Polars engine ISO temporal comparison**: comparing Cypher temporal values (`time({...}) > time({...})`, `date({...}) < date({...})`) gave a wrong answer under `engine='polars'` — the cypher→gfql lowering renders the constructors to ISO strings (`'10:00+01:00'`), and the polars engine compared them **lexicographically** (wrong across timezones/precision; pandas parses them temporally). The lowering now detects an ISO date/datetime/time string-literal operand in a comparison and raises an honest `NotImplementedError` rather than returning a silently-wrong result (native temporal-typed comparison is a tracked follow-up). Surfaced by the TCK run (`expr-temporal7`). **With this, the native polars engine has ZERO wrong-answers across the full Cypher TCK** — every scenario either matches pandas or honestly declines. +- **GFQL Polars engine NaN comparison semantics**: comparisons over a NaN computed inside polars (e.g. `0.0/0.0 > 1`) returned polars' answer, which treats NaN as the *largest* value (`NaN > 1` → True) — but IEEE-754 / Python / pandas / Neo4j-Cypher compare any NaN as **false** (and `!=` as **true**). The expr lowering now masks float comparisons to the IEEE answer (`& ~is_nan` for `< > <= >= =`, `| is_nan` for `<> !=`), gated by conservative float-operand inference so int/string/bool comparisons are untouched (no `is_nan()` on non-float). Note: input NaN from `pandas`→`polars` is already converted to null (`nan_to_null`), so this only affects NaN produced by in-query float math. Surfaced by the TCK run (`expr-comparison2-5`). +- **GFQL Polars engine numeric-vs-string comparison**: comparing a numeric value to a string (e.g. `n.val > 'a'`, `0.0/0.0 > 'a'`) crashed under `engine='polars'` with `ComputeError: cannot compare string with numeric type` (pandas/cypher return a value/null). The lowering now detects a numeric↔string comparison (in both the expression path and the folded filter-predicate path) and raises an honest `NotImplementedError` instead of crashing. Surfaced by the TCK run (`expr-comparison2-5-4`). +- **GFQL Polars engine label match on the `labels` List column**: a label match (`MATCH (n:Label)`) that targets the reserved `labels` List column (e.g. a label with no one-hot `label__X` column — typed-schema unknown labels, OPTIONAL MATCH to a non-existent label) crashed under `engine='polars'` with `InvalidOperationError: cannot cast List type (inner: 'String', to: 'String')` — `filter_by_dict` lowered it to a scalar `==` that tried to cast the List to String. Now uses `list.contains` for List-dtype columns (correct Cypher label-membership: `Label ∈ n.labels`; empty for a non-existent label, matching pandas). Surfaced by the TCK run (`match7-28`, `firstparty-typed-schema1-3`). +- **GFQL Polars engine OPTIONAL MATCH null-fill decline**: a multi-clause `OPTIONAL MATCH` needing null-row fill (some seed rows unmatched) raised a misleading `GFQLValidationError` ("null-row alignment could not recover matched seed identities") under `engine='polars'` — the alignment machinery (matched-id meta, `.iloc` slicing, per-segment concat) is pandas-centric and the polars OPTIONAL MATCH doesn't populate the `_cypher_entity_projection_meta["ids"]` it needs. Now raises an honest `NotImplementedError` (use `engine='pandas'`). Surfaced by the TCK run (`match7-7`, `expr-graph4-4`). +- **GFQL Polars engine OPTIONAL MATCH absent-entity rendering**: an OPTIONAL MATCH miss returning a whole entity (`OPTIONAL MATCH (n) RETURN n` with no match) rendered as `'()'` under `engine='polars'` instead of `null` — the native entity-text expression did not nullify absent rows (whose alias marker column is null). Now mirrors the pandas renderer's `_nullify_missing_alias_rows`; a real property-less node still renders `()`. Surfaced by the TCK run (`match7-1`). +- **GFQL Cypher `WITH`-scalar `MATCH` re-entry on the Polars engine**: a bounded `MATCH ... WITH ... MATCH ...` query (carrying a scalar across MATCH clauses) crashed under `engine='polars'` with `AttributeError: 'DataFrame' object has no attribute 'iloc'` — the engine-agnostic re-entry broadcast (`cypher/reentry/execution.py`) used pandas `.iloc`/`.assign`/`.drop(columns=)` on a Polars frame. Made the scalar-row extraction and constant-column broadcast engine-aware (`prefix_rows.row(i, named=True)` + `with_columns(pl.lit(...))` for Polars). The re-entry now completes; any downstream RETURN op the Polars engine doesn't yet render natively raises an honest `NotImplementedError` instead of the crash. +- **GFQL Polars engine predicate pandas-bridge removed (NO-CHEATING) + wider native coverage**: `filter_by_dict` on `engine='polars'` previously evaluated any predicate it couldn't lower natively by converting the column to pandas (`.to_pandas()`), running the predicate's pandas callable, and carrying the mask back — a silent polars→pandas bridge that misrepresented pandas semantics as polars. Removed it: an unsupported predicate now raises `NotImplementedError` (use `engine='pandas'`). To keep common queries native, expanded `predicate_to_expr` lowering to cover `AllOf` (conjunction — e.g. `WHERE n.val > 20 AND n.val < 90` folds to `AllOf[GT, LT]`, lowered recursively), `IsNull`/`IsNA`→`is_null()`, `NotNull`/`NotNA`→`is_not_null()`, and case-insensitive `STARTS WITH`/`ENDS WITH` (anchored `(?i)` regex on the escaped literal). Surfaced from the source-mined optimization review (`pygraphistry4` opportunity #6). +- **GFQL Cypher `UNION DISTINCT` on the Polars engine**: `RETURN … UNION RETURN …` (distinct) crashed under `engine='polars'`/`'polars-gpu'` with `AttributeError: 'DataFrame' object has no attribute 'drop_duplicates'` — the union de-dup in `gfql_unified._execute_compiled_query` called the pandas-only `drop_duplicates` on a Polars frame. Routed through a new engine-aware `Engine.df_unique` (Polars `unique(maintain_order=True)`; pandas/cuDF `drop_duplicates(keep='first')`), matching the existing `row/frame_ops.distinct` convention. Surfaced by the cross-repo TCK conformance run (`tck-gfql`, `TEST_POLARS=1`). +- **GFQL Cypher 3-valued boolean over null literals on the Polars engine**: `null AND null`, `null OR null`, and `NOT null` crashed under `engine='polars'` with `InvalidOperationError: bitand operation not supported for dtype null` / `dtype Null not supported in 'not' operation` — a bare `null` literal lowered to a Null-dtype Polars expression, on which `&`/`|`/`~` are undefined. The boolean lowering now casts AND/OR/NOT operands to `pl.Boolean` first, so Cypher Kleene 3-valued logic (`true AND null = null`, `false OR null = null`, `NOT null = null`) evaluates instead of raising; casting a real Boolean column is a no-op. Surfaced by the TCK run (`expr-boolean1/2/4`). +- **GFQL Polars engine temporal arithmetic decline**: `a.time + duration({minutes: 6})` (in `RETURN`, `ORDER BY`, or `WHERE`) silently became **string concatenation** under `engine='polars'` — Cypher `duration({...})` translates to an ISO duration string literal (`'PT6M'`), and the expr lowering applied `+` to two strings, so e.g. an `ORDER BY` sorted lexicographically on the concatenated text (wrong order). The lowering now raises `NotImplementedError` when `+`/`-` has an ISO-duration string-literal operand (`^-?P(?=[0-9T])`, which doesn't misfire on ordinary strings); the pandas engine handles temporal arithmetic. Surfaced by the TCK run (`with-orderby2`). +- **GFQL Polars engine temporal-constructor property decline**: a standalone property projection over a column holding Cypher temporal-constructor text (`date({year: 1910, month: 5, day: 6})`, `datetime({...})`, … — how Cypher/TCK store temporal values) leaked the raw constructor string under `engine='polars'` instead of the ISO form (`'1910-05-06'`) the pandas projection produces via `_normalize_temporal_constructor_series`. That normalizer is not yet ported natively, so both projection paths (`engine_polars.projection` final result projection and `row_pipeline.select_polars` `WITH`/`RETURN`) now detect temporal-constructor String columns and raise `NotImplementedError` (use `engine='pandas'`) rather than emit a wrong rendering. The detection scans String columns only (numeric/bool projections pay nothing). Surfaced by the TCK run (`with-orderby1-33`+). Whole-entity `RETURN a` over a temporal property is unaffected (it flattens and renders via `render_entity_text`). +- **GFQL Polars engine heterogeneous-column handling (via the `validate`/`warn` convention)**: converting a pandas frame with a mixed-type object column (e.g. `int` and `str` together — legal for dynamically-typed Cypher properties in pandas, but unrepresentable in polars/Arrow) to `engine='polars'` surfaced a cryptic `pyarrow.lib.ArrowInvalid: Could not convert 'xx' with type str: tried to convert to int64` from deep inside polars construction. `Engine.df_to_engine` now handles it per the repo-wide `validate`/`warn` convention: **strict** (the compute-path default) raises a clear `NotImplementedError` naming the offending column(s) and pointing at `engine='pandas'` (no silent coercion by default — that would change comparison semantics), while **`validate='autofix'`** coerces the column(s) to string and warns (matching the cuDF converter and the `plot()`/`upload()` boundary). Surfaced by the TCK run (`expr-comparison2`, `match-where5`, `with-where5`). +- **GFQL cuDF→polars conversion is dtype-lossless (via Arrow)**: converting a cuDF frame to `engine='polars'`/`'polars-gpu'` routed cuDF → pandas → polars, which double-converts **and is lossy** — a cuDF nullable `Int64`/`boolean` degraded through pandas to `float64`+NaN / `object`, so the polars frame started with the wrong dtype before the query ran. Now converts cuDF → **Arrow** → polars (cuDF's native interchange, near-zero-copy on the polars side), preserving dtypes and nulls. For `engine='polars-gpu'` this also removes a device→host→device round trip for the host frame. Verified on dgx (cuDF): `Int64`/`boolean`/`String` + nulls preserved for both polars and polars-gpu targets. - **GFQL `ne()`/`<>` and `IN`/membership on NULL follow openCypher/SQL 3-valued logic (pandas + cuDF)**: a `ne()` / `<>` filter or a list-membership (`IN`) filter over a NULL cell now EXCLUDES the row — per 3VL, `null <> x` and `null IN [...]` evaluate to `null` (not a match) — consistent with `eq`/`gt`/`lt` (which already dropped nulls) and with the cuDF/polars engines. Previously the pandas `NE` predicate kept null rows (`NaN != x` → True) and cuDF matched a null cell against a `None`/`NaN` list element. Behavior change for `ne()` / `NOT IN` on nullable columns under the default pandas engine. (Broader openCypher null-semantics tracked in #1664.) ### Infrastructure @@ -26,6 +58,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Performance - **GFQL temporal-detection dtype gate (#1650)**: `order_detect_temporal_mode` now short-circuits for numeric/bool/complex columns, which can never hold temporal *text*, instead of running an `astype(str)` + multi-regex `fullmatch` scan on every comparison. Eliminates spurious row-wise stringification in `where_rows`/comparison paths whose output never contains entity-text. Byte-identical results; measured `where_rows` speedups ~3.1× (pandas) and ~4.4–13.3× (cuDF, scaling with row count). Does not address whole-entity `RETURN a` text rendering, which is tracked separately. - **GFQL generic single-hop fast path (perf, pandas + cuDF)**: a single `MATCH (n)` (node-only) or `MATCH (a {f})-[e]->(b)` (1-hop) — the dominant tabular/crossfilter + basic-graph-query shapes — now skip the forward/backward/combine BFS machinery in the generic engine: node-only returns the filtered node table; 1-hop returns the edges whose endpoints pass the node filters + those endpoint nodes. Same VALUES + node/edge sets as before (345-case adversarial golden: only dtype differs; hop/chain suites; gated to pandas/cuDF — dask/spark keep the full path). **~100× faster on pandas** (node filter 204→2 ms @10M; graph query similarly near-raw); cuDF stays on the resident frame (a couple semi-joins instead of the BFS + ~31 drop_duplicates), capturing the GPU semijoin win. **Minor behavior change:** the 1-hop now PRESERVES node-attribute dtypes (int stays int) instead of the full machinery's spurious int→float merge upcast — making pandas/cuDF consistent with the polars engine. The fast path is automatically skipped when a GFQL `policy` is installed, so the `prechain`/`postchain`/`postload` hooks (which can observe intermediate state and deny execution) always fire on the full path — keeping the optimization observationally transparent. Differential-verified equivalent to the full BFS path across 440 random well-formed graph×query cases plus targeted edge cases: it drops edges to nodes absent from the node table and dedups duplicate node ids (matching the full path's join semantics), and a NaN node id never validates a NaN edge endpoint. +- **GFQL generic node-only MATCH fast path (perf, all engines)**: a single `MATCH (n)` (no edge hop) — the dominant tabular/crossfilter shape (`MATCH (n) WHERE/RETURN …`, histograms, filters, table search) — now returns the filtered node table directly + empty edges, skipping the forward/backward/combine BFS machinery in the generic engine (pandas + cuDF). Byte-identical (345-case adversarial golden + hop/chain suites). **~100× faster on pandas at scale** (node filter 204→2 ms @10M, 0.36 ms @1M); cuDF ~0.8 ms @1M / 2.3 ms @10M (op stays on the resident frame). The 1-hop shape is left to the per-engine path (the generic node-merge upcasts int→float, so a join-free generic 1-hop would change dtypes). +- **GFQL polars unconstrained 1-hop fast path (perf)**: a single `MATCH (a)-[e]->(b)` where both nodes are unconstrained (no filter/name/query) and the edge has no match/name/query — the basic graph-query shape and the viz edge-crossfilter MATCH (`MATCH ()-[e]->() WHERE e.x RETURN e`, WHERE/RETURN run later) — now returns ALL edges + their endpoint nodes directly (direction-independent; isolated nodes excluded), skipping forward/backward/combine. Byte-identical (full polars conformance + row-pipeline parity + adversarial graphs: dup/self-loop/cycle/isolated). **~9× faster polars `[n,e,n]`: 95.6→10.3 ms @1M, 855→99 ms @10M.** +- **GFQL polars single-hop fast path (perf)**: a single `MATCH (a)-[e]->(b)` where both nodes have no name/query and the edge has no match/name/query — the basic graph query AND the "filter then expand" viz crossfilter (`MATCH (a {f})-[e]->(b)`, src/dst/both filters) — returns the edges whose endpoints pass the node filters + those endpoint nodes directly (isolated/dead-end excluded), skipping forward/backward/combine. (Unconstrained: all edges, any direction; filtered: forward/reverse — filtered-undirected falls through.) Byte-identical (full polars conformance + row-pipeline parity + adversarial src/dst/both/reverse on dup/self-loop/cycle/isolated). **~9× faster polars `[n,e,n]` (95.6→10.3 ms @1M, 855→99 ms @10M); filtered graph query similarly near-raw.** +- **GFQL polars node-only MATCH fast path (perf)**: a single `MATCH (n)` traversal (no edge hop) — the dominant tabular/crossfilter shape (`MATCH (n) WHERE/RETURN …`, histograms, filters, search) — now returns the filtered node table directly and skips the entire forward/backward/combine + `collect_all` (a ~2.5 ms fixed cost that dominated small/interactive queries). Byte-identical (full polars conformance + row-pipeline parity). **Moves the polars>pandas crossover BELOW 100K** for the real product workloads: e.g. categorical histogram 0.68→1.70× @100K and 1.38→7.62× @1M; node filter 2.44→13.85× @1M; timeline 2.55→8.12× @1M (vs pandas). +- **GFQL polars-GPU in-memory executor (perf+stability)**: the GPU target now collects with cudf-polars' `pl.GPUEngine(executor="in-memory")` instead of the default streaming `engine="gpu"`. GFQL results fit in device memory (the in-memory engine's regime), where it is both faster (semijoin 1.33×, antijoin 2.58×, unique 1.49× @10M) and far more STABLE — the default streaming executor spiked bimodally to ~1 s on the same 10M semijoin (median ~360 ms), while in-memory holds ~30 ms with max ~30. Parity preserved (polars-gpu == polars, 39 tests). NOTE: gfql chains are not GPU-compute-bound (host orchestration + the eager single-hop fast paths dominate), so this is a correctness/stability fix for GPU-collect paths, not a chain-GPU speedup. +- **GFQL polars chain combine collect-once (perf)**: the native polars `chain()` now builds the whole forward/backward COMBINE (combine_nodes/edges + endpoint materialization + alias names) as ONE deferred `pl.LazyFrame` plan over the already-materialized hop frames and collects ONCE, instead of ~a dozen eager ops that each internally `lazy().op().collect()`. Stable order columns restore the eager `g._nodes`/`g._edges` row order (lazy joins don't preserve it) so trailing `LIMIT`/`SKIP` is unaffected — byte-identical (full polars conformance + row-pipeline parity). NO recompute (inputs are materialized; unlike whole-chain fusion). ~5% faster polars 1-hop chain (95.6→90.3 ms @1M, 897→855 ms @10M); GPU-target neutral. - **GFQL hop/chain redundant-dedup removal (perf)**: dropped the explicit `.unique()` dedup pass that fed only an `.isin()` membership test in the generic traversal — `_filter_edges_by_endpoint`, the undirected combine masks, and the per-hop wavefront filter (`compute/hop.py`, `compute/chain.py`). `isin(s) == isin(s.unique())` by set membership, so this is byte-identical (verified across 345 adversarial graph×query cases: dup/parallel edges, self-loops, isolated/dead-end nodes, cycles, undirected, multi-hop, fixed-point, min/max hops, names, filters, seeds). Each removed `.unique()` is one fewer kernel launch on GPU, where launch latency — not compute — dominates small/mid traversals: cuDF 1-hop chain ~126→103 ms @1M edges (~18% faster), pandas unaffected within noise. - **GFQL row-expression parse memoization**: `parse_expr()` (the GFQL row-expression parser) now memoizes its result per expression string. Complements the Cypher-query parse memo (PR #1652); profiling showed that once whole-query parsing is cached, the residual fixed per-call compile cost is dominated by `parse_expr`, which is invoked ~4× per query compile (each RETURN/WHERE/WITH expression) and rebuilt a Lark transformer on every call. `parse_expr` is a pure function of the expression text (no params/schema) returning a tree of frozen (immutable) dataclasses, so identical expressions — re-parsed on every compile and recurring across queries (e.g. `a.val > 50`) — are served from an `lru_cache(maxsize=1024)`. Measured (dgx-spark): stacked on the query-parse memo, the fixed per-call cost of a repeated string Cypher query drops a further ~4.5 ms → ~1.8 ms (≈ parity with the equivalent native chain). The non-str/empty guard stays outside the cache; only successful parses are cached (invalid input re-raises every call). diff --git a/benchmarks/gfql/cypher_row_pipeline.py b/benchmarks/gfql/cypher_row_pipeline.py new file mode 100644 index 0000000000..295ccc09f5 --- /dev/null +++ b/benchmarks/gfql/cypher_row_pipeline.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Benchmark the native polars GFQL row pipeline vs pandas for cypher queries. + +Phase 2 of the polars engine enables cypher RETURN / LIMIT / SKIP / DISTINCT / +single-entity WHERE on ``engine='polars'`` (before this increment these raised +NotImplementedError on polars). The heavy traversal + frame ops (filter, dedup, +slice) run natively in polars; only the final row-wise entity-text projection is +host-bridged to pandas. So polars wins most where a row op reduces the set +before projection (LIMIT, selective WHERE, DISTINCT), and is closest to neutral +on a full-table whole-entity RETURN (projection dominates, bridge roundtrip). + +Reports median latency and the polars speedup (pandas_ms / polars_ms; > 1 means +polars wins). On a shared host, interleave is implicit (pandas then polars +back-to-back per query); for regression-grade claims run several times and +compare distributions (see plans/gfql-polars-engine memory). + +Example:: + + python benchmarks/gfql/cypher_row_pipeline.py --runs 7 --warmup 2 \ + --sizes 10000,100000,1000000 --output /tmp/cypher-row.md +""" + +from __future__ import annotations + +import argparse +import statistics +import time +from dataclasses import dataclass +from typing import Callable, List, Optional, Tuple + +import numpy as np +import pandas as pd + +import graphistry + +# (name, cypher) — exercised on both engines via g.gfql(cypher, engine=...) +# Native = frame ops (rows/limit/skip/distinct) run in polars; Bridged = the +# cypher expression engine (select/order_by/group_by) runs host-bridged to pandas. +WORKLOADS: List[Tuple[str, str]] = [ + # native frame-op path + ("RETURN n LIMIT 10", "MATCH (n) RETURN n LIMIT 10"), + ("RETURN n SKIP/LIMIT", "MATCH (n) RETURN n SKIP 5 LIMIT 100"), + ("WHERE > RETURN LIMIT", "MATCH (n) WHERE n.score > 90 RETURN n LIMIT 50"), + ("RETURN DISTINCT n", "MATCH (n) RETURN DISTINCT n"), + ("WHERE > RETURN n", "MATCH (n) WHERE n.score > 50 RETURN n"), + ("RETURN n (full)", "MATCH (n) RETURN n"), + ("rel RETURN m LIMIT", "MATCH (n)-[e]->(m) RETURN m LIMIT 100"), + # host-bridged expression path + ("select n.score", "MATCH (n) RETURN n.score"), + ("select 2 cols", "MATCH (n) RETURN n.score, n.kind"), + ("order_by", "MATCH (n) RETURN n.score ORDER BY n.score DESC"), + ("where+select+limit", "MATCH (n) WHERE n.score > 50 RETURN n.score ORDER BY n.score LIMIT 100"), + ("group_by count", "MATCH (n) RETURN n.kind, count(n) AS c"), +] + + +@dataclass +class ResultRow: + workload: str + n_nodes: int + n_edges: int + pandas_ms: Optional[float] + polars_ms: Optional[float] + error: Optional[str] = None + + @property + def speedup(self) -> Optional[float]: + if self.pandas_ms and self.polars_ms: + return self.pandas_ms / self.polars_ms + return None + + +def make_graph(n_nodes: int, n_edges: int, seed: int = 0): + rng = np.random.default_rng(seed) + nodes = pd.DataFrame({ + "id": np.arange(n_nodes), + "kind": rng.choice(["x", "y", "z"], size=n_nodes), + "score": rng.integers(0, 100, size=n_nodes), + }) + edges = pd.DataFrame({ + "s": rng.integers(0, n_nodes, size=n_edges), + "d": rng.integers(0, n_nodes, size=n_edges), + "rel": rng.choice(["r1", "r2", "r3"], size=n_edges), + }) + return graphistry.nodes(nodes, "id").edges(edges, "s", "d") + + +def timeit(fn: Callable[[], object], runs: int, warmup: int) -> float: + for _ in range(warmup): + fn() + samples = [] + for _ in range(runs): + t0 = time.perf_counter() + fn() + samples.append((time.perf_counter() - t0) * 1000.0) + return statistics.median(samples) + + +def _polars_graph(g): + """Same graph with node/edge frames already in polars, so the polars runs + don't pay a per-call pandas->polars input coercion that the pandas runs avoid + (a real deployment keeps the graph in its engine's native frame type).""" + from graphistry.Engine import Engine, df_to_engine + return g.nodes(df_to_engine(g._nodes, Engine.POLARS), g._node).edges( + df_to_engine(g._edges, Engine.POLARS), g._source, g._destination) + + +def run(sizes: List[Tuple[int, int]], runs: int, warmup: int) -> List[ResultRow]: + rows: List[ResultRow] = [] + for n_nodes, n_edges in sizes: + g_pd = make_graph(n_nodes, n_edges) + g_pl = _polars_graph(g_pd) + for name, query in WORKLOADS: + try: + pandas_ms = timeit(lambda: g_pd.gfql(query, engine="pandas"), runs, warmup) + polars_ms = timeit(lambda: g_pl.gfql(query, engine="polars"), runs, warmup) + rows.append(ResultRow(name, n_nodes, n_edges, pandas_ms, polars_ms)) + except Exception as exc: # noqa: BLE001 - bench harness reports, never crashes the sweep + rows.append(ResultRow(name, n_nodes, n_edges, None, None, error=f"{type(exc).__name__}: {exc}")) + return rows + + +def to_markdown(rows: List[ResultRow]) -> str: + lines = [ + "| workload | nodes | edges | pandas_ms | polars_ms | speedup |", + "|----------|-------|-------|-----------|-----------|---------|", + ] + for r in rows: + if r.error: + lines.append(f"| {r.workload} | {r.n_nodes} | {r.n_edges} | ERROR | ERROR | {r.error} |") + else: + lines.append( + f"| {r.workload} | {r.n_nodes} | {r.n_edges} | " + f"{r.pandas_ms:.1f} | {r.polars_ms:.1f} | {r.speedup:.2f}x |" + ) + return "\n".join(lines) + + +def _parse_sizes(text: str) -> List[Tuple[int, int]]: + # "nodes:edges,nodes:edges" or "nodes" (edges defaults to 5x nodes) + out: List[Tuple[int, int]] = [] + for chunk in text.split(","): + chunk = chunk.strip() + if not chunk: + continue + if ":" in chunk: + nn, ne = chunk.split(":") + out.append((int(nn), int(ne))) + else: + nn = int(chunk) + out.append((nn, nn * 5)) + return out + + +def main() -> None: + try: + import polars # noqa: F401 + except ImportError: + raise SystemExit("polars is not installed; install with `pip install polars`") + + parser = argparse.ArgumentParser(description="Benchmark GFQL cypher row pipeline pandas vs polars.") + parser.add_argument("--runs", type=int, default=7) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument( + "--sizes", + default="10000,100000,1000000", + help="Comma list of node counts (edges=5x) or nodes:edges pairs.", + ) + parser.add_argument("--output", default="", help="Optional path to write the markdown table.") + args = parser.parse_args() + + rows = run(_parse_sizes(args.sizes), args.runs, args.warmup) + table = to_markdown(rows) + print(table) + if args.output: + with open(args.output, "w") as fh: + fh.write(table + "\n") + print(f"\nwrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gfql/multihop_polars_bench.py b/benchmarks/gfql/multihop_polars_bench.py new file mode 100644 index 0000000000..d583cec69c --- /dev/null +++ b/benchmarks/gfql/multihop_polars_bench.py @@ -0,0 +1,70 @@ +"""Multi-hop polars-vs-pandas chain benchmark (validation for the native multi-hop port). + +Confirms native multi-hop polars is on-par-or-better than pandas on non-trivial graphs. +Run on dgx (CPU lane is the relevant comparison; cudf/polars-gpu optional). +""" +import time +import numpy as np +import pandas as pd +import graphistry +from graphistry.compute.ast import n, e_forward, e_undirected + + +def _graph(nn, ne, seed=0): + rng = np.random.default_rng(seed) + nd = pd.DataFrame({"id": np.arange(nn), "k": rng.integers(0, 5, nn)}) + ed = pd.DataFrame({ + "s": rng.integers(0, nn, ne), + "d": rng.integers(0, nn, ne), + "w": rng.integers(0, 100, ne), + }) + return graphistry.nodes(nd, "id").edges(ed, "s", "d") + + +def _time(g, ch, engine, reps=3): + g.chain(ch, engine=engine) # warm + best = float("inf") + for _ in range(reps): + t = time.perf_counter() + out = g.chain(ch, engine=engine) + dt = time.perf_counter() - t + best = min(best, dt) + ne = 0 if out._edges is None else len(out._edges) + nn = 0 if out._nodes is None else len(out._nodes) + return best * 1000, nn, ne + + +CHAINS = { + "fwd-hops2": [n({"id": [0]}), e_forward(hops=2), n()], + "fwd-hops3": [n({"id": [0]}), e_forward(hops=3), n()], + "maxhops4": [n({"id": [0]}), e_forward(max_hops=4), n()], + "sandwiched": [n({"id": [0]}), e_forward(), n(), e_forward(hops=2), n()], + "fwd-tofixed": [n({"id": [0]}), e_forward(to_fixed_point=True), n()], + "und-hops2": [n({"id": [0]}), e_undirected(hops=2), n()], + "und-maxhops3": [n({"id": [0]}), e_undirected(max_hops=3), n()], +} + +for (nn, ne) in [(10_000, 50_000), (100_000, 500_000), (500_000, 2_000_000)]: + g = _graph(nn, ne) + print(f"\n=== graph nn={nn:,} ne={ne:,} ===") + for name, ch in CHAINS.items(): + engines = ["pandas", "polars"] + try: + import cudf # noqa: F401 + engines.append("cudf") + except Exception: + pass + row = [] + ref = None + for eng in engines: + try: + ms, rnn, rne = _time(g, ch, eng) + if eng == "pandas": + ref = ms + spd = f"{ref/ms:.2f}x" if (ref and eng != "pandas") else "1.00x" + row.append(f"{eng}={ms:.1f}ms({spd},n={rnn},e={rne})") + except NotImplementedError: + row.append(f"{eng}=NIE") + except Exception as ex: + row.append(f"{eng}=ERR:{type(ex).__name__}") + print(f" {name:12s} " + " ".join(row)) diff --git a/benchmarks/gfql/pandas_vs_polars.py b/benchmarks/gfql/pandas_vs_polars.py new file mode 100644 index 0000000000..e4c6cbbfad --- /dev/null +++ b/benchmarks/gfql/pandas_vs_polars.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Benchmark the native polars GFQL engine vs pandas for hop() and chain(). + +Compares ``engine='pandas'`` against ``engine='polars'`` on synthetic random +graphs across a size sweep, for representative hop and single-hop chain +workloads. Reports a markdown table of median latency and the polars speedup +(pandas_ms / polars_ms; > 1 means polars wins). + +Polars wins at scale (joins amortize its fixed per-call overhead); the crossover +is typically ~50-100k rows. On a shared host, interleave is implicit (each +workload times pandas then polars back-to-back per size). + +Example:: + + python benchmarks/gfql/pandas_vs_polars.py --runs 7 --warmup 2 \ + --sizes 10000,100000,500000 --output /tmp/pandas-vs-polars.md +""" + +from __future__ import annotations + +import argparse +import statistics +import time +from dataclasses import dataclass +from typing import Callable, List, Optional, Tuple + +import numpy as np +import pandas as pd + +import graphistry +from graphistry.compute.ast import n, e_forward + +# (name, builder) — builder takes (graphistry_graph, engine_str) -> Plottable +WORKLOADS: List[Tuple[str, Callable]] = [ + ("hop1", lambda g, eng: g.hop(hops=1, engine=eng)), + ("hop2", lambda g, eng: g.hop(hops=2, engine=eng)), + ("chain n-e-n", lambda g, eng: g.chain([n(), e_forward(), n()], engine=eng)), + ("chain filter", lambda g, eng: g.chain([n({"kind": "x"}), e_forward({"rel": "r1"}), n()], engine=eng)), + ("chain 2-edge", lambda g, eng: g.chain([n({"kind": "x"}), e_forward(), n(), e_forward(), n()], engine=eng)), +] + + +@dataclass +class ResultRow: + workload: str + n_nodes: int + n_edges: int + pandas_ms: Optional[float] + polars_ms: Optional[float] + error: Optional[str] = None + + @property + def speedup(self) -> Optional[float]: + if self.pandas_ms and self.polars_ms: + return self.pandas_ms / self.polars_ms + return None + + +def make_graph(n_nodes: int, n_edges: int, seed: int = 0): + rng = np.random.default_rng(seed) + nodes = pd.DataFrame({ + "id": np.arange(n_nodes), + "kind": rng.choice(["x", "y", "z"], size=n_nodes), + "score": rng.integers(0, 100, size=n_nodes), + }) + edges = pd.DataFrame({ + "s": rng.integers(0, n_nodes, size=n_edges), + "d": rng.integers(0, n_nodes, size=n_edges), + "rel": rng.choice(["r1", "r2", "r3"], size=n_edges), + }) + return graphistry.nodes(nodes, "id").edges(edges, "s", "d") + + +def timeit(fn: Callable[[], object], runs: int, warmup: int) -> float: + for _ in range(warmup): + fn() + samples = [] + for _ in range(runs): + t0 = time.perf_counter() + fn() + samples.append((time.perf_counter() - t0) * 1000.0) + return statistics.median(samples) + + +def _polars_graph(g): + """Graph with frames already in polars so polars runs don't pay a per-call + pandas->polars input coercion the pandas runs avoid (real deployments keep + the graph in the engine's native frame type).""" + from graphistry.Engine import Engine, df_to_engine + return g.nodes(df_to_engine(g._nodes, Engine.POLARS), g._node).edges( + df_to_engine(g._edges, Engine.POLARS), g._source, g._destination) + + +def run(sizes: List[Tuple[int, int]], runs: int, warmup: int) -> List[ResultRow]: + rows: List[ResultRow] = [] + for n_nodes, n_edges in sizes: + g_pd = make_graph(n_nodes, n_edges) + g_pl = _polars_graph(g_pd) + for name, fn in WORKLOADS: + try: + pandas_ms = timeit(lambda: fn(g_pd, "pandas"), runs, warmup) + polars_ms = timeit(lambda: fn(g_pl, "polars"), runs, warmup) + rows.append(ResultRow(name, n_nodes, n_edges, pandas_ms, polars_ms)) + except Exception as exc: # noqa: BLE001 - bench harness reports, never crashes the sweep + rows.append(ResultRow(name, n_nodes, n_edges, None, None, error=f"{type(exc).__name__}: {exc}")) + return rows + + +def to_markdown(rows: List[ResultRow]) -> str: + lines = [ + "| workload | nodes | edges | pandas_ms | polars_ms | speedup |", + "|----------|-------|-------|-----------|-----------|---------|", + ] + for r in rows: + if r.error: + lines.append(f"| {r.workload} | {r.n_nodes} | {r.n_edges} | ERROR | ERROR | {r.error} |") + else: + lines.append( + f"| {r.workload} | {r.n_nodes} | {r.n_edges} | " + f"{r.pandas_ms:.1f} | {r.polars_ms:.1f} | {r.speedup:.2f}x |" + ) + return "\n".join(lines) + + +def _parse_sizes(text: str) -> List[Tuple[int, int]]: + # "nodes:edges,nodes:edges" or "nodes" (edges defaults to 5x nodes) + out: List[Tuple[int, int]] = [] + for chunk in text.split(","): + chunk = chunk.strip() + if not chunk: + continue + if ":" in chunk: + nn, ne = chunk.split(":") + out.append((int(nn), int(ne))) + else: + nn = int(chunk) + out.append((nn, nn * 5)) + return out + + +def main() -> None: + try: + import polars # noqa: F401 + except ImportError: + raise SystemExit("polars is not installed; install with `pip install polars`") + + parser = argparse.ArgumentParser(description="Benchmark GFQL pandas vs native polars engine.") + parser.add_argument("--runs", type=int, default=7) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument( + "--sizes", + default="1000,10000,100000", + help="Comma list of node counts (edges=5x) or nodes:edges pairs.", + ) + parser.add_argument("--output", default="", help="Optional path to write the markdown table.") + args = parser.parse_args() + + rows = run(_parse_sizes(args.sizes), args.runs, args.warmup) + table = to_markdown(rows) + print(table) + if args.output: + with open(args.output, "w") as fh: + fh.write(table + "\n") + print(f"\nwrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/bin/coverage_audit.py b/bin/coverage_audit.py index 9c9b864f81..980e8fdd6f 100755 --- a/bin/coverage_audit.py +++ b/bin/coverage_audit.py @@ -76,7 +76,23 @@ class AuditProfile: next_triage: Tuple[str, ...] +POLARS_TARGET_PATTERNS = ( + "graphistry/compute/gfql/lazy/**/*.py", +) + PROFILES: Dict[str, AuditProfile] = { + "gfql-polars": AuditProfile( + name="gfql-polars", + title="GFQL Polars Engine Coverage Audit", + coverage_basis="line coverage from `coverage.py` over the polars lane (bin/test-polars.sh)", + target_patterns=POLARS_TARGET_PATTERNS, + source_paths=GFQL_SOURCE_PATHS, + default_pytest_args=GFQL_DEFAULT_PYTEST_ARGS, + next_triage=( + "The native polars engine files are exercised only by the polars lane (engine='polars').", + "Raise floors as coverage grows; keep parity/NIE behavior gated by the conformance suite.", + ), + ), "gfql": AuditProfile( name="gfql", title="GFQL Coverage Audit", diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 0a0338044d..89c7f3c243 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -2,12 +2,29 @@ set -ex # Run from project root -# - Args get passed to pytest phase -# Non-zero exit code on fail +# - Extra args are passed through to the pytest phase +# - Set POLARS_COV=1 to collect coverage over graphistry/compute; the coverage +# data file location is taken from $COVERAGE_FILE (as the CI py3.12 lane sets it) +# - Non-zero exit code on fail # Assume [polars,test] installed python -m pytest --version -python -B -m pytest -vv \ +# Single source of truth for the polars test file list (CI reuses this script). +POLARS_TEST_FILES=( graphistry/tests/compute/test_polars.py + graphistry/tests/compute/gfql/test_engine_polars_hop.py + graphistry/tests/compute/gfql/test_engine_polars_chain.py + graphistry/tests/compute/gfql/test_engine_polars_row_pipeline.py + graphistry/tests/compute/gfql/test_engine_polars_cypher_conformance.py + graphistry/tests/compute/gfql/test_engine_polars_conformance_matrix.py + graphistry/tests/compute/gfql/test_conformance_ledger.py +) + +COV_ARGS=() +if [ -n "${POLARS_COV:-}" ]; then + COV_ARGS=(--cov=graphistry/compute --cov-report=) +fi + +python -B -m pytest -vv "${COV_ARGS[@]}" "${POLARS_TEST_FILES[@]}" "$@" diff --git a/graphistry/Engine.py b/graphistry/Engine.py index bc39199c49..a3d93c2939 100644 --- a/graphistry/Engine.py +++ b/graphistry/Engine.py @@ -7,24 +7,38 @@ from typing_extensions import Literal from enum import Enum +from graphistry.models.types import ValidationParam + class Engine(Enum): PANDAS = 'pandas' CUDF = 'cudf' DASK = 'dask' DASK_CUDF = 'dask_cudf' + POLARS = 'polars' + # GPU execution TARGET of the lazy Polars engine (cudf_polars): frames stay + # ``pl.DataFrame`` (handled exactly like POLARS in all frame ops); only the + # lazy ``.collect()`` runs on GPU. Explicit opt-in only — AUTO never selects it. + POLARS_GPU = 'polars-gpu' + +# Engines whose frames use the polars API (unique/with_columns/...) rather than the +# pandas API (drop_duplicates/assign/...). POLARS_GPU is the GPU execution target of +# the same lazy Polars engine — frames stay ``pl.DataFrame``, so it shares the path. +POLARS_ENGINES = (Engine.POLARS, Engine.POLARS_GPU) class EngineAbstract(Enum): PANDAS = Engine.PANDAS.value CUDF = Engine.CUDF.value DASK = Engine.DASK.value DASK_CUDF = Engine.DASK_CUDF.value + POLARS = Engine.POLARS.value + POLARS_GPU = Engine.POLARS_GPU.value AUTO = 'auto' # Type alias for engine parameter - accepts both enum values and string literals # Includes 'auto' for automatic detection -EngineAbstractType = Union[EngineAbstract, Literal['pandas', 'cudf', 'dask', 'dask_cudf', 'auto']] +EngineAbstractType = Union[EngineAbstract, Literal['pandas', 'cudf', 'dask', 'dask_cudf', 'polars', 'polars-gpu', 'auto']] DataframeLike = Any # pdf, cudf, ddf, dgdf DataframeLocalLike = Any # pdf, cudf @@ -123,12 +137,25 @@ def df_to_pdf(df, engine: Engine): return df.compute() raise ValueError('Only engines pandas/cudf supported') -def _cudf_from_pandas_best_effort(df: pd.DataFrame): +def _cudf_from_pandas_best_effort(df: pd.DataFrame, *, validate: Optional[ValidationParam] = None, warn: bool = True): + """pandas -> cuDF honoring the repo-wide ``validate``/``warn`` convention. + + Default (``validate=None`` -> ``autofix``): best-effort per-column coercion of + mixed-type object columns to string (numeric-looking columns kept numeric), warning + once. ``strict``/``strict-fast``: raise ``ArrowConversionError`` instead of coercing + (matching the plot()/upload() boundary and the polars converter's strict mode). + ``validate=False`` == autofix but suppresses the warning. + """ import cudf + from graphistry.validate.common import normalize_validation_params + validate_mode, warn = normalize_validation_params('autofix' if validate is None else validate, warn) try: return cudf.from_pandas(df) - except Exception: + except Exception as e: + if validate_mode in ('strict', 'strict-fast'): + from graphistry.exceptions import ArrowConversionError + raise ArrowConversionError(columns=_mixed_type_object_columns(df), original_error=e) from e failed_cols: List[str] = [] out_gdf = cudf.from_pandas(df[[]]) for col in df.columns: @@ -154,7 +181,7 @@ def _cudf_from_pandas_best_effort(df: pd.DataFrame): failed_cols.append(str(col)) string_df = pd.DataFrame({col: series.astype("string")}) out_gdf[col] = cudf.from_pandas(string_df)[col] - if failed_cols: + if failed_cols and warn: warnings.warn( "Best-effort pandas->cuDF coercion converted mixed-type columns to string dtype: " + ", ".join(f"{col}[{df[col].dtype}]" for col in failed_cols), @@ -163,7 +190,48 @@ def _cudf_from_pandas_best_effort(df: pd.DataFrame): return out_gdf -def df_to_engine(df, engine: Engine): +def is_polars_df(df: Any) -> bool: + """True if ``df`` is a polars DataFrame or LazyFrame. + + Import-light module-name check (polars is an optional dependency, so we avoid importing + it just to ``isinstance``). ``type(df).__module__`` starts with ``polars.`` for both + ``pl.DataFrame`` and ``pl.LazyFrame``. Single source of truth — the gfql engine had this + reimplemented in 5 places.""" + return df is not None and "polars" in type(df).__module__ + + +def active_frames_are_polars(g: Any) -> bool: + """True if ``g``'s active table (nodes, else edges) is a polars frame.""" + if g._nodes is not None: + return is_polars_df(g._nodes) + return is_polars_df(g._edges) + + +def _pl_nan_to_null(df): + """Convert NaN -> null in float columns of a polars frame. + + Matches ``pl.from_pandas(nan_to_null=True)`` (the pandas-input path) so a *native* + polars / Arrow / cuDF input carrying genuine NaN is treated as MISSING like the pandas + oracle (which skipna/dropna's NaN). Without this, ``engine='polars'`` on a frame with a + real NaN keeps rows a filter/aggregation should drop (silent divergence from pandas). + No-op when there are no float columns.""" + import polars as pl + float_cols = [c for c, dt in df.schema.items() if dt in (pl.Float32, pl.Float64)] + if not float_cols: + return df + return df.with_columns([pl.col(c).fill_nan(None) for c in float_cols]) + + +def df_to_engine(df, engine: Engine, *, validate: Optional[ValidationParam] = None, warn: bool = True): + """Convert ``df`` to ``engine``'s frame type. + + ``validate``/``warn`` (see ``graphistry.models.types.ValidationParam``) control how + mixed-type object columns that cannot go to Arrow are handled. ``validate=None`` + (default) means "use the engine's own default": cuDF defaults ``autofix`` (best-effort + coerce mixed cols to string + warn, its shipped behavior), polars defaults ``strict`` + (parity-or-raise). ``strict``/``strict-fast`` raise ``ArrowConversionError`` (cuDF) or + ``NotImplementedError`` (polars); ``autofix`` coerces + warns on both. + """ if engine == Engine.PANDAS: if isinstance(df, pd.DataFrame): return df @@ -201,7 +269,7 @@ def df_to_engine(df, engine: Engine): return df if not isinstance(df, pd.DataFrame): df = df_to_engine(df, Engine.PANDAS) - return _cudf_from_pandas_best_effort(df) + return _cudf_from_pandas_best_effort(df, validate=validate, warn=warn) elif engine == Engine.DASK: import dask.dataframe as dd if isinstance(df, dd.DataFrame): @@ -209,7 +277,127 @@ def df_to_engine(df, engine: Engine): if not isinstance(df, pd.DataFrame): df = df_to_engine(df, Engine.PANDAS) return dd.from_pandas(df, npartitions=1) - raise ValueError(f'Only engines pandas/cudf/dask supported, got: {engine}') + elif engine in POLARS_ENGINES: + import polars as pl + if isinstance(df, pl.DataFrame): + return _pl_nan_to_null(df) + if isinstance(df, pl.LazyFrame): + # Collect via the target-aware lazy collect so the executor knobs apply: + # engine=POLARS_GPU collects on the cudf-polars GPU executor (gpu_executor()), + # engine=POLARS honors cpu_streaming(). A bare df.collect() would materialize on + # the CPU default executor and ignore both — the POLARS_ENGINES branch does not + # otherwise distinguish POLARS from POLARS_GPU for a LazyFrame input. Function-local + # import: lazy/__init__ imports no heavy deps and never imports Engine (no cycle). + from graphistry.compute.gfql.lazy import collect as _lazy_collect, target_mode, ExecutionTarget + _tgt = ExecutionTarget.GPU if engine == Engine.POLARS_GPU else ExecutionTarget.CPU + with target_mode(_tgt): + return _pl_nan_to_null(_lazy_collect(df)) + if isinstance(df, pa.Table): + return _pl_nan_to_null(pl.from_arrow(df)) + pl_validate: ValidationParam = 'strict' if validate is None else validate + if isinstance(df, pd.DataFrame): + return _pl_from_pandas(df, validate=pl_validate, warn=warn) + # cuDF (device) -> Arrow -> polars: a single host copy via cuDF's native + # interchange, not the cuDF -> pandas -> polars double-convert. Besides the + # extra hop, the pandas detour is lossy (cuDF nullable Int64/boolean -> + # pandas float+NaN/object), whereas Arrow preserves dtypes and nulls. + # (Skipping the host round trip entirely for polars-gpu is a deeper + # follow-up: cudf_polars still ingests a host polars frame today.) + if 'cudf' in str(type(df).__module__): + import cudf + if isinstance(df, cudf.DataFrame): + return _pl_nan_to_null(pl.from_arrow(df.to_arrow())) + # dask/spark and anything else: route through pandas + return _pl_from_pandas(df_to_engine(df, Engine.PANDAS), validate=pl_validate, warn=warn) + raise ValueError(f'Only engines pandas/cudf/dask/polars supported, got: {engine}') + + +def _mixed_type_object_columns(df) -> List[str]: + """Object columns holding >1 Python scalar type among non-null values. + + Cypher properties are dynamically typed, so a pandas object column can hold + e.g. ``int`` and ``str`` together; polars/Arrow cannot represent that in one + column. Used to name the offender in the honest ``NotImplementedError``.""" + bad: List[str] = [] + for col in df.columns: + if str(getattr(df[col], "dtype", "")) != "object": + continue + types = set() + for value in df[col].to_numpy(): + if value is None or (isinstance(value, float) and value != value): # None / NaN + continue + types.add(type(value)) + if len(types) > 1: + bad.append(str(col)) + break + return bad + + +def _pl_from_pandas(df, *, validate: ValidationParam = 'strict', warn: bool = True): + """``pl.from_pandas`` participating in the repo-wide ``validate``/``warn`` convention. + + Polars/Arrow cannot represent a mixed-type (e.g. int+str) object column, which + pandas allows for dynamically-typed Cypher properties. This mirrors the plot()/ + upload() ``validate`` knob (``graphistry.models.types.ValidationParam``), but — + unlike the display/upload boundary, which defaults ``autofix`` — the compute path + defaults ``strict``: silently string-coercing a mixed column diverges from the + pandas oracle (pandas keeps it ``object`` + Python-compares), a wrong-answer risk. + So parity-or-raise is the safe default; ``autofix`` is an explicit opt-in. + + - ``strict``/``strict-fast`` (default): raise ``NotImplementedError`` (use + ``engine='pandas'``, or ``validate='autofix'`` to coerce), instead of surfacing a + cryptic ``pyarrow.lib.ArrowInvalid`` from deep inside construction. + - ``autofix`` (or ``validate=False``): coerce the offending mixed-type object + column(s) to string and, if ``warn``, emit a ``RuntimeWarning`` — matching + ``_cudf_from_pandas_best_effort``. + """ + import polars as pl + from graphistry.validate.common import normalize_validation_params + validate_mode, warn = normalize_validation_params(validate, warn) + try: + return pl.from_pandas(df) + except Exception as e: + bad = _mixed_type_object_columns(df) + if validate_mode == 'autofix' and bad: + fixed = df.astype({col: 'string' for col in bad}) + try: + out = pl.from_pandas(fixed) + except Exception: + out = None + if out is not None: + if warn: + warnings.warn( + "engine='polars': coerced mixed-type column(s) to string for " + f"Arrow conversion: {bad}. Convert explicitly or use " + "engine='pandas' for control.", + RuntimeWarning, + ) + return out + hint = f" (mixed-type column(s): {bad})" if bad else "" + raise NotImplementedError( + "engine='polars' cannot represent a heterogeneous/mixed-type column" + f"{hint}; use engine='pandas', or validate='autofix' to coerce to string" + ) from e + +def _pl_concat(frames, *, ignore_index: bool = True, sort: bool = False, **_kw): + """pandas/cudf-signature-compatible row concat for polars. + + polars has no row index and no ``sort`` kwarg; ``ignore_index``/``sort`` are + accepted-and-ignored so generic call sites (hop/chain) work unchanged. + ``vertical_relaxed`` aligns to a common supertype like pandas does for + mixed-but-compatible column dtypes. + """ + import polars as pl + frames = list(frames) + if len(frames) == 0: + return pl.DataFrame() + if len(frames) == 1: + return frames[0] + if isinstance(frames[0], pl.Series): + # Series concat only supports the default vertical strategy. + return pl.concat(frames) + return pl.concat(frames, how="vertical_relaxed") + def df_concat(engine: Engine): if engine == Engine.PANDAS: @@ -217,9 +405,11 @@ def df_concat(engine: Engine): elif engine == Engine.CUDF: import cudf return cudf.concat + elif engine in POLARS_ENGINES: + return _pl_concat elif engine == Engine.DASK: raise NotImplementedError("DASK is an input format, not a compute engine — use engine='auto' or engine='pandas'") - raise ValueError(f'Only engines pandas/cudf supported, got: {engine}') + raise ValueError(f'Only engines pandas/cudf/polars supported, got: {engine}') def align_shared_column_dtypes( @@ -290,7 +480,22 @@ def df_cons(engine: Engine): elif engine == Engine.CUDF: import cudf return cudf.DataFrame - raise ValueError(f'Only engines pandas/cudf supported, got: {engine}') + elif engine in POLARS_ENGINES: + import polars as pl + return pl.DataFrame + raise ValueError(f'Only engines pandas/cudf/polars supported, got: {engine}') + + +def df_unique(df, engine: Engine): + """Row-dedupe keeping first occurrence, engine-aware. + + pandas/cuDF use ``drop_duplicates``; polars uses ``unique(maintain_order=True)`` + (``maintain_order`` matches ``drop_duplicates(keep='first')`` — same convention as + ``compute/gfql/row/frame_ops.distinct``). Avoids calling the pandas-only + ``drop_duplicates`` on a polars frame (the UNION DISTINCT crash).""" + if engine in POLARS_ENGINES: + return df.unique(maintain_order=True) + return df.drop_duplicates(ignore_index=True) def s_cons(engine: Engine): if engine == Engine.PANDAS: @@ -298,7 +503,10 @@ def s_cons(engine: Engine): elif engine == Engine.CUDF: import cudf return cudf.Series - raise ValueError(f'Only engines pandas/cudf supported, got: {engine}') + elif engine in POLARS_ENGINES: + import polars as pl + return pl.Series + raise ValueError(f'Only engines pandas/cudf/polars supported, got: {engine}') def s_sqrt(engine: Engine): if engine == Engine.PANDAS: diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 891c3eb3ac..624a0ffc93 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -2,7 +2,7 @@ import pandas as pd from typing import Any, Dict, List, Optional, Union from typing_extensions import Literal -from graphistry.Engine import Engine, EngineAbstract, EngineAbstractType, resolve_engine, df_to_engine, df_concat, safe_merge +from graphistry.Engine import Engine, EngineAbstract, EngineAbstractType, POLARS_ENGINES, resolve_engine, df_to_engine, df_concat, safe_merge from graphistry.Plottable import Plottable from graphistry.util import setup_logger from graphistry.utils.json import JSONVal @@ -86,12 +86,25 @@ def _is_already_correct(df: Any) -> bool: if not has_cudf: return True # cudf unavailable — skip coercion; downstream handles gracefully return 'cudf' in type_mod + elif engine in POLARS_ENGINES: + return 'polars' in type_mod return True if g._edges is not None and not _is_already_correct(g._edges): g = g.edges(df_to_engine(g._edges, engine), g._source, g._destination) if g._nodes is not None and not _is_already_correct(g._nodes): g = g.nodes(df_to_engine(g._nodes, engine), g._node) + # A NATIVE-polars input is "already correct" and skips df_to_engine above, so it never + # gets the NaN->null normalization the pandas->polars path does (pl.from_pandas nan_to_null). + # Without it, engine='polars' on a frame carrying real NaN keeps rows a filter/aggregation + # should drop (silent divergence from the pandas oracle, which treats NaN as missing). + # _pl_nan_to_null is idempotent, so re-running it on a just-converted frame is a no-op. + if engine in POLARS_ENGINES: + from graphistry.Engine import _pl_nan_to_null, is_polars_df + if g._edges is not None and is_polars_df(g._edges): + g = g.edges(_pl_nan_to_null(g._edges), g._source, g._destination) + if g._nodes is not None and is_polars_df(g._nodes): + g = g.nodes(_pl_nan_to_null(g._nodes), g._node) return g @@ -215,16 +228,25 @@ def materialize_nodes( ) node_id = g._node if g._node is not None else "id" if _safe_len(g._edges) == 0: - empty_nodes_df = ( - g._edges[[g._source]] - .rename(columns={g._source: node_id}) - .reset_index(drop=True) - ) + if engine_concrete in POLARS_ENGINES: + empty_nodes_df = g._edges.select(g._source).rename({g._source: node_id}) + else: + empty_nodes_df = ( + g._edges[[g._source]] + .rename(columns={g._source: node_id}) + .reset_index(drop=True) + ) return g.nodes(empty_nodes_df, node_id) concat_fn = df_concat(engine_concrete) concat_df = concat_fn([g._edges[g._source], g._edges[g._destination]]) - nodes_df = concat_df.rename(node_id).drop_duplicates().to_frame().reset_index(drop=True) + if engine_concrete in POLARS_ENGINES: + # polars Series has no row index and uses .unique() (== pandas drop_duplicates + # keep-first with maintain_order) + .to_frame(); .drop_duplicates()/.reset_index + # are pandas-only and raise on a polars Series (edges-only graph under engine='polars'). + nodes_df = concat_df.rename(node_id).unique(maintain_order=True).to_frame() + else: + nodes_df = concat_df.rename(node_id).drop_duplicates().to_frame().reset_index(drop=True) return g.nodes(nodes_df, node_id) def _single_direction_degree(self, key_col: str, col: str) -> "Plottable": diff --git a/graphistry/compute/ast.py b/graphistry/compute/ast.py index d024568137..7f53199a62 100644 --- a/graphistry/compute/ast.py +++ b/graphistry/compute/ast.py @@ -1751,6 +1751,28 @@ def drop_cols(cols: Iterable[str]) -> ASTCall: return ASTCall("drop_cols", {"cols": list(cols)}) +def count_table( + table: str = "nodes", + source: Optional[str] = None, + alias: str = "count(*)", +) -> ASTCall: + """Count matched rows without materializing them (fast path for a lone ``count(*)``). + + Emitted by the Cypher lowering when a RETURN is exactly ``count(*)`` over a + single node/edge pattern (no DISTINCT, GROUP BY, row-level WHERE, UNWIND, + paging, or multi-relationship binding). Produces a one-row table + ``{alias: n}`` where ``n`` is the height of the active ``table`` (or, when a + ``source`` alias-mask column is present, the count of its truthy rows). This + avoids the full-frame materialize + constant-key ``group_by`` the general + aggregate path performs — the win that turns count(*) from O(N) into a single + reduction. See plans/gfql-engine-followups (BEAT LADYBUG). + """ + params: Dict[str, Any] = {"table": table, "alias": alias} + if source is not None: + params["source"] = source + return ASTCall("count_table", params) + + def group_by( keys: Iterable[str], aggregations: Iterable[Sequence[Any]], diff --git a/graphistry/compute/chain.py b/graphistry/compute/chain.py index 0edd42b6b1..96044f5c64 100644 --- a/graphistry/compute/chain.py +++ b/graphistry/compute/chain.py @@ -1,6 +1,6 @@ import logging from typing import Any, Dict, Union, cast, List, Tuple, Sequence, Optional, TYPE_CHECKING -from graphistry.Engine import Engine, EngineAbstract, align_shared_column_dtypes, df_concat, df_to_engine, resolve_engine, safe_map_series, safe_row_concat, s_na +from graphistry.Engine import Engine, EngineAbstract, POLARS_ENGINES, align_shared_column_dtypes, df_concat, df_to_engine, resolve_engine, safe_map_series, safe_row_concat, s_na from graphistry.compute.dataframe_utils import dbg_df from graphistry.Plottable import Plottable @@ -796,7 +796,46 @@ def chain( if isinstance(engine, str): engine = EngineAbstract(engine) from graphistry.compute.ComputeMixin import _coerce_input_formats # lazy — avoids circular import - self = _coerce_input_formats(self, resolve_engine(engine, self)) + engine_concrete_early = resolve_engine(engine, self) + if engine_concrete_early in (Engine.POLARS, Engine.POLARS_GPU): + # Clean dependency errors BEFORE _coerce_input_formats (which imports polars to + # coerce frames) so the user sees actionable install guidance, not a raw ImportError + # deep in coercion / the lazy engine. Both engines need polars; polars-gpu also needs + # the RAPIDS cudf_polars stack (checked here so it's consistent regardless of whether a + # given query reaches a GPU collect, and reads as an install issue rather than the + # genuine not-GPU-capable signal from lazy._engine_for). + try: + import polars # noqa: F401 + except ImportError as e: + raise ImportError( + f"GFQL engine={engine_concrete_early.value!r} requires the 'polars' package; " + "install it with `pip install polars` (or use engine='pandas')." + ) from e + if engine_concrete_early == Engine.POLARS_GPU: + import importlib.util + if importlib.util.find_spec("cudf_polars") is None: + raise ImportError( + "GFQL engine='polars-gpu' requires the RAPIDS cudf_polars stack (NVIDIA GPU). " + "Install RAPIDS/cudf_polars, or use engine='polars' for native CPU execution." + ) + self = _coerce_input_formats(self, engine_concrete_early) + + if engine_concrete_early in POLARS_ENGINES: + # Native polars chain lives in a dedicated dispatched module so the + # production pandas/cuDF orchestration below stays untouched (see + # no-silent-fallback policy). Correctness gated by differential parity. + # POLARS_GPU = the same lazy engine with the GPU execution target. + # (Dependency guards for polars / cudf_polars are above, pre-coercion.) + if validate_schema: + Chain(ops if not isinstance(ops, Chain) else ops.chain).validate(collect_all=False) + from graphistry.compute.gfql.lazy.engine.polars.chain import chain_polars + from graphistry.compute.gfql.lazy import target_mode, ExecutionTarget + # NO pandas fallback here (no-silent-fallback policy): chain_polars raises + # NotImplementedError for deferred features (var-length/multi-hop edges, + # undirected multi-edge); that honest signal propagates to the caller. + _tgt = ExecutionTarget.GPU if engine_concrete_early == Engine.POLARS_GPU else ExecutionTarget.CPU + with target_mode(_tgt): + return chain_polars(self, ops, start_nodes=start_nodes) if policy: from graphistry.compute.gfql.call.executor import _thread_local as call_thread_local diff --git a/graphistry/compute/chain_let.py b/graphistry/compute/chain_let.py index e8583ccaff..1832bb9feb 100644 --- a/graphistry/compute/chain_let.py +++ b/graphistry/compute/chain_let.py @@ -580,7 +580,12 @@ def chain_let_impl(g: Plottable, dag: ASTLet, else: raise policy_error elif binding_error is not None: - # Wrap in RuntimeError with context + # Honest engine-capability declines propagate as-is so a DAG caller can catch + # NotImplementedError and retry on engine='pandas' (matching the chain surface); + # don't bury them in a RuntimeError. + if isinstance(binding_error, NotImplementedError): + raise binding_error + # Wrap other failures in RuntimeError with context raise RuntimeError( f"Failed to execute node '{node_name}' in DAG. " f"Error: {type(binding_error).__name__}: {str(binding_error)}" diff --git a/graphistry/compute/gfql/call/executor.py b/graphistry/compute/gfql/call/executor.py index 0934847999..eb6b0aa0bf 100644 --- a/graphistry/compute/gfql/call/executor.py +++ b/graphistry/compute/gfql/call/executor.py @@ -6,9 +6,11 @@ import threading import time -from typing import Dict, Any, cast, Optional, TYPE_CHECKING, Tuple +import warnings +from typing import Dict, Any, cast, Optional, TYPE_CHECKING, Set, Tuple from graphistry.Plottable import Plottable from graphistry.Engine import Engine +from graphistry.compute.gfql.lazy import call_mode from graphistry.compute.gfql.call.validation import validate_call_params from graphistry.compute.gfql.row.pipeline import ( execute_row_pipeline_call, @@ -50,16 +52,110 @@ def _run_policy_hook(handler: Any, policy_context: 'PolicyContext', data_size: O return None -def _execute_validated_call(g: Plottable, function: str, validated_params: Dict[str, Any]) -> Any: +from graphistry.Engine import active_frames_are_polars as _active_frames_are_polars + +# Off-engine call() modality bridge (PHASE 12). A GFQL call() that runs a Plottable-method +# ANALYTIC (umap / hypergraph / compute_cugraph / compute_igraph / layout_* / collapse / ...) +# has NO native polars impl and never will — it runs eagerly on pandas/cuDF. Under a polars +# engine, call_mode()='auto' (default) BRIDGES: run the analytic off-engine on pandas (polars) +# / cuDF (polars-gpu), then coerce the result back to polars losslessly (Arrow). call_mode()= +# 'strict' DECLINES (parity-or-NIE) for benchmark integrity / a hard memory ceiling. This is +# DELIBERATELY narrower than CHAIN traversal/filter/row ops, which stay parity-or-NIE (a bridge +# there hides a missing impl + cheats a benchmark). The distinction is MECHANICAL, not a curated +# list: row-pipeline calls (is_row_pipeline_call) and native-polars degree calls are dispatched +# BEFORE the analytic path in _execute_validated_call, so everything reaching the bridge is by +# construction a non-native eager analytic. Mirrors the GRAPHISTRY_CUDF_SAME_PATH_MODE auto/strict +# precedent. (Follow-ups tracked in plan PHASE 12: G3 otel attribution, G4 queryable flag, G5 size guard.) +_OFFENGINE_BRIDGE_WARNED: Set[str] = set() + + +def _compute_engine_for_offengine_call(engine: Engine, function: str) -> Engine: + """Modality an off-engine analytic runs on under a polars engine. + + ``polars`` -> pandas; ``polars-gpu`` -> cuDF (on-device). polars-gpu is GPU-or-error: + if cuDF is unavailable we do NOT silently drop a GPU analytic to host pandas (which + would break the engine contract and risk a unified-memory OOM) — we decline. + """ + if engine == Engine.POLARS_GPU: + try: + import cudf # type: ignore # noqa: F401 + except Exception as e: # cuDF/GPU stack missing + raise NotImplementedError( + f"GFQL engine='polars-gpu' call '{function}' runs on cuDF (on device), but " + f"cuDF is unavailable ({type(e).__name__}). polars-gpu does not silently fall " + "back to host pandas. Install the cuDF/GPU stack, or use engine='pandas'." + ) + return Engine.CUDF + return Engine.PANDAS + + +def _bridge_graph_for_offengine_call(g: Plottable, function: str, engine: Engine) -> Plottable: + """Bridge ``g`` to the compute modality for an off-engine analytic, or decline (strict). + + ``call_mode()='strict'`` raises ``NotImplementedError`` (honest decline). ``'auto'`` (default) + converts ``g``'s frames polars->pandas (or polars->cuDF for polars-gpu) so the analytic gets + the frame type it expects, warns ONCE per (process, function), and returns the bridged graph. + The caller coerces the analytic's result back to polars (``ensure_engine_match``, lossless). + """ + if call_mode() == "strict": + raise NotImplementedError( + f"GFQL engine='{engine.value}' does not natively support call '{function}' " + "(it runs on pandas/cuDF); call_mode='strict' declines the off-engine bridge. " + "Use engine='pandas', or set call_mode='auto' (the default) to run it off-engine." + ) + compute_engine = _compute_engine_for_offengine_call(engine, function) + # Convert the frames EXPLICITLY via df_to_engine — not ensure_engine_match, whose + # resolve_engine(AUTO, ...) detection classifies a polars frame as PANDAS (polars isn't a + # resolve_engine target), so it would treat the polars input as "already pandas" and no-op. + # df_to_engine is a genuine no-op when the frame is already compute_engine's type. + from graphistry.Engine import df_to_engine + bridged = g + if g._nodes is not None: + bridged = bridged.nodes(df_to_engine(g._nodes, compute_engine), g._node) + if g._edges is not None: + bridged = bridged.edges( + df_to_engine(g._edges, compute_engine), g._source, g._destination, edge=g._edge + ) + if function not in _OFFENGINE_BRIDGE_WARNED: + _OFFENGINE_BRIDGE_WARNED.add(function) + warnings.warn( + f"GFQL call '{function}' has no native polars implementation; running it off-engine " + f"on {compute_engine.value} and coercing the result back to '{engine.value}' " + "(lossless via Arrow). Set call_mode='strict' to decline instead.", + RuntimeWarning, + stacklevel=3, + ) + return bridged + + +def _execute_validated_call(g: Plottable, function: str, validated_params: Dict[str, Any], engine: Engine) -> Any: if is_row_pipeline_call(function): return execute_row_pipeline_call(g, function, validated_params) + # NATIVE polars degree calls (get_degrees / get_indegrees / get_outdegrees): pure + # groupby/count over edge endpoints — NO pandas bridge (see NO-CHEATING). Reached by + # the let()/ref() DAG surface (and the schema-changer chain path); the native chain + # surface routes the same ops through polars.chain._try_native_row_op. The result is + # polars, so it passes the no-bridge guard in execute_call (and ensure_engine_match is + # then a no-op). Other Plottable-method calls have no native polars impl and stay + # declined by that guard. + if _active_frames_are_polars(g) and function in ("get_degrees", "get_indegrees", "get_outdegrees"): + from graphistry.compute.gfql.lazy.engine.polars import degrees as _pl_degrees + return getattr(_pl_degrees, function + "_polars")(g, **validated_params) + if not hasattr(g, function): raise AttributeError( f"Plottable has no method '{function}'. " f"This should not happen if safelist is properly configured." ) + # Off-engine analytic (no native polars impl — the row-pipeline + native-degree paths + # above already returned for the native ops). Under a polars engine, bridge to pandas/cuDF + # (call_mode='auto', no-op if frames already match) or decline (call_mode='strict'). Gated on + # engine, not frame type, so strict declines every off-engine analytic consistently. + if engine in (Engine.POLARS, Engine.POLARS_GPU): + g = _bridge_graph_for_offengine_call(g, function, engine) + method = getattr(g, function) return method(**validated_params) @@ -130,7 +226,7 @@ def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: En if function == 'hypergraph': hypergraph_returns_dataframe = validated_params.get('return_as', 'graph') != 'graph' - result = _execute_validated_call(g, function, validated_params) + result = _execute_validated_call(g, function, validated_params, engine) execution_time = time.perf_counter() - start_time # Ensure result is a Plottable (most methods return self or new Plottable) @@ -144,8 +240,11 @@ def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: En suggestion="Only methods that return Plottable objects are allowed" ) - # Ensure result matches requested engine (defensive coercion) - # Schema-changing operations (UMAP, hypergraph) may alter DataFrame types + # Ensure result matches requested engine (defensive coercion). Schema-changing ops + # (UMAP, hypergraph) may alter DataFrame types. Under a polars engine, an off-engine + # analytic ran on pandas/cuDF via the call_mode='auto' bridge (see _execute_validated_call); + # coerce its result back to polars losslessly (Arrow). call_mode='strict' already declined + # upstream with NotImplementedError (propagated cleanly below). if _is_plottable_like(result): result = ensure_engine_match(cast(Plottable, result), engine) result = apply_call_schema_effect(g, cast(Plottable, result), function, validated_params) @@ -204,6 +303,13 @@ def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: En ) from error if isinstance(error, GFQLTypeError): raise error + if isinstance(error, NotImplementedError) and engine in (Engine.POLARS, Engine.POLARS_GPU): + # Honest engine-capability decline — propagate as-is so the DAG surface matches the chain + # surface's NotImplementedError. Sources: call_mode='strict' off-engine decline, and the + # polars-gpu GPU-or-error when cuDF is unavailable (_bridge_graph_for_offengine_call). + # Gated to the polars engines: a pandas/cudf NIE (e.g. fa2_layout requiring a GPU) must + # still fall through to the GFQLTypeError(E303) wrapper below, not leak as a bare NIE. + raise error if error is not None: raise GFQLTypeError( ErrorCode.E303, diff --git a/graphistry/compute/gfql/call/validation.py b/graphistry/compute/gfql/call/validation.py index 8ad5ae9813..3d4778ae3c 100644 --- a/graphistry/compute/gfql/call/validation.py +++ b/graphistry/compute/gfql/call/validation.py @@ -491,6 +491,19 @@ def _semi_apply_mark_added_node_cols(params: Dict[str, object]) -> Set[str]: description='Drop duplicate rows from active row table', ), + 'count_table': _safelist_entry( + {'table', 'source', 'alias'}, + param_validators={ + 'table': lambda v: v in ['nodes', 'edges'], + 'source': is_string_or_none, + 'alias': is_non_empty_string, + }, + description='Count matched rows (node/edge table height or source-alias mask) into a one-row table, without materializing the frame — fast path for a lone count(*)', + schema_effects=_schema_effects( + adds_node_cols=lambda p: [p.get('alias', 'count(*)')], + ), + ), + 'get_degrees': _safelist_entry( {'col', 'degree_in', 'degree_out', 'engine'}, param_validators={ diff --git a/graphistry/compute/gfql/cypher/call_procedures.py b/graphistry/compute/gfql/cypher/call_procedures.py index bc6207375d..d4e3acffe7 100644 --- a/graphistry/compute/gfql/cypher/call_procedures.py +++ b/graphistry/compute/gfql/cypher/call_procedures.py @@ -622,7 +622,7 @@ def compile_cypher_call( def _graph_row_result(base_graph: Plottable, rows_df: DataFrameT) -> Plottable: out = base_graph.bind() out._nodes = rows_df - edges_df = getattr(base_graph, "_edges", None) + edges_df = base_graph._edges if edges_df is not None: out._edges = cast(DataFrameT, edges_df[:0]) else: diff --git a/graphistry/compute/gfql/cypher/lowering.py b/graphistry/compute/gfql/cypher/lowering.py index 93bcd30b72..c79a4b3548 100644 --- a/graphistry/compute/gfql/cypher/lowering.py +++ b/graphistry/compute/gfql/cypher/lowering.py @@ -13,6 +13,7 @@ ASTObject, ASTNode, anti_semi_apply, + count_table, distinct, drop_cols, e_forward, @@ -1236,7 +1237,7 @@ def _rewrite(node_in: ExprNode) -> ExprNode: if isinstance(node_in, PropertyAccessExpr) and isinstance(node_in.value, Identifier): if "." not in node_in.value.name and node_in.value.name in alias_targets: return node_in - return PropertyAccessExpr(cast(ExprNode, _rewrite(node_in.value)), node_in.property) + return PropertyAccessExpr(_rewrite(node_in.value), node_in.property) if isinstance(node_in, Identifier) and "." not in node_in.name and node_in.name in alias_targets: target = alias_targets[node_in.name] prop = NODE_IDENTITY_COLUMN if isinstance(target, ASTNode) else "__gfql_edge_index_0__" @@ -2308,6 +2309,62 @@ def _match_relationship_count(clause: MatchClause) -> int: return sum(1 for element in _match_pattern_elements(clause) if isinstance(element, RelationshipPattern)) +def _is_pure_count_star_shortcircuit( + *, + aggregate_specs: Sequence[_AggregateSpec], + pre_items: Sequence[Tuple[str, Any]], + row_steps: Sequence[ASTObject], + query: CypherQuery, + binding_row_aliases: AbstractSet[str], + relationship_count: int, + active_match_alias: Optional[str], + alias_targets: Mapping[str, ASTObject], +) -> bool: + """True when a RETURN is exactly ``count(*)`` over a single node/edge scan. + + Guards the count_table fast path (skip the full-frame materialize + constant- + key group_by): the count then equals the height (or source-mask sum) of the + active table. Requires a lone non-DISTINCT ``count(*)`` with no group keys, + row-level WHERE, UNWIND, or multi-relationship binding, and a plain + ``rows(table=nodes|edges[, source])`` as the only prior step. Post-aggregate + exprs (``count(*) + 1``) compose fine: the count lands in a temp column and + the trailing ``select`` applies the expr, same as the group_by path. + Sound only for a pure node scan (``relationship_count == 0``) or a single + relationship counted on its edge alias (``relationship_count == 1`` with an + ``ASTEdge`` active alias) — exactly the cases + ``_reject_unsound_relationship_multiplicity_aggregates`` permits; any other + shape (node-alias-over-relationship, multi-hop paths) falls through to the + general aggregate path (which counts bindings or rejects as unsound). + """ + if len(aggregate_specs) != 1: + return False + agg = aggregate_specs[0] + if agg.func != "count" or agg.expr_text is not None or agg.distinct: + return False + if pre_items or binding_row_aliases or query.unwinds: + return False + # Exactly the initial rows() step: any row-level WHERE or UNWIND would have + # appended further steps, so len == 1 proves the count is over the raw scan. + if len(row_steps) != 1: + return False + base = row_steps[0] + if not (isinstance(base, ASTCall) and base.function == "rows"): + return False + if base.params.get("table") not in ("nodes", "edges"): + return False + if base.params.get("binding_ops") is not None or base.params.get("alias_endpoints") is not None: + return False + if relationship_count == 0: + return True + if ( + relationship_count == 1 + and active_match_alias is not None + and isinstance(alias_targets.get(active_match_alias), ASTEdge) + ): + return True + return False + + def _reject_unsound_relationship_multiplicity_aggregates_common( *, aggregate_specs: Sequence[_AggregateSpec], @@ -2928,7 +2985,7 @@ def _stitch_patterns( merged = _merge_node_patterns(left_end, cast(NodePattern, reversed_right[0])) return tuple(left[:-1]) + (merged,) + tuple(reversed_right[1:]) if _node_can_join(left_start, right_end): - merged = _merge_node_patterns(cast(NodePattern, right_end), left_start) + merged = _merge_node_patterns(right_end, left_start) return tuple(right[:-1]) + (merged,) + tuple(left[1:]) if _node_can_join(left_start, right_start): reversed_right = _reverse_pattern(right) @@ -5948,8 +6005,8 @@ def lower_match_query( stack: List[BooleanExpr] = [query.where.expr_tree] while stack: cur = stack.pop() - expr_left = cast(Optional[BooleanExpr], cur.left) - expr_right = cast(Optional[BooleanExpr], cur.right) + expr_left = cur.left + expr_right = cur.right if cur.op in {"or", "xor"} and ((expr_left is not None and _where_expr_tree_pattern_predicates(expr_left)) or (expr_right is not None and _where_expr_tree_pattern_predicates(expr_right))): raise _unsupported_at_span("Cypher WHERE pattern predicates mixed with OR/XOR are not yet supported for cartesian MATCH patterns", field="where", value=where_expr_upper, span=query.where.span) if expr_left is not None: @@ -6573,6 +6630,35 @@ def _lower_general_row_projection( if len(pre_items) > 0: row_steps.append(with_(pre_items, extend=bindings_row_path)) row_steps.append(group_by(key_names, aggregations, key_prefixes=alias_key_prefixes)) + elif _is_pure_count_star_shortcircuit( + aggregate_specs=aggregate_specs, + pre_items=pre_items, + row_steps=row_steps, + query=query, + binding_row_aliases=binding_row_aliases, + relationship_count=relationship_count, + active_match_alias=active_match_alias, + alias_targets=alias_targets, + ): + # Fast path: count_table reads the scanned table's height (or the + # source-alias mask sum) with one reduction — no full-frame + # materialize + constant-key group_by. It replaces the sole rows() + # step and produces the same ``count(*)`` column the group_by would, + # so the trailing identity projection (elif not key_names, below) is + # unchanged. empty_result_row is belt-and-suspenders here (count_table + # always emits a 1-row result), kept for parity with the group_by path. + base_rows = cast(ASTCall, row_steps[0]) + count_alias = aggregate_specs[0].output_name + row_steps = [ + count_table( + table=cast(str, base_rows.params.get("table", "nodes")), + source=cast(Optional[str], base_rows.params.get("source")), + alias=count_alias, + ) + ] + available_columns = {count_alias} + empty_aggregate_row = _empty_aggregate_row(aggregate_specs) + empty_result_row = empty_aggregate_row else: global_key = _fresh_temp_name(temp_names, "__cypher_group__") row_steps.append(with_([(global_key, 1)] + pre_items)) diff --git a/graphistry/compute/gfql/cypher/procedures/networkx.py b/graphistry/compute/gfql/cypher/procedures/networkx.py index 1a59b6499e..b57a82eff1 100644 --- a/graphistry/compute/gfql/cypher/procedures/networkx.py +++ b/graphistry/compute/gfql/cypher/procedures/networkx.py @@ -317,7 +317,7 @@ def _networkx_graph( assert graph_with_nodes._destination is not None nodes_pdf = as_pandas_frame(graph_with_nodes._nodes, (graph_with_nodes._node,)) - edges_df = getattr(graph_with_nodes, "_edges", None) + edges_df = graph_with_nodes._edges edge_columns = (graph_with_nodes._source, graph_with_nodes._destination) edges_pdf = pd.DataFrame(columns=list(edge_columns)) if edges_df is None else as_pandas_frame(edges_df, edge_columns) @@ -355,7 +355,7 @@ def networkx_edge_rows(base_graph: Plottable, compiled_call: CompiledCypherProce assert graph_with_nodes._source is not None assert graph_with_nodes._destination is not None edge_columns = (graph_with_nodes._source, graph_with_nodes._destination) - edges_df = getattr(graph_with_nodes, "_edges", None) + edges_df = graph_with_nodes._edges edges_pdf = pd.DataFrame(columns=list(edge_columns)) if edges_df is None else as_pandas_frame(edges_df, edge_columns) scores = _networkx_algorithm_result(compiled_call, graph_nx, params) diff --git a/graphistry/compute/gfql/cypher/reentry/execution.py b/graphistry/compute/gfql/cypher/reentry/execution.py index d0d332612d..6028a60d21 100644 --- a/graphistry/compute/gfql/cypher/reentry/execution.py +++ b/graphistry/compute/gfql/cypher/reentry/execution.py @@ -20,13 +20,40 @@ REENTRY_DUPLICATE_CARRIED_ROWS_REASON = "duplicate_carried_node_rows" +from graphistry.Engine import is_polars_df as _is_polars_df + + +def _reentry_row(prefix_rows: Any, row_index: int) -> Any: + """One prefix row as a col->scalar mapping, engine-aware (``row[col]`` works for + both the pandas Series and the polars named-row dict).""" + if _is_polars_df(prefix_rows): + return prefix_rows.row(row_index, named=True) + return prefix_rows.iloc[row_index] + + +def _assign_constant_columns(df: Any, values: Dict[str, Any]) -> Any: + """Broadcast scalar ``values`` as constant columns, engine-aware.""" + if not values: + return df + if _is_polars_df(df): + import polars as pl + return df.with_columns([pl.lit(v).alias(k) for k, v in values.items()]) + return df.assign(**values) + + +def _drop_columns(df: Any, cols: Sequence[str]) -> Any: + if _is_polars_df(df): + return df.drop(list(cols)) + return df.drop(columns=list(cols)) + + def _bind_reentry_graph(graph: Plottable, node_rows: Optional[DataFrameT], *, empty_edges: bool = False) -> Plottable: out = graph.bind() out._nodes = node_rows if empty_edges: - edges_df = getattr(graph, "_edges", None) + edges_df = graph._edges if edges_df is not None: - out._edges = cast(DataFrameT, edges_df.iloc[0:0]) + out._edges = cast(DataFrameT, edges_df.head(0) if _is_polars_df(edges_df) else edges_df.iloc[0:0]) return out @@ -58,8 +85,8 @@ def apply_optional_reentry_null_fill( reentry_plan: Optional[ReentryPlan] = None, ) -> Plottable: """Null-fill result rows for prefix rows that the optional reentry didn't match.""" - prefix_df = getattr(prefix_result, "_nodes", None) - result_df = getattr(result, "_nodes", None) + prefix_df = prefix_result._nodes + result_df = result._nodes if prefix_df is None or len(prefix_df) == 0: return result @@ -196,7 +223,7 @@ def compiled_query_reentry_state( ) output_name = plan.reentry_alias_name carried_columns = tuple(plan.scalar_columns) - prefix_rows = cast(Optional[DataFrameT], getattr(prefix_result, "_nodes", None)) + prefix_rows = cast(Optional[DataFrameT], prefix_result._nodes) prefix_alias_values: Optional[SeriesT] = None if prefix_rows is not None and output_name in prefix_rows.columns: prefix_alias_values = cast(SeriesT, prefix_rows[output_name]) @@ -215,8 +242,8 @@ def compiled_query_reentry_state( non_null_mask = cast(SeriesT, prefix_alias_values.notna()) has_non_null_ids = bool(non_null_mask.any()) if hasattr(non_null_mask, "any") else True if not has_non_null_ids: - base_nodes = cast(Optional[DataFrameT], getattr(base_graph, "_nodes", None)) - id_column = cast(Optional[str], getattr(base_graph, "_node", None)) + base_nodes = cast(Optional[DataFrameT], base_graph._nodes) + id_column = base_graph._node if base_nodes is None or id_column is None or id_column not in base_nodes.columns: raise reentry_validation_error( "Cypher MATCH after WITH could not recover the base node table for re-entry", @@ -253,7 +280,7 @@ def compiled_query_reentry_state( value=output_name, suggestion=REENTRY_WHOLE_ROW_SUGGESTION, ) - base_nodes = getattr(base_graph, "_nodes", None) + base_nodes = base_graph._nodes if base_nodes is None or id_column not in base_nodes.columns: raise reentry_validation_error( "Cypher MATCH after WITH could not recover the base node table for re-entry", @@ -263,7 +290,7 @@ def compiled_query_reentry_state( concrete_engine = resolve_engine(cast(Any, engine), base_graph) carried_ids, aligned_prefix_rows = aligned_reentry_rows( ids=cast(SeriesT, ids), - prefix_rows=getattr(prefix_result, "_nodes", None), + prefix_rows=prefix_result._nodes, output_name=output_name, ) carried_node_ids = cast(DataFrameT, df_cons(concrete_engine)({id_column: carried_ids})) @@ -316,7 +343,7 @@ def union_scalar_reentry_results( """Union per-row suffix results from a multi-row scalar prefix.""" node_frames = [] for r in row_results: - nodes = getattr(r, "_nodes", None) + nodes = r._nodes if nodes is not None and len(cast(Any, nodes)) > 0: node_frames.append(nodes) if node_frames: @@ -324,7 +351,7 @@ def union_scalar_reentry_results( concat = df_concat(concrete_engine) node_rows = cast(DataFrameT, concat(node_frames, ignore_index=True)) else: - base_nodes = getattr(base_graph, "_nodes", None) + base_nodes = base_graph._nodes node_rows = cast(DataFrameT, base_nodes.iloc[0:0]) if base_nodes is not None else None return _bind_reentry_graph(base_graph, node_rows) @@ -336,7 +363,7 @@ def compiled_query_scalar_reentry_state( carried_columns: Sequence[str], row_index: int = 0, ) -> Tuple[Plottable, Optional[DataFrameT]]: - prefix_rows = getattr(prefix_result, "_nodes", None) + prefix_rows = prefix_result._nodes if prefix_rows is None: raise reentry_validation_error( "Cypher MATCH after WITH scalar-only prefix stages could not recover prefix rows", @@ -344,7 +371,7 @@ def compiled_query_scalar_reentry_state( suggestion="Project scalar columns directly before MATCH re-entry.", ) prefix_row_count = len(prefix_rows) - base_nodes = getattr(base_graph, "_nodes", None) + base_nodes = base_graph._nodes if prefix_row_count == 0: if base_nodes is None: return base_graph, None @@ -370,14 +397,15 @@ def compiled_query_scalar_reentry_state( value=missing_column, suggestion="Project the scalar column explicitly before MATCH re-entry.", ) - row = prefix_rows.iloc[row_index] + row = _reentry_row(prefix_rows, row_index) node_rows = cast( DataFrameT, - base_nodes.assign( - **{ + _assign_constant_columns( + base_nodes, + { _reentry_hidden_column_name(output_name): row[output_name] for output_name in carried_columns - } + }, ), ) return _bind_reentry_graph(base_graph, node_rows), None @@ -392,7 +420,7 @@ def freeform_broadcast_row_to_nodes( row_index: int, ) -> Plottable: """Broadcast one free-form prefix row's hidden carries onto the base nodes.""" - row = prefix_rows.iloc[row_index] + row = _reentry_row(prefix_rows, row_index) broadcast_values: Dict[str, Any] = { _reentry_hidden_column_name(col): row[col] for col in plan.scalar_columns @@ -407,11 +435,11 @@ def freeform_broadcast_row_to_nodes( if broadcast_values: existing_hidden = [c for c in base_nodes.columns if isinstance(c, str) and c.startswith("__cypher_reentry_")] node_rows = ( - cast(DataFrameT, base_nodes.drop(columns=existing_hidden)) + cast(DataFrameT, _drop_columns(base_nodes, existing_hidden)) if existing_hidden else base_nodes ) - node_rows = cast(DataFrameT, node_rows.assign(**broadcast_values)) + node_rows = cast(DataFrameT, _assign_constant_columns(node_rows, broadcast_values)) else: node_rows = cast(DataFrameT, base_nodes) @@ -425,8 +453,8 @@ def compiled_query_freeform_reentry_state( plan: ReentryPlan, ) -> Tuple[Plottable, Optional[DataFrameT]]: """Build the single-row dispatch state for free-form intermediate MATCH.""" - prefix_rows = getattr(prefix_result, "_nodes", None) - base_nodes = getattr(base_graph, "_nodes", None) + prefix_rows = prefix_result._nodes + base_nodes = base_graph._nodes if base_nodes is None: raise reentry_validation_error( "Cypher MATCH after WITH (free-form intermediate MATCH; #1263) " diff --git a/graphistry/compute/gfql/cypher/result_postprocess.py b/graphistry/compute/gfql/cypher/result_postprocess.py index bf2ce497f0..05e4b26d63 100644 --- a/graphistry/compute/gfql/cypher/result_postprocess.py +++ b/graphistry/compute/gfql/cypher/result_postprocess.py @@ -7,6 +7,7 @@ from graphistry.Plottable import Plottable from graphistry.compute.typing import DataFrameT, SeriesT +from graphistry.Engine import is_polars_df from graphistry.compute.gfql.series_str_compat import is_non_textual_scalar_dtype from .lowering import ResultProjectionColumn, ResultProjectionPlan @@ -292,8 +293,22 @@ def apply_result_projection( Cypher-display-string column; the reentry / OPTIONAL-MATCH null-fill machinery (which still assumes a single-column entity value) opts out via this flag until it is unified onto the structured path. + + For ``engine='polars'`` the native projection lives in gfql.lazy.engine.polars (not this + pandas-audited module); it renders natively or raises NotImplementedError — NO + pandas bridge (no-silent-fallback policy). """ - rows_df = cast(DataFrameT, getattr(result, "_nodes", None)) + rows_df = result._nodes + if is_polars_df(rows_df): + from graphistry.compute.gfql.lazy.engine.polars.projection import apply_result_projection_polars + return apply_result_projection_polars(result, projection, structured=structured) + return _apply_result_projection_pandas(result, projection, structured=structured) + + +def _apply_result_projection_pandas( + result: Plottable, projection: ResultProjectionPlan, *, structured: bool = True +) -> Plottable: + rows_df = cast(Optional[DataFrameT], result._nodes) if rows_df is None: return result alias_rows_df = _projection_alias_rows(rows_df, alias=projection.alias) @@ -309,7 +324,7 @@ def apply_result_projection( if source_rows_df is None or source_alias not in source_rows_df.columns: raise ValueError(f"whole-row projection source alias not found: {source_alias!r}") source_projection = projection if source_alias == projection.alias else replace(projection, alias=source_alias) - id_column = getattr(result, "_node" if source_projection.table == "nodes" else "_edge", None) + id_column = result._node if source_projection.table == "nodes" else result._edge flat_columns = ( _flat_entity_columns(source_rows_df, source_projection, column.output_name, id_column) if structured @@ -321,9 +336,17 @@ def apply_result_projection( projected_data.update(flat_columns) output_columns.extend(flat_columns.keys()) elif structured: - # No fields to flatten: the synthesized absent-entity row (OPTIONAL miss - # / reentry no-match, a single ``{alias: None}`` column) or a field-less - # real entity. Emit the single-column text form (renders to None / []). + # ⚠️ REGRESSION GUARD — DO NOT REMOVE (#1650). This fallback fixes + # two regressions: top-level OPTIONAL-MATCH miss, and + # OPTIONAL-WITH-reentry no-match. It looks redundant but is not. + # No flattenable fields: the entity has no materialized property/id + # columns on this frame. This is the synthesized null/absent-entity + # row (top-level OPTIONAL-MATCH miss / reentry no-match, built by + # _apply_empty_result_row as a single ``{alias: None}`` column). There + # is nothing to flatten, so emit the single-column text form (which + # renders to ``None`` here) — preserving the legacy shape the + # OPTIONAL/reentry machinery consumes. Real rows always carry flat + # field columns and take the branch above. projected_data[column.output_name] = ( _format_node_entities(source_rows_df, source_projection) if source_projection.table == "nodes" @@ -385,7 +408,7 @@ def apply_result_projection( out._nodes = projected_nodes if projected_entity_meta: setattr(out, "_cypher_entity_projection_meta", projected_entity_meta) - edges_df = getattr(result, "_edges", None) + edges_df = result._edges if edges_df is not None: out._edges = edges_df[:0] return out diff --git a/graphistry/compute/gfql/lazy/__init__.py b/graphistry/compute/gfql/lazy/__init__.py new file mode 100644 index 0000000000..1a5df26c60 --- /dev/null +++ b/graphistry/compute/gfql/lazy/__init__.py @@ -0,0 +1,256 @@ +"""Lazy / deferred GFQL execution framework. + +Lazy engines build a deferred plan and execute it once on a chosen **execution target** +(CPU / GPU now; remote backends later). This package is the shared framework — the target +abstraction + collect-once helpers — that each lazy backend (``lazy/engine/polars``, future +``lazy/engine/duckdb``) plugs into. Per-backend *lowering* lives under +``lazy/engine//`` and is NOT shared (polars builds ``pl.LazyFrame``, DuckDB would +build relations). + +Design (from the GPU benchmark + architecture review): ``engine='polars'`` and +``engine='polars-gpu'`` are ONE lazy polars engine with two ``.collect()`` targets, not two +engines. The win requires ONE plan collected ONCE (transfer-once, fused) — per-op eager +collect loses on GPU (repeated H2D) — so backends accumulate a plan and call +:func:`collect` / :func:`collect_all` at the materialization boundary, on the active target. +""" +from __future__ import annotations + +import contextvars +from enum import Enum +from typing import TYPE_CHECKING, Any, List, Optional +from typing_extensions import Literal + +if TYPE_CHECKING: + import polars as pl + + +class ExecutionTarget(Enum): + CPU = "cpu" + GPU = "gpu" # polars GPU backend (cudf_polars) + + +_TARGET: "contextvars.ContextVar[ExecutionTarget]" = contextvars.ContextVar( + "gfql_lazy_target", default=ExecutionTarget.CPU +) + + +def active_target() -> ExecutionTarget: + return _TARGET.get() + + +class target_mode: + """Context manager selecting the execution target for an enclosed lazy run.""" + + def __init__(self, target: ExecutionTarget) -> None: + self._target = target + self._token: Optional[contextvars.Token] = None + + def __enter__(self) -> "target_mode": + self._token = _TARGET.set(self._target) + return self + + def __exit__(self, *exc: Any) -> None: + if self._token is not None: + _TARGET.reset(self._token) + + +import os as _os + +# --- polars execution config ----------------------------------------------------- +# Two knobs, each resolved LIVE at call time (not frozen at import), in order: +# (1) Python override (set_*, None = unset) > (2) env var > (3) default. +# NOTE: overrides are PROCESS-GLOBAL module state (unlike _TARGET, a ContextVar that +# target_mode sets transiently) — a startup-time global default, not per-request/thread; +# a concurrent set_cpu_streaming() affects all in-flight GFQL queries in the process. +_cpu_streaming_override: Optional[bool] = None +_gpu_executor_override: "Optional[GpuExecutor]" = None + +#: Valid cudf-polars GPU collect executors (public; see :func:`set_gpu_executor`). +GPU_EXECUTORS = ("in-memory", "streaming") +GpuExecutor = Literal["in-memory", "streaming"] + + +def cpu_streaming() -> bool: + """Does the CPU polars collect use the STREAMING executor? (default off) + + Streaming is faster + more stable on big traversal joins (isolated 80M-edge 2-hop semijoin + 1669→1040 ms, ~1.6×; end-to-end chain dilutes to ~1.04–1.11× since forward/backward/combine + overhead is unaffected), parity-identical. Opt-in because small/interactive sizes REGRESS + (~0.86× at 100K) from streaming overhead. + Resolution: :func:`set_cpu_streaming` override > ``$GFQL_POLARS_CPU_STREAMING`` (``"1"``) > ``False``. + """ + if _cpu_streaming_override is not None: + return _cpu_streaming_override + return _os.environ.get("GFQL_POLARS_CPU_STREAMING", "0") == "1" + + +def set_cpu_streaming(value: Optional[bool]) -> None: + """Enable/disable the CPU streaming-collect from Python. ``None`` resets to env/default.""" + global _cpu_streaming_override + _cpu_streaming_override = None if value is None else bool(value) + + +def gpu_executor() -> GpuExecutor: + """cudf-polars GPU collect executor: ``'in-memory'`` (default) or ``'streaming'``. + + 'in-memory' is fast + stable for results fitting device memory (the GFQL regime — see + :func:`_engine_for`); 'streaming' is the opt-in escape hatch for larger-than-device results + (in-memory would OOM), slower/less stable on small work. ONLY these two values are accepted + (``GPU_EXECUTORS``); a size-aware 'auto' is a possible future addition but NOT selectable + today (``set_gpu_executor('auto')`` raises). + Resolution: :func:`set_gpu_executor` override > ``$GFQL_POLARS_GPU_EXECUTOR`` > ``'in-memory'`` + (an invalid env value also resolves to 'in-memory'). + """ + if _gpu_executor_override is not None: + return _gpu_executor_override + raw = _os.environ.get("GFQL_POLARS_GPU_EXECUTOR", "in-memory").strip().lower() + return "streaming" if raw == "streaming" else "in-memory" + + +def set_gpu_executor(value: Optional[str]) -> None: + """Select the GPU collect executor from Python (``'in-memory'`` | ``'streaming'``). + + ``None`` resets to env/default; an invalid value raises ``ValueError`` (the Python + setter is strict, unlike the env path which falls back to 'in-memory').""" + global _gpu_executor_override + if value is None: + _gpu_executor_override = None + return + v = value.strip().lower() + if v not in GPU_EXECUTORS: + raise ValueError(f"gpu_executor must be one of {GPU_EXECUTORS}, got {value!r}") + _gpu_executor_override = "streaming" if v == "streaming" else "in-memory" + + +# --- off-engine call() modality policy ------------------------------------------- +# A GFQL call() may run a Plottable-method ANALYTIC (umap/hypergraph/compute_cugraph/ +# compute_igraph/layout_*/collapse/...) with NO native polars impl, ever — it runs eagerly on +# pandas/cuDF. Under engine='polars'|'polars-gpu' this knob decides: +# 'auto' (default): BRIDGE — run off-engine on pandas (polars) / cuDF (polars-gpu), coerce +# back to polars losslessly (Arrow), warn once per (process, function). +# 'strict': DECLINE with NotImplementedError — benchmark integrity (no hidden modality +# switch) or a hard memory ceiling. +# DELIBERATELY distinct from CHAIN traversal/filter/row ops, which stay parity-or-NIE (a bridge +# there would hide a missing impl + cheat a benchmark). Mirrors the +# GRAPHISTRY_CUDF_SAME_PATH_MODE auto/strict precedent. +_call_mode_override: "Optional[CallMode]" = None + +#: Valid off-engine call() modality modes (public; see :func:`set_call_mode`). +CALL_MODES = ("auto", "strict") +CallMode = Literal["auto", "strict"] + + +def call_mode() -> CallMode: + """Off-engine ``call()`` analytic policy under a polars engine: ``'auto'`` | ``'strict'``. + + ``'auto'`` (default) bridges an analytic with no native polars impl (umap / hypergraph / + compute_cugraph / ...) to pandas/cuDF, runs it, and coerces the result back to polars + losslessly (warn once per function). ``'strict'`` declines with ``NotImplementedError`` + (no hidden modality switch — for benchmarking / a hard memory ceiling). + Resolution: :func:`set_call_mode` override > ``$GFQL_POLARS_CALL_MODE`` > ``'auto'`` + (an invalid env value also resolves to 'auto'). + """ + if _call_mode_override is not None: + return _call_mode_override + raw = _os.environ.get("GFQL_POLARS_CALL_MODE", "auto").strip().lower() + return "strict" if raw == "strict" else "auto" + + +def set_call_mode(value: Optional[str]) -> None: + """Select the off-engine call() policy from Python (``'auto'`` | ``'strict'``). + + ``None`` resets to env/default; an invalid value raises ``ValueError`` (the Python + setter is strict, unlike the env path which falls back to 'auto').""" + global _call_mode_override + if value is None: + _call_mode_override = None + return + v = value.strip().lower() + if v not in CALL_MODES: + raise ValueError(f"call_mode must be one of {CALL_MODES}, got {value!r}") + _call_mode_override = "strict" if v == "strict" else "auto" + + +# Public surface. Explicit (no dedicated types.py): the quasi-public types +# (ExecutionTarget/GpuExecutor/CallMode + tuples) are few and co-located with their +# validators/accessors; a types module is deferred until a 2nd lazy backend (duckdb) +# needs to share ExecutionTarget (mirrors the lazy-restructure defer). +__all__ = [ + "ExecutionTarget", "active_target", "target_mode", "collect", "collect_all", + "GpuExecutor", "GPU_EXECUTORS", "gpu_executor", "set_gpu_executor", + "CallMode", "CALL_MODES", "call_mode", "set_call_mode", + "cpu_streaming", "set_cpu_streaming", +] + + +def _engine_for(target: ExecutionTarget) -> "Optional[pl.GPUEngine]": + """Polars collect engine for a target. ``None`` = default (CPU streaming/in-mem). + + GPU uses the cudf-polars IN-MEMORY executor, not the default streaming `engine="gpu"` + (`DefaultSingletonEngine`): GFQL results fit device memory — the in-memory regime — and it + is FASTER (semijoin 1.33×, antijoin 2.58×, unique 1.49× @10M) and STABLE (streaming spiked + bimodally to ~1 s on the same semijoin; in-memory holds ~30 ms). ``raise_on_fail=True`` is + the NO-CHEATING contract for the GPU target: any non-GPU-executable plan node RAISES — + never silently run on CPU and reported as GPU. (raise_on_fail=False looked honest — the + fallback stays in Polars, not a pandas bridge — but it makes polars-gpu indistinguishable + from polars whenever the plan isn't fully GPU-capable, mislabeling CPU work as GPU; hard + raise forces the truth: polars-gpu is GPU-or-error, use polars for CPU.) Larger-than-device + inputs OOM rather than stream — acceptable (in-scope gfql graphs fit); revisit if that changes.""" + if target == ExecutionTarget.GPU: + import polars as pl + # RAPIDS/cudf_polars-not-installed is checked at the chain dispatch (pre-coercion) so + # engine='polars-gpu' gets a clean install error there; here we only build the engine — + # a genuinely non-GPU-capable plan is reported via _gpu_raise. Executor defaults to + # in-memory; GFQL_POLARS_GPU_EXECUTOR=streaming opts into streaming for + # larger-than-device results (see gpu_executor()). + executor = gpu_executor() + return pl.GPUEngine(executor=executor, raise_on_fail=True) + return None + + +def _gpu_raise(exc: Exception) -> "NotImplementedError": + """Translate a cudf-polars GPU failure (raise_on_fail=True, non-GPU-executable plan node) + into the NO-CHEATING error: a clear NotImplementedError chained from the polars error + (not a cryptic ComputeError), pointing at the CPU engine.""" + return NotImplementedError( + "GFQL engine='polars-gpu': this query plan is not fully GPU-executable on the " + "installed cudf-polars (NO-CHEATING: we raise rather than silently run it on CPU " + "and call it a GPU result). Rerun with engine='polars' for native CPU execution. " + f"Underlying GPU error: {type(exc).__name__}: {exc}" + ) + + +def collect(lf: "pl.LazyFrame") -> "pl.DataFrame": + """Collect one polars LazyFrame on the active target (CPU/GPU).""" + eng = _engine_for(active_target()) + if eng is not None: + try: + return lf.collect(engine=eng) # pragma: no cover # GPU-target collect (no GPU in CI) + except NotImplementedError: + raise + except Exception as ex: # pragma: no cover # GPU-target collect (no GPU in CI) + raise _gpu_raise(ex) from ex + return lf.collect(engine="streaming") if cpu_streaming() else lf.collect() + + +def collect_all(lfs: "List[pl.LazyFrame]") -> "List[pl.DataFrame]": + """Collect several LazyFrames in ONE pass on the active target: common subplans (e.g. the + edge table) materialize/transfer once — the whole point of going lazy on GPU. Falls back to + per-frame collect if the installed polars lacks ``collect_all``.""" + import polars as pl + eng = _engine_for(active_target()) + if hasattr(pl, "collect_all"): + try: + if eng is not None: + return pl.collect_all(lfs, engine=eng) # pragma: no cover # GPU-target collect (no GPU in CI) + return pl.collect_all(lfs, engine="streaming") if cpu_streaming() else pl.collect_all(lfs) + except TypeError: + # older signature without engine= — collect individually on target + pass + except NotImplementedError: # pragma: no cover # GPU-target collect (no GPU in CI) + raise + except Exception as ex: # pragma: no cover # GPU-target collect (no GPU in CI) + if eng is not None: + raise _gpu_raise(ex) from ex + raise + return [collect(lf) for lf in lfs] diff --git a/graphistry/compute/gfql/lazy/engine/__init__.py b/graphistry/compute/gfql/lazy/engine/__init__.py new file mode 100644 index 0000000000..2f8f64514c --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/__init__.py @@ -0,0 +1,4 @@ +"""Per-backend lazy GFQL engines. Each backend lowers GFQL to its own deferred +plan representation (polars: ``pl.LazyFrame``; future duckdb: relations) and +executes via the shared target/collect framework in ``graphistry.compute.gfql.lazy``. +The lowering is per-backend; only the target/collect framework is shared.""" diff --git a/graphistry/compute/gfql/lazy/engine/polars/__init__.py b/graphistry/compute/gfql/lazy/engine/polars/__init__.py new file mode 100644 index 0000000000..d4c07083d9 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/__init__.py @@ -0,0 +1,14 @@ +"""Lazy Polars GFQL backend. + +Builds one ``pl.LazyFrame`` plan per query and collects ONCE on the active +execution target (CPU or GPU) — so polars (CPU) and polars-gpu are two TARGETS of +this single engine, not two engines. DRY: reuses the cypher-AST -> ``pl.Expr`` +lowering from this package (``lower_expr`` / ``predicate_to_expr`` / agg / +select / order_by lowering) verbatim — only the materialization strategy differs +(eager ``.collect()`` per op -> lazy plan + collect-once). +""" + +from .hop_eager import hop_polars +from .chain import chain_polars + +__all__ = ["hop_polars", "chain_polars"] diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py new file mode 100644 index 0000000000..10d15c0d21 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -0,0 +1,685 @@ +"""Native Polars chain() — Phase 1, vectorized. + +Reimplements the chain forward/backward/combine orchestration in polars, reusing the polars hop +for edge steps. Vectorization-first: set ops are semi/anti joins, alias tags are join-based flag +columns — no Python id lists or ``is_in(python_list)``. Parity-or-NIE contract: differential +parity vs the pandas chain gates correctness; unsupported shapes raise NotImplementedError +(no silent pandas fallback). Deferred: variable-length/multi-hop edge sub-cases, some +undirected multi-edge combos, node query=. +""" +from typing import Any, List, Optional, Tuple + +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge +from .hop_eager import ensure_nodes_polars +from .dtypes import is_lazy, colnames, endpoint_ids +from .degrees import get_degrees_polars, get_indegrees_polars, get_outdegrees_polars +from .predicates import filter_by_dict_polars + + +def _semi(df, ids_df, df_col, id_col): + """Rows of df whose df_col is present in ids_df[id_col] (vectorized semi-join).""" + return df.join(ids_df.select(id_col).unique(), left_on=df_col, right_on=id_col, how="semi") + + +def _align_seed_dtype(seed, node_col, ref_nodes): + """Cast the seed's node-id column to the node table's id dtype. start_nodes can arrive with a + divergent dtype (e.g. empty crossfilter selection defaults float64 vs int64 ids); polars won't + auto-cast, so the combine semi-join raises SchemaError where pandas joins fine. No-op when the + column is absent or already matches.""" + import polars as pl + if seed is None or node_col not in seed.columns: + return seed + ndt = ref_nodes.schema[node_col] + if seed.schema[node_col] == ndt: + return seed + return seed.with_columns(pl.col(node_col).cast(ndt)) + + +def _align_edge_endpoints(g, node_col, src, dst): + """Cast edge endpoint columns to the node-id dtype so join keys match. + + polars won't auto-cast int↔float join keys, and a null endpoint promotes its column to float + while int node ids stay int — endpoint↔node-id joins then raise SchemaError where pandas joins + fine. The hop casts internally; the chain (fast paths + combine) did not. Returns + ``(aligned_g, restore)``: restore = original (src_dtype, dst_dtype) to put back on the OUTPUT + edges (matching pandas' dtype), or None when already matched (common case — no table copy).""" + import polars as pl + ndt = g._nodes.schema[node_col] + sdt, ddt = g._edges.schema[src], g._edges.schema[dst] + if sdt == ndt and ddt == ndt: + return g, None + aligned = g.edges(g._edges.with_columns([pl.col(src).cast(ndt), pl.col(dst).cast(ndt)]), src, dst) + return aligned, (sdt, ddt) + + +def _restore_edge_dtypes(edges, src, dst, restore): + """Restore output edge endpoint dtypes recorded by :func:`_align_edge_endpoints`.""" + if restore is None: + return edges + import polars as pl + sdt, ddt = restore + return edges.with_columns([pl.col(src).cast(sdt), pl.col(dst).cast(ddt)]) + + +def _exec(op: ASTObject, g: Plottable, prev_wf, target_wf, intermediate_universe=None) -> Plottable: + import polars as pl + + node_col = g._node + assert node_col is not None + if isinstance(op, ASTNode): + if op.query is not None: + raise NotImplementedError("polars chain engine does not yet support node query=") + base = prev_wf if prev_wf is not None else g._nodes + nodes = filter_by_dict_polars(base, op.filter_dict) + if target_wf is not None: + nodes = _semi(nodes, target_wf, node_col, node_col) + if op._name is not None: + nodes = nodes.with_columns(pl.lit(True).alias(op._name)) + return g.nodes(nodes, node_col).edges(g._edges.clear(), g._source, g._destination) + + if isinstance(op, ASTEdge): + from graphistry.compute.gfql.lazy.engine.polars.hop import hop_lazy_or_eager + g_step = hop_lazy_or_eager( + g, + nodes=prev_wf, + hops=op.hops, + min_hops=op.min_hops, + max_hops=op.max_hops, + to_fixed_point=op.to_fixed_point, + direction=op.direction, + edge_match=op.edge_match, + source_node_match=op.source_node_match, + destination_node_match=op.destination_node_match, + source_node_query=op.source_node_query, + destination_node_query=op.destination_node_query, + edge_query=op.edge_query, + label_node_hops=op.label_node_hops, + label_edge_hops=op.label_edge_hops, + label_seeds=op.label_seeds, + output_min_hops=op.output_min_hops, + output_max_hops=op.output_max_hops, + include_zero_hop_seed=op.include_zero_hop_seed, + return_as_wave_front=True, + target_wave_front=target_wf, + intermediate_universe=intermediate_universe, + # Chain wants pandas' LABELED min_hops node policy (null attrs on source-side + # endpoints; mirrors ASTEdge.execute's auto hop-labels). Direct base.hop( + # engine='polars', min_hops=...) leaves this False -> full attrs (pandas parity). + min_hops_label_policy=True, + ) + if op._name is not None: + g_step = g_step.edges( + g_step._edges.with_columns(pl.lit(True).alias(op._name)), + g._source, g._destination, + ) + return g_step + + raise NotImplementedError(f"polars chain engine does not support op {type(op).__name__}") + + +def _is_native_multihop(op: ASTObject) -> bool: + """Multi-hop shapes the native combine supports: hops=N / max_hops (fwd/rev/undirected), + to_fixed_point (fwd/rev), min_hops>1 (fwd/rev, finite max), optionally with match/name. + Deferred combos (undirected to_fixed_point / undirected min_hops>1, output slicing, hop + labels, *_query, include_zero_hop_seed, prune_to_endpoints) return False so the guard + NIEs rather than risk silent divergence — inline comments give per-case reasons.""" + if not isinstance(op, ASTEdge): + return False + if op.is_simple_single_hop(): + return False + if op.direction not in ("forward", "reverse", "undirected"): + return False + # to_fixed_point IS native fwd/rev: recompute re-runs the forward tfp hop over the + # backward-pruned subgraph (hop's fixed-point detection guarantees termination), same + # path-aware combine as hops=N. decline (NIE): UNDIRECTED tfp needs pandas' connected- + # components + 2-core seed retention (hop.py:817-887), not reproducible in the vectorized hop. + if op.direction == "undirected" and op.to_fixed_point: + return False + # min_hops>1 IS native fwd/rev with finite max_hops (NON-anti-joined BFS + layered + # backward-tree walk). decline (NIE): undirected min_hops>1 and min_hops+to_fixed_point. + if op.min_hops is not None and op.min_hops > 1 and (op.direction == "undirected" or op.to_fixed_point): + return False + if op.output_min_hops is not None or op.output_max_hops is not None: + return False + if op.label_node_hops is not None or op.label_edge_hops is not None or op.label_seeds: + return False + if op.source_node_query is not None or op.destination_node_query is not None or op.edge_query is not None: + return False + if op.include_zero_hop_seed or op.prune_to_endpoints: + return False + return True + + +class _LazyShim: + """Track B collect-once shim: carries _nodes/_edges as LazyFrames (+ col names) so the eager + combine helpers run lazily over already-materialized hop frames without Plottable rebinds.""" + __slots__ = ("_nodes", "_edges", "_node", "_source", "_destination", "_edge") + + def __init__(self, nodes_lf, edges_lf, node, source, destination, edge): + self._nodes = nodes_lf + self._edges = edges_lf + self._node = node + self._source = source + self._destination = destination + self._edge = edge + + @staticmethod + def step(p): + nd = p._nodes.lazy() if p._nodes is not None else None + ed = p._edges.lazy() if p._edges is not None else None + return _LazyShim(nd, ed, None, None, None, None) + + +def _combine_edges(g, steps, label_steps, has_multihop=False): + import polars as pl + src, dst, node_col, edge_id = g._source, g._destination, g._node, g._edge + assert src is not None and dst is not None and node_col is not None and edge_id is not None + + frames = [] + for idx, (op, g_step) in enumerate(steps): + edges_df = g_step._edges + if edges_df is None: + continue + if not is_lazy(edges_df) and edges_df.height == 0: + continue + if has_multihop or (isinstance(op, ASTEdge) and not op.is_simple_single_hop()): + # has_multihop: every edge step was already recomputed path-valid (forward re-exec over + # its backward-pruned subgraph), so append ALL ids and skip the prev/next endpoint + # semijoin below — it would drop intermediate-hop edges (and diverged by an edge/node + # vs pandas on mixed single+multi chains). Port of pandas combine_steps has_multihop + # branch: recompute all steps, concat as-is, no semijoin. + frames.append(edges_df.select(pl.col(edge_id))) + continue + prev_nodes = label_steps[idx - 1][1]._nodes if idx > 0 else g._nodes + next_nodes = label_steps[idx + 1][1]._nodes if idx + 1 < len(label_steps) else None + direction = op.direction if isinstance(op, ASTEdge) else "forward" + + if direction == "undirected" and prev_nodes is not None and next_nodes is not None: + fwd = _semi(_semi(edges_df, prev_nodes, src, node_col), next_nodes, dst, node_col) + rev = _semi(_semi(edges_df, prev_nodes, dst, node_col), next_nodes, src, node_col) + edges_df = pl.concat([fwd, rev], how="vertical_relaxed").unique(subset=[edge_id]) + else: + prev_col, next_col = (dst, src) if direction == "reverse" else (src, dst) + if prev_nodes is not None: + edges_df = _semi(edges_df, prev_nodes, prev_col, node_col) + if next_nodes is not None: + edges_df = _semi(edges_df, next_nodes, next_col, node_col) + frames.append(edges_df.select(pl.col(edge_id))) + + if not frames: + out_ids = g._edges.select(pl.col(edge_id)).limit(0) + else: + out_ids = pl.concat(frames, how="vertical_relaxed").unique(subset=[edge_id]) + + out = g._edges.join(out_ids, on=edge_id, how="semi") + + for op, g_step in label_steps: + if op._name is not None and isinstance(op, ASTEdge) and g_step._edges is not None and op._name in colnames(g_step._edges): + named = g_step._edges.filter(pl.col(op._name)).select(pl.col(edge_id)).with_columns(pl.lit(True).alias(op._name)) + out = out.join(named, on=edge_id, how="left").with_columns(pl.col(op._name).fill_null(False)) + return out + + +def _combine_nodes(g, steps): + import polars as pl + node_col = g._node + assert node_col is not None + frames = [ + g_step._nodes.select(pl.col(node_col)) + for _, g_step in steps + if g_step._nodes is not None and node_col in colnames(g_step._nodes) + ] + if frames: + ids = pl.concat(frames, how="vertical_relaxed").unique(subset=[node_col]) + else: + ids = g._nodes.select(pl.col(node_col)).limit(0) + return g._nodes.join(ids, on=node_col, how="semi") + + +def _apply_node_names(out, g, steps): + """Tag node aliases on the FINAL node frame (after endpoint materialization). A node carries + the alias iff it matched the named step in the backward-PRUNED frame (dead-end matches + excluded) AND, when followed by an edge step, participates in that edge's PRUNED edges. + Pruned ``steps`` (not forward frames) are essential — forward frames over-include and would + tag nodes absent from the final graph.""" + import polars as pl + node_col, src, dst = g._node, g._source, g._destination + assert node_col is not None and src is not None and dst is not None + step_list = list(steps) + for idx, (op, g_step) in enumerate(step_list): + if op._name is None or not isinstance(op, ASTNode) or g_step._nodes is None: + continue + if op._name not in colnames(g_step._nodes): + continue + named = g_step._nodes.filter(pl.col(op._name)).select(pl.col(node_col)).unique() + if idx + 1 < len(step_list): + next_op, next_step = step_list[idx + 1] + if isinstance(next_op, ASTEdge) and next_step._edges is not None and (is_lazy(next_step._edges) or next_step._edges.height > 0): + e = next_step._edges + if next_op.direction == "forward": + part = e.select(pl.col(src).alias(node_col)) + elif next_op.direction == "reverse": + part = e.select(pl.col(dst).alias(node_col)) + else: + part = endpoint_ids(e, src, dst, node_col) + named = named.join(part.unique(), on=node_col, how="semi") + flag = named.with_columns(pl.lit(True).alias(op._name)) + out = out.join(flag, on=node_col, how="left").with_columns(pl.col(op._name).fill_null(False)) + return out + + +def _call_native_on_polars(op) -> bool: + """Whether a row-pipeline call has a native polars implementation (no bridge).""" + from graphistry.compute.ast import ASTCall + from graphistry.compute.gfql.row.pipeline import _POLARS_NATIVE_ROW_PIPELINE_CALLS + if not isinstance(op, ASTCall): + return False + if op.function not in _POLARS_NATIVE_ROW_PIPELINE_CALLS: + return False + if op.function == "rows" and ( + op.params.get("binding_ops") is not None + or op.params.get("alias_endpoints") is not None + ): + return False + return True + + +def _run_calls_polars(g_cur, calls, start_nodes, base_graph, middle): + """Execute a boundary run of ASTCall ops on a polars graph. + + Mirrors ``chain._handle_boundary_calls``: threads the row-pipeline context attrs and applies + the named-middle → ``rows(binding_ops=...)`` rewrite. Each call runs natively via + ``_try_native_row_op``; a row op with no native impl raises NotImplementedError (no pandas + fallback) rather than secretly running the pandas row pipeline. + """ + from graphistry.compute.ast import ASTCall, ASTNode as _ASTNode, ASTEdge as _ASTEdge, rows as rows_fn + from graphistry.compute.chain import serialize_binding_ops + + calls = list(calls) + if not calls: + return g_cur + + if start_nodes is not None: + setattr(g_cur, "_gfql_start_nodes", start_nodes) + setattr(g_cur, "_gfql_rows_base_graph", base_graph) + setattr(g_cur, "_gfql_shortest_path_backend", getattr(g_cur, "_gfql_shortest_path_backend", "auto")) + + if ( + middle + and any(getattr(op, "_name", None) is not None for op in middle) + and isinstance(calls[0], ASTCall) + and calls[0].function == "rows" + and calls[0].params.get("binding_ops") is None + and calls[0].params.get("source") is None + and calls[0].params.get("alias_endpoints") is None + and all(isinstance(op, (_ASTNode, _ASTEdge)) for op in middle) + ): + calls = [rows_fn(binding_ops=serialize_binding_ops(middle))] + list(calls[1:]) + + # Per-op NATIVE-OR-DEFER. Ops that don't lower: + # - non-native ROW op (correlated-subquery semi_apply/anti_semi_apply/join_apply): + # parity-or-NIE — it SHOULD be polars-native; a bridge would hide that. + # - ANALYTIC (compute_cugraph/compute_igraph/layout_*/collapse/get_topological_levels/...), + # no native impl by nature: route via execute_call, which applies the off-engine modality + # policy (call_mode='auto' bridges to pandas/cuDF + coerces back; 'strict' declines) — the + # SAME path as the DAG/let() surface, keeping surfaces consistent. Row-vs-analytic split + # is MECHANICAL (is_row_pipeline_call), not curated. (umap/hypergraph never reach here — + # the generic chain routes schema-changers straight to execute_call.) + from graphistry.compute.gfql.row.pipeline import is_row_pipeline_call + for op in calls: + native = _try_native_row_op(g_cur, op) + if native is not None: + g_cur = native + continue + fn = getattr(op, "function", None) + if fn is not None and not is_row_pipeline_call(fn): + from graphistry.compute.gfql.call.executor import execute_call + from graphistry.compute.gfql.lazy import active_target, ExecutionTarget + from graphistry.Engine import Engine as _Engine + _eng = _Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else _Engine.POLARS + g_cur = execute_call(g_cur, fn, getattr(op, "params", {}) or {}, _eng) + continue + raise NotImplementedError( + f"polars engine does not yet natively support cypher row op " + f"{getattr(op, 'function', op)!r}; use engine='pandas' for this query " + f"(no pandas fallback; parity-or-error by design)" + ) + return g_cur + + +def _try_native_row_op(g_cur, op): + """Run a row-pipeline call natively on polars, or return None to defer (NIE).""" + from graphistry.Engine import Engine + from .row_pipeline import select_polars, with_columns_polars, order_by_polars, group_by_polars, unwind_polars, where_rows_polars + + fn = getattr(op, "function", None) + if _call_native_on_polars(op): + # frame ops (rows/limit/skip/distinct/drop_cols) — engine-polymorphic + return op.execute(g=g_cur, prev_node_wavefront=None, target_wave_front=None, engine=Engine.POLARS) + if fn in ("select", "return_"): + return select_polars(g_cur, op.params.get("items", [])) + if fn == "with_": + # extend=True (WITH keeping existing columns) -> with_columns; extend=False (full + # re-projection) -> select. Both decline (NIE) on an unlowerable item. + if op.params.get("extend", False): + return with_columns_polars(g_cur, op.params.get("items", [])) + return select_polars(g_cur, op.params.get("items", [])) + if fn == "where_rows": + return where_rows_polars(g_cur, op.params.get("filter_dict"), op.params.get("expr")) + if fn == "order_by": + return order_by_polars(g_cur, op.params.get("keys", [])) + if fn == "group_by": + return group_by_polars(g_cur, op.params.get("keys", []), op.params.get("aggregations", [])) + if fn == "unwind": + return unwind_polars(g_cur, op.params.get("expr", ""), op.params.get("as_", "value")) + if fn == "get_degrees": + return get_degrees_polars( + g_cur, + col=op.params.get("col", "degree"), + degree_in=op.params.get("degree_in", "degree_in"), + degree_out=op.params.get("degree_out", "degree_out"), + ) + if fn == "get_indegrees": + return get_indegrees_polars(g_cur, col=op.params.get("col", "degree_in")) + if fn == "get_outdegrees": + return get_outdegrees_polars(g_cur, col=op.params.get("col", "degree_out")) + return None + + +def chain_polars(self: Plottable, ops, start_nodes: Optional[Any] = None) -> Plottable: + from graphistry.compute.ast import ASTCall + from graphistry.compute.chain import Chain, _get_boundary_calls + + if isinstance(ops, Chain): + ops = ops.chain + ops = list(ops) + + if len(ops) == 0: + return self + + # Reject duplicate alias names (node/edge aliases scoped separately; mirrors the pandas + # combine_steps E201 guard). A reused name like [n(name='a'), e(), n(name='a')] would + # produce a malformed schema (colliding a/a_right join cols) — wrong where pandas declines. + for _alias_type in (ASTNode, ASTEdge): + _seen: dict = {} + for _idx, _op in enumerate(ops): + _name = getattr(_op, "_name", None) + if _name is not None and isinstance(_op, _alias_type): + if _name in _seen: + from graphistry.compute.exceptions import GFQLValidationError, ErrorCode + raise GFQLValidationError( + code=ErrorCode.E201, + message=f"Duplicate alias name '{_name}' in chain (steps {_seen[_name]} and {_idx})", + suggestion="Use distinct alias names for each step in the chain", + ) + _seen[_name] = _idx + + has_call = any(isinstance(op, ASTCall) for op in ops) + has_traversal = any(isinstance(op, (ASTNode, ASTEdge)) for op in ops) + + if not has_call: + return _chain_traversal_polars(self, ops, start_nodes) + + if not has_traversal: + # Pure call chain (e.g. let() bodies): no traversal, just run the calls. + return _run_calls_polars(self, ops, start_nodes, base_graph=self, middle=[]) + + prefix, middle, suffix = _get_boundary_calls(ops) + + # has_traversal is True here, so middle is non-empty. + has_call_in_middle = any(isinstance(op, ASTCall) for op in middle) + has_traversal_in_middle = any(isinstance(op, (ASTNode, ASTEdge)) for op in middle) + if has_call_in_middle and has_traversal_in_middle: + from graphistry.compute.exceptions import GFQLValidationError, ErrorCode + raise GFQLValidationError( + code=ErrorCode.E201, + message="Cannot mix call() operations with n()/e() traversals in interior of chain", + suggestion="call() operations are only allowed at chain boundaries (start/end).", + ) + + if prefix: + # decline (NIE): leading call() yields a row table the following traversal would have to + # re-enter as a graph. pandas cascades via _chain_impl, but it's not a cypher shape + # (MATCH comes first) and the polars traversal doesn't yet consume a row-table input. + raise NotImplementedError( + "polars chain engine does not yet support call() before a traversal; " + "use engine='pandas' for this chain." + ) + + g_cur = _chain_traversal_polars(self, middle, start_nodes) + if suffix: + g_cur = _run_calls_polars(g_cur, suffix, start_nodes, base_graph=self, middle=middle) + return g_cur + + +def _chain_traversal_polars(self: Plottable, ops, start_nodes: Optional[Any] = None) -> Plottable: + import polars as pl + from graphistry.compute.chain import Chain + + if isinstance(ops, Chain): + ops = ops.chain + ops = list(ops) + + if len(ops) == 0: + return self + + # Node-only fast path: single MATCH (n) — the dominant tabular/viz/crossfilter shape. + # Result is just the filtered node table + empty edges, so skip forward/backward/combine + + # collect_all (~2.5 ms fixed cost at small sizes — the interactive crossfilter regime). + # Byte-identical: the one-node-step combine yields filtered g._nodes in order + empty edges + # + the alias flag on every matched node. + if len(ops) == 1 and isinstance(ops[0], ASTNode) and ops[0].query is None: + op0 = ops[0] + g0 = ensure_nodes_polars(self) + nc = g0._node + assert nc is not None and g0._source is not None and g0._destination is not None + nodes = filter_by_dict_polars(g0._nodes, op0.filter_dict) + if start_nodes is not None: + from graphistry.Engine import Engine as _E, df_to_engine as _d2e + seed = _align_seed_dtype(_d2e(start_nodes, _E.POLARS), nc, g0._nodes) + nodes = _semi(nodes, seed, nc, nc) + if op0._name is not None: + nodes = nodes.with_columns(pl.lit(True).alias(op0._name)) + return g0.nodes(nodes, nc).edges(g0._edges.clear(), g0._source, g0._destination) + + if isinstance(ops[0], ASTEdge): + ops = [ASTNode()] + ops + if isinstance(ops[-1], ASTEdge): + ops = ops + [ASTNode()] + + if any( + isinstance(op, ASTEdge) and not op.is_simple_single_hop() and not _is_native_multihop(op) + for op in ops + ): + raise NotImplementedError( + "polars chain engine supports single-hop and multi-hop edges " + "(hops=N / max_hops fwd/rev/undirected; to_fixed_point and min_hops>1 " + "fwd/rev); deferred features (undirected to_fixed_point / undirected " + "min_hops>1, output slicing, hop labels, *_query, " + "include_zero_hop_seed, prune_to_endpoints) require engine='pandas'." + ) + + edge_ops = [op for op in ops if isinstance(op, ASTEdge)] + # Undirected edges in multi-edge chains: NATIVE for single-hop (backward pass threads BOTH + # endpoints — see override below) and fixed-length multi-hop (generic backward hop + + # path-aware recompute, same as directed). decline (NIE): undirected carrying + # include_zero_hop_seed / *_query (unverified combine semantics); undirected to_fixed_point + # is NIE'd upstream by _is_native_multihop (needs components/2-core). + if len(edge_ops) > 1: + _undirected = [op for op in edge_ops if op.direction == "undirected"] + _undirected_unsupported = any( + op.include_zero_hop_seed + or op.source_node_query is not None + or op.destination_node_query is not None + or op.edge_query is not None + for op in _undirected + ) + if _undirected and _undirected_unsupported: + raise NotImplementedError( + "polars chain engine supports single-hop and fixed-length multi-hop " + "undirected edges in multi-edge chains; deferred undirected sub-cases — " + "include_zero_hop_seed or *_query — require engine='pandas'." + ) + + # Single-hop fast path: [n(), e, n()] with no names/queries/matches — the basic + # "filter then expand" crossfilter (`MATCH (a {f})-[e]->(b)`). Result = edges whose + # endpoints pass the node filters + those endpoint nodes (isolated/dead-ends excluded); + # one hop means the backward pass prunes nothing more, so skip forward/backward/combine. + # Byte-identical vs pandas (verified: src/dst/both filters, reverse, dup/self-loop/cycle/ + # isolated). Undirected fast-paths only when UNCONSTRAINED; filtered-undirected (OR of + # both directions) falls through to the full path. + def _fp_node(op): + return isinstance(op, ASTNode) and op._name is None and op.query is None + + def _plain_edge(op): + return (isinstance(op, ASTEdge) and op.is_simple_single_hop() + and op.edge_match is None and op.source_node_match is None + and op.destination_node_match is None and op._name is None + and op.source_node_query is None and op.destination_node_query is None + and op.edge_query is None and not op.include_zero_hop_seed) + + if start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) and _fp_node(ops[2]): + n0, e1, n2 = ops + unconstrained = not n0.filter_dict and not n2.filter_dict + if unconstrained or e1.direction in ("forward", "reverse"): + gf = ensure_nodes_polars(self) + ncol, scol, dcol = gf._node, gf._source, gf._destination + assert ncol is not None and scol is not None and dcol is not None + gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) + edges = gf._edges + if not unconstrained: + from_col, to_col = (scol, dcol) if e1.direction == "forward" else (dcol, scol) + if n0.filter_dict: + from_ids = filter_by_dict_polars(gf._nodes, n0.filter_dict).select(pl.col(ncol)) + edges = edges.join(from_ids, left_on=from_col, right_on=ncol, how="semi") + if n2.filter_dict: + to_ids = filter_by_dict_polars(gf._nodes, n2.filter_dict).select(pl.col(ncol)) + edges = edges.join(to_ids, left_on=to_col, right_on=ncol, how="semi") + endpoints = endpoint_ids(edges, scol, dcol, ncol) + nodes = gf._nodes.join(endpoints.unique(), on=ncol, how="semi") + return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) + + if start_nodes is not None: + from graphistry.Engine import Engine, df_to_engine + start_nodes = df_to_engine(start_nodes, Engine.POLARS) + + g = ensure_nodes_polars(self) + assert g._node is not None and g._source is not None and g._destination is not None + start_nodes = _align_seed_dtype(start_nodes, g._node, g._nodes) + g, _endpoint_restore = _align_edge_endpoints(g, g._node, g._source, g._destination) + if g._edge is None: + EID = "__gfql_edge_index__" + g = g.edges(g._edges.with_row_index(EID), g._source, g._destination, edge=EID) + added_edge_index = True + else: + EID = g._edge + added_edge_index = False + + node_col, src, dst = g._node, g._source, g._destination + assert node_col is not None and src is not None and dst is not None + + # Forward pass. + g_stack: List[Plottable] = [] + for i, op in enumerate(ops): + prev = start_nodes if i == 0 else g_stack[-1]._nodes + g_stack.append(_exec(op, g, prev, None)) + + # Backward pass. + g_rev: List[Plottable] = [] + for op, g_step in zip(reversed(ops), reversed(g_stack)): + prev_loop = g_stack[-1] if len(g_rev) == 0 else g_rev[-1] + if len(g_rev) == len(g_stack) - 1: + prev_orig = None + else: + prev_orig = g_stack[-(len(g_rev) + 2)] + prev_wf = prev_loop._nodes + target_wf = prev_orig._nodes if prev_orig is not None else None + # OUTPUT universe = FULL g._nodes (reverse hop output not truncated / keeps node columns); + # the INTERMEDIATE-hop gate universe is decoupled. Multi-hop reverse: pass the forward + # wavefront (g_step._nodes, pre-widening) so intermediate hops can't wander outside + # forward-reached nodes (pandas chain.py:1104 feeds g_step as the multi-hop reverse node + # table). Single-hop reverse: None -> gate = all_nodes (vacuous), matching pandas + # use_fast_backward (full g._nodes). + _iu = g_step._nodes if (isinstance(op, ASTEdge) and not op.is_simple_single_hop()) else None + g_step_full = g_step.nodes(g._nodes, g._node) + rev = _exec(op.reverse(), g_step_full, prev_wf, target_wf, intermediate_universe=_iu) + # Undirected single-hop backward threading: the generic hop returns a ONE-SIDED + # (TO-side) wavefront; pandas' fast backward branch (chain.py:1090-1098) threads BOTH + # endpoints of surviving edges. One-sided drops an intermediate node reachable only as + # the frontier-side endpoint of a sibling edge, and the combine then drops its incident + # edge (fuzz seed-18: >1 undirected edge + intermediate node filters loses a node vs + # pandas). Edge set already matches (both orientations joined), so override ONLY the + # node frame — with FULL node columns so the next backward node step can still filter. + if (isinstance(op, ASTEdge) and op.direction == "undirected" + and op.is_simple_single_hop() and rev._edges is not None): + _both = endpoint_ids(rev._edges, src, dst, node_col).unique(subset=[node_col]) + rev = rev.nodes(g._nodes.join(_both, on=node_col, how="semi"), node_col) + g_rev.append(rev) + + steps: List[Tuple[ASTObject, Plottable]] = list(zip(ops, list(reversed(g_rev)))) + label_steps: List[Tuple[ASTObject, Plottable]] = list(zip(ops, g_stack)) + + # Native multi-hop: port of pandas combine_steps has_multihop branch (chain.py:201-209). + # The per-step single-hop endpoint semijoin in _combine_edges would drop intermediate-hop + # edges (observed e=3 vs pandas e=10). Instead RE-EXECUTE the FORWARD hop over each step's + # backward-pruned edge subgraph, seeded from the forward-pass entry wavefront: backward + # prune = "reaches a valid endpoint", forward re-exec = "reachable from seed", and their + # composition along the BFS frontier is exactly the valid-path edges (NOT a flat + # forward∩backward intersection — the reverted shortcut). Only the EDGE combine sees the recomputed + # frames (pandas gates on kind=='edges'); node combine + name tagging keep the original + # backward-pruned steps. + edge_steps: List[Tuple[ASTObject, Plottable]] = steps + has_multihop = any(isinstance(op, ASTEdge) and not op.is_simple_single_hop() for op, _ in steps) + if has_multihop: + # pandas recomputes EVERY step when any is multi-hop (chain.py:201-209), then concats + # with NO per-step semijoin. Mirror exactly: recompute ALL ASTEdge steps and let + # _combine_edges append-all (has_multihop=True). Recomputing only the multi-hop step + + # semijoining the single-hop steps diverged by an edge/node on mixed chains + # (fuzz seeds 24/48). + edge_steps = [] + for idx, (op, g_step) in enumerate(steps): + if isinstance(op, ASTEdge): + prev_src = label_steps[idx - 1][1]._nodes if idx > 0 else g_step._nodes + prev_wf = ( + _semi(g._nodes, prev_src, node_col, node_col) + if prev_src is not None else None + ) + g_sub = g.edges(g_step._edges, src, dst, edge=g._edge) + edge_steps.append((op, _exec(op, g_sub, prev_wf, None))) + else: + edge_steps.append((op, g_step)) + + # Track B: build the WHOLE combine (combine_nodes/edges + endpoint + names) as ONE deferred + # plan over already-materialized hop frames and collect ONCE — collapsing the ~dozen eager + # combine ops (each internally lazy().op().collect()) into a single fused pass on the active + # target. NO recompute (inputs materialized; distinct from the disproven full-chain fusion). + # Stable order columns restore the eager g._nodes/g._edges order (lazy joins don't preserve + # it) so a trailing row pipeline's LIMIT/SKIP is unaffected — byte-identical to eager combine. + from graphistry.compute.util import generate_safe_column_name + from graphistry.compute.gfql.lazy import collect_all + NORD = generate_safe_column_name("__gfql_norder__", g._nodes, prefix="__gfql_", suffix="__") + EORD = generate_safe_column_name("__gfql_eorder__", g._edges, prefix="__gfql_", suffix="__") + g_lz = _LazyShim(g._nodes.with_row_index(NORD).lazy(), g._edges.with_row_index(EORD).lazy(), + node_col, src, dst, g._edge) + steps_lz = [(op, _LazyShim.step(p)) for op, p in steps] + edge_steps_lz = [(op, _LazyShim.step(p)) for op, p in edge_steps] + label_lz = [(op, _LazyShim.step(p)) for op, p in label_steps] + + final_nodes = _combine_nodes(g_lz, steps_lz) + final_edges = _combine_edges(g_lz, edge_steps_lz, label_lz, has_multihop) + # Endpoint (lazy: always compute; maintain_order keeps the semi-join order). + endpoints = endpoint_ids(final_edges, src, dst, node_col).unique(subset=[node_col]) + missing = endpoints.join(final_nodes.select(pl.col(node_col)), on=node_col, how="anti") + extra = g_lz._nodes.join(missing, on=node_col, how="semi") + final_nodes = pl.concat([final_nodes, extra], how="diagonal_relaxed").unique( + subset=[node_col], maintain_order=True) + final_nodes = _apply_node_names(final_nodes, g_lz, steps_lz) + + final_nodes = final_nodes.sort(NORD).drop(NORD) + final_edges = final_edges.sort(EORD).drop(EORD) + if added_edge_index: + final_edges = final_edges.drop(EID) + final_edges, final_nodes = collect_all([final_edges, final_nodes]) + final_edges = _restore_edge_dtypes(final_edges, src, dst, _endpoint_restore) + return self.nodes(final_nodes, node_col).edges(final_edges, src, dst) diff --git a/graphistry/compute/gfql/lazy/engine/polars/degrees.py b/graphistry/compute/gfql/lazy/engine/polars/degrees.py new file mode 100644 index 0000000000..37b01ff3bf --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/degrees.py @@ -0,0 +1,117 @@ +"""Native polars graph-degree helpers (get_degrees / get_indegrees / get_outdegrees). + +Extracted from chain.py (node-degree computation, not traversal orchestration). Parity with +ComputeMixin.get_degrees & friends; pure polars, no pandas bridge (plan.md NO-CHEATING). +Reached via .get_degrees() etc. dispatch and the GFQL CALL executor. +""" +from typing import Optional + +from graphistry.Plottable import Plottable +from .dtypes import is_lazy, colnames, col_dtype +from .hop_eager import ensure_nodes_polars + + +def _endpoint_counts(edges, key_col: str, node_dt, node_col: str, alias: str): + """group_by-count on ONE edge endpoint, key cast to node-id dtype so left-join keys match + when node/endpoint dtypes diverge (polars won't auto-cast int<->float keys; pandas merges fine).""" + import polars as pl + return edges.group_by(pl.col(key_col).cast(node_dt).alias(node_col)).agg( + pl.len().alias(alias) + ) + + +def get_degrees_polars( + g: Plottable, + col: str = "degree", + degree_in: str = "degree_in", + degree_out: str = "degree_out", + engine: Optional[str] = None, +) -> Plottable: + """Native ``get_degrees`` — parity with ComputeMixin.get_degrees. + + group_by-count per endpoint, left-joined onto the node table (materialized from edges when + absent). Oracle contract: isolated and src-only/dst-only nodes get 0; self-loops double-count + (one in + one out); all three columns Int32. Empty edges need no special case — left-join + + fill_null(0) yields all-zero, matching the pandas empty-edges branch. The ``engine`` safelist + sub-param is accepted and ignored (execution already committed to polars, parity-equal). + """ + import polars as pl + + g = ensure_nodes_polars(g) + node_col, src, dst = g._node, g._source, g._destination + assert node_col is not None and src is not None and dst is not None + assert g._nodes is not None and g._edges is not None + nodes, edges = g._nodes, g._edges + + node_dt = col_dtype(nodes, node_col) + in_counts = _endpoint_counts(edges, dst, node_dt, node_col, degree_in) + out_counts = _endpoint_counts(edges, src, node_dt, node_col, degree_out) + + # Drop pre-existing degree columns (mirrors the pandas keep-subset) so a re-run + # overwrites instead of producing *_right join collisions. + drop_cols = [c for c in colnames(nodes) if c in (degree_in, degree_out, col)] + base = nodes.drop(drop_cols) if drop_cols else nodes + + out = ( + base.join(in_counts, on=node_col, how="left") + .join(out_counts, on=node_col, how="left") + .with_columns( + pl.col(degree_in).fill_null(0).cast(pl.Int32), + pl.col(degree_out).fill_null(0).cast(pl.Int32), + ) + .with_columns( + (pl.col(degree_in) + pl.col(degree_out)).cast(pl.Int32).alias(col) + ) + ) + return g.nodes(out, node_col) + + +def _single_direction_degree_polars(g: Plottable, key_col: str, col: str) -> Plottable: + """Shared body for get_indegrees / get_outdegrees — parity with + ComputeMixin._single_direction_degree. group_by-count on ONE endpoint (destination for in, + source for out) left-joined onto the node table; isolated / opposite-endpoint-only nodes get + 0; a self-loop counts ONCE per direction (not double-counted like the get_degrees total); + column is Int32; count key cast to node-id dtype (polars won't auto-cast int<->float join + keys). Empty-edges mirrors the oracle EXACTLY and differs from get_degrees: no edges AND the + target column already exists -> node table returned UNCHANGED (pandas keeps pre-existing + values); otherwise materialize all-zero Int32. + """ + import polars as pl + + g = ensure_nodes_polars(g) + node_col = g._node + assert node_col is not None + assert g._nodes is not None and g._edges is not None + nodes, edges = g._nodes, g._edges + + if is_lazy(edges): + edges_empty = edges.select(pl.len()).collect().item() == 0 + else: + edges_empty = edges.height == 0 + if edges_empty: + if col in colnames(nodes): + return g.nodes(nodes, node_col) + return g.nodes(nodes.with_columns(pl.lit(0).cast(pl.Int32).alias(col)), node_col) + + counts = _endpoint_counts(edges, key_col, col_dtype(nodes, node_col), node_col, col) + # Drop a pre-existing degree column (mirrors the pandas keep-subset) so a re-run + # overwrites instead of a *_right join collision. + base = nodes.drop(col) if col in colnames(nodes) else nodes + out = base.join(counts, on=node_col, how="left").with_columns( + pl.col(col).fill_null(0).cast(pl.Int32) + ) + return g.nodes(out, node_col) + + +def get_indegrees_polars(g: Plottable, col: str = "degree_in") -> Plottable: + """Native ``get_indegrees`` (parity with ComputeMixin.get_indegrees): in-degree = count of + edges whose DESTINATION endpoint is the node.""" + assert g._destination is not None, "Missing destination binding; set via .bind() or .edges()" + return _single_direction_degree_polars(g, g._destination, col) + + +def get_outdegrees_polars(g: Plottable, col: str = "degree_out") -> Plottable: + """Native ``get_outdegrees`` (parity with ComputeMixin.get_outdegrees): out-degree = count of + edges whose SOURCE endpoint is the node.""" + assert g._source is not None, "Missing source binding; set via .bind() or .edges()" + return _single_direction_degree_polars(g, g._source, col) diff --git a/graphistry/compute/gfql/lazy/engine/polars/dtypes.py b/graphistry/compute/gfql/lazy/engine/polars/dtypes.py new file mode 100644 index 0000000000..3f5533c43f --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/dtypes.py @@ -0,0 +1,82 @@ +"""Shared polars dtype classifiers for the native polars GFQL engine. + +Encodes the cross-type / NaN-guard correctness CONTRACT (which dtypes are numeric, float, +string-like) used by predicate lowering, expression lowering, and result projection — ONE +definition so the guards can't silently diverge when a dtype is added or a classification is +fixed at one site. Polars imported lazily (optional dependency), per engine convention. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional, TypeVar, Union + +if TYPE_CHECKING: + import polars as pl + PolarsFrame = Union["pl.DataFrame", "pl.LazyFrame"] + # eager-in→eager-out / lazy-in→lazy-out (a Union return would type-error at call sites) + PolarsT = TypeVar("PolarsT", "pl.DataFrame", "pl.LazyFrame") + + +def is_int(dt: "Optional[pl.DataType]") -> bool: + """Signed/unsigned integer dtype (not bool, not float).""" + import polars as pl + return dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64, + pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64) + + +def is_float(dt: "Optional[pl.DataType]") -> bool: + import polars as pl + return dt in (pl.Float32, pl.Float64) + + +def is_numeric(dt: "Optional[pl.DataType]") -> bool: + """Integer or float — the operand types polars arithmetic/comparison accepts.""" + return is_int(dt) or is_float(dt) + + +def is_stringlike(dt: "Optional[pl.DataType]") -> bool: + """String / Categorical / Enum — all compare/order like strings and all raise vs a + numeric operand in polars (so all must trip the cross-type guard).""" + import polars as pl + if dt == pl.String: + return True + for name in ("Categorical", "Enum"): + t = getattr(pl, name, None) + if t is not None and (dt == t or isinstance(dt, t)): + return True + return False + + +# --- frame-shape helpers (lazy/eager agnostic), shared by chain orchestration + degree +# helpers so frame introspection is uniform across DataFrame-vs-LazyFrame ------------ + +def is_lazy(df: "PolarsFrame") -> bool: + """True for a ``pl.LazyFrame`` (vs an eager ``pl.DataFrame``).""" + import polars as pl + return isinstance(df, pl.LazyFrame) + + +def colnames(df: "PolarsFrame") -> List[str]: + """Column names for an eager or lazy polars frame (no collect for lazy).""" + return df.collect_schema().names() if is_lazy(df) else df.columns + + +def col_dtype(df: "PolarsFrame", col: str) -> "pl.DataType": + """One column's dtype for an eager or lazy polars frame (no collect for lazy).""" + return (df.collect_schema() if is_lazy(df) else df.schema)[col] + + +def endpoint_ids(frame: "PolarsT", src: str, dst: str, out_col: str, + dtype: "Optional[pl.DataType]" = None) -> "PolarsT": + """One-column frame of edge endpoints (src stacked on dst) as ``out_col`` — the engine's + node-id-universe builder, shared by hop/hop_eager/chain; eager/lazy agnostic. ``dtype`` casts + both sides to the node-id join dtype (polars won't coerce int/float join keys like pandas). + NOT deduplicated: each caller applies its own ``.unique(...)`` variant, preserved verbatim + from pre-refactor sites (plain vs ``subset=`` are equivalent on this one-column output — + kept per-site for a byte-identical diff, not semantics).""" + import polars as pl + + def _side(c: str) -> "pl.Expr": + e = pl.col(c) + return (e.cast(dtype) if dtype is not None else e).alias(out_col) + return pl.concat([frame.select(_side(src)), frame.select(_side(dst))], + how="vertical_relaxed") diff --git a/graphistry/compute/gfql/lazy/engine/polars/hop.py b/graphistry/compute/gfql/lazy/engine/polars/hop.py new file mode 100644 index 0000000000..e9ee3e48b1 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/hop.py @@ -0,0 +1,17 @@ +"""Polars hop entry point — the single place both ``.hop()`` dispatch and ``chain_polars`` +go through (also the #1658 seeded-index hook site). + +The unified ``hop_polars`` (hop_eager.py) runs a single bounded hop as ONE lazy +collect-once plan (the GPU path) and everything else (multi-hop / min_hops / +to_fixed_point) on the eager BFS loop — formerly a separate lazy twin lived in this +module; its setup/gates/epilogue sections are now shared inside ``hop_polars``. +""" +from typing import Any, Optional + +from graphistry.Plottable import Plottable +from graphistry.compute.gfql.lazy.engine.polars.hop_eager import hop_polars + + +def hop_lazy_or_eager(self: Plottable, nodes: Optional[Any] = None, hops: Optional[int] = 1, **kwargs: Any) -> Plottable: + """Run the polars hop: lazy collect-once for a single bounded hop, eager loop otherwise.""" + return hop_polars(self, nodes, hops, **kwargs) diff --git a/graphistry/compute/gfql/lazy/engine/polars/hop_eager.py b/graphistry/compute/gfql/lazy/engine/polars/hop_eager.py new file mode 100644 index 0000000000..a6de4481d1 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/hop_eager.py @@ -0,0 +1,432 @@ +"""Native Polars hop() — Phase 1, vectorized. + +Supports forward/reverse/undirected, integer hops + to_fixed_point, default and +return_as_wave_front seed semantics, edge/source/destination match predicates, and +target_wave_front (chain reverse pass). Vectorization-first: BFS frontier/visited/allowed sets +are polars frames advanced by semi/anti joins — no per-element Python work, no +``.to_list()``/``is_in(python_list)`` ping-pong; each hop is one big join. Parity-or-NIE: +pandas is the oracle; not-yet-ported features (hop labeling, min_hops>1 outside the chain +policy, output_min/max slicing, *_query, prune_to_endpoints, include_zero_hop_seed) raise +NotImplementedError. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Tuple + +from graphistry.Plottable import Plottable +from graphistry.compute.util import generate_safe_column_name +from .dtypes import endpoint_ids +from .predicates import filter_by_dict_polars + +if TYPE_CHECKING: + import polars as pl + from .dtypes import PolarsT + + +def _unsupported(**kwargs: Any) -> None: + unsupported = [k for k, v in kwargs.items() if v is not None and v is not False] + if unsupported: + raise NotImplementedError( + "polars hop engine (Phase 1) does not yet support: " + + ", ".join(sorted(unsupported)) + + ". Use engine='pandas' or extend graphistry/compute/gfql/lazy/engine/polars/hop_eager.py." + ) + + +def ensure_nodes_polars(g: Plottable) -> Plottable: + """Materialize a polars node table from edges when absent (native — avoids the + pandas-idiom ``materialize_nodes`` path, which uses drop_duplicates/reset_index).""" + import polars as pl + + if g._nodes is not None: + return g + src, dst = g._source, g._destination + assert src is not None and dst is not None and g._edges is not None + node_id = g._node if g._node is not None else "id" + ids = endpoint_ids(g._edges, src, dst, node_id).unique() + return g.nodes(ids, node_id) + + +def _hop_setup_columns( + edges: "pl.DataFrame", all_nodes: "pl.DataFrame", node_col: str, edge_binding: Optional[str], +) -> Tuple[str, str, str, str, "pl.DataFrame", bool, "pl.DataType"]: + """Shared eager+lazy hop setup: safe (FROM, TO, NID, EID) names, the edge-id frame (reuse + `edge_binding` like the chain's __gfql_edge_index__, else synthesize a row index), and the + node-id join dtype. Identical on pl.DataFrame and pl.LazyFrame (previously duplicated + verbatim in hop.py; now de-duplicated).""" + FROM = generate_safe_column_name("__gfql_from__", edges, prefix="__gfql_", suffix="__") + TO = generate_safe_column_name("__gfql_to__", edges, prefix="__gfql_", suffix="__") + NID = generate_safe_column_name("__gfql_nid__", all_nodes, prefix="__gfql_", suffix="__") + if edge_binding is not None and edge_binding in edges.columns: + EID, edges_idx, synth_eid = edge_binding, edges, False + else: + EID = generate_safe_column_name("__gfql_eid__", edges, prefix="__gfql_", suffix="__") + edges_idx, synth_eid = edges.with_row_index(EID), True + return FROM, TO, NID, EID, edges_idx, synth_eid, all_nodes.schema[node_col] + + +def _build_hop_pairs( + frame: "PolarsT", direction: str, src: str, dst: str, + node_dtype: "pl.DataType", FROM: str, TO: str, EID: str, +) -> "PolarsT": + """Directed-(FROM,TO,EID) builder with join-key dtype aligned (polars won't coerce int/float + join keys like pandas). `frame` = edge-id frame, eager or lazy; select/concat identical on both.""" + import polars as pl + + def _p(s: str, d: str) -> "PolarsT": + return frame.select(pl.col(s).cast(node_dtype).alias(FROM), + pl.col(d).cast(node_dtype).alias(TO), pl.col(EID)) + if direction == "forward": + return _p(src, dst) + if direction == "reverse": + return _p(dst, src) + return pl.concat([_p(src, dst), _p(dst, src)], how="vertical_relaxed") + + +def _min_hops_labeled_node_output( + all_nodes: "pl.DataFrame", needed: "pl.DataFrame", labeled: "pl.DataFrame", + node_col: str, NID: str, +) -> "pl.DataFrame": + """min_hops CHAIN wavefront node frame, mirroring pandas' labeled (track_node_hops) hop: + FULL-attribute rows for hop-LABELED nodes (retained-path destinations; rich_nodes inner-merge, + hop.py:944) + id-only NULL-attribute stubs for remaining retained-edge endpoints (source-side + nodes pandas adds via the edge-endpoint concat, hop.py:994-1011, non-id cols NaN) — so a + downstream attribute filter rejects those source-side endpoints (fuzz seed-48: reverse n5/n7, + kind->NaN, dropped by `kind=y`). Used ONLY under min_hops_label_policy; a direct hop takes the + plain full-attr join, matching pandas' un-labeled direct hop (hop.py:978-983).""" + import polars as pl + full = all_nodes.join( + needed.join(labeled, on=NID, how="semi").rename({NID: node_col}), on=node_col, how="semi") + stub_ids = needed.join(labeled, on=NID, how="anti").rename({NID: node_col}) + if stub_ids.height == 0: + return full + stub = stub_ids.with_columns( + [pl.lit(None, dtype=all_nodes.schema[c]).alias(c) for c in all_nodes.columns if c != node_col] + ).select(all_nodes.columns) + return pl.concat([full, stub], how="vertical_relaxed") + + +def hop_polars( + self: Plottable, + nodes: Optional[Any] = None, + hops: Optional[int] = 1, + *, + min_hops: Optional[int] = None, + max_hops: Optional[int] = None, + output_min_hops: Optional[int] = None, + output_max_hops: Optional[int] = None, + label_node_hops: Optional[str] = None, + label_edge_hops: Optional[str] = None, + label_seeds: bool = False, + to_fixed_point: bool = False, + direction: str = "forward", + edge_match: Optional[dict] = None, + source_node_match: Optional[dict] = None, + destination_node_match: Optional[dict] = None, + source_node_query: Optional[str] = None, + destination_node_query: Optional[str] = None, + edge_query: Optional[str] = None, + return_as_wave_front: bool = False, + include_zero_hop_seed: bool = False, + target_wave_front: Optional[Any] = None, + intermediate_universe: Optional[Any] = None, + min_hops_label_policy: bool = False, +) -> Plottable: + import polars as pl + from graphistry.Engine import Engine, df_to_engine + + if direction not in ("forward", "reverse", "undirected"): + raise ValueError( + f'Invalid direction: "{direction}", must be one of: ' + '"forward" (default), "reverse", "undirected"' + ) + + _unsupported( + # min_hops>1 is NATIVE fwd/rev with finite max_hops, but ONLY in the CHAIN context + # (min_hops_label_policy=True): the layered walk + NON-anti-joined BFS port pandas' + # CHAIN/labeled min_hops policy (hop.py:509-776 + track_node_hops node-output rule). + # decline (NIE): a DIRECT base.hop(engine='polars', min_hops>1) needs pandas' UN-labeled + # direct-hop node-output (hop.py:978-983) + the chain's target_wave_front threading — + # without them it silently drops genuinely-reachable nodes; use chain()/gfql() or + # engine='pandas'. UNDIRECTED min_hops>1 (2-core/components, hop.py:817-887) and + # min_hops+to_fixed_point also stay deferred. + min_hops=min_hops if (min_hops is not None and min_hops > 1 + and (direction == "undirected" or to_fixed_point + or not min_hops_label_policy)) else None, + output_min_hops=output_min_hops, + output_max_hops=output_max_hops, + label_node_hops=label_node_hops, + label_edge_hops=label_edge_hops, + label_seeds=label_seeds or None, + source_node_query=source_node_query, + destination_node_query=destination_node_query, + edge_query=edge_query, + include_zero_hop_seed=include_zero_hop_seed or None, + ) + + if target_wave_front is not None and nodes is None: + raise ValueError("target_wave_front requires nodes to target against (for intermediate hops)") + + if nodes is not None: + nodes = df_to_engine(nodes, Engine.POLARS) + if target_wave_front is not None: + target_wave_front = df_to_engine(target_wave_front, Engine.POLARS) + + g = ensure_nodes_polars(self) + + node_col = g._node + src = g._source + dst = g._destination + assert node_col is not None and src is not None and dst is not None + assert g._edges is not None and g._nodes is not None + edges = g._edges + all_nodes = g._nodes + + if edge_match is not None: + edges = filter_by_dict_polars(edges, edge_match) + + resolved_max_hops = max_hops if max_hops is not None else hops + if to_fixed_point: + resolved_max_hops = None + elif not isinstance(resolved_max_hops, int): + raise ValueError( + f"Must provide integer hops when to_fixed_point is False, received: {resolved_max_hops}" + ) + + FROM, TO, NID, EID, edges_idx, synth_eid, node_dtype = _hop_setup_columns( + edges, all_nodes, node_col, g._edge) + pairs = _build_hop_pairs(edges_idx, direction, src, dst, node_dtype, FROM, TO, EID) + + def _idframe(df: "pl.DataFrame", col: str) -> "pl.DataFrame": + return df.select(pl.col(col).cast(node_dtype).alias(NID)).unique() + + # --- SINGLE BOUNDED HOP (the dominant case — every chain edge): ONE lazy plan, ONE + # collect_all on the active target (GPU: edge table read/transferred once — the 2.84x + # collect-once path, formerly the separate hop.py twin). Placed BEFORE the eager gate + # construction: the seed/gate/target id-frame .unique()s must stay INSIDE the lazy plan + # (eagerly materializing them over chain wavefronts cost +5-14% at 1M-10M edges — A/B + # measured, twice). Multi-hop/min_hops/to_fixed_point use the eager loop below: the + # early-break + revisit bookkeeping need per-hop materialization, and for hops>=2 an + # unrolled lazy plan recomputes the big edge-join per hop (polars CSE doesn't dedup it). + if (not to_fixed_point and resolved_max_hops == 1 + and not (min_hops is not None and min_hops > 1 and direction in ("forward", "reverse"))): + from graphistry.compute.gfql.lazy import collect_all + edges_lf = edges_idx.lazy() + pairs_lf = _build_hop_pairs(edges_lf, direction, src, dst, node_dtype, FROM, TO, EID) + + def _idframe_lf(lf: "pl.LazyFrame", col: str) -> "pl.LazyFrame": + return lf.select(pl.col(col).cast(node_dtype).alias(NID)).unique() + + allowed_source_lf = ( + _idframe_lf(filter_by_dict_polars(nodes if nodes is not None else all_nodes, + source_node_match).lazy(), node_col) + if source_node_match is not None else None + ) + allowed_dest_lf = ( + _idframe_lf(filter_by_dict_polars(all_nodes, destination_node_match).lazy(), node_col) + if destination_node_match is not None else None + ) + target_final_lf = (_idframe_lf(target_wave_front.lazy(), node_col) + if target_wave_front is not None else None) + frontier_lf = _idframe_lf((nodes if nodes is not None else all_nodes).lazy(), node_col) + if allowed_source_lf is not None: + frontier_lf = frontier_lf.join(allowed_source_lf, on=NID, how="semi") + hop_edges_lf = pairs_lf.join(frontier_lf.rename({NID: FROM}), on=FROM, how="semi") + if target_final_lf is not None: # single hop IS the last hop -> final gate only + hop_edges_lf = hop_edges_lf.join(target_final_lf.rename({NID: TO}), on=TO, how="semi") + if allowed_dest_lf is not None: + hop_edges_lf = hop_edges_lf.join(allowed_dest_lf.rename({NID: TO}), on=TO, how="semi") + visited_edges_lf = hop_edges_lf.select(pl.col(EID)).unique(subset=[EID]) + out_edges_lf = edges_lf.join(visited_edges_lf, on=EID, how="semi") + if synth_eid: + out_edges_lf = out_edges_lf.drop(EID) + # Node set == one eager iteration: DESTINATIONS ∪ (FROM side unless wavefront); then + # ∪ retained-edge endpoints unless wavefront-with-seeds (the wave front IS the + # destinations). Empty out_edges yields empty endpoints — no height guard needed. + dest_lf = hop_edges_lf.select(pl.col(TO).alias(NID)).unique() + needed_lf = (dest_lf if return_as_wave_front else pl.concat( + [hop_edges_lf.select(pl.col(FROM).alias(NID)), dest_lf], + how="vertical_relaxed").unique(subset=[NID])) + if not (return_as_wave_front and nodes is not None): + endpoints_lf = endpoint_ids(out_edges_lf, src, dst, NID, node_dtype).unique(subset=[NID]) + needed_lf = pl.concat([needed_lf, endpoints_lf], how="vertical_relaxed").unique(subset=[NID]) + out_nodes_lf = all_nodes.lazy().join(needed_lf.rename({NID: node_col}), on=node_col, how="semi") + out_edges_c, out_nodes_c = collect_all([out_edges_lf, out_nodes_lf]) + return g.nodes(out_nodes_c, node_col).edges(out_edges_c, src, dst) + + allowed_source = ( + _idframe(filter_by_dict_polars(nodes if (nodes is not None and not to_fixed_point and resolved_max_hops == 1) else all_nodes, source_node_match), node_col) + if source_node_match is not None else None + ) + allowed_dest = ( + _idframe(filter_by_dict_polars(all_nodes, destination_node_match), node_col) + if destination_node_match is not None else None + ) + target_final = _idframe(target_wave_front, node_col) if target_wave_front is not None else None + # Intermediate-hop target gate (pandas hop.py:319-349,529-533): a NON-final hop's TO-node must + # land in target_wave_front UNION the intermediate universe. The chain passes the multi-hop + # reverse step's FORWARD WAVEFRONT as intermediate_universe (the only valid intermediate path + # nodes); standalone hops pass None -> all_nodes, gate vacuous (a standalone hop may pass + # through any node). Decoupled from the OUTPUT universe (all_nodes at materialization) so a + # reduced gate never truncates returned nodes. Only the FINAL hop is gated by target alone. + if target_final is not None: + _univ = _idframe(intermediate_universe, node_col) if intermediate_universe is not None else _idframe(all_nodes, node_col) + target_intermediate = pl.concat([target_final, _univ], how="vertical_relaxed").unique(subset=[NID]) + else: + target_intermediate = None + + # min_hops>1 (fwd/rev, finite max): pandas runs a NON-anti-joined BFS (wavefront carries + # REVISITS, hop.py:535,620) so a cycle keeps bumping max_reached_hop until max_hops — what + # lets the lower bound be satisfied on cyclic graphs. The anti-joined (shortest-path) + # frontier used for plain hops stops after one pass and under-counts the bound. + min_hops_active = ( + min_hops is not None and min_hops > 1 and direction in ("forward", "reverse") and not to_fixed_point + ) + # Per-hop label column (populated only when min_hops_active; computed unconditionally so it + # is a plain str — every use is guarded by `if min_hops_active`). + HOP = generate_safe_column_name("__gfql_hop__", edges, prefix="__gfql_", suffix="__") + max_reached_hop = 0 + reached_for_attrs = None # set in the min_hops gate: the reached-destination set (full attrs) + + empty_ids = all_nodes.select(pl.col(node_col).cast(node_dtype).alias(NID)).clear() + + seed = _idframe(nodes if nodes is not None else all_nodes, node_col) + frontier = seed # DataFrame[NID] + visited_nodes = empty_ids # DataFrame[NID] + visited_edge_frames = [] # collect per-hop EID frames; concat once at end + current_hop = 0 + first = True + + while True: + if not to_fixed_point and resolved_max_hops is not None and current_hop >= resolved_max_hops: + break + if frontier.height == 0: + break + current_hop += 1 + + frontier_iter = frontier if allowed_source is None else frontier.join(allowed_source, on=NID, how="semi") + hop_edges = pairs.join(frontier_iter.rename({NID: FROM}), on=FROM, how="semi") + + is_last = not to_fixed_point and resolved_max_hops is not None and current_hop >= resolved_max_hops + if target_final is not None: + assert target_intermediate is not None # set together with target_final + gate = target_final if is_last else target_intermediate + hop_edges = hop_edges.join(gate.rename({NID: TO}), on=TO, how="semi") + if allowed_dest is not None: + hop_edges = hop_edges.join(allowed_dest.rename({NID: TO}), on=TO, how="semi") + + if first and not return_as_wave_front: + visited_nodes = hop_edges.select(pl.col(FROM).alias(NID)).unique() + first = False + + if min_hops_active: + if hop_edges.height > 0: + max_reached_hop = current_hop + visited_edge_frames.append( + hop_edges.select(pl.col(EID)).with_columns(pl.lit(current_hop, dtype=pl.Int64).alias(HOP)) + ) + else: + visited_edge_frames.append(hop_edges.select(pl.col(EID))) + + cand = hop_edges.select(pl.col(TO).alias(NID)).unique() + new_frontier = cand.join(visited_nodes, on=NID, how="anti") + visited_nodes = pl.concat([visited_nodes, new_frontier], how="vertical_relaxed").unique(subset=[NID]) + if min_hops_active: + # Advance with ALL destinations (revisits, hop.py:620) so a cycle-re-entered node + # re-traverses its edges, but terminate once the cumulative reachable set stops + # growing (hop.py:617). max_reached_hop (set above when the hop had any edge) is then + # the closure hop the 3-case gate compares to min_hops. + if new_frontier.height == 0: + break + frontier = cand + else: + frontier = new_frontier + + if visited_edge_frames: + visited_edges = pl.concat(visited_edge_frames, how="vertical_relaxed").unique(subset=[EID]) + else: + visited_edges = edges_idx.select(pl.col(EID)).clear() + + if min_hops_active: + assert min_hops is not None + # visited_nodes here = the loop's cumulative reached-DEST accumulation = pandas + # matches_nodes (hop.py:609-621). `reached` drives the seed-strip; `labeled` the attr carry. + reached = visited_nodes + labeled = empty_ids # nodes that get a retained-path HOP LABEL (-> full attributes) + # Port of the pandas min_hops gate (hop.py:623-776). Three cases on max_reached_hop / goal: + if max_reached_hop < min_hops: + # genuinely can't reach the lower bound (hop.py:623-629) -> empty. + visited_edges = edges_idx.select(pl.col(EID)).clear() + reached = empty_ids + else: + edge_hops = pl.concat(visited_edge_frames, how="vertical_relaxed").group_by(EID).agg(pl.col(HOP).min()) + edge_rec = pairs.join(edge_hops, on=EID, how="inner") # FROM, TO, EID, HOP (first-traversal) + goal = edge_rec.filter(pl.col(HOP) >= min_hops) + if goal.height == 0: + # max_reached_hop>=min_hops but NO edge labeled >=min_hops: a cyclic REVISIT + # satisfied the bound (no NEW node at that depth). pandas skips the prune + # (hop.py:660 else) -> return the UNPRUNED ball; reached = full loop accumulation. + # With no layered prune, track_node_hops keeps ALL BFS-reached node rows (seed + # labels may reset to NA but the ROW remains -> full attrs via the rich-node + # merge), so every reached node is "labeled". edge_hops is already grouped by + # EID, so its key column IS the distinct retained set. + visited_edges = edge_hops.select(pl.col(EID)) + labeled = reached + else: + # layered backward-TREE walk (hop.py:688-724): descend level-by-level by edge + # label, keep edges whose TO is a current target, reset targets = their FROM. + current_targets = goal.select(pl.col(TO).alias(NID)).unique() + valid_node = current_targets + # node hop-labels (hop.py:676-723): goal DESTINATIONS labeled, plus the FROM of a + # retained edge at level>=2 (label=level-1). A FROM only at level 1 (seed/source + # side, e.g. seed-48 reverse n5/n7) gets NO label -> null attrs later. + labeled = current_targets + max_edge_hop = int(edge_hops.select(pl.col(HOP).max()).item()) + valid_edge_frames = [] + for level in range(max_edge_hop, 0, -1): + lvl = edge_rec.filter(pl.col(HOP) == level) + reaching = lvl.join(current_targets.rename({NID: TO}), on=TO, how="semi") + valid_edge_frames.append(reaching.select(pl.col(EID))) + current_targets = reaching.select(pl.col(FROM).alias(NID)).unique() + valid_node = pl.concat([valid_node, current_targets], how="vertical_relaxed").unique(subset=[NID]) + if level >= 2: + labeled = pl.concat([labeled, current_targets], how="vertical_relaxed").unique(subset=[NID]) + visited_edges = ( + pl.concat(valid_edge_frames, how="vertical_relaxed").unique(subset=[EID]) + if valid_edge_frames else edges_idx.select(pl.col(EID)).clear() + ) + # matches_nodes ∩ valid_node_series (hop.py:783) — restrict reached to the pruned tree. + reached = reached.join(valid_node, on=NID, how="semi") + + # min_hops node output = endpoints (src ∪ dst) of RETAINED edges, MINUS seeds not + # genuinely re-reached at >=min_hops. Pandas' labeled hop keeps a node iff its + # retained-path label is in [output_min..output_max] OR it's a retained-edge endpoint + # (hop.py:1117-1140 output-slice mask; no slicing collapses to exactly the endpoints), + # THEN strips SEEDS absent from matches_nodes (hop.py:1144-1170). So an endpoint seed + # never re-reached on a >=min path drops (fuzz seed-404 reverse n1), a seed re-reached + # at >=min keeps (seed-24 forward n2/n5), a reached non-endpoint seed drops (seed-24 n0). + # The polars chain runs this hop WITHOUT labels, so replicate the labeled+stripped result. + ep = pairs.join(visited_edges, on=EID, how="semi") + visited_nodes = endpoint_ids(ep, FROM, TO, NID).unique(subset=[NID]) + if nodes is not None: # seeds provided (chain wavefront) -> strip unreached seeds + unreached_seeds = seed.join(reached, on=NID, how="anti") + visited_nodes = visited_nodes.join(unreached_seeds, on=NID, how="anti") + reached_for_attrs = labeled + + out_edges = edges_idx.join(visited_edges, on=EID, how="semi") + if synth_eid: + out_edges = out_edges.drop(EID) + + # Final node set: reached ∪ (edge endpoints, unless wavefront-with-seeds). + needed = visited_nodes + materialize_endpoints = not (return_as_wave_front and nodes is not None) + if out_edges.height > 0 and materialize_endpoints: + endpoints = endpoint_ids(out_edges, src, dst, NID, node_dtype).unique(subset=[NID]) + needed = pl.concat([needed, endpoints], how="vertical_relaxed").unique(subset=[NID]) + + if min_hops_active and reached_for_attrs is not None and nodes is not None and min_hops_label_policy: + out_nodes = _min_hops_labeled_node_output(all_nodes, needed, reached_for_attrs, node_col, NID) + else: + out_nodes = all_nodes.join(needed.rename({NID: node_col}), on=node_col, how="semi") + + return g.nodes(out_nodes, node_col).edges(out_edges, src, dst) diff --git a/graphistry/compute/gfql/lazy/engine/polars/predicates.py b/graphistry/compute/gfql/lazy/engine/polars/predicates.py new file mode 100644 index 0000000000..0ef1ed3ca0 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/predicates.py @@ -0,0 +1,358 @@ +"""Vectorized polars filter_by_dict for the native polars GFQL engine. + +Common comparison/membership/string/null predicates lower to native polars expressions. +NO-CHEATING contract: no pandas bridge — a predicate with no native lowering raises +NotImplementedError (bridging one column would misrepresent pandas semantics as polars and +break columnar/GPU assumptions; use engine='pandas'). All filtering is one vectorized +``df.filter(expr)`` — no per-row work, no Python materialization. +""" +from __future__ import annotations + +import operator +import re +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +from graphistry.compute.predicates.ASTPredicate import ASTPredicate +from graphistry.compute.filter_by_dict import resolve_filter_column +from .dtypes import is_numeric as _dtype_numeric, is_stringlike as _dtype_stringlike + +if TYPE_CHECKING: + import datetime + import polars as pl + +# Comparison-predicate RHS: genuinely dynamic (Cypher properties are dynamically typed) — a +# python scalar or a GFQL/py temporal matched structurally by type(val).__name__ +# (DateValue/TemporalValue/…, never imported here). +CmpValue = Union[int, float, str, bool, "datetime.date", "datetime.time", Any] + +# Python-`re` features Rust regex (polars) rejects or evaluates differently: lookaround +# ((?=/(?!/(?<=/(?) — ComputeError or silent +# difference, so such patterns decline (NIE), never guess; pandas evaluates them with Python `re`. +_REGEX_RUST_INCOMPAT = re.compile(r"\(\? bool: + return _REGEX_RUST_INCOMPAT.search(pat) is not None + + +def _homogeneous_scalar_category(opts: List[Any]) -> Optional[str]: + """The single value-category (num/str/bool) of a literal list, else None (mixed/empty). + polars ``is_in`` raises on a cross-type list, so a non-homogeneous IN must decline (NIE).""" + def _cat(o: Any) -> Optional[str]: + if isinstance(o, bool): + return "bool" + if isinstance(o, (int, float)): + return "num" + if isinstance(o, str): + return "str" + return None + cats = {_cat(o) for o in opts} + return next(iter(cats)) if (len(cats) == 1 and None not in cats) else None + + +# Comparison callables predicates declare (op = staticmethod(operator.gt) etc.); pl.Expr +# implements Python rich comparison so op(lhs, rhs) builds exactly `lhs > rhs`/... . Ops outside +# this whitelist have no proven lowering and fall through to the decline paths. +_CMP_OPS = frozenset({operator.gt, operator.lt, operator.ge, operator.le, operator.eq, operator.ne}) + + +def _cmp_expr( + col_expr: "pl.Expr", + op: Callable[[Any, Any], Any], + val: CmpValue, + dtype: "Optional[pl.DataType]" = None, +) -> "Optional[pl.Expr]": + import datetime as _dt + + # Native temporal SAFE SUBSET: DateValue (Cypher date('YYYY-MM-DD') / p.gt(date(...))) vs a + # NAIVE pl.Datetime column. pandas oracle: s.dt.date compared to the tz-free python date + # (DateValue._parsed); col_expr.dt.date() pl.lit(date) is the exact equivalent — same + # calendar-date truncation, no tz on either side, parity provable. REQUIRE dtype (from the + # frame schema) to prove naive Datetime, else fall through to the decline. All else declines: + # tz-tagged DateTimeValue (always carries tz; pandas tz_localize/convert), TimeValue, + # tz-aware columns, raw python datetimes — each risks a silent tz/semantic mismatch. + if type(val).__name__ == "DateValue": + import polars as pl + d = getattr(val, "_parsed", None) + if ( + isinstance(dtype, pl.Datetime) and dtype.time_zone is None + and isinstance(d, _dt.date) and not isinstance(d, _dt.datetime) + ): + if op in _CMP_OPS: + return op(col_expr.dt.date(), pl.lit(d)) + # naive-Datetime parity unprovable here -> fall through to the decline below. + + # decline (NIE): remaining temporal values (datetime/time/TemporalValue etc.) have no SAFE + # polars-literal comparison — returning `col > TemporalValue` would be a non-None broken expr + # erroring at df.filter (or silently misordering). Tracked feature gap; numeric/string + # vals unaffected. + if isinstance(val, (_dt.date, _dt.datetime, _dt.time)) or type(val).__name__ in ( + "TemporalValue", "Timestamp", "Timedelta", "datetime64", + "DateTimeValue", "TimeValue", "DateValue", + ): + return None + # Narrow residual: no IEEE NaN mask here (unlike the WHERE/row-pipeline _nan_guard) — on a + # GENUINE polars NaN, `col > x` keeps the row (NaN = largest) where pandas drops it. + # Unreachable on standard ingestion: from_pandas/df_to_engine convert NaN→null (nan_to_null) + # and filter_by_dict runs on INGESTED columns (no in-query float math — that's the WHERE + # path). Only a natively-built polars frame with raw NaN diverges; documented, not guarded, + # to keep the lowering simple. (Mirrors the documented integer 0/0 column-compare residual.) + if op in _CMP_OPS: + return op(col_expr, val) + return None + + +def _inline_regex_flag_prefix(case: bool, flags: int) -> str: + """Translate pandas-style `re` flags int + case bool to a Rust-regex inline prefix like + ``(?im)`` (empty when nothing applies), keeping the polars regex lowering faithful to pandas.""" + letters = "" + if not case or (flags & re.IGNORECASE): + letters += "i" + if flags & re.MULTILINE: + letters += "m" + if flags & re.DOTALL: + letters += "s" + if flags & re.VERBOSE: + letters += "x" + return f"(?{letters})" if letters else "" + + +def predicate_to_expr(col: str, pred: ASTPredicate, dtype: "Optional[pl.DataType]" = None) -> "Optional[pl.Expr]": + """Lower an ASTPredicate to a polars boolean expression, or None if unsupported. ``dtype`` = + schema dtype of ``col`` (None if unknown), letting the temporal lowering prove a NAIVE + Datetime column before lowering a DateValue comparison (else decline).""" + import polars as pl + + c = pl.col(col) + name = type(pred).__name__ + + op = getattr(pred, "op", None) + if op is not None and hasattr(pred, "val"): + expr = _cmp_expr(c, op, pred.val, dtype) + if expr is not None: + return expr + + if name == "Between" and hasattr(pred, "lower") and hasattr(pred, "upper"): + lo, hi = pred.lower, pred.upper + if isinstance(lo, (int, float)) and isinstance(hi, (int, float)): + if getattr(pred, "inclusive", True): + return (c >= lo) & (c <= hi) + return (c > lo) & (c < hi) + # Temporal Between: pandas evaluates as GE/LE (inclusive) or GT/LT (exclusive) + # sub-predicates, and for DateValue bounds each is the date-truncated compare _cmp_expr + # already lowers with proven parity — composing the two endpoint compares adds NO new + # proof obligation. Both endpoints must be DateValue over a naive Datetime or _cmp_expr + # returns None -> honest NIE (tz-aware DateTimeValue, TimeValue, raw datetime, mixed + # bounds, non-Datetime dtype all decline this way — never a silent mismatch). + inclusive = getattr(pred, "inclusive", True) + lo_expr = _cmp_expr(c, operator.ge if inclusive else operator.gt, lo, dtype) + hi_expr = _cmp_expr(c, operator.le if inclusive else operator.lt, hi, dtype) + if lo_expr is not None and hi_expr is not None: + return lo_expr & hi_expr + + if name == "IsIn" and hasattr(pred, "options"): + opts = list(pred.options) + # Only non-empty single-category literal lists are parity-safe: is_in raises ComputeError + # on a cross-type list ([1, 'a'] over Int), so mixed/empty declines (NIE) — matching the + # row-pipeline _lower_in discipline. + if opts and _homogeneous_scalar_category(opts) is not None: + return c.is_in(opts) + return None + + if name == "AllOf" and hasattr(pred, "predicates"): + # Conjunction (n.val > 20 AND n.val < 90 folds to AllOf[GT, LT]): lower each child and + # AND them; if ANY child can't lower, the whole predicate can't (caller NIEs). + child_exprs = [predicate_to_expr(col, p, dtype) for p in pred.predicates] + if child_exprs and all(e is not None for e in child_exprs): + lowered: "List[pl.Expr]" = [e for e in child_exprs if e is not None] + combined = lowered[0] + for e in lowered[1:]: + combined = combined & e + return combined + return None + + if name in ("IsNull", "IsNA"): + return c.is_null() + if name in ("NotNull", "NotNA"): + return c.is_not_null() + + if name == "IsLeapYear": + # pandas s.dt.is_leap_year and polars expr.dt.is_leap_year() are the identical Boolean + # over the calendar year (Gregorian rule incl. 1900-not-leap / 2000-leap) — parity + # provable. Require naive Datetime or Date from the schema (a tz derives the year in + # LOCAL time; year-boundary equality unproven); else decline (NIE). Mirrors the + # naive-Datetime guard used for DateValue. + if (isinstance(dtype, pl.Datetime) and dtype.time_zone is None) or dtype == pl.Date: + return c.dt.is_leap_year() + return None + + if name in ("IsMonthStart", "IsMonthEnd", "IsQuarterStart", + "IsQuarterEnd", "IsYearStart", "IsYearEnd"): + # Temporal boundary predicates: polars has no is_month_start/... BOOLEAN accessor + # (month_start()/month_end() ROLL to a Datetime), but the pandas oracle with freq=None is + # a pure calendar-field test (is_month_start = day==1; is_month_end = day==days_in_month; + # quarter/year add a month-set / month==N). polars dt.day()/dt.month()/dt.days_in_month() + # extract the SAME fields (both proleptic Gregorian, days_in_month leap-aware), so each + # compose is BIT-IDENTICAL on non-null rows — a proven derivation, not a guess. pandas + # returns False for NaT; polars comparison yields null -> fill_null(False) restores + # parity. Require naive Datetime or Date (tz shifts wall-clock fields), like IsLeapYear; + # else honest NIE. + if (isinstance(dtype, pl.Datetime) and dtype.time_zone is None) or dtype == pl.Date: + day = c.dt.day() + month = c.dt.month() + if name == "IsMonthStart": + expr = day == 1 + elif name == "IsMonthEnd": + expr = day == c.dt.days_in_month() + elif name == "IsQuarterStart": + expr = (day == 1) & month.is_in([1, 4, 7, 10]) + elif name == "IsQuarterEnd": + expr = (day == c.dt.days_in_month()) & month.is_in([3, 6, 9, 12]) + elif name == "IsYearStart": + expr = (day == 1) & (month == 1) + else: # IsYearEnd + expr = (day == c.dt.days_in_month()) & (month == 12) + return expr.fill_null(False) + return None + + if name == "Contains" and hasattr(pred, "pat") and isinstance(pred.pat, str): + case = getattr(pred, "case", True) + regex = getattr(pred, "regex", True) + flags = getattr(pred, "flags", 0) + if not regex: + # Literal substring must NOT be regex-interpreted (metachars like ./*/( over-match). + # polars has no literal+case flag, so lowercase both sides for the case-insensitive + # literal (matches pandas). + if case: + return c.str.contains(pred.pat, literal=True) + return c.str.to_lowercase().str.contains(pred.pat.lower(), literal=True) + # Regex: mirror pandas case/flags via a Rust inline flag prefix. decline (NIE): + # Python-re-only features (lookaround, backreferences) — no silent wrong answer. + if _regex_rust_incompatible(pred.pat): + return None + prefix = _inline_regex_flag_prefix(case, flags) + return c.str.contains(f"{prefix}{pred.pat}", literal=False) + + if name in ("Startswith", "Endswith") and hasattr(pred, "pat") and isinstance(pred.pat, str): + if getattr(pred, "case", True): + return c.str.starts_with(pred.pat) if name == "Startswith" else c.str.ends_with(pred.pat) + # Case-insensitive: anchored (?i) regex on the escaped literal — matches the pandas + # boundary predicate's lowercase-both-sides semantics for a single str pat. + anchored = f"(?i)^{re.escape(pred.pat)}" if name == "Startswith" else f"(?i){re.escape(pred.pat)}$" + return c.str.contains(anchored, literal=False) + + if name in ("Startswith", "Endswith") and hasattr(pred, "pat") and isinstance(pred.pat, (tuple, list)): + # pandas boundary predicates accept a tuple of prefixes/suffixes (match if ANY) — OR-fold. + case = getattr(pred, "case", True) + parts = [] + for p in pred.pat: + if not isinstance(p, str): + return None + if case: + parts.append(c.str.starts_with(p) if name == "Startswith" else c.str.ends_with(p)) + else: + anc = f"(?i)^{re.escape(p)}" if name == "Startswith" else f"(?i){re.escape(p)}$" + parts.append(c.str.contains(anc, literal=False)) + if not parts: + return pl.lit(False) + expr = parts[0] + for p in parts[1:]: + expr = expr | p + return expr + + if name in ("Match", "Fullmatch") and hasattr(pred, "pat") and isinstance(pred.pat, str): + # pandas str.match = anchored at START; str.fullmatch = anchored BOTH ends. decline (NIE): + # custom regex flags beyond case — avoids a flag-semantics gap. + if getattr(pred, "flags", 0): + return None + prefix = "" if getattr(pred, "case", True) else "(?i)" + body = f"(?:{pred.pat})" + anchored = f"{prefix}^{body}" if name == "Match" else f"{prefix}^{body}$" + return c.str.contains(anchored, literal=False) + + return None + + +def _is_membership(value: Any) -> bool: + return isinstance(value, (list, tuple, set, frozenset)) + + +def _is_cross_type_predicate(df: "pl.DataFrame", col: str, pred: ASTPredicate) -> bool: + """True iff the predicate compares a numeric column to a string value (or vice versa): + polars raises `cannot compare string with numeric type` (an uncatchable Rust panic when + nested); pandas/cypher return a value/null. Recurses into AllOf (fold of x>a AND x