diff --git a/ai_code_notes/README.md b/ai_code_notes/README.md index f327e85d52..3305aa4a3c 100644 --- a/ai_code_notes/README.md +++ b/ai_code_notes/README.md @@ -136,6 +136,35 @@ When adding a new guide: - **USER_TESTING_PLAYBOOK.md** [TODO]: AI-driven testing workflows - **Load when**: Starting new tasks, creating commits, fixing code quality issues +## ๐Ÿ“š Documentation Building + +### Quick Commands +```bash +# Build HTML docs only (fastest) +cd docs && ./html.sh + +# Full CI-like build with Docker +cd docs && ./ci.sh + +# Build without notebook validation +cd docs && VALIDATE_NOTEBOOK_EXECUTION=0 ./ci.sh + +# Build specific format +cd docs && DOCS_FORMAT=html ./ci.sh +``` + +### Architecture +- **Sphinx**: Main doc generator with MyST for Markdown support +- **nbsphinx**: Converts notebooks to docs (execution disabled by default) +- **Docker**: Consistent build environment using `sphinxdoc/sphinx:8.0.2` +- **Validation**: Strict mode (`-W`), notebook structure checks, optional execution + +### Key Paths +- `docs/source/`: Sphinx source files (.rst, .md) +- `demos/`: Notebooks mounted to docs build +- `docs/docker/build-docs.sh`: CI validation script +- `docs/test_notebooks/`: Test-specific notebooks + ## ๐Ÿงช Testing Quick Reference ### Docker Commands (Recommended) diff --git a/ai_code_notes/gfql/README.md b/ai_code_notes/gfql/README.md new file mode 100644 index 0000000000..16d84b3c0e --- /dev/null +++ b/ai_code_notes/gfql/README.md @@ -0,0 +1,211 @@ +# GFQL AI Assistant Guide + +Guide for AI assistants working with GFQL (Graph Frame Query Language) in PyGraphistry. + +## ๐ŸŽฏ Quick Reference + +### Essential GFQL Operations +```python +# Node matching +n() # All nodes +n({"type": "person"}) # Filter by property +n({"age": gt(30)}) # With predicate +n(name="result") # Named results + +# Edge traversal +e_forward() # Forward direction +e_reverse() # Reverse direction +e() or e_undirected() # Both directions +e_forward(hops=2) # Multi-hop +e_forward(to_fixed_point=True) # All reachable + +# Chaining +g.chain([n(), e_forward(), n()]) # Pattern matching +``` + +### Key Predicates +- Comparison: `gt()`, `lt()`, `ge()`, `le()`, `eq()`, `ne()` +- Membership: `is_in([...])` +- Range: `between(lower, upper)` +- String: `contains()`, `startswith()`, `endswith()` +- Null: `is_null()`, `not_null()` +- Temporal: `is_month_start()`, `is_year_end()`, etc. + +### Performance Tips +- Filter early in the chain +- Use specific hop counts vs `to_fixed_point` +- Prefer `filter_dict` over `query` strings +- Use appropriate engine: `pandas` (CPU) or `cudf` (GPU) + +## ๐Ÿ“‹ When to Use GFQL + +### Use GFQL When +- Performing graph traversals or path queries +- Finding patterns in connected data +- Need efficient multi-hop operations +- Working with node/edge dataframes + +### Use Pandas/Aggregations When +- Need sorting (`sort_values()`) +- Need limiting (`head()`, `tail()`) +- Aggregating results (`groupby()`, `count()`) +- Complex transformations + +## ๐Ÿš€ Common Patterns + +### User 360 Query +```python +# Customer's recent activity +g.chain([ + n({"customer_id": "C123"}), + e_forward({ + "type": is_in(["purchase", "view", "support"]), + "timestamp": gt(pd.Timestamp.now() - pd.Timedelta(days=30)) + }) +]) +``` + +### Cyber Security Pattern +```python +# Lateral movement detection +g.chain([ + n({"status": "compromised"}), + e_forward({"type": "login", "success": True}, hops=3), + n({"criticality": "high"}, name="at_risk") +]) +``` + +### Business Intelligence +```python +# Cross-sell opportunities +g.chain([ + n({"product_id": "P123"}), + e_reverse({"type": "purchased"}), + n({"type": "customer"}), + e_forward({"type": "purchased"}), + n({"product_id": ne("P123")}, name="also_bought") +]) +``` + +## ๐Ÿ”ง Code Style Guidelines + +### Preferred Style +```python +# โœ… Good - Clean, code-golfed chains +g.chain([n({"type": "user"}), e({"active": True}), n()]) + +# โŒ Avoid - Overly verbose +result = g.chain([ + n(filter_dict={"type": "user"}), + e_forward(edge_match={"active": True}, hops=1), + n(filter_dict={}) +]) +``` + +### Naming Conventions +- Use descriptive names for `name` parameters +- Keep filter keys consistent with dataframe columns +- Use snake_case for all identifiers + +## ๐Ÿ› Common Errors and Fixes + +### Schema Errors +```python +# โŒ Wrong - Column doesn't exist +n({"username": "Alice"}) + +# โœ… Fix - Use correct column name +n({"name": "Alice"}) +``` + +### Type Errors +```python +# โŒ Wrong - String predicate on number +n({"age": contains("30")}) + +# โœ… Fix - Use numeric predicate +n({"age": gt(30)}) +``` + +### Temporal Errors +```python +# โŒ Wrong - Raw string for datetime +n({"created": gt("2024-01-01")}) + +# โœ… Fix - Use proper datetime +n({"created": gt(pd.Timestamp("2024-01-01"))}) +``` + +## ๐Ÿ“ Natural Language to GFQL + +### Translation Patterns +- "recent" โ†’ `gt(pd.Timestamp.now() - pd.Timedelta(days=N))` +- "between X and Y" โ†’ `between(X, Y)` +- "any of" โ†’ `is_in([...])` +- "connected to" โ†’ `e()` or `e_undirected()` +- "from X to Y" โ†’ X with `e_forward()` to Y +- "within N hops" โ†’ `hops=N` + +### Example Translations + +**NL**: "Find all employees who report to managers in NYC" +```python +g.chain([ + n({"type": "employee"}), + e_forward({"type": "reports_to"}), + n({"type": "manager", "office": "NYC"}) +]) +``` + +**NL**: "Show me high-value customers from last week" +```python +g.chain([ + n({"customer_tier": "high_value"}), + e_forward({ + "type": "purchase", + "date": gt(pd.Timestamp.now() - pd.Timedelta(days=7)) + }) +]) +``` + +## ๐Ÿ”„ Cypher to GFQL + +### Basic Mappings +| Cypher | GFQL | +|--------|------| +| `(n)` | `n()` | +| `(n:Label)` | `n({"type": "Label"})` | +| `-[r]->` | `e_forward()` | +| `<-[r]-` | `e_reverse()` | +| `-[r*2]-` | `e_forward(hops=2)` | +| `WHERE n.prop = val` | `n({"prop": val})` | + +### Unsupported in GFQL +- `OPTIONAL MATCH` - Handle nulls in post-processing +- `WITH` clauses - Use intermediate chains +- `ORDER BY/LIMIT` - Use pandas after +- `CREATE/DELETE` - GFQL is read-only + +## ๐Ÿงช Validation Checklist + +Before generating GFQL: +1. โœ“ Check column names exist in schema +2. โœ“ Verify predicate types match column types +3. โœ“ Ensure temporal values use proper types +4. โœ“ Validate operation names (n, e_forward, etc.) +5. โœ“ Check chain structure is valid + +## ๐Ÿ“š Additional Resources + +- Full specifications in: `AI_PROGRESS/gfql_llm_specs/` + - `gfql_language_spec.md` - Complete language specification + - `gfql_wire_protocol_spec.md` - JSON wire format + - `cypher_to_gfql_mapping_spec.md` - Cypher translation + +## ๐ŸŽฏ Key Takeaways + +1. **GFQL is functional**: Chain operations, don't mutate +2. **Filter early**: Put selective conditions first +3. **Think patterns**: Focus on graph patterns, not procedures +4. **Post-process**: Use pandas for sorting/aggregating +5. **Code golf**: Keep queries concise and elegant \ No newline at end of file diff --git a/demos/demos_databases_apis/hypernetx/hypernetx.ipynb b/demos/demos_databases_apis/hypernetx/hypernetx.ipynb index 0f84845470..be208ee842 100644 --- a/demos/demos_databases_apis/hypernetx/hypernetx.ipynb +++ b/demos/demos_databases_apis/hypernetx/hypernetx.ipynb @@ -6,22 +6,7 @@ "colab_type": "text", "id": "bdxNPN0PhlsS" }, - "source": [ - "# HyperNetX + Graphistry = ๐Ÿ’ช๐Ÿ’ช๐Ÿ’ช\n", - "\n", - "You can quickly explore HyperNetX graphs using Graphistry through the below sample class.\n", - "\n", - "PNNL's [HyperNetX](https://github.com/pnnl/HyperNetX) is a new Python library for manipulating hypergraphs: graphs where an edge may connect any number of nodes. The below helper class converts HyperNetX graphs into 2-edge graphs (node/edge property tables as Panda dataframes ) in two different modes:\n", - "\n", - "* `hypernetx_to_graphistry_bipartite(hnx_graph)`: \n", - " * Turn every hyperedge and hypernode into nodes. They form a bipartite graph: whenever a hyperedge includes a hypernode, create an edge from the hyperedge's node to the hypernode's node.\n", - " * ex: Hyperedge `0: ['a', 'b', 'c']` => edge `(0, 'a')`, edge `(0, 'b')`, edge `(0, 'c')`\n", - "* `hypernetx_to_graphistry_nodes(hnx_graph)`:\n", - " * Turn every hypernode into a node, and whenever two hypernodes share the same hyperedge, create an edge between their corresponding nodes\n", - " * To emphasize that edges are undirect, the library sets the edge curvature to 0 (straight)\n", - " * ex: Hyperedge `0: ['a', 'b', 'c']` => edge `('a', 'b')`, edge `('a', 'c')`, edge `('b', 'c')`\n", - "\n" - ] + "source": "# HyperNetX + Graphistry = [POWER]\n\nYou can quickly explore HyperNetX graphs using Graphistry through the below sample class.\n\nPNNL's [HyperNetX](https://github.com/pnnl/HyperNetX) is a new Python library for manipulating hypergraphs: graphs where an edge may connect any number of nodes. The below helper class converts HyperNetX graphs into 2-edge graphs (node/edge property tables as Panda dataframes ) in two different modes:\n\n* `hypernetx_to_graphistry_bipartite(hnx_graph)`: \n * Turn every hyperedge and hypernode into nodes. They form a bipartite graph: whenever a hyperedge includes a hypernode, create an edge from the hyperedge's node to the hypernode's node.\n * ex: Hyperedge `0: ['a', 'b', 'c']` => edge `(0, 'a')`, edge `(0, 'b')`, edge `(0, 'c')`\n* `hypernetx_to_graphistry_nodes(hnx_graph)`:\n * Turn every hypernode into a node, and whenever two hypernodes share the same hyperedge, create an edge between their corresponding nodes\n * To emphasize that edges are undirect, the library sets the edge curvature to 0 (straight)\n * ex: Hyperedge `0: ['a', 'b', 'c']` => edge `('a', 'b')`, edge `('a', 'c')`, edge `('b', 'c')`" }, { "cell_type": "markdown", @@ -720,4 +705,4 @@ }, "nbformat": 4, "nbformat_minor": 1 -} +} \ No newline at end of file diff --git a/demos/demos_databases_apis/memgraph/visualizing_iam_dataset.ipynb b/demos/demos_databases_apis/memgraph/visualizing_iam_dataset.ipynb index 9d2658d427..1153d7d7ac 100644 --- a/demos/demos_databases_apis/memgraph/visualizing_iam_dataset.ipynb +++ b/demos/demos_databases_apis/memgraph/visualizing_iam_dataset.ipynb @@ -289,7 +289,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "![Screenshot](https://github.com/karmenrabar/pygraphistry_images/blob/main/memgraphlab.png?raw=true)" + "![Memgraph Lab Schema](https://github.com/karmenrabar/pygraphistry_images/blob/main/memgraphlab.png?raw=true)" ] }, { @@ -339,15 +339,13 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "Screenshot:" - ] + "source": "Screenshot - All Access View:" }, { "cell_type": "markdown", "metadata": {}, "source": [ - "![Screenshot](https://github.com/karmenrabar/pygraphistry_images/blob/main/allaccess.png?raw=true)" + "![All Access View](https://github.com/karmenrabar/pygraphistry_images/blob/main/allaccess.png?raw=true)" ] }, { @@ -399,15 +397,13 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "Screenshot:\n" - ] + "source": "Screenshot - Carl's Direct Access:" }, { "cell_type": "markdown", "metadata": {}, "source": [ - "![Screenshot](https://github.com/karmenrabar/pygraphistry_images/blob/main/access3.png?raw=true)" + "![Carl Direct Access](https://github.com/karmenrabar/pygraphistry_images/blob/main/access3.png?raw=true)" ] }, { @@ -461,15 +457,13 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "Screenshot:" - ] + "source": "Screenshot - Carl's All File Access:" }, { "cell_type": "markdown", "metadata": {}, "source": [ - "![Screenshot](https://github.com/karmenrabar/pygraphistry_images/blob/main/access2.png?raw=true)" + "![Carl All File Access](https://github.com/karmenrabar/pygraphistry_images/blob/main/access2.png?raw=true)" ] }, { @@ -523,15 +517,13 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "Screenshot:" - ] + "source": "Screenshot - All Persons File Access:" }, { "cell_type": "markdown", "metadata": {}, "source": [ - "![Screenshot](https://github.com/karmenrabar/pygraphistry_images/blob/main/access.png?raw=true)" + "![All Persons File Access](https://github.com/karmenrabar/pygraphistry_images/blob/main/access.png?raw=true)" ] }, { @@ -578,4 +570,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/demos/gfql/gfql_validation_fundamentals.ipynb b/demos/gfql/gfql_validation_fundamentals.ipynb new file mode 100644 index 0000000000..f6851f9ee4 --- /dev/null +++ b/demos/gfql/gfql_validation_fundamentals.ipynb @@ -0,0 +1,2285 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GFQL Validation Fundamentals\n", + "\n", + "Learn the basics of validating GFQL queries to catch errors early and build robust graph applications.\n", + "\n", + "## What You'll Learn\n", + "- How to validate GFQL query syntax\n", + "- Understanding validation error messages\n", + "- Basic schema validation with DataFrames\n", + "- Common syntax errors and how to fix them\n", + "\n", + "## Prerequisites\n", + "- Basic Python knowledge\n", + "- PyGraphistry installed (`pip install graphistry[ai]`)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup and Imports\n", + "\n", + "First, let's import the necessary modules and check our PyGraphistry version." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": [ + "#", + " ", + "C", + "o", + "r", + "e", + " ", + "i", + "m", + "p", + "o", + "r", + "t", + "s", + "\n", + "i", + "m", + "p", + "o", + "r", + "t", + " ", + "p", + "a", + "n", + "d", + "a", + "s", + " ", + "a", + "s", + " ", + "p", + "d", + "\n", + "i", + "m", + "p", + "o", + "r", + "t", + " ", + "g", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + "\n", + "\n", + "#", + " ", + "V", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "i", + "m", + "p", + "o", + "r", + "t", + "s", + "\n", + "f", + "r", + "o", + "m", + " ", + "g", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + ".", + "c", + "o", + "m", + "p", + "u", + "t", + "e", + ".", + "g", + "f", + "q", + "l", + ".", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + " ", + "i", + "m", + "p", + "o", + "r", + "t", + " ", + "(", + "\n", + " ", + " ", + " ", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "s", + "y", + "n", + "t", + "a", + "x", + ",", + "\n", + " ", + " ", + " ", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "s", + "c", + "h", + "e", + "m", + "a", + ",", + "\n", + " ", + " ", + " ", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "q", + "u", + "e", + "r", + "y", + ",", + "\n", + " ", + " ", + " ", + " ", + "e", + "x", + "t", + "r", + "a", + "c", + "t", + "_", + "s", + "c", + "h", + "e", + "m", + "a", + "_", + "f", + "r", + "o", + "m", + "_", + "d", + "a", + "t", + "a", + "f", + "r", + "a", + "m", + "e", + "s", + "\n", + ")", + "\n", + "\n", + "#", + " ", + "C", + "h", + "e", + "c", + "k", + " ", + "v", + "e", + "r", + "s", + "i", + "o", + "n", + "\n", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + "P", + "y", + "G", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + " ", + "v", + "e", + "r", + "s", + "i", + "o", + "n", + ":", + " ", + "{", + "g", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + ".", + "_", + "_", + "v", + "e", + "r", + "s", + "i", + "o", + "n", + "_", + "_", + "}", + "\"", + ")", + "\n", + "p", + "r", + "i", + "n", + "t", + "(", + "\"", + "\\", + "n", + "V", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "f", + "u", + "n", + "c", + "t", + "i", + "o", + "n", + "s", + " ", + "a", + "v", + "a", + "i", + "l", + "a", + "b", + "l", + "e", + ":", + "\"", + ")", + "\n", + "p", + "r", + "i", + "n", + "t", + "(", + "\"", + "-", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "s", + "y", + "n", + "t", + "a", + "x", + "(", + ")", + ":", + " ", + "C", + "h", + "e", + "c", + "k", + " ", + "q", + "u", + "e", + "r", + "y", + " ", + "s", + "y", + "n", + "t", + "a", + "x", + "\"", + ")", + "\n", + "p", + "r", + "i", + "n", + "t", + "(", + "\"", + "-", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "s", + "c", + "h", + "e", + "m", + "a", + "(", + ")", + ":", + " ", + "C", + "h", + "e", + "c", + "k", + " ", + "q", + "u", + "e", + "r", + "y", + " ", + "a", + "g", + "a", + "i", + "n", + "s", + "t", + " ", + "d", + "a", + "t", + "a", + " ", + "s", + "c", + "h", + "e", + "m", + "a", + "\"", + ")", + "\n", + "p", + "r", + "i", + "n", + "t", + "(", + "\"", + "-", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "q", + "u", + "e", + "r", + "y", + "(", + ")", + ":", + " ", + "C", + "o", + "m", + "b", + "i", + "n", + "e", + "d", + " ", + "s", + "y", + "n", + "t", + "a", + "x", + " ", + "+", + " ", + "s", + "c", + "h", + "e", + "m", + "a", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + "\"", + ")" + ], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Syntax Validation\n", + "\n", + "GFQL queries must follow specific syntax rules. Let's start with validating query syntax." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example 1: Valid query syntax\n", + "valid_query = [\n", + " {\"type\": \"n\"},\n", + " {\"type\": \"e_forward\", \"hops\": 1},\n", + " {\"type\": \"n\", \"filter\": {\"type\": {\"eq\": \"customer\"}}}\n", + "]\n", + "\n", + "# Validate syntax\n", + "issues = validate_syntax(valid_query)\n", + "\n", + "print(\"Query:\", valid_query)\n", + "print(f\"\\nValidation issues: {len(issues)}\")\n", + "if not issues:\n", + " print(\"[OK] Query syntax is valid!\")\n", + "else:\n", + " for issue in issues:\n", + " print(f\"- {issue.level}: {issue.message}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Common Syntax Errors\n", + "\n", + "Let's look at common syntax errors and how validation catches them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example 2: Invalid operation type\n", + "invalid_query_1 = [\n", + " {\"type\": \"node\"}, # Should be \"n\"\n", + " {\"type\": \"e_forward\"}\n", + "]\n", + "\n", + "issues = validate_syntax(invalid_query_1)\n", + "print(\"Query with invalid operation type:\")\n", + "print(invalid_query_1)\n", + "print(f\"\\nValidation found {len(issues)} issue(s):\")\n", + "for issue in issues:\n", + " print(f\"- {issue.level}: {issue.message}\")\n", + " if issue.operation_index is not None:\n", + " print(f\" At operation {issue.operation_index}: {invalid_query_1[issue.operation_index]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example 3: Invalid filter structure\n", + "invalid_query_2 = [\n", + " {\"type\": \"n\", \"filter\": {\"name\": \"Alice\"}} # Missing operator\n", + "]\n", + "\n", + "issues = validate_syntax(invalid_query_2)\n", + "print(\"Query with invalid filter:\")\n", + "print(invalid_query_2)\n", + "print(f\"\\nValidation found {len(issues)} issue(s):\")\n", + "for issue in issues:\n", + " print(f\"- {issue.level}: {issue.message}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example 4: Semantic warning - orphaned edges\n", + "warning_query = [\n", + " {\"type\": \"e_forward\", \"hops\": 1} # Edge without starting node\n", + "]\n", + "\n", + "issues = validate_syntax(warning_query)\n", + "print(\"Query with semantic warning:\")\n", + "print(warning_query)\n", + "print(f\"\\nValidation found {len(issues)} issue(s):\")\n", + "for issue in issues:\n", + " print(f\"- {issue.level.upper()}: {issue.message}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Understanding Validation Issues\n", + "\n", + "Validation issues have different levels and provide helpful information." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": [ + "#", + " ", + "L", + "e", + "t", + "'", + "s", + " ", + "e", + "x", + "a", + "m", + "i", + "n", + "e", + " ", + "a", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "i", + "s", + "s", + "u", + "e", + " ", + "i", + "n", + " ", + "d", + "e", + "t", + "a", + "i", + "l", + "\n", + "f", + "r", + "o", + "m", + " ", + "g", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + ".", + "c", + "o", + "m", + "p", + "u", + "t", + "e", + ".", + "g", + "f", + "q", + "l", + ".", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + " ", + "i", + "m", + "p", + "o", + "r", + "t", + " ", + "V", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + "I", + "s", + "s", + "u", + "e", + "\n", + "\n", + "#", + " ", + "C", + "r", + "e", + "a", + "t", + "e", + " ", + "a", + " ", + "q", + "u", + "e", + "r", + "y", + " ", + "w", + "i", + "t", + "h", + " ", + "m", + "u", + "l", + "t", + "i", + "p", + "l", + "e", + " ", + "i", + "s", + "s", + "u", + "e", + "s", + "\n", + "c", + "o", + "m", + "p", + "l", + "e", + "x", + "_", + "i", + "n", + "v", + "a", + "l", + "i", + "d", + " ", + "=", + " ", + "[", + "\n", + " ", + " ", + " ", + " ", + "{", + "\"", + "t", + "y", + "p", + "e", + "\"", + ":", + " ", + "\"", + "n", + "\"", + "}", + ",", + "\n", + " ", + " ", + " ", + " ", + "{", + "\"", + "t", + "y", + "p", + "e", + "\"", + ":", + " ", + "\"", + "e", + "d", + "g", + "e", + "\"", + "}", + ",", + " ", + " ", + "#", + " ", + "I", + "n", + "v", + "a", + "l", + "i", + "d", + " ", + "t", + "y", + "p", + "e", + "\n", + " ", + " ", + " ", + " ", + "{", + "\"", + "t", + "y", + "p", + "e", + "\"", + ":", + " ", + "\"", + "n", + "\"", + ",", + " ", + "\"", + "f", + "i", + "l", + "t", + "e", + "r", + "\"", + ":", + " ", + "{", + "\"", + "s", + "c", + "o", + "r", + "e", + "\"", + ":", + " ", + "{", + "\"", + "g", + "r", + "e", + "a", + "t", + "e", + "r", + "\"", + ":", + " ", + "5", + "}", + "}", + "}", + " ", + " ", + "#", + " ", + "I", + "n", + "v", + "a", + "l", + "i", + "d", + " ", + "o", + "p", + "e", + "r", + "a", + "t", + "o", + "r", + "\n", + "]", + "\n", + "\n", + "i", + "s", + "s", + "u", + "e", + "s", + " ", + "=", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "e", + "_", + "s", + "y", + "n", + "t", + "a", + "x", + "(", + "c", + "o", + "m", + "p", + "l", + "e", + "x", + "_", + "i", + "n", + "v", + "a", + "l", + "i", + "d", + ")", + "\n", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + "F", + "o", + "u", + "n", + "d", + " ", + "{", + "l", + "e", + "n", + "(", + "i", + "s", + "s", + "u", + "e", + "s", + ")", + "}", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "i", + "s", + "s", + "u", + "e", + "s", + ":", + "\\", + "n", + "\"", + ")", + "\n", + "\n", + "f", + "o", + "r", + " ", + "i", + ",", + " ", + "i", + "s", + "s", + "u", + "e", + " ", + "i", + "n", + " ", + "e", + "n", + "u", + "m", + "e", + "r", + "a", + "t", + "e", + "(", + "i", + "s", + "s", + "u", + "e", + "s", + ")", + ":", + "\n", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + "I", + "s", + "s", + "u", + "e", + " ", + "{", + "i", + "+", + "1", + "}", + ":", + "\"", + ")", + "\n", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + " ", + " ", + "L", + "e", + "v", + "e", + "l", + ":", + " ", + "{", + "i", + "s", + "s", + "u", + "e", + ".", + "l", + "e", + "v", + "e", + "l", + "}", + "\"", + ")", + "\n", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + " ", + " ", + "M", + "e", + "s", + "s", + "a", + "g", + "e", + ":", + " ", + "{", + "i", + "s", + "s", + "u", + "e", + ".", + "m", + "e", + "s", + "s", + "a", + "g", + "e", + "}", + "\"", + ")", + "\n", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + " ", + " ", + "O", + "p", + "e", + "r", + "a", + "t", + "i", + "o", + "n", + " ", + "i", + "n", + "d", + "e", + "x", + ":", + " ", + "{", + "i", + "s", + "s", + "u", + "e", + ".", + "o", + "p", + "e", + "r", + "a", + "t", + "i", + "o", + "n", + "_", + "i", + "n", + "d", + "e", + "x", + "}", + "\"", + ")", + "\n", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + " ", + " ", + "F", + "i", + "e", + "l", + "d", + ":", + " ", + "{", + "i", + "s", + "s", + "u", + "e", + ".", + "f", + "i", + "e", + "l", + "d", + "}", + "\"", + ")", + "\n", + " ", + " ", + " ", + " ", + "i", + "f", + " ", + "i", + "s", + "s", + "u", + "e", + ".", + "s", + "u", + "g", + "g", + "e", + "s", + "t", + "i", + "o", + "n", + ":", + "\n", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + "f", + "\"", + " ", + " ", + "S", + "u", + "g", + "g", + "e", + "s", + "t", + "i", + "o", + "n", + ":", + " ", + "{", + "i", + "s", + "s", + "u", + "e", + ".", + "s", + "u", + "g", + "g", + "e", + "s", + "t", + "i", + "o", + "n", + "}", + "\"", + ")", + "\n", + " ", + " ", + " ", + " ", + "p", + "r", + "i", + "n", + "t", + "(", + ")" + ], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple Schema Validation\n", + "\n", + "Now let's validate queries against actual data schemas." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create sample data\n", + "nodes_df = pd.DataFrame({\n", + " 'id': [1, 2, 3, 4, 5],\n", + " 'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],\n", + " 'type': ['customer', 'customer', 'product', 'product', 'customer'],\n", + " 'score': [100, 85, 95, 120, 110]\n", + "})\n", + "\n", + "edges_df = pd.DataFrame({\n", + " 'src': [1, 2, 3, 4, 5],\n", + " 'dst': [3, 4, 1, 2, 3],\n", + " 'weight': [1.0, 2.5, 0.8, 1.2, 3.0],\n", + " 'timestamp': pd.to_datetime(['2024-01-01', '2024-01-02', '2024-01-03', '2024-01-04', '2024-01-05'])\n", + "})\n", + "\n", + "print(\"Nodes DataFrame:\")\n", + "print(nodes_df)\n", + "print(\"\\nEdges DataFrame:\")\n", + "print(edges_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract schema from DataFrames\n", + "schema = extract_schema_from_dataframes(nodes_df, edges_df)\n", + "\n", + "print(\"Extracted Schema:\")\n", + "print(f\"\\nNode columns: {list(schema.node_columns.keys())}\")\n", + "print(f\"Edge columns: {list(schema.edge_columns.keys())}\")\n", + "\n", + "# Show column types\n", + "print(\"\\nNode column types:\")\n", + "for col, dtype in schema.node_columns.items():\n", + " print(f\" {col}: {dtype}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Valid query using existing columns\n", + "schema_valid_query = [\n", + " {\"type\": \"n\", \"filter\": {\"type\": {\"eq\": \"customer\"}}},\n", + " {\"type\": \"e_forward\"},\n", + " {\"type\": \"n\", \"filter\": {\"score\": {\"gte\": 100}}}\n", + "]\n", + "\n", + "# Validate against schema\n", + "issues = validate_schema(schema_valid_query, schema)\n", + "\n", + "print(\"Query using valid columns:\")\n", + "print(schema_valid_query)\n", + "print(f\"\\nSchema validation issues: {len(issues)}\")\n", + "if not issues:\n", + " print(\"[OK] Query is valid for this schema!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Column Not Found Errors\n", + "\n", + "The most common schema error is referencing non-existent columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query with non-existent column\n", + "invalid_column_query = [\n", + " {\"type\": \"n\", \"filter\": {\"category\": {\"eq\": \"VIP\"}}} # 'category' doesn't exist\n", + "]\n", + "\n", + "issues = validate_schema(invalid_column_query, schema)\n", + "\n", + "print(\"Query with non-existent column:\")\n", + "print(invalid_column_query)\n", + "print(f\"\\nValidation found {len(issues)} issue(s):\")\n", + "for issue in issues:\n", + " print(f\"\\n- {issue.level}: {issue.message}\")\n", + " if issue.suggestion:\n", + " print(f\" Suggestion: {issue.suggestion}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Type Mismatch Errors\n", + "\n", + "Validation also catches when you use the wrong predicate type for a column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# String predicate on numeric column\n", + "type_mismatch_query = [\n", + " {\"type\": \"n\", \"filter\": {\"score\": {\"contains\": \"100\"}}} # 'contains' is for strings\n", + "]\n", + "\n", + "issues = validate_schema(type_mismatch_query, schema)\n", + "\n", + "print(\"Query with type mismatch:\")\n", + "print(type_mismatch_query)\n", + "print(f\"\\nValidation found {len(issues)} issue(s):\")\n", + "for issue in issues:\n", + " print(f\"\\n- {issue.level}: {issue.message}\")\n", + " if issue.suggestion:\n", + " print(f\" Suggestion: {issue.suggestion}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complete Example: Building a Query Step by Step\n", + "\n", + "Let's build a query incrementally, validating at each step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 1: Start with finding customers\n", + "query_v1 = [\n", + " {\"type\": \"n\", \"filter\": {\"type\": {\"eq\": \"customer\"}}}\n", + "]\n", + "\n", + "issues = validate_query(query_v1, nodes_df=nodes_df, edges_df=edges_df)\n", + "print(\"Step 1 - Find customers:\")\n", + "print(f\"Issues: {len(issues)}\")\n", + "print(\"[OK] Valid!\" if not issues else \"[X] Has issues\")\n", + "\n", + "# Step 2: Add edge traversal\n", + "query_v2 = [\n", + " {\"type\": \"n\", \"filter\": {\"type\": {\"eq\": \"customer\"}}},\n", + " {\"type\": \"e_forward\", \"hops\": 1}\n", + "]\n", + "\n", + "issues = validate_query(query_v2, nodes_df=nodes_df, edges_df=edges_df)\n", + "print(\"\\nStep 2 - Add edge traversal:\")\n", + "print(f\"Issues: {len(issues)}\")\n", + "print(\"[OK] Valid!\" if not issues else \"[X] Has issues\")\n", + "\n", + "# Step 3: Complete with destination filter\n", + "query_v3 = [\n", + " {\"type\": \"n\", \"filter\": {\"type\": {\"eq\": \"customer\"}}},\n", + " {\"type\": \"e_forward\", \"hops\": 1},\n", + " {\"type\": \"n\", \"filter\": {\"type\": {\"eq\": \"product\"}}}\n", + "]\n", + "\n", + "issues = validate_query(query_v3, nodes_df=nodes_df, edges_df=edges_df)\n", + "print(\"\\nStep 3 - Add destination filter:\")\n", + "print(f\"Issues: {len(issues)}\")\n", + "print(\"[OK] Valid!\" if not issues else \"[X] Has issues\")\n", + "\n", + "print(\"\\nFinal query finds: Customers connected to products\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Reference\n", + "\n", + "### Error Levels\n", + "- **error**: Query will fail if executed\n", + "- **warning**: Query may work but has potential issues\n", + "\n", + "### Common Fixes\n", + "1. **Invalid operation type**: Use `n`, `e_forward`, `e_reverse`, or `e`\n", + "2. **Missing operator**: Add comparison operator like `eq`, `gte`, `contains`\n", + "3. **Column not found**: Check available columns with `schema.node_columns`\n", + "4. **Type mismatch**: Use numeric operators for numbers, string operators for text" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#", + "#", + " ", + "S", + "u", + "m", + "m", + "a", + "r", + "y", + " ", + "&", + " ", + "N", + "e", + "x", + "t", + " ", + "S", + "t", + "e", + "p", + "s", + "\n", + "\n", + "Y", + "o", + "u", + "'", + "v", + "e", + " ", + "l", + "e", + "a", + "r", + "n", + "e", + "d", + " ", + "t", + "h", + "e", + " ", + "f", + "u", + "n", + "d", + "a", + "m", + "e", + "n", + "t", + "a", + "l", + "s", + " ", + "o", + "f", + " ", + "G", + "F", + "Q", + "L", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + ":", + "\n", + "-", + " ", + "[OK]", + " ", + "S", + "y", + "n", + "t", + "a", + "x", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "c", + "a", + "t", + "c", + "h", + "e", + "s", + " ", + "s", + "t", + "r", + "u", + "c", + "t", + "u", + "r", + "a", + "l", + " ", + "e", + "r", + "r", + "o", + "r", + "s", + "\n", + "-", + " ", + "[OK]", + " ", + "S", + "c", + "h", + "e", + "m", + "a", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "e", + "n", + "s", + "u", + "r", + "e", + "s", + " ", + "c", + "o", + "l", + "u", + "m", + "n", + "s", + " ", + "e", + "x", + "i", + "s", + "t", + " ", + "a", + "n", + "d", + " ", + "t", + "y", + "p", + "e", + "s", + " ", + "m", + "a", + "t", + "c", + "h", + "\n", + "-", + " ", + "[OK]", + " ", + "C", + "o", + "m", + "b", + "i", + "n", + "e", + "d", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "p", + "r", + "o", + "v", + "i", + "d", + "e", + "s", + " ", + "c", + "o", + "m", + "p", + "r", + "e", + "h", + "e", + "n", + "s", + "i", + "v", + "e", + " ", + "c", + "h", + "e", + "c", + "k", + "i", + "n", + "g", + "\n", + "-", + " ", + "[OK]", + " ", + "C", + "l", + "e", + "a", + "r", + " ", + "e", + "r", + "r", + "o", + "r", + " ", + "m", + "e", + "s", + "s", + "a", + "g", + "e", + "s", + " ", + "h", + "e", + "l", + "p", + " ", + "f", + "i", + "x", + " ", + "i", + "s", + "s", + "u", + "e", + "s", + " ", + "q", + "u", + "i", + "c", + "k", + "l", + "y", + "\n", + "\n", + "#", + "#", + "#", + " ", + "N", + "e", + "x", + "t", + " ", + "S", + "t", + "e", + "p", + "s", + "\n", + "1", + ".", + " ", + "*", + "*", + "A", + "d", + "v", + "a", + "n", + "c", + "e", + "d", + " ", + "P", + "a", + "t", + "t", + "e", + "r", + "n", + "s", + "*", + "*", + ":", + " ", + "L", + "e", + "a", + "r", + "n", + " ", + "c", + "o", + "m", + "p", + "l", + "e", + "x", + " ", + "q", + "u", + "e", + "r", + "i", + "e", + "s", + " ", + "a", + "n", + "d", + " ", + "m", + "u", + "l", + "t", + "i", + "-", + "h", + "o", + "p", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + "\n", + "2", + ".", + " ", + "*", + "*", + "L", + "L", + "M", + " ", + "I", + "n", + "t", + "e", + "g", + "r", + "a", + "t", + "i", + "o", + "n", + "*", + "*", + ":", + " ", + "U", + "s", + "e", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "f", + "o", + "r", + " ", + "A", + "I", + "-", + "g", + "e", + "n", + "e", + "r", + "a", + "t", + "e", + "d", + " ", + "q", + "u", + "e", + "r", + "i", + "e", + "s", + "\n", + "3", + ".", + " ", + "*", + "*", + "P", + "r", + "o", + "d", + "u", + "c", + "t", + "i", + "o", + "n", + " ", + "U", + "s", + "e", + "*", + "*", + ":", + " ", + "I", + "m", + "p", + "l", + "e", + "m", + "e", + "n", + "t", + " ", + "v", + "a", + "l", + "i", + "d", + "a", + "t", + "i", + "o", + "n", + " ", + "i", + "n", + " ", + "y", + "o", + "u", + "r", + " ", + "a", + "p", + "p", + "l", + "i", + "c", + "a", + "t", + "i", + "o", + "n", + "s", + "\n", + "\n", + "#", + "#", + "#", + " ", + "R", + "e", + "s", + "o", + "u", + "r", + "c", + "e", + "s", + "\n", + "-", + " ", + "[", + "G", + "F", + "Q", + "L", + " ", + "D", + "o", + "c", + "u", + "m", + "e", + "n", + "t", + "a", + "t", + "i", + "o", + "n", + "]", + "(", + "h", + "t", + "t", + "p", + "s", + ":", + "/", + "/", + "d", + "o", + "c", + "s", + ".", + "g", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + ".", + "c", + "o", + "m", + "/", + "g", + "f", + "q", + "l", + "/", + ")", + "\n", + "-", + " ", + "[", + "G", + "F", + "Q", + "L", + " ", + "L", + "a", + "n", + "g", + "u", + "a", + "g", + "e", + " ", + "S", + "p", + "e", + "c", + "i", + "f", + "i", + "c", + "a", + "t", + "i", + "o", + "n", + "]", + "(", + "h", + "t", + "t", + "p", + "s", + ":", + "/", + "/", + "d", + "o", + "c", + "s", + ".", + "g", + "r", + "a", + "p", + "h", + "i", + "s", + "t", + "r", + "y", + ".", + "c", + "o", + "m", + "/", + "g", + "f", + "q", + "l", + "/", + "s", + "p", + "e", + "c", + "/", + "l", + "a", + "n", + "g", + "u", + "a", + "g", + "e", + "/", + ")" + ], + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/docs/docker/build-docs.sh b/docs/docker/build-docs.sh index 5ff97c342a..99eb003f0f 100755 --- a/docs/docker/build-docs.sh +++ b/docs/docker/build-docs.sh @@ -15,9 +15,31 @@ build_epub() { build_pdf() { sphinx-build -b latex -d /docs/doctrees . /docs/_build/latexpdf cd /docs/_build/latexpdf - # Run pdflatex twice to resolve cross-references, using batchmode for non-interactive build + + # Temporarily disable exit on error for pdflatex + # LaTeX may exit with code 1 due to warnings (e.g., multiply-defined labels) + # but still generate a valid PDF + set +e + pdflatex -file-line-error -interaction=nonstopmode PyGraphistry.tex + FIRST_EXIT=$? + pdflatex -file-line-error -interaction=nonstopmode PyGraphistry.tex + SECOND_EXIT=$? + + # Re-enable exit on error + set -e + + # If PDF was generated, consider it a success even if there were warnings + if [ -f PyGraphistry.pdf ]; then + if [ $SECOND_EXIT -ne 0 ]; then + echo "WARNING: pdflatex exited with code $SECOND_EXIT but PDF was generated successfully" + fi + return 0 + else + echo "ERROR: PDF generation failed" + return 1 + fi } # Build docs first diff --git a/docs/source/api/gfql/exceptions.rst b/docs/source/api/gfql/exceptions.rst new file mode 100644 index 0000000000..9f45347049 --- /dev/null +++ b/docs/source/api/gfql/exceptions.rst @@ -0,0 +1,7 @@ +graphistry.compute.gfql.exceptions module +========================================= + +.. automodule:: graphistry.compute.gfql.exceptions + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/api/gfql/index.rst b/docs/source/api/gfql/index.rst index 877b019aee..0d7172f85a 100644 --- a/docs/source/api/gfql/index.rst +++ b/docs/source/api/gfql/index.rst @@ -9,6 +9,8 @@ GFQL API Reference ast chain edge + exceptions hop node predicates + validate diff --git a/docs/source/api/gfql/validate.rst b/docs/source/api/gfql/validate.rst new file mode 100644 index 0000000000..21f6fca9f1 --- /dev/null +++ b/docs/source/api/gfql/validate.rst @@ -0,0 +1,7 @@ +graphistry.compute.gfql.validate module +======================================= + +.. automodule:: graphistry.compute.gfql.validate + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/gfql/index.rst b/docs/source/gfql/index.rst index fd21b5f2ba..e8b26a10db 100644 --- a/docs/source/gfql/index.rst +++ b/docs/source/gfql/index.rst @@ -10,6 +10,7 @@ See also: .. toctree:: :maxdepth: 1 + :caption: User Guide about overview @@ -21,3 +22,10 @@ See also: predicates/quick datetime_filtering wire_protocol_examples + +.. toctree:: + :maxdepth: 2 + :caption: Developer Resources + + spec/index + validation/index diff --git a/docs/source/gfql/spec/cypher_mapping.md b/docs/source/gfql/spec/cypher_mapping.md new file mode 100644 index 0000000000..d511d22e81 --- /dev/null +++ b/docs/source/gfql/spec/cypher_mapping.md @@ -0,0 +1,258 @@ +(gfql-spec-cypher-mapping)= + +# Cypher to GFQL Python & Wire Protocol Mapping + +## Introduction + +This specification shows how to translate Cypher queries to both GFQL Python code and Wire Protocol JSON, enabling: +- Migration from Cypher-based systems +- Two-stage LLM synthesis: Text โ†’ Cypher โ†’ GFQL +- Language-agnostic API integration +- Secure query generation without code execution + +## Conceptual Framework + +### Translation Scenarios + +When translating from Cypher, you'll encounter three scenarios: + +**1. Direct Translation** - Most pattern matching maps cleanly to pure GFQL +**2. Hybrid Approach** - Post-processing operations (RETURN clauses) use dataframes +**3. GFQL Advantages** - Some capabilities go beyond what Cypher offers + +### What Translates Directly +- Graph patterns: `(a)-[r]->(b)` โ†’ chain operations +- Property filters: WHERE clauses embed into operations +- Path traversals: Variable-length paths use `hops` parameter +- Pattern composition: Multiple patterns become sequential operations + +### What Requires DataFrames +- Aggregations: COUNT, SUM, AVG โ†’ pandas operations +- Projections: RETURN specific columns โ†’ DataFrame selection +- Sorting/limiting: ORDER BY, LIMIT โ†’ DataFrame methods +- Joins: Multiple disconnected patterns โ†’ pandas merge + +### GFQL Advantages Beyond Cypher +- **Rich edge properties**: Query edges as first-class entities +- **Dataframe-native**: Zero-cost transitions between graph and tabular operations +- **GPU acceleration**: Massively parallel execution on NVIDIA hardware +- **Heterogeneous graphs**: No schema constraints on types or properties + +## Quick Example + +**Cypher:** +```cypher +MATCH (p:Person)-[r:FOLLOWS]->(q:Person) +WHERE p.age > 30 +``` + +**Python:** +```python +g.chain([ + n({"type": "Person", "age": gt(30)}, name="p"), + e_forward({"type": "FOLLOWS"}, name="r"), + n({"type": "Person"}, name="q") +]) +``` + +**Wire Protocol:** +```json +{"type": "Chain", "chain": [ + {"type": "Node", "filter_dict": {"type": "Person", "age": {"type": "GT", "val": 30}}, "name": "p"}, + {"type": "Edge", "direction": "forward", "edge_match": {"type": "FOLLOWS"}, "name": "r"}, + {"type": "Node", "filter_dict": {"type": "Person"}, "name": "q"} +]} + +## Pattern Translations + +### Node Patterns + +| Cypher | Python | Wire Protocol | +|--------|--------|---------------| +| `(n)` | `n()` | `{"type": "Node"}` | +| `(n:Label)` | `n({"type": "Label"})` | `{"type": "Node", "filter_dict": {"type": "Label"}}` | +| `(n {prop: val})` | `n({"prop": val})` | `{"type": "Node", "filter_dict": {"prop": val}}` | +| `(n:Person) WHERE n.age > 30` | `n({"type": "Person", "age": gt(30)})` | `{"type": "Node", "filter_dict": {"type": "Person", "age": {"type": "GT", "val": 30}}}` | + +### Edge Patterns + +| Cypher | Python | Wire Protocol (compact) | +|--------|--------|-------------------------| +| `-[]->` | `e_forward()` | `{"type": "Edge", "direction": "forward"}` | +| `-[r:KNOWS]->` | `e_forward({"type": "KNOWS"}, name="r")` | `{"type": "Edge", "direction": "forward", "edge_match": {"type": "KNOWS"}, "name": "r"}` | +| `<-[r]-` | `e_reverse(name="r")` | `{"type": "Edge", "direction": "reverse", "name": "r"}` | +| `-[r]-` | `e(name="r")` | `{"type": "Edge", "direction": "undirected", "name": "r"}` | +| `-[*2]->` | `e_forward(hops=2)` | `{"type": "Edge", "direction": "forward", "hops": 2}` | +| `-[*1..3]->` | `e_forward(hops=3)` | `{"type": "Edge", "direction": "forward", "hops": 3}` | +| `-[*]->` | `e_forward(to_fixed_point=True)` | `{"type": "Edge", "direction": "forward", "to_fixed_point": true}` | +| `-[r:BOUGHT {amount: gt(100)}]->` | `e_forward({"type": "BOUGHT", "amount": gt(100)}, name="r")` | `{"type": "Edge", "direction": "forward", "edge_match": {"type": "BOUGHT", "amount": {"type": "GT", "val": 100}}, "name": "r"}` | + +### Predicates + +| Cypher | Python | Wire Protocol | +|--------|--------|---------------| +| `n.age > 30` | `gt(30)` | `{"type": "GT", "val": 30}` | +| `n.age >= 50` | `ge(50)` | `{"type": "GE", "val": 50}` | +| `n.age < 100` | `lt(100)` | `{"type": "LT", "val": 100}` | +| `n.age <= 50` | `le(50)` | `{"type": "LE", "val": 50}` | +| `n.status = 'active'` | `"active"` | `"active"` | +| `n.status <> 'deleted'` | `ne("deleted")` | `{"type": "NE", "val": "deleted"}` | +| `n.id IN [1,2,3]` | `is_in([1,2,3])` | `{"type": "IsIn", "options": [1,2,3]}` | +| `n.score BETWEEN 0 AND 100` | `between(0, 100)` | `{"type": "Between", "lower": 0, "upper": 100}` | +| `n.name =~ '^A.*'` | `match("^A.*")` | `{"type": "Match", "pattern": "^A.*"}` | +| `n.text CONTAINS 'search'` | `contains("search")` | `{"type": "Contains", "pattern": "search"}` | +| `n.name STARTS WITH 'Dr'` | `startswith("Dr")` | `{"type": "Startswith", "pattern": "Dr"}` | +| `n.email ENDS WITH '.com'` | `endswith(".com")` | `{"type": "Endswith", "pattern": ".com"}` | +| `n.val IS NULL` | `is_null()` | `{"type": "IsNull"}` | +| `n.val IS NOT NULL` | `not_null()` | `{"type": "NotNull"}` | + +## Complete Examples + +### Friend of Friend + +**Cypher:** +```cypher +MATCH (u:User {name: 'Alice'})-[:FRIEND*2]->(fof:User) +WHERE fof.active = true +``` + +**Python:** +```python +g.chain([ + n({"type": "User", "name": "Alice"}), + e_forward({"type": "FRIEND"}, hops=2), + n({"type": "User", "active": True}, name="fof") +]) +``` + +**Wire Protocol:** +```json +{"type": "Chain", "chain": [ + {"type": "Node", "filter_dict": {"type": "User", "name": "Alice"}}, + {"type": "Edge", "direction": "forward", "edge_match": {"type": "FRIEND"}, "hops": 2}, + {"type": "Node", "filter_dict": {"type": "User", "active": true}, "name": "fof"} +]} + +### Fraud Detection + +**Cypher:** +```cypher +MATCH (a:Account)-[t:TRANSFER]->(b:Account) +WHERE t.amount > 10000 AND t.date > date('2024-01-01') +``` + +**Python:** +```python +g.chain([ + n({"type": "Account"}), + e_forward({ + "type": "TRANSFER", + "amount": gt(10000), + "date": gt(date(2024,1,1)) + }, name="t"), + n({"type": "Account"}) +]) +``` + +**Wire Protocol:** +```json +{"type": "Chain", "chain": [ + {"type": "Node", "filter_dict": {"type": "Account"}}, + {"type": "Edge", "direction": "forward", "edge_match": { + "type": "TRANSFER", + "amount": {"type": "GT", "val": 10000}, + "date": {"type": "GT", "val": {"type": "date", "value": "2024-01-01"}} + }, "name": "t"}, + {"type": "Node", "filter_dict": {"type": "Account"}} +]} +``` + +### Complex Aggregation Example + +**Cypher:** +```cypher +MATCH (u:User)-[t:TRANSACTION]->(m:Merchant) +WHERE t.date > date('2024-01-01') +RETURN m.category, count(*) as cnt, sum(t.amount) as total +ORDER BY total DESC +LIMIT 10 +``` + +**Python:** +```python +# Step 1: Graph pattern +result = g.chain([ + n({"type": "User"}), + e_forward({"type": "TRANSACTION", "date": gt(date(2024,1,1))}, name="trans"), + n({"type": "Merchant"}) +]) + +# Step 2: DataFrame operations +trans_df = result._edges[result._edges["trans"]] +merchant_df = result._nodes +analysis = (trans_df + .merge(merchant_df, left_on=g._destination, right_on=g._node) + .groupby('category') + .agg(cnt=('amount', 'count'), total=('amount', 'sum')) + .nlargest(10, 'total')) +``` + +**Note:** Wire protocol returns the filtered graph; aggregations require client-side processing. + +## DataFrame Operations Mapping + +| Cypher Feature | Python DataFrame Operation | Notes | +|----------------|---------------------------|--------| +| `RETURN a, b, c` | `df[['a', 'b', 'c']]` | Column selection | +| `RETURN DISTINCT` | `df.drop_duplicates()` | Remove duplicates | +| `ORDER BY x DESC` | `df.sort_values('x', ascending=False)` | Sort results | +| `LIMIT 10` | `df.head(10)` | Limit rows | +| `count(*)` | `len(df)` or `df.groupby(...).size()` | Count rows | +| `sum(n.val)` | `df['val'].sum()` or `df.groupby(...).agg(sum)` | Aggregation | +| `collect(n.x)` | `df.groupby(...).agg(list)` | Collect to list | +| Named patterns | `df[df['pattern_name']]` | Boolean column filtering | + +## Key Differences + +| Feature | Python | Wire Protocol | +|---------|--------|---------------| +| **Temporal values** | `pd.Timestamp()`, `date()` | `{"type": "date", "value": "..."}` | +| **Direct equality** | `"active"` | `"active"` (same) | +| **Comparisons** | `gt(30)` | `{"type": "GT", "val": 30}` | +| **Collections** | `is_in([...])` | `{"type": "IsIn", "options": [...]}` | + +## Not Supported +- `OPTIONAL MATCH` - No equivalent (would need outer joins) +- `CREATE`, `DELETE`, `SET` - GFQL is read-only +- `WITH` clauses - Requires intermediate variables +- Multiple `MATCH` patterns - Use separate chains or joins + +## Best Practices + +1. **Direct Translation First**: Try pure GFQL before adding DataFrame operations +2. **Use Named Patterns**: Label important results with `name=` for easy access +3. **Filter Early**: Apply selective node filters before traversing edges +4. **Type Consistency**: Ensure wire protocol types match expected column types +5. **Validate JSON**: Test wire protocol against schema before sending + +## LLM Integration Guide + +When building translators: + +``` +Given Cypher: {cypher_query} + +Generate both: +1. Python: Human-readable GFQL code +2. Wire Protocol: JSON for API calls + +Rules: +- (n:Label) โ†’ Python: n({"type": "Label"}) โ†’ JSON: {"type": "Node", "filter_dict": {"type": "Label"}} +- WHERE โ†’ Embed as predicates in both formats +- Aggregations โ†’ Note as requiring DataFrame post-processing +``` + +## See Also +- {ref}`gfql-spec-wire-protocol` - Full wire protocol specification +- {ref}`gfql-spec-language` - Language specification +- {ref}`gfql-spec-python-embedding` - Python implementation details \ No newline at end of file diff --git a/docs/source/gfql/spec/index.md b/docs/source/gfql/spec/index.md new file mode 100644 index 0000000000..4e1c3f6b41 --- /dev/null +++ b/docs/source/gfql/spec/index.md @@ -0,0 +1,23 @@ +(gfql-specifications)= + +# GFQL Specifications + +This section contains formal specifications for GFQL (Graph Frame Query Language), designed to support both human understanding and automated tooling, including LLM-based code synthesis. + +```{toctree} +:maxdepth: 1 + +language +python_embedding +wire_protocol +cypher_mapping +``` + +## Overview + +- {ref}`gfql-spec-language` - Complete formal grammar, operations, predicates, and type system +- {ref}`gfql-spec-python-embedding` - Python-specific implementation with pandas/cuDF +- {ref}`gfql-spec-wire-protocol` - JSON serialization format for client-server communication +- {ref}`gfql-spec-cypher-mapping` - Cypher to GFQL translations with both Python and wire protocol + +These specifications are optimized for text-to-GFQL synthesis, Cypher-to-GFQL pipelines, query validation, and schema-aware code generation. \ No newline at end of file diff --git a/docs/source/gfql/spec/language.md b/docs/source/gfql/spec/language.md new file mode 100644 index 0000000000..d862bdb64c --- /dev/null +++ b/docs/source/gfql/spec/language.md @@ -0,0 +1,366 @@ +(gfql-spec-language)= + +# GFQL Language Specification + +## Introduction + +GFQL (Graph Frame Query Language) is a DataFrame-native graph query language designed for expressing graph patterns and traversals on tabular data. It operates on node and edge DataFrames, providing a functional, composable approach to graph querying with native GPU acceleration support. + +### Design Principles +- **Dataframe-native**: Type-safe functional bulk operations over dataframe libraries like pandas, cuDF +- **Declarative**: Focus on what to retrieve, and give the engine freedom to optimize how +- **Accessible**: Designed for both human readability and machine generation, and building on intuitions from popular tabular and graph systems +- **Performance-oriented**: Vectorized operations by default, including GPU acceleration +- **Embeddable**: Similar to DuckDB, can be embedded in different languages, and initially focused on Python data ecosystem +- **Computer-tier**: Decoupling from storage enables flexible execution - embedded locally or via remote acceleration servers + +### Language Forms + +GFQL exists in three complementary forms: + +1. **Core Language**: Abstract graph pattern matching language defined by this specification +2. **Embedded DSL**: Host language implementations (currently Python with pandas/cuDF) +3. **Wire Protocol**: JSON serialization for client-server communication (see Wire Protocol spec) + +This specification focuses on the core language concepts. Examples use Python syntax for concreteness, but the patterns apply to any embedding. + +## Language Overview + +### Core Concepts + +1. **Graph Model**: Graphs consist of node and edge dataframes + - Edges: DataFrame with source and destination columns + - Nodes: DataFrame with unique identifier column + - Column names are user-defined globals for the graph: + - Node ID attribute: `g._node` (e.g., "node_id", "id") + - Edge source attribute: `g._source` (e.g., "source", "from") + - Edge destination attribute: `g._destination` (e.g., "destination", "to") + - GFQL infers nodes from edge references when only edges are provided + +2. **GFQL Programs**: GFQL programs are declarative graph-to-graph transformations + - Enable use cases like search, filter, enrich, and traverse + - Express *what* to find (ex: Cypher), not *how* to find it (ex: Gremlin) + +3. **Chains**: Path pattern expressions for matching graph structures + - Express graph patterns as sequences of node and edge matching operations + - Similar to Cypher patterns but decomposed into composable steps + - Define paths through the graph: start nodes โ†’ edges โ†’ end nodes + - Each operation refines the pattern match based on previous results + +4. **Operations**: Act on graph entities (nodes and edges) + - Node matchers: Filter and select nodes + - Edge matchers: Traverse relationships + - Operations work on the graph structure itself + +5. **Predicates**: Act on attributes of nodes and edges + - Filter based on property values + - Comparison, membership, string matching, temporal checks + - Composable within operations to build complex conditions + +6. **Values**: Type system matching modern data formats + - Scalars: numbers, strings, booleans, null + - Temporal: ISO datetimes, dates, times with timezone support + - Collections: lists for membership tests + - Compatible with JSON, Arrow, and DataFrame type systems + +## Formal Grammar + +```{code-block} ebnf +:caption: GFQL Grammar in Extended Backus-Naur Form + +(* Entry point *) +query ::= chain + +(* Chain - path pattern expression *) +chain ::= "[" operation ("," operation)* "]" + +(* Operations *) +operation ::= node_matcher | edge_matcher + +(* Node Matcher *) +node_matcher ::= "n(" node_params? ")" +node_params ::= filter_dict ("," name_param)? ("," query_param)? + | name_param ("," query_param)? + | query_param + +(* Edge Matchers *) +edge_matcher ::= edge_forward | edge_reverse | edge_undirected +edge_forward ::= "e_forward(" edge_params? ")" +edge_reverse ::= "e_reverse(" edge_params? ")" +edge_undirected ::= ("e" | "e_undirected") "(" edge_params? ")" + +(* Parameters *) +edge_params ::= edge_match_params ("," hop_params)? ("," node_filter_params)? ("," name_param)? + +filter_dict ::= "{" (property_filter ("," property_filter)*)? "}" +property_filter ::= string ":" (value | predicate) + +hop_params ::= "hops=" integer | "to_fixed_point=True" +node_filter_params ::= source_filter ("," dest_filter)? +source_filter ::= "source_node_match=" filter_dict | "source_node_query=" string +dest_filter ::= "destination_node_match=" filter_dict | "destination_node_query=" string + +name_param ::= "name=" string +query_param ::= "query=" string +edge_query_param ::= "edge_query=" string +edge_match_params ::= filter_dict | edge_query_param + +(* Predicates *) +predicate ::= comparison | membership | range | null_check | string_pred | temporal_pred + +comparison ::= ("gt" | "lt" | "ge" | "le" | "eq" | "ne") "(" value ")" +membership ::= "is_in(" "[" value ("," value)* "]" ")" +range ::= "between(" value "," value ("," "inclusive=" boolean)? ")" +null_check ::= "is_null()" | "not_null()" | "is_na()" | "not_na()" +string_pred ::= string_match | string_check +string_match ::= "contains(" string ("," "case=" boolean)? ("," "regex=" boolean)? ")" + | "match(" string ("," "case=" boolean)? ")" + | ("startswith" | "endswith") "(" string ")" +string_check ::= ("isalpha" | "isnumeric" | "isdigit" | "isalnum" + | "isupper" | "islower") "()" +temporal_pred ::= temporal_check "()" +temporal_check ::= "is_month_start" | "is_month_end" | "is_quarter_start" + | "is_quarter_end" | "is_year_start" | "is_year_end" | "is_leap_year" + +(* Values *) +value ::= scalar | temporal_value | collection +scalar ::= number | string | boolean | null +temporal_value ::= datetime_value | date_value | time_value +datetime_value ::= "pd.Timestamp(" string ("," "tz=" string)? ")" + | "datetime(" datetime_args ")" +date_value ::= "date(" date_args ")" +time_value ::= "time(" time_args ")" +collection ::= "[" (value ("," value)*)? "]" + +(* Primitives *) +string ::= '"' [^"]* '"' | "'" [^']* "'" +number ::= integer | float +integer ::= ["-"]? [0-9]+ +float ::= ["-"]? [0-9]+ "." [0-9]+ +boolean ::= "True" | "False" +null ::= "None" +datetime_args ::= integer ("," integer)* +date_args ::= integer "," integer "," integer +time_args ::= integer "," integer ("," integer)? +``` + +## Operations + +### Node Matcher: `n()` + +Filters nodes based on attributes. + +**Syntax**: `n(filter_dict?, name?, query?)` + +**Parameters**: +- `filter_dict`: Dictionary of attribute filters +- `name`: Optional string label for results +- `query`: Pandas query string expression + +**Examples**: +```python +n() # All nodes +n({"type": "person"}) # Nodes where type='person' +n({"age": gt(30)}) # Nodes where age > 30 +n(name="important") # Label matching nodes +n(query="age > 30 and status == 'active'") # Query string +``` + +### Edge Matchers + +#### Forward Traversal: `e_forward()` + +Traverses edges in forward direction (source โ†’ destination). + +**Syntax**: `e_forward(edge_match?, hops?, to_fixed_point?, source_node_match?, destination_node_match?, name?)` + +**Parameters**: +- `edge_match`: Edge attribute filters +- `hops`: Number of hops (default: 1) +- `to_fixed_point`: Continue until no new nodes (default: False) +- `source_node_match`: Filters for source nodes +- `destination_node_match`: Filters for destination nodes +- `name`: Optional label + +**Examples**: +```python +e_forward() # One hop forward +e_forward(hops=2) # Two hops forward +e_forward(to_fixed_point=True) # All reachable nodes +e_forward({"type": "follows"}) # Only 'follows' edges +e_forward(source_node_match={"active": True}) # From active nodes +``` + +#### Reverse Traversal: `e_reverse()` + +Traverses edges in reverse direction (destination โ†’ source). + +**Syntax**: Same as `e_forward()` + +#### Undirected Traversal: `e()` or `e_undirected()` + +Traverses edges in both directions. + +**Syntax**: Same as `e_forward()` + +## Predicates + +### Comparison Predicates + +```python +gt(value) # Greater than +lt(value) # Less than +ge(value) # Greater than or equal +le(value) # Less than or equal +eq(value) # Equal +ne(value) # Not equal +``` + +### Membership Predicate + +```python +is_in([value1, value2, ...]) # Value in list +``` + +### Range Predicate + +```python +between(lower, upper, inclusive=True) # Value in range +``` + +### String Predicates + +Pattern matching predicates: +```python +contains(pat, case=True, regex=True) # Contains pattern (substring or regex) +startswith(prefix) # Starts with prefix (case-sensitive) +endswith(suffix) # Ends with suffix (case-sensitive) +match(pat, case=True) # Matches regex from start of string +``` + +String type checking predicates: +```python +isalpha() # Alphabetic characters only +isnumeric() # Numeric characters only +isdigit() # Digits only +isalnum() # Alphanumeric +isupper() # All uppercase +islower() # All lowercase +``` + +### Null Predicates + +```python +is_null() # Is null/None +not_null() # Is not null/None +is_na() # Is NaN (numeric) +not_na() # Is not NaN +``` + +### Temporal Predicates + +```python +is_month_start() # First day of month +is_month_end() # Last day of month +is_quarter_start() # First day of quarter +is_quarter_end() # Last day of quarter +is_year_start() # First day of year +is_year_end() # Last day of year +is_leap_year() # Is leap year +``` + +## Type System + +### Value Types + +1. **Scalars** + - `number`: int, float + - `string`: Text values + - `boolean`: True/False + - `null`: None + +2. **Temporal Types** + - `datetime`: Timestamp with optional timezone + - `date`: Calendar date + - `time`: Time of day + +3. **Collections** + - `list`: Ordered sequence of values + +### Type Coercion + +GFQL performs automatic type coercion: +- Python datetime โ†’ pandas Timestamp +- Numeric types โ†’ appropriate precision +- Collections โ†’ lists for `is_in()` + +## Execution Model + +### Declarative Pattern Matching + +GFQL follows a declarative execution model similar to Neo4j's Cypher: + +1. **Pattern Declaration**: Chains express path patterns in the graph + - Users declare graph patterns as sequences of node and edge constraints + - Patterns specify *what* paths to match, not *how* to find them + - The engine optimizes pattern matching based on data characteristics + +2. **Set-Based Operations**: All operations work on sets of entities + - No explicit iteration or traversal order + - Results include all matching patterns in the graph + - Current GFQL engines use a novel bulk-oriented execution model that is asymptotically faster than traditional iterative approaches used for Cypher, but this is not a requirement of the language itself + +3. **Lazy Evaluation**: Chains define pattern transformations without immediate execution + - Allows engines to optimize path finding and pattern matching strategies +\ +### Result Access + +Query execution returns filtered node and edge datasets. In the Python embedding: + +```python +result = g.chain([...]) +nodes_df = result._nodes # Filtered nodes +edges_df = result._edges # Filtered edges +``` + +### Named Results + +Operations with `name` parameter add boolean columns to mark matched entities: + +```python +result = g.chain([ + n({"type": "person"}, name="people"), + e_forward(name="connections"), + n({"active": True}, name="active_targets") +]) + +# Access all matched nodes and edges: +all_nodes = result._nodes +all_edges = result._edges + +# Access specific matched nodes/edges using pandas filtering: +people_nodes = result._nodes[result._nodes["people"]] +connection_edges = result._edges[result._edges["connections"]] +active_nodes = result._nodes[result._nodes["active_targets"]] + +# Or using standard pandas query syntax: +people_nodes = result._nodes.query("people == True") +``` + +This pattern is essential for extracting specific subsets from complex graph traversals. + +## Best Practices + +1. **Use specific filters early**: Filter nodes before traversing edges +2. **Limit hops**: Use reasonable hop limits to avoid explosion +3. **Name important results**: Use `name` parameter for analysis +4. **Prefer filter_dict**: More efficient than query strings +5. **Use appropriate predicates**: Match predicate to column type + +## See Also + +- {ref}`gfql-spec-python-embedding` - Python implementation details +- {ref}`gfql-spec-wire-protocol` - JSON serialization format +- {ref}`gfql-spec-cypher-mapping` - Cypher to GFQL translation with wire protocol +- [GFQL Quick Reference](../quick.rst) - Comprehensive examples and usage patterns +- [GFQL Validation Guide](../validation/fundamentals.rst) - Learn validation basics \ No newline at end of file diff --git a/docs/source/gfql/spec/python_embedding.md b/docs/source/gfql/spec/python_embedding.md new file mode 100644 index 0000000000..27be847e4b --- /dev/null +++ b/docs/source/gfql/spec/python_embedding.md @@ -0,0 +1,195 @@ +(gfql-spec-python-embedding)= + +# GFQL Python Embedding + +This document describes the Python-specific implementation of GFQL using pandas and cuDF dataframes. + +## Graph Construction + +In Python, graphs are created with user-defined column names: + +```python +import graphistry +assert 'src_col' in df.columns and 'dst_col' in df.columns +g = graphistry.edges(df, source='src_col', destination='dst_col') + +# Optional; GFQL infers node existence when only edges are provided +assert 'node_col' in df.columns +g2 = graphistry.nodes(df, node='node_col') +``` + +### Schema Access + +The graph schema is accessible via attributes: +- `g._node`: Node ID column name +- `g._source`: Edge source column name +- `g._destination`: Edge destination column name + +Graph nodes can be generically accessed using these attributes: +- `g._nodes`: Node DataFrame +- `g._nodes[g._node]`: Node ID column +- `g._nodes[[attr for attr in g._nodes.columns if attr != g._node]]`: All other node attributes + +Graph edges can be accessed similarly: +- `g._edges`: Edge DataFrame +- `g._edges[g._source]`: Edge source column +- `g._edges[g._destination]`: Edge destination column +- `g._edges[[attr for attr in g._edges.columns if attr not in [g._source, g._destination]]]`: All other edge attributes + +## Query Execution + +```python +from graphistry import n, e_forward + +# Execute a chain +result = g.chain([ + n({"type": "person"}), + e_forward(), + n() +]) + +# Access results +nodes_df = result._nodes # Filtered nodes DataFrame +edges_df = result._edges # Filtered edges DataFrame +``` + +## Engine Selection + +GFQL supports multiple execution engines: + +- **pandas**: CPU execution (default) +- **cudf**: GPU acceleration +- **auto**: Automatic selection based on data type + +```python +# Force specific engine +g.chain([...], engine='cudf') # GPU execution +g.chain([...], engine='pandas') # CPU execution +g.chain([...], engine='auto') # Auto-select +``` + +## Python-Specific Values + +### Temporal Values + +```python +import pandas as pd + +# Timestamps +pd.Timestamp('2023-01-01') +pd.Timestamp.now() + +# Time deltas +pd.Timedelta(days=30) +pd.Timedelta(hours=24) +``` + +### DataFrame Operations + +Results can be further processed using standard pandas operations: + +```python +# Using boolean columns from named operations +people_nodes = result._nodes[result._nodes["people"]] + +# Using pandas query +active_nodes = result._nodes.query("active == True") + +# Standard pandas operations +result._nodes.groupby('type').size() +``` + +## Common Errors and Validation + +### Type Mismatches + +```python +# Wrong - String predicate on numeric column +n({"age": contains("3")}) + +# Correct - Use numeric predicate +n({"age": gt(30)}) + +# Wrong - String comparison on datetime +n({"created": gt("2024-01-01")}) + +# Correct - Use proper datetime type +n({"created": gt(pd.Timestamp("2024-01-01"))}) +``` + +### Schema Validation + +```python +# Check available columns before querying +print(g._nodes.columns) # ['id', 'type', 'name'] + +# Wrong - Column doesn't exist +g.chain([n({"username": "Alice"})]) # KeyError + +# Correct - Use existing column +g.chain([n({"name": "Alice"})]) +``` + +### Unsupported Operations + +```python +# Wrong - Can't aggregate in chain +# g.chain([n(), e(), count()]) + +# Correct - Aggregate after chain +result = g.chain([n(), e()]) +count = len(result._edges) + +# Wrong - OPTIONAL MATCH not supported +# No direct GFQL equivalent + +# Correct - Handle optionality in post-processing +result = g.chain([n(), e_forward()]) +# Check for nodes without edges +nodes_with_edges = result._nodes[result._nodes[g._node].isin(result._edges[g._source])] +``` + +## Best Practices + +### Query Construction +```python +# Good: Build queries programmatically +node_filters = {"type": "User"} +if min_age: + node_filters["age"] = gt(min_age) +g.chain([n(node_filters)]) + +# Avoid: Hardcoded query strings +g.chain([n(query=f"type == 'User' and age > {min_age}")]) # SQL injection risk +``` + +### Memory Efficiency +```python +# Good: Filter early and use named results +result = g.chain([ + n({"active": True}, name="active_users"), # Filter first + e_forward({"recent": True}) +]) +# Only access what you need +active_users = result._nodes[result._nodes["active_users"]] + +# Avoid: Loading everything then filtering +all_nodes = g._nodes +active = all_nodes[all_nodes["active"] == True] # Loads entire graph +``` + +### GPU Best Practices +```python +# Check GPU memory before large operations +if engine == 'cudf': + import cudf + print(f"GPU memory: {cudf.cuda.cuda.get_memory_info()}") + +# Convert results back to pandas if needed for compatibility +result_pandas = result._nodes.to_pandas() if hasattr(result._nodes, 'to_pandas') else result._nodes +``` + +## See Also + +- {ref}`gfql-spec-language` - Core language specification +- [GFQL Quick Reference](../quick.rst) - Python API examples \ No newline at end of file diff --git a/docs/source/gfql/spec/wire_protocol.md b/docs/source/gfql/spec/wire_protocol.md new file mode 100644 index 0000000000..ca8cdc0654 --- /dev/null +++ b/docs/source/gfql/spec/wire_protocol.md @@ -0,0 +1,455 @@ +(gfql-spec-wire-protocol)= + +# GFQL Wire Protocol Specification + +## Introduction + +The GFQL Wire Protocol defines the JSON serialization format for GFQL queries, enabling: +- Client-server communication +- Query persistence and storage +- Cross-language interoperability between Python, JavaScript, and other clients +- Configuration-driven query generation + +### Design Principles +- **Type Safety**: Tagged dictionaries preserve type information +- **Self-Describing**: Each object includes type metadata +- **Extensible**: Schema supports future additions +- **Round-Trip Safe**: Lossless serialization/deserialization + +## Protocol Overview + +### Message Structure + +All GFQL wire protocol messages are JSON objects with a `type` field: + +```json +{ + "type": "MessageType", + ...additional fields... +} +``` + +### Supported Message Types +- `Chain`: Complete query chain +- `Node`: Node matcher operation +- `Edge`: Edge traversal operation +- Predicates: `GT`, `LT`, `EQ`, `IsIn`, `Between`, etc. +- Temporal values: `datetime`, `date`, `time` + +## JSON Schema + +```{code-block} json +:caption: Complete JSON Schema for GFQL Wire Protocol + +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://graphistry.com/schemas/gfql/wire-protocol.json", + + "definitions": { + "Chain": { + "type": "object", + "properties": { + "type": {"const": "Chain"}, + "chain": { + "type": "array", + "items": {"$ref": "#/definitions/Operation"}, + "minItems": 1 + } + }, + "required": ["type", "chain"], + "additionalProperties": false + }, + + "Operation": { + "oneOf": [ + {"$ref": "#/definitions/NodeOperation"}, + {"$ref": "#/definitions/EdgeOperation"} + ] + }, + + "NodeOperation": { + "type": "object", + "properties": { + "type": {"const": "Node"}, + "filter_dict": {"$ref": "#/definitions/FilterDict"}, + "query": {"type": "string"}, + "name": {"type": "string"} + }, + "required": ["type"], + "additionalProperties": false + }, + + "EdgeOperation": { + "type": "object", + "properties": { + "type": {"const": "Edge"}, + "direction": { + "enum": ["forward", "reverse", "undirected"] + }, + "edge_match": {"$ref": "#/definitions/FilterDict"}, + "edge_query": {"type": "string"}, + "hops": { + "type": "integer", + "minimum": 1, + "default": 1 + }, + "to_fixed_point": { + "type": "boolean", + "default": false + }, + "source_node_match": {"$ref": "#/definitions/FilterDict"}, + "source_node_query": {"type": "string"}, + "destination_node_match": {"$ref": "#/definitions/FilterDict"}, + "destination_node_query": {"type": "string"}, + "name": {"type": "string"} + }, + "required": ["type", "direction"], + "additionalProperties": false + } + } +} +``` + +## Operation Serialization + +### Node Operation + +**Python**: +```python +n({"type": "person", "age": gt(30)}, name="adults") +``` + +**Wire Format**: +```json +{ + "type": "Node", + "filter_dict": { + "type": "person", + "age": { + "type": "GT", + "val": 30 + } + }, + "name": "adults" +} +``` + +### Edge Operation + +**Python**: +```python +e_forward( + {"type": "transaction"}, + hops=2, + source_node_match={"active": True}, + name="txns" +) +``` + +**Wire Format**: +```json +{ + "type": "Edge", + "direction": "forward", + "edge_match": { + "type": "transaction" + }, + "hops": 2, + "source_node_match": { + "active": true + }, + "name": "txns" +} +``` + +### Chain + +**Python**: +```python +chain([ + n({"id": "Alice"}), + e_forward({"type": "friend"}), + n({"status": "active"}) +]) +``` + +**Wire Format**: +```json +{ + "type": "Chain", + "chain": [ + { + "type": "Node", + "filter_dict": {"id": "Alice"} + }, + { + "type": "Edge", + "direction": "forward", + "edge_match": {"type": "friend"} + }, + { + "type": "Node", + "filter_dict": {"status": "active"} + } + ] +} +``` + +## Predicate Serialization + +### Comparison Predicates + +```json +{"type": "GT", "val": 100} +{"type": "LT", "val": 50.5} +{"type": "GE", "val": "2024-01-01"} +{"type": "LE", "val": true} +{"type": "EQ", "val": "active"} +{"type": "NE", "val": null} +``` + +### Between Predicate + +```json +{ + "type": "Between", + "lower": 10, + "upper": 20, + "inclusive": true +} +``` + +### IsIn Predicate + +```json +{ + "type": "IsIn", + "options": ["A", "B", "C"] +} +``` + +### String Predicates + +```json +{"type": "Contains", "pattern": "search"} +{"type": "Startswith", "pattern": "prefix"} +{"type": "Endswith", "pattern": "suffix"} +{"type": "Match", "pattern": "^[A-Z]+\\d+$"} +``` + +### Null Predicates + +```json +{"type": "IsNull"} +{"type": "NotNull"} +{"type": "IsNA"} +{"type": "NotNA"} +``` + +### Temporal Check Predicates + +```json +{"type": "IsMonthStart"} +{"type": "IsYearEnd"} +{"type": "IsLeapYear"} +``` + +## Type Serialization + +### Scalar Types + +```json +"hello world" // string +42 // integer +3.14159 // float +true // boolean +null // null +``` + +### Temporal Types + +#### DateTime +```json +{ + "type": "datetime", + "value": "2024-01-15T10:30:00", + "timezone": "America/New_York" // Optional, defaults to "UTC" +} +``` + +#### Date +```json +{ + "type": "date", + "value": "2024-01-15" +} +``` + +#### Time +```json +{ + "type": "time", + "value": "14:30:00.123456" +} +``` + +**Note**: The `timezone` field is optional for DateTime values and defaults to "UTC" if omitted. This ensures consistent behavior across systems while allowing explicit timezone specification when needed. + +## Examples + +### User 360 Query + +**Python**: +```python +g.chain([ + n({"customer_id": "C123"}), + e_forward({ + "type": "purchase", + "timestamp": gt(pd.Timestamp("2024-01-01")) + }) +]) +``` + +**Wire Format**: +```json +{ + "type": "Chain", + "chain": [ + { + "type": "Node", + "filter_dict": { + "customer_id": "C123" + } + }, + { + "type": "Edge", + "direction": "forward", + "edge_match": { + "type": "purchase", + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2024-01-01T00:00:00", + "timezone": "UTC" + } + } + } + } + ] +} +``` + +### Cyber Security Pattern + +**Python**: +```python +g.chain([ + n({"ip": is_in(["192.168.1.100", "192.168.1.101"])}), + e_forward( + edge_query="port IN [22, 23, 3389]", + to_fixed_point=True + ), + n({"type": "server", "critical": True}) +]) +``` + +**Wire Format**: +```json +{ + "type": "Chain", + "chain": [ + { + "type": "Node", + "filter_dict": { + "ip": { + "type": "IsIn", + "options": ["192.168.1.100", "192.168.1.101"] + } + } + }, + { + "type": "Edge", + "direction": "forward", + "edge_query": "port IN [22, 23, 3389]", + "to_fixed_point": true + }, + { + "type": "Node", + "filter_dict": { + "type": "server", + "critical": true + } + } + ] +} +``` + +## Error Handling + +### Error Response Format + +```json +{ + "type": "Error", + "error_type": "ValidationError", + "message": "Invalid predicate type 'GT' for string column 'name'", + "details": { + "column": "name", + "predicate": "GT", + "expected_types": ["string"], + "actual_type": "comparison" + } +} +``` + +### Error Types + +1. **ValidationError**: Schema validation failed +2. **ParseError**: JSON parsing failed +3. **TypeError**: Type mismatch +4. **SemanticError**: Logical inconsistency +5. **UnsupportedError**: Feature not supported + +## Protocol Extensions + +### Future Considerations + +The protocol is designed to support future extensions: + +1. **Versioning**: Add `"version"` field for protocol versioning +2. **Metadata**: Add `"metadata": {}` for additional context +3. **Streaming**: Support for partial results +4. **Transactions**: Batch multiple operations +5. **Optimization Hints**: Engine-specific parameters + +### Custom Predicates + +New predicates can be added by extending the schema: + +```json +{ + "definitions": { + "CustomPredicate": { + "type": "object", + "properties": { + "type": {"const": "CustomPredicateName"}, + ...custom fields... + } + } + } +} +``` + +## Best Practices + +1. **Always include type fields**: Every object must have a `type` +2. **Use ISO formats**: Dates and times in ISO 8601 +3. **Handle timezones consistently**: Include timezone for datetime values when precision matters (defaults to UTC) +4. **Validate before sending**: Use JSON Schema validation +5. **Handle unknown fields**: Ignore unrecognized fields for compatibility + +## See Also + +- {ref}`gfql-spec-language` - Language specification +- {ref}`gfql-spec-cypher-mapping` - Cypher to GFQL translation with wire protocol examples \ No newline at end of file diff --git a/docs/source/gfql/validation/advanced.rst b/docs/source/gfql/validation/advanced.rst new file mode 100644 index 0000000000..1c4d456151 --- /dev/null +++ b/docs/source/gfql/validation/advanced.rst @@ -0,0 +1,143 @@ +Advanced GFQL Validation Patterns +================================= + +Deep dive into complex GFQL validation scenarios, performance considerations, and advanced patterns. + +.. note:: + Run the interactive examples yourself in + `demos/gfql/gfql_validation_advanced.ipynb `_. + +Prerequisites +------------- + +* Complete :doc:`fundamentals` first +* Experience writing GFQL queries +* Understanding of graph traversal concepts + +Complex Multi-Hop Queries +------------------------- + +Validate queries with multiple hops and complex traversal patterns. + +.. code-block:: python + + # Multi-hop with bounded traversal + query = [ + {"type": "n", "filter": {"type": {"eq": "user"}}}, + {"type": "e_forward", "hops": 2}, # 2-hop traversal + {"type": "n", "filter": {"risk_score": {"gt": 50}}} + ] + +Named Operations +^^^^^^^^^^^^^^^^ + +Use named operations for complex patterns: + +.. code-block:: python + + query = [ + {"type": "n", "name": "start_users", "filter": {"type": {"eq": "user"}}}, + {"type": "e_forward", "filter": {"rel_type": {"eq": "purchased"}}}, + {"type": "n", "name": "products"}, + {"type": "e_reverse", "filter": {"rel_type": {"eq": "viewed"}}}, + {"type": "n", "name": "viewers"} + ] + +Advanced Predicates +------------------- + +Temporal Predicates +^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + query = [ + {"type": "n", "filter": { + "created_at": { + "gt": {"type": "datetime", "value": "2024-01-10T00:00:00Z"} + } + }} + ] + +Nested Predicates +^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + query = [ + {"type": "n", "filter": { + "_and": [ + {"type": {"in": ["user", "payment"]}}, + {"_or": [ + {"risk_score": {"gte": 75}}, + {"tags": {"contains": "urgent"}} + ]} + ] + }} + ] + +Performance Considerations +-------------------------- + +Bounded vs Unbounded Hops +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Always specify hop limits for better performance: + +.. code-block:: python + + # [OK] Good - bounded + {"type": "e_forward", "hops": 3} + + # [WARNING] Warning - unbounded + {"type": "e_forward"} # No hop limit + +Query Complexity Estimation +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Monitor query complexity to prevent performance issues in production. + +Schema Evolution +---------------- + +Handle schema changes gracefully: + +.. code-block:: python + + def create_compatible_query(query, column_mapping): + """Update query to use new column names.""" + # Implementation to map old columns to new ones + pass + +Custom Validation +----------------- + +Extend validation for domain-specific requirements: + +.. code-block:: python + + def validate_business_rules(query, schema): + """Add custom business rule validation.""" + custom_issues = [] + + # Check for sensitive columns without filters + # Warn about expensive patterns + # Enforce domain-specific constraints + + return custom_issues + +Best Practices +-------------- + +1. **Multi-hop queries**: Always specify hop limits +2. **Complex predicates**: Use nested AND/OR for sophisticated filtering +3. **Schema evolution**: Plan for column changes +4. **Custom validation**: Extend for business rules +5. **Performance**: Consider query complexity + +Next Steps +---------- + +* :doc:`llm` - LLM integration patterns +* :doc:`production` - Production deployment +* :doc:`../spec/language` - Language specification \ No newline at end of file diff --git a/docs/source/gfql/validation/fundamentals.rst b/docs/source/gfql/validation/fundamentals.rst new file mode 100644 index 0000000000..34a85ad71e --- /dev/null +++ b/docs/source/gfql/validation/fundamentals.rst @@ -0,0 +1,99 @@ +GFQL Validation Fundamentals +============================ + +Learn the basics of validating GFQL queries to catch errors early and build robust graph applications. + +.. note:: + This guide is accompanied by an interactive Jupyter notebook. To run the examples yourself, see + `demos/gfql/gfql_validation_fundamentals.ipynb `_. + +What You'll Learn +----------------- + +* How to validate GFQL query syntax +* Understanding validation error messages +* Basic schema validation with DataFrames +* Common syntax errors and how to fix them + +Prerequisites +------------- + +* Basic Python knowledge +* PyGraphistry installed (``pip install graphistry[ai]``) + +Quick Start +----------- + +.. code-block:: python + + from graphistry.compute.gfql.validate import validate_syntax, validate_query + + # Validate query syntax + query = [ + {"type": "n", "filter": {"type": {"eq": "customer"}}}, + {"type": "e_forward"}, + {"type": "n"} + ] + + issues = validate_syntax(query) + if not issues: + print("[OK] Query syntax is valid!") + +Key Concepts +------------ + +Error Levels +^^^^^^^^^^^^ + +* **error**: Query will fail if executed +* **warning**: Query may work but has potential issues + +Common Validation Functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``validate_syntax(query)``: Check query structure and syntax +* ``validate_schema(query, schema)``: Validate against data schema +* ``validate_query(query, nodes_df, edges_df)``: Combined validation + +Common Errors and Fixes +----------------------- + +Invalid Operation Type +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + # [X] Wrong + [{"type": "node"}] # Should be "n" + + # [OK] Correct + [{"type": "n"}] + +Missing Operator +^^^^^^^^^^^^^^^^ + +.. code-block:: python + + # [X] Wrong + {"filter": {"name": "Alice"}} # Missing operator + + # [OK] Correct + {"filter": {"name": {"eq": "Alice"}}} + +Column Not Found +^^^^^^^^^^^^^^^^ + +Always validate against your schema to catch column name errors early. + +Next Steps +---------- + +* :doc:`advanced` - Complex queries and multi-hop validation +* :doc:`llm` - AI integration patterns +* :doc:`production` - Production deployment patterns + +See Also +-------- + +* :doc:`../spec/language` - Complete language specification +* :doc:`../overview` - GFQL overview \ No newline at end of file diff --git a/docs/source/gfql/validation/index.rst b/docs/source/gfql/validation/index.rst new file mode 100644 index 0000000000..20400e8044 --- /dev/null +++ b/docs/source/gfql/validation/index.rst @@ -0,0 +1,26 @@ +GFQL Validation Guide +===================== + +Learn how to validate GFQL queries for syntax correctness, schema compatibility, and production use. + +.. toctree:: + :maxdepth: 1 + :caption: Validation Topics + + fundamentals + advanced + llm + production + +Overview +-------- + +The GFQL validation framework provides comprehensive tools for ensuring query correctness: + +* **Syntax Validation** - Check query structure without data +* **Schema Validation** - Verify queries against actual data schemas +* **Combined Validation** - Full validation with helpful error messages +* **LLM Integration** - Structured formats for AI code generation +* **Production Patterns** - Caching, monitoring, and CI/CD integration + +Start with :doc:`fundamentals` to learn the basics, then explore advanced topics based on your needs. \ No newline at end of file diff --git a/docs/source/gfql/validation/llm.rst b/docs/source/gfql/validation/llm.rst new file mode 100644 index 0000000000..b5560f6c67 --- /dev/null +++ b/docs/source/gfql/validation/llm.rst @@ -0,0 +1,147 @@ +GFQL Validation for LLMs +======================== + +Learn how to integrate GFQL validation with Large Language Models and automation pipelines. + +.. note:: + Explore the complete examples in + `demos/gfql/gfql_validation_llm.ipynb `_. + +Target Audience +--------------- + +* AI/ML Engineers building GFQL generation systems +* Developers integrating LLMs with graph queries +* Teams building automated query generation pipelines + +JSON Serialization +------------------ + +Convert validation results to structured formats for LLMs: + +.. code-block:: python + + def validation_issue_to_dict(issue): + return { + "level": issue.level, + "message": issue.message, + "operation_index": issue.operation_index, + "suggestion": issue.suggestion + } + +Error Categorization +-------------------- + +Prioritize fixes for LLM processing: + +.. code-block:: python + + categories = { + "critical": [], # Must fix - syntax errors + "important": [], # Should fix - schema errors + "suggested": [] # Nice to fix - warnings + } + +Automated Fix Suggestions +------------------------- + +Generate actionable suggestions: + +.. code-block:: python + + fixes = [ + { + "action": "replace", + "path": "[0].type", + "old_value": "node", + "new_value": "n" + } + ] + +LLM Integration Pipeline +------------------------ + +.. code-block:: python + + class GFQLValidationPipeline: + def __init__(self, schema=None, max_iterations=3): + self.schema = schema + self.max_iterations = max_iterations + + def validate_and_report(self, query): + # Validate syntax and schema + # Create comprehensive report + # Generate fix suggestions + pass + + def create_llm_prompt(self, report): + # Format validation feedback for LLM + pass + +Prompt Engineering +------------------ + +System Prompt Template +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: text + + You are a GFQL expert. + + GFQL Rules: + 1. Queries are JSON arrays of operations + 2. Valid types: "n", "e_forward", "e_reverse", "e" + 3. Filters use operators: eq, ne, gt, gte, lt, lte + 4. Complex filters use _and, _or + + Available columns: + Nodes: [id, name, type, score] + Edges: [src, dst, weight] + +Iterative Refinement +-------------------- + +.. code-block:: python + + for iteration in range(max_iterations): + report = pipeline.validate_and_report(query) + + if report["valid"]: + break + + # LLM fixes based on validation feedback + query = llm.fix_query(query, report["fixes"]) + +Best Practices +-------------- + +1. **Structured Formats**: Always use JSON for LLM consumption +2. **Error Prioritization**: Fix critical โ†’ important โ†’ suggested +3. **Schema Context**: Provide available columns to LLMs +4. **Iterative Approach**: Allow multiple refinement rounds +5. **Rate Limiting**: Implement for production APIs + +Integration Checklist +--------------------- + +* [OK] Serialize validation issues to JSON +* [OK] Implement fix suggestion generation +* [OK] Create iterative validation pipeline +* [OK] Provide schema context in prompts +* [OK] Handle rate limiting and retries +* [OK] Log validation metrics + +Next Steps +---------- + +* Integrate with real LLM providers (OpenAI, Anthropic) +* Build production validation pipelines +* Create domain-specific templates +* Monitor generation accuracy + +See Also +-------- + +* :doc:`production` - Production patterns +* :doc:`../spec/language` - Language specification +* :doc:`../spec/cypher_mapping` - Cypher to GFQL mapping \ No newline at end of file diff --git a/docs/source/gfql/validation/production.rst b/docs/source/gfql/validation/production.rst new file mode 100644 index 0000000000..6d61092bc8 --- /dev/null +++ b/docs/source/gfql/validation/production.rst @@ -0,0 +1,226 @@ +GFQL Validation in Production +============================= + +Production-ready patterns for GFQL validation in platform engineering and DevOps contexts. + +.. note:: + See complete implementation examples in + `demos/gfql/gfql_validation_production.ipynb `_. + +Target Audience +--------------- + +* Platform Engineers +* DevOps Teams +* Backend Developers +* System Architects + +Plottable Integration +--------------------- + +Seamlessly validate queries against Plottable objects: + +.. code-block:: python + + from graphistry.compute.gfql.validate import extract_schema + + class PlottableValidator: + def __init__(self, plottable): + self.plottable = plottable + self.schema = extract_schema(plottable) + + def validate(self, query): + return validate_query( + query, + nodes_df=self.plottable._nodes, + edges_df=self.plottable._edges + ) + +Performance & Caching +--------------------- + +Schema Caching +^^^^^^^^^^^^^^ + +.. code-block:: python + + from functools import lru_cache + + class CachedSchemaValidator: + def __init__(self, cache_size=1000, ttl_seconds=3600): + self._schema_cache = {} + self._query_cache = lru_cache(maxsize=cache_size)( + self._validate_uncached + ) + +Batch Validation +^^^^^^^^^^^^^^^^ + +.. code-block:: python + + def batch_validate_queries(queries, plottable): + """Validate multiple queries efficiently.""" + schema = extract_schema_from_plottable(plottable) + + results = [] + for query in queries: + issues = validate_query(query, plottable._nodes, plottable._edges) + results.append({ + "valid": len(issues) == 0, + "issues": issues + }) + + return results + +Testing Patterns +---------------- + +pytest Fixtures +^^^^^^^^^^^^^^^ + +.. code-block:: python + + @pytest.fixture + def sample_data(): + nodes = pd.DataFrame({ + 'id': [1, 2, 3], + 'type': ['A', 'B', 'A'] + }) + edges = pd.DataFrame({ + 'src': [1, 2], + 'dst': [2, 3] + }) + return nodes, edges + + def test_valid_query(sample_data): + nodes, edges = sample_data + query = [{"type": "n", "filter": {"type": {"eq": "A"}}}] + issues = validate_query(query, nodes, edges) + assert len(issues) == 0 + +CI/CD Integration +----------------- + +GitHub Actions +^^^^^^^^^^^^^^ + +.. code-block:: yaml + + name: GFQL Query Validation + + on: + pull_request: + paths: + - 'queries/**/*.json' + + jobs: + validate-queries: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Validate GFQL queries + run: python scripts/validate_queries.py queries/ + +Pre-commit Hooks +^^^^^^^^^^^^^^^^ + +.. code-block:: yaml + + # .pre-commit-config.yaml + repos: + - repo: local + hooks: + - id: validate-gfql + name: Validate GFQL Queries + entry: python scripts/validate_gfql_hook.py + language: system + files: '\.(json|py)$' + +Monitoring & Logging +-------------------- + +.. code-block:: python + + class ValidationMonitor: + def log_validation(self, query, issues, elapsed_ms, context=None): + log_data = { + "timestamp": datetime.utcnow().isoformat(), + "validation_time_ms": elapsed_ms, + "errors": len([i for i in issues if i.level == "error"]), + "warnings": len([i for i in issues if i.level == "warning"]), + "context": context or {} + } + + if errors: + logger.error("GFQL validation failed", extra=log_data) + +API Integration +--------------- + +Flask Example +^^^^^^^^^^^^^ + +.. code-block:: python + + @app.route('/api/v1/validate', methods=['POST']) + def validate_gfql(): + data = request.get_json() + query = data.get('query') + + issues = validate_syntax(query) + + return jsonify({ + 'valid': not any(i.level == 'error' for i in issues), + 'issues': [issue_to_dict(i) for i in issues] + }) + +Security Considerations +----------------------- + +.. code-block:: python + + class SecureValidator: + def __init__(self, max_query_size=1000, rate_limit_per_minute=100): + self.max_query_size = max_query_size + self.rate_limit_per_minute = rate_limit_per_minute + + def validate_secure(self, query, user_id): + # Check rate limit + # Check query size + # Sanitize query + # Validate + +Production Checklist +-------------------- + +* [OK] **Plottable Integration**: Use ``extract_schema_from_plottable()`` +* [OK] **Caching**: Implement schema and query result caching +* [OK] **Batch Processing**: Validate multiple queries efficiently +* [OK] **Testing**: Comprehensive test coverage +* [OK] **CI/CD**: Automated validation in pipelines +* [OK] **Monitoring**: Track metrics and error patterns +* [OK] **API Design**: RESTful endpoints with error handling +* [OK] **Security**: Rate limiting and sanitization + +Performance Guidelines +---------------------- + +1. Cache schemas with appropriate TTL +2. Use batch validation for multiple queries +3. Monitor p95 validation times +4. Set reasonable query size limits + +Next Steps +---------- + +* Implement production validation service +* Set up monitoring dashboards +* Create runbooks for common issues +* Establish SLOs for validation performance + +See Also +-------- + +* :doc:`../spec/wire_protocol` - Wire protocol specification +* `PyGraphistry API Reference `_ +* `Production Deployment Guide `_ \ No newline at end of file diff --git a/docs/source/graphistry.compute.gfql.rst b/docs/source/graphistry.compute.gfql.rst new file mode 100644 index 0000000000..f42403baea --- /dev/null +++ b/docs/source/graphistry.compute.gfql.rst @@ -0,0 +1,29 @@ +graphistry.compute.gfql package +=============================== + +Submodules +---------- + +graphistry.compute.gfql.exceptions module +----------------------------------------- + +.. automodule:: graphistry.compute.gfql.exceptions + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.gfql.validate module +--------------------------------------- + +.. automodule:: graphistry.compute.gfql.validate + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.compute.gfql + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/graphistry.compute.predicates.rst b/docs/source/graphistry.compute.predicates.rst new file mode 100644 index 0000000000..badd0deb38 --- /dev/null +++ b/docs/source/graphistry.compute.predicates.rst @@ -0,0 +1,85 @@ +graphistry.compute.predicates package +===================================== + +Submodules +---------- + +graphistry.compute.predicates.ASTPredicate module +------------------------------------------------- + +.. automodule:: graphistry.compute.predicates.ASTPredicate + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.categorical module +------------------------------------------------ + +.. automodule:: graphistry.compute.predicates.categorical + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.comparison module +----------------------------------------------- + +.. automodule:: graphistry.compute.predicates.comparison + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.from\_json module +----------------------------------------------- + +.. automodule:: graphistry.compute.predicates.from_json + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.is\_in module +------------------------------------------- + +.. automodule:: graphistry.compute.predicates.is_in + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.numeric module +-------------------------------------------- + +.. automodule:: graphistry.compute.predicates.numeric + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.str module +---------------------------------------- + +.. automodule:: graphistry.compute.predicates.str + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.temporal module +--------------------------------------------- + +.. automodule:: graphistry.compute.predicates.temporal + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.predicates.types module +------------------------------------------ + +.. automodule:: graphistry.compute.predicates.types + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.compute.predicates + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.compute.rst b/docs/source/graphistry.compute.rst new file mode 100644 index 0000000000..1291c74966 --- /dev/null +++ b/docs/source/graphistry.compute.rst @@ -0,0 +1,136 @@ +graphistry.compute package +========================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + graphistry.compute.gfql + graphistry.compute.predicates + +Submodules +---------- + +graphistry.compute.ASTSerializable module +----------------------------------------- + +.. automodule:: graphistry.compute.ASTSerializable + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.ComputeMixin module +-------------------------------------- + +.. automodule:: graphistry.compute.ComputeMixin + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.ast module +----------------------------- + +.. automodule:: graphistry.compute.ast + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.ast\_temporal module +--------------------------------------- + +.. automodule:: graphistry.compute.ast_temporal + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.chain module +------------------------------- + +.. automodule:: graphistry.compute.chain + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.chain\_remote module +--------------------------------------- + +.. automodule:: graphistry.compute.chain_remote + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.chain\_validate module +----------------------------------------- + +.. automodule:: graphistry.compute.chain_validate + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.cluster module +--------------------------------- + +.. automodule:: graphistry.compute.cluster + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.collapse module +---------------------------------- + +.. automodule:: graphistry.compute.collapse + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.conditional module +------------------------------------- + +.. automodule:: graphistry.compute.conditional + :members: + :undoc-members: + :show-inheritance: + + +graphistry.compute.filter\_by\_dict module +------------------------------------------ + +.. automodule:: graphistry.compute.filter_by_dict + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.hop module +----------------------------- + +.. automodule:: graphistry.compute.hop + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.python\_remote module +---------------------------------------- + +.. automodule:: graphistry.compute.python_remote + :members: + :undoc-members: + :show-inheritance: + +graphistry.compute.typing module +-------------------------------- + +.. automodule:: graphistry.compute.typing + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: graphistry.compute + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.gib.rst b/docs/source/graphistry.layout.gib.rst new file mode 100644 index 0000000000..094027e2bc --- /dev/null +++ b/docs/source/graphistry.layout.gib.rst @@ -0,0 +1,69 @@ +graphistry.layout.gib package +============================= + +Submodules +---------- + +graphistry.layout.gib.gib module +-------------------------------- + +.. automodule:: graphistry.layout.gib.gib + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.gib.layout\_bulk module +----------------------------------------- + +.. automodule:: graphistry.layout.gib.layout_bulk + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.gib.layout\_non\_bulk module +---------------------------------------------- + +.. automodule:: graphistry.layout.gib.layout_non_bulk + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.gib.partition module +-------------------------------------- + +.. automodule:: graphistry.layout.gib.partition + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.gib.partitioned\_layout module +------------------------------------------------ + +.. automodule:: graphistry.layout.gib.partitioned_layout + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.gib.style module +---------------------------------- + +.. automodule:: graphistry.layout.gib.style + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.gib.treemap module +------------------------------------ + +.. automodule:: graphistry.layout.gib.treemap + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout.gib + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.graph.rst b/docs/source/graphistry.layout.graph.rst new file mode 100644 index 0000000000..72d559ad11 --- /dev/null +++ b/docs/source/graphistry.layout.graph.rst @@ -0,0 +1,61 @@ +graphistry.layout.graph package +=============================== + +Submodules +---------- + +graphistry.layout.graph.edge module +----------------------------------- + +.. automodule:: graphistry.layout.graph.edge + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.graph.edgeBase module +--------------------------------------- + +.. automodule:: graphistry.layout.graph.edgeBase + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.graph.graph module +------------------------------------ + +.. automodule:: graphistry.layout.graph.graph + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.graph.graphBase module +---------------------------------------- + +.. automodule:: graphistry.layout.graph.graphBase + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.graph.vertex module +------------------------------------- + +.. automodule:: graphistry.layout.graph.vertex + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.graph.vertexBase module +----------------------------------------- + +.. automodule:: graphistry.layout.graph.vertexBase + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout.graph + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.modularity_weighted.rst b/docs/source/graphistry.layout.modularity_weighted.rst new file mode 100644 index 0000000000..b0d1ce7b07 --- /dev/null +++ b/docs/source/graphistry.layout.modularity_weighted.rst @@ -0,0 +1,21 @@ +graphistry.layout.modularity\_weighted package +============================================== + +Submodules +---------- + +graphistry.layout.modularity\_weighted.modularity\_weighted module +------------------------------------------------------------------ + +.. automodule:: graphistry.layout.modularity_weighted.modularity_weighted + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout.modularity_weighted + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.ring.rst b/docs/source/graphistry.layout.ring.rst new file mode 100644 index 0000000000..ccb99e3daa --- /dev/null +++ b/docs/source/graphistry.layout.ring.rst @@ -0,0 +1,45 @@ +graphistry.layout.ring package +============================== + +Submodules +---------- + +graphistry.layout.ring.categorical module +----------------------------------------- + +.. automodule:: graphistry.layout.ring.categorical + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.ring.continuous module +---------------------------------------- + +.. automodule:: graphistry.layout.ring.continuous + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.ring.time module +---------------------------------- + +.. automodule:: graphistry.layout.ring.time + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.ring.util module +---------------------------------- + +.. automodule:: graphistry.layout.ring.util + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout.ring + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.rst b/docs/source/graphistry.layout.rst new file mode 100644 index 0000000000..db8ea71359 --- /dev/null +++ b/docs/source/graphistry.layout.rst @@ -0,0 +1,42 @@ +graphistry.layout package +========================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + graphistry.layout.gib + graphistry.layout.graph + graphistry.layout.modularity_weighted + graphistry.layout.ring + graphistry.layout.sugiyama + graphistry.layout.utils + +Submodules +---------- + +graphistry.layout.circle module +------------------------------- + +.. automodule:: graphistry.layout.circle + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.fa2 module +---------------------------- + +.. automodule:: graphistry.layout.fa2 + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.sugiyama.rst b/docs/source/graphistry.layout.sugiyama.rst new file mode 100644 index 0000000000..41b83f7cb1 --- /dev/null +++ b/docs/source/graphistry.layout.sugiyama.rst @@ -0,0 +1,21 @@ +graphistry.layout.sugiyama package +================================== + +Submodules +---------- + +graphistry.layout.sugiyama.sugiyamaLayout module +------------------------------------------------ + +.. automodule:: graphistry.layout.sugiyama.sugiyamaLayout + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout.sugiyama + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.layout.utils.rst b/docs/source/graphistry.layout.utils.rst new file mode 100644 index 0000000000..de1d80140d --- /dev/null +++ b/docs/source/graphistry.layout.utils.rst @@ -0,0 +1,69 @@ +graphistry.layout.utils package +=============================== + +Submodules +---------- + +graphistry.layout.utils.dummyVertex module +------------------------------------------ + +.. automodule:: graphistry.layout.utils.dummyVertex + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.utils.geometry module +--------------------------------------- + +.. automodule:: graphistry.layout.utils.geometry + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.utils.layer module +------------------------------------ + +.. automodule:: graphistry.layout.utils.layer + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.utils.layoutVertex module +------------------------------------------- + +.. automodule:: graphistry.layout.utils.layoutVertex + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.utils.poset module +------------------------------------ + +.. automodule:: graphistry.layout.utils.poset + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.utils.rectangle module +---------------------------------------- + +.. automodule:: graphistry.layout.utils.rectangle + :members: + :undoc-members: + :show-inheritance: + +graphistry.layout.utils.routing module +-------------------------------------- + +.. automodule:: graphistry.layout.utils.routing + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.layout.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.models.compute.rst b/docs/source/graphistry.models.compute.rst new file mode 100644 index 0000000000..9e3c8c3bde --- /dev/null +++ b/docs/source/graphistry.models.compute.rst @@ -0,0 +1,45 @@ +graphistry.models.compute package +================================= + +Submodules +---------- + +graphistry.models.compute.chain\_remote module +---------------------------------------------- + +.. automodule:: graphistry.models.compute.chain_remote + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.compute.dbscan module +--------------------------------------- + +.. automodule:: graphistry.models.compute.dbscan + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.compute.features module +----------------------------------------- + +.. automodule:: graphistry.models.compute.features + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.compute.umap module +------------------------------------- + +.. automodule:: graphistry.models.compute.umap + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.models.compute + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.models.gfql.coercions.rst b/docs/source/graphistry.models.gfql.coercions.rst new file mode 100644 index 0000000000..a9656046b9 --- /dev/null +++ b/docs/source/graphistry.models.gfql.coercions.rst @@ -0,0 +1,29 @@ +graphistry.models.gfql.coercions package +======================================== + +Submodules +---------- + +graphistry.models.gfql.coercions.numeric module +----------------------------------------------- + +.. automodule:: graphistry.models.gfql.coercions.numeric + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.gfql.coercions.temporal module +------------------------------------------------ + +.. automodule:: graphistry.models.gfql.coercions.temporal + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.models.gfql.coercions + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.models.gfql.rst b/docs/source/graphistry.models.gfql.rst new file mode 100644 index 0000000000..d4e3f3784d --- /dev/null +++ b/docs/source/graphistry.models.gfql.rst @@ -0,0 +1,19 @@ +graphistry.models.gfql package +============================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + graphistry.models.gfql.coercions + graphistry.models.gfql.types + +Module contents +--------------- + +.. automodule:: graphistry.models.gfql + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.models.gfql.types.rst b/docs/source/graphistry.models.gfql.types.rst new file mode 100644 index 0000000000..4bc70c1121 --- /dev/null +++ b/docs/source/graphistry.models.gfql.types.rst @@ -0,0 +1,45 @@ +graphistry.models.gfql.types package +==================================== + +Submodules +---------- + +graphistry.models.gfql.types.guards module +------------------------------------------ + +.. automodule:: graphistry.models.gfql.types.guards + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.gfql.types.numeric module +------------------------------------------- + +.. automodule:: graphistry.models.gfql.types.numeric + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.gfql.types.predicates module +---------------------------------------------- + +.. automodule:: graphistry.models.gfql.types.predicates + :members: + :undoc-members: + :show-inheritance: + +graphistry.models.gfql.types.temporal module +-------------------------------------------- + +.. automodule:: graphistry.models.gfql.types.temporal + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.models.gfql.types + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.models.rst b/docs/source/graphistry.models.rst new file mode 100644 index 0000000000..d232b6533e --- /dev/null +++ b/docs/source/graphistry.models.rst @@ -0,0 +1,30 @@ +graphistry.models package +========================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + graphistry.models.compute + graphistry.models.gfql + +Submodules +---------- + +graphistry.models.ModelDict module +---------------------------------- + +.. automodule:: graphistry.models.ModelDict + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.plugins.rst b/docs/source/graphistry.plugins.rst new file mode 100644 index 0000000000..48a4dc1ca2 --- /dev/null +++ b/docs/source/graphistry.plugins.rst @@ -0,0 +1,53 @@ +graphistry.plugins package +========================== + +Submodules +---------- + +graphistry.plugins.cugraph module +--------------------------------- + +.. automodule:: graphistry.plugins.cugraph + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins.graphviz module +---------------------------------- + +.. automodule:: graphistry.plugins.graphviz + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins.igraph module +-------------------------------- + +.. automodule:: graphistry.plugins.igraph + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins.kusto module +------------------------------- + +.. automodule:: graphistry.plugins.kusto + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins.spanner module +--------------------------------- + +.. automodule:: graphistry.plugins.spanner + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.plugins + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.plugins_types.rst b/docs/source/graphistry.plugins_types.rst new file mode 100644 index 0000000000..99c7cdddbe --- /dev/null +++ b/docs/source/graphistry.plugins_types.rst @@ -0,0 +1,53 @@ +graphistry.plugins\_types package +================================= + +Submodules +---------- + +graphistry.plugins\_types.cugraph\_types module +----------------------------------------------- + +.. automodule:: graphistry.plugins_types.cugraph_types + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins\_types.graphviz\_types module +------------------------------------------------ + +.. automodule:: graphistry.plugins_types.graphviz_types + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins\_types.hypergraph module +------------------------------------------- + +.. automodule:: graphistry.plugins_types.hypergraph + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins\_types.kusto\_types module +--------------------------------------------- + +.. automodule:: graphistry.plugins_types.kusto_types + :members: + :undoc-members: + :show-inheritance: + +graphistry.plugins\_types.spanner\_types module +----------------------------------------------- + +.. automodule:: graphistry.plugins_types.spanner_types + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.plugins_types + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.render.rst b/docs/source/graphistry.render.rst new file mode 100644 index 0000000000..b41ec88b22 --- /dev/null +++ b/docs/source/graphistry.render.rst @@ -0,0 +1,21 @@ +graphistry.render package +========================= + +Submodules +---------- + +graphistry.render.resolve\_render\_mode module +---------------------------------------------- + +.. automodule:: graphistry.render.resolve_render_mode + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.render + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.utils.rst b/docs/source/graphistry.utils.rst new file mode 100644 index 0000000000..080c73e39d --- /dev/null +++ b/docs/source/graphistry.utils.rst @@ -0,0 +1,45 @@ +graphistry.utils package +======================== + +Submodules +---------- + +graphistry.utils.json module +---------------------------- + +.. automodule:: graphistry.utils.json + :members: + :undoc-members: + :show-inheritance: + +graphistry.utils.lazy\_import module +------------------------------------ + +.. automodule:: graphistry.utils.lazy_import + :members: + :undoc-members: + :show-inheritance: + +graphistry.utils.plottable\_memoize module +------------------------------------------ + +.. automodule:: graphistry.utils.plottable_memoize + :members: + :undoc-members: + :show-inheritance: + +graphistry.utils.requests module +-------------------------------- + +.. automodule:: graphistry.utils.requests + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/graphistry.validate.rst b/docs/source/graphistry.validate.rst new file mode 100644 index 0000000000..b02efd093a --- /dev/null +++ b/docs/source/graphistry.validate.rst @@ -0,0 +1,21 @@ +graphistry.validate package +=========================== + +Submodules +---------- + +graphistry.validate.validate\_encodings module +---------------------------------------------- + +.. automodule:: graphistry.validate.validate_encodings + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: graphistry.validate + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000000..7adb957d3b --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,8 @@ +pygraphistry +============ + +.. toctree:: + :maxdepth: 4 + + graphistry + versioneer diff --git a/docs/source/notebooks/gfql.rst b/docs/source/notebooks/gfql.rst index d03accb479..a967eae076 100644 --- a/docs/source/notebooks/gfql.rst +++ b/docs/source/notebooks/gfql.rst @@ -7,6 +7,7 @@ GFQL Graph queries :titlesonly: Intro to graph queries with hop and chain <../demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb> + GFQL Validation Fundamentals <../demos/gfql/gfql_validation_fundamentals.ipynb> DateTime Filtering Examples <../demos/gfql/temporal_predicates.ipynb> GPU Benchmarking <../demos/gfql/benchmark_hops_cpu_gpu.ipynb> GFQL Remote mode <../demos/gfql/gfql_remote.ipynb> diff --git a/docs/source/versioneer.rst b/docs/source/versioneer.rst new file mode 100644 index 0000000000..f8155c0e4d --- /dev/null +++ b/docs/source/versioneer.rst @@ -0,0 +1,7 @@ +versioneer module +================= + +.. automodule:: versioneer + :members: + :undoc-members: + :show-inheritance: diff --git a/graphistry/compute/chain_validate.py b/graphistry/compute/chain_validate.py new file mode 100644 index 0000000000..8e113a06aa --- /dev/null +++ b/graphistry/compute/chain_validate.py @@ -0,0 +1,127 @@ +"""Enhanced chain function with validation support.""" + +from typing import Union, List, Optional +from graphistry.Plottable import Plottable +from graphistry.compute.chain import chain as chain_original, Chain +from graphistry.compute.ast import ASTObject +from graphistry.compute.gfql.validate import ( + validate_query, extract_schema, format_validation_errors, + ValidationIssue, Schema +) +from graphistry.compute.gfql.exceptions import GFQLValidationError +from graphistry.Engine import EngineAbstract +import logging + +logger = logging.getLogger(__name__) + + +def chain_with_validation( + self: Plottable, + ops: Union[List[ASTObject], Chain], + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + validate: bool = True, + validate_mode: str = 'warn', # 'warn', 'error', or 'silent' + validate_schema: bool = True +) -> Plottable: + """ + Chain operations with optional validation. + + This is a wrapper around the original chain function that adds validation support. + + Args: + self: Plottable instance + ops: List of operations or Chain object + engine: Engine to use + validate: Whether to perform validation + validate_mode: How to handle validation issues - + 'warn' (Log warnings but continue - default), + 'error' (Raise exception on first error), + 'silent' (Collect issues but don't log/raise) + validate_schema: Whether to validate against data schema if available + + Returns: + Plottable result + + Raises: + GFQLValidationError: If validate_mode='error' and validation fails + """ + if not validate: + return chain_original(self, ops, engine) + + # Perform validation + if validate_schema and (self._nodes is not None or self._edges is not None): + # Validate with schema + issues = validate_query(ops, self._nodes, self._edges) + else: + # Syntax validation only + from graphistry.compute.gfql.validate import validate_syntax + issues = validate_syntax(ops) + + # Handle validation results based on mode + if issues: + errors = [i for i in issues if i.level == 'error'] + warnings = [i for i in issues if i.level == 'warning'] + + if validate_mode == 'error' and errors: + # Raise on first error + error_msg = format_validation_errors(errors[:1]) + raise GFQLValidationError(error_msg) + + elif validate_mode == 'warn': + # Log all issues + if errors: + logger.error("GFQL Validation Errors:\n%s", format_validation_errors(errors)) + if warnings: + logger.warning("GFQL Validation Warnings:\n%s", format_validation_errors(warnings)) + + # For 'silent' mode, issues are available but not logged + + # Store validation results for access + if hasattr(self, '_last_validation_issues'): + self._last_validation_issues = issues + + # Execute the chain + return chain_original(self, ops, engine) + + +def validate_chain( + self: Plottable, + ops: Union[List[ASTObject], Chain], + return_issues: bool = False +) -> Union[bool, List[ValidationIssue]]: + """ + Validate a chain without executing it. + + Args: + self: Plottable instance + ops: Operations to validate + return_issues: If True, return list of issues; if False, return bool + + Returns: + If return_issues=False: True if valid, False otherwise + If return_issues=True: List of ValidationIssue objects + """ + if self._nodes is not None or self._edges is not None: + issues = validate_query(ops, self._nodes, self._edges) + else: + from graphistry.compute.gfql.validate import validate_syntax + issues = validate_syntax(ops) + + if return_issues: + return issues + else: + errors = [i for i in issues if i.level == 'error'] + return len(errors) == 0 + + +def get_chain_schema(self: Plottable) -> "Schema": + """ + Extract schema from Plottable for validation purposes. + + Args: + self: Plottable instance + + Returns: + Schema object with column information + """ + return extract_schema(self) diff --git a/graphistry/compute/gfql/__init__.py b/graphistry/compute/gfql/__init__.py new file mode 100644 index 0000000000..1df331d90f --- /dev/null +++ b/graphistry/compute/gfql/__init__.py @@ -0,0 +1,45 @@ +"""GFQL validation and related utilities.""" + +from graphistry.compute.gfql.validate import ( + ValidationIssue, + Schema, + validate_syntax, + validate_schema, + validate_query, + extract_schema, + extract_schema_from_dataframes, + format_validation_errors, + suggest_fixes +) + +from graphistry.compute.gfql.exceptions import ( + GFQLException, + GFQLValidationError, + GFQLSyntaxError, + GFQLSchemaError, + GFQLTypeError, + GFQLColumnNotFoundError +) + +__all__ = [ + # Validation classes + 'ValidationIssue', + 'Schema', + + # Validation functions + 'validate_syntax', + 'validate_schema', + 'validate_query', + 'extract_schema', + 'extract_schema_from_dataframes', + 'format_validation_errors', + 'suggest_fixes', + + # Exceptions + 'GFQLException', + 'GFQLValidationError', + 'GFQLSyntaxError', + 'GFQLSchemaError', + 'GFQLTypeError', + 'GFQLColumnNotFoundError' +] diff --git a/graphistry/compute/gfql/exceptions.py b/graphistry/compute/gfql/exceptions.py new file mode 100644 index 0000000000..fd44068bf6 --- /dev/null +++ b/graphistry/compute/gfql/exceptions.py @@ -0,0 +1,62 @@ +"""GFQL-specific exceptions for validation and error handling.""" + +from typing import Optional, Dict, Any + + +class GFQLException(Exception): + """Base exception for all GFQL-related errors.""" + + def __init__(self, message: str, context: Optional[Dict[str, Any]] = None): + self.context = context or {} + super().__init__(message) + + def __str__(self) -> str: + if self.context: + context_str = ", ".join(f"{k}={v}" for k, v in self.context.items()) + return f"{super().__str__()} ({context_str})" + return super().__str__() + + +class GFQLValidationError(GFQLException, ValueError): + """Base validation error. Inherits from ValueError for backwards compatibility.""" + pass + + +class GFQLSyntaxError(GFQLValidationError): + """Raised when GFQL query has invalid syntax.""" + pass + + +class GFQLSchemaError(GFQLValidationError): + """Raised when GFQL query references non-existent columns or has type mismatches.""" + pass + + +class GFQLSemanticError(GFQLValidationError): + """Raised when GFQL query is syntactically valid but semantically incorrect.""" + pass + + +class GFQLTypeError(GFQLSchemaError): + """Raised when a predicate is applied to incompatible column type.""" + + def __init__(self, column: str, column_type: str, predicate: str, expected_type: str): + message = f"Column '{column}' has type '{column_type}' but predicate '{predicate}' expects '{expected_type}'" + super().__init__(message, { + 'column': column, + 'column_type': column_type, + 'predicate': predicate, + 'expected_type': expected_type + }) + + +class GFQLColumnNotFoundError(GFQLSchemaError): + """Raised when a referenced column doesn't exist in the schema.""" + + def __init__(self, column: str, table: str, available_columns: list): + message = f"Column '{column}' not found in {table} data" + super().__init__(message, { + 'column': column, + 'table': table, + 'available_columns': available_columns + }) diff --git a/graphistry/compute/gfql/validate.py b/graphistry/compute/gfql/validate.py new file mode 100644 index 0000000000..9b87d92409 --- /dev/null +++ b/graphistry/compute/gfql/validate.py @@ -0,0 +1,650 @@ +"""GFQL query validation utilities for syntax and schema checking.""" + +from typing import List, Optional, Dict, Union, Any, Tuple, TYPE_CHECKING +import pandas as pd + +from graphistry.compute.chain import Chain +from graphistry.compute.ast import ASTNode, ASTEdge, ASTObject + +if TYPE_CHECKING: + from graphistry.Plottable import Plottable +from graphistry.compute.predicates.ASTPredicate import ASTPredicate +from graphistry.compute.predicates.numeric import NumericASTPredicate +from graphistry.compute.predicates.str import ( + Contains, Startswith, Endswith, Match, + IsNumeric, IsAlpha, IsDigit, IsLower, IsUpper, + IsSpace, IsAlnum, IsDecimal, IsTitle +) +from graphistry.compute.predicates.temporal import ( + IsMonthStart, IsMonthEnd, IsQuarterStart, IsQuarterEnd, + IsYearStart, IsYearEnd, IsLeapYear +) +from graphistry.util import setup_logger + +logger = setup_logger(__name__) + + +class ValidationIssue: + """Represents a validation issue (error or warning).""" + + def __init__(self, + level: str, # 'error' or 'warning' + message: str, + operation_index: Optional[int] = None, + field: Optional[str] = None, + suggestion: Optional[str] = None, + error_type: Optional[str] = None): + self.level = level + self.message = message + self.operation_index = operation_index + self.field = field + self.suggestion = suggestion + self.error_type = error_type + + def __repr__(self) -> str: + parts = [f"{self.level.upper()}: {self.message}"] + if self.operation_index is not None: + parts.append(f"at operation {self.operation_index}") + if self.field: + parts.append(f"field: {self.field}") + if self.suggestion: + parts.append(f"Suggestion: {self.suggestion}") + return " | ".join(parts) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'level': self.level, + 'message': self.message, + 'operation_index': self.operation_index, + 'field': self.field, + 'suggestion': self.suggestion, + 'error_type': self.error_type + } + + +class Schema: + """Represents the schema of node and edge dataframes.""" + + def __init__(self, + node_columns: Optional[Dict[str, str]] = None, + edge_columns: Optional[Dict[str, str]] = None): + self.node_columns = node_columns or {} + self.edge_columns = edge_columns or {} + + def __repr__(self) -> str: + return (f"Schema(nodes={list(self.node_columns.keys())}, " + f"edges={list(self.edge_columns.keys())})") + + +# Error message templates +ERROR_MESSAGES = { + # Syntax errors + 'INVALID_CHAIN_TYPE': { + 'message': 'Chain must be a list of operations', + 'suggestion': 'Wrap your operations in a list: [n(), e(), n()]' + }, + 'INVALID_OPERATION': { + 'message': 'Operation at index {index} is not a valid GFQL operation', + 'suggestion': ('Use n() for nodes, ' + 'e()/e_forward()/e_reverse() for edges') + }, + 'INVALID_FILTER_KEY': { + 'message': 'Invalid filter key format: {key}', + 'suggestion': 'Filter keys must be strings' + }, + 'INVALID_HOPS': { + 'message': 'Hops must be a positive integer or None', + 'suggestion': ('Use hops=2 for specific count, ' + 'or to_fixed_point=True for unbounded') + }, + + # Schema errors + 'COLUMN_NOT_FOUND': { + 'message': 'Column "{column}" not found in {table} data', + 'suggestion': 'Available columns: {available}' + }, + 'TYPE_MISMATCH': { + 'message': ('Column "{column}" is {actual_type} but ' + 'predicate expects {expected_type}'), + 'suggestion': 'Use appropriate predicate for {actual_type} columns' + }, + 'INVALID_PREDICATE': { + 'message': ('Predicate {predicate} cannot be used with ' + '{column_type} columns'), + 'suggestion': 'Use {suggested_predicates} for {column_type} columns' + }, + + # Semantic errors + 'ORPHANED_EDGE': { + 'message': 'Edge operation at index {index} not connected to nodes', + 'suggestion': ('Add node operations before/after edge: ' + 'n() -> e() -> n()') + }, + 'UNBOUNDED_HOPS_WARNING': { + 'message': ('Unbounded hops (to_fixed_point=True) may be slow ' + 'on large graphs'), + 'suggestion': ('Consider using specific hop count for better ' + 'performance') + }, + 'EMPTY_CHAIN': { + 'message': 'Chain is empty', + 'suggestion': 'Add at least one operation: [n()]' + }, + 'INVALID_EDGE_DIRECTION': { + 'message': 'Invalid edge direction: {direction}', + 'suggestion': 'Use "forward", "reverse", or "undirected"' + } +} + + +def _format_error(error_key: str, **kwargs) -> Tuple[str, str]: + """Format error message with context.""" + template = ERROR_MESSAGES.get(error_key, { + 'message': f'Unknown error: {error_key}', + 'suggestion': 'Check documentation for valid syntax' + }) + message = template['message'].format(**kwargs) + suggestion = template['suggestion'].format(**kwargs) + return message, suggestion + + +def validate_syntax(chain: Union[Chain, List]) -> List[ValidationIssue]: + """ + Validate GFQL query syntax without requiring data. + + Args: + chain: GFQL chain or list of operations + + Returns: + List of validation issues (errors and warnings) + """ + issues = [] + + # Convert to list if Chain object + if isinstance(chain, Chain): + operations = chain.chain + elif isinstance(chain, list): + operations = chain + else: + message, suggestion = _format_error('INVALID_CHAIN_TYPE') + issues.append(ValidationIssue( + 'error', message, suggestion=suggestion, + error_type='INVALID_CHAIN_TYPE')) + return issues + + # Check empty chain + if not operations: + message, suggestion = _format_error('EMPTY_CHAIN') + issues.append(ValidationIssue( + 'error', message, suggestion=suggestion, + error_type='EMPTY_CHAIN')) + return issues + + # Validate each operation + for i, op in enumerate(operations): + # Check if valid operation type + if not isinstance(op, ASTObject): + message, suggestion = _format_error('INVALID_OPERATION', index=i) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + suggestion=suggestion, error_type='INVALID_OPERATION')) + continue + + # Validate nodes + if isinstance(op, ASTNode): + if op.filter_dict: + for key, value in op.filter_dict.items(): + if not isinstance(key, str): + message, suggestion = _format_error( + 'INVALID_FILTER_KEY', key=key) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field=str(key), suggestion=suggestion, + error_type='INVALID_FILTER_KEY')) + + # Validate edges + elif isinstance(op, ASTEdge): + # Check hops + if (op.hops is not None + and (not isinstance(op.hops, int) or op.hops < 1)): + message, suggestion = _format_error('INVALID_HOPS') + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field='hops', suggestion=suggestion, + error_type='INVALID_HOPS')) + + # Check unbounded hops warning + if op.to_fixed_point: + message, suggestion = _format_error('UNBOUNDED_HOPS_WARNING') + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='UNBOUNDED_HOPS_WARNING')) + + # Check edge filters + for filter_name, filter_dict in [ + ('edge_match', op.edge_match), + ('source_node_match', op.source_node_match), + ('destination_node_match', op.destination_node_match) + ]: + if filter_dict: + for key, value in filter_dict.items(): + if not isinstance(key, str): + message, suggestion = _format_error( + 'INVALID_FILTER_KEY', key=key) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field=f"{filter_name}.{key}", + suggestion=suggestion, + error_type='INVALID_FILTER_KEY')) + + # Check semantic issues + issues.extend(_validate_semantics(operations)) + + return issues + + +def _validate_semantics(operations: List[ASTObject]) -> List[ValidationIssue]: + """Validate semantic correctness of operation sequence.""" + issues = [] + + # Check for orphaned edges (edges not between nodes) + for i, op in enumerate(operations): + if isinstance(op, ASTEdge): + # Check if first or last operation + if i == 0 or i == len(operations) - 1: + # Edge at boundary - likely orphaned + message, suggestion = _format_error('ORPHANED_EDGE', index=i) + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='ORPHANED_EDGE')) + # Check if between two edges + elif (i > 0 and isinstance(operations[i - 1], ASTEdge) + and i < len(operations) - 1 + and isinstance(operations[i + 1], ASTEdge)): + message, suggestion = _format_error('ORPHANED_EDGE', index=i) + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='ORPHANED_EDGE')) + + return issues + + +def validate_schema(chain: Union[Chain, List], + schema: Schema) -> List[ValidationIssue]: + """ + Validate query against data schema. + + Args: + chain: GFQL chain or list of operations + schema: Schema object with column information + + Returns: + List of validation issues + """ + issues = [] + + # First do syntax validation + syntax_issues = validate_syntax(chain) + # Only keep errors from syntax validation for schema validation + issues.extend([issue for issue in syntax_issues if issue.level == 'error']) + + if any(issue.level == 'error' for issue in issues): + return issues # Don't do schema validation if syntax errors + + # Convert to list if Chain object + operations = chain.chain if isinstance(chain, Chain) else chain + + # Validate each operation against schema + for i, op in enumerate(operations): + if isinstance(op, ASTNode): + issues.extend(_validate_node_schema(op, schema, i)) + elif isinstance(op, ASTEdge): + issues.extend(_validate_edge_schema(op, schema, i)) + + return issues + + +def _validate_node_schema(node: ASTNode, schema: Schema, + op_index: int) -> List[ValidationIssue]: + """Validate node operation against schema.""" + issues = [] + + if node.filter_dict and schema.node_columns: + for col, predicate in node.filter_dict.items(): + # Check column exists + if col not in schema.node_columns: + available = list(schema.node_columns.keys())[:5] + if len(schema.node_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='node', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=col, suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + # Check predicate type compatibility + if isinstance(predicate, ASTPredicate): + col_type = schema.node_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index)) + + return issues + + +def _validate_edge_schema(edge: ASTEdge, schema: Schema, + op_index: int) -> List[ValidationIssue]: + """Validate edge operation against schema.""" + issues = [] + + # Validate edge filters + if edge.edge_match and schema.edge_columns: + for col, predicate in edge.edge_match.items(): + if col not in schema.edge_columns: + available = list(schema.edge_columns.keys())[:5] + if len(schema.edge_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='edge', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"edge_match.{col}", suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + if isinstance(predicate, ASTPredicate): + col_type = schema.edge_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index, + field_prefix="edge_match.")) + + # Validate source/dest node filters against node schema + for filter_name, filter_dict in [ + ('source_node_match', edge.source_node_match), + ('destination_node_match', edge.destination_node_match) + ]: + if filter_dict and schema.node_columns: + for col, predicate in filter_dict.items(): + if col not in schema.node_columns: + available = list(schema.node_columns.keys())[:5] + if len(schema.node_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='node', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{filter_name}.{col}", suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + if isinstance(predicate, ASTPredicate): + col_type = schema.node_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index, + field_prefix=f"{filter_name}.")) + + return issues + + +def _validate_predicate_type(predicate: ASTPredicate, column: str, + column_type: str, op_index: int, + field_prefix: str = "") -> List[ValidationIssue]: + """Validate predicate is appropriate for column type.""" + issues = [] + + # Map pandas/numpy dtypes to categories + type_category = _get_type_category(column_type) + + # Define string predicate types + STRING_PREDICATES = ( + Contains, Startswith, Endswith, Match, + IsNumeric, IsAlpha, IsDigit, IsLower, IsUpper, + IsSpace, IsAlnum, IsDecimal, IsTitle + ) + + # Define temporal predicate types + TEMPORAL_PREDICATES = ( + IsMonthStart, IsMonthEnd, IsQuarterStart, IsQuarterEnd, + IsYearStart, IsYearEnd, IsLeapYear + ) + + # Check predicate compatibility + if (isinstance(predicate, NumericASTPredicate) + and type_category not in ['numeric', 'temporal']): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='numeric') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + elif (isinstance(predicate, STRING_PREDICATES) + and type_category != 'string'): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='string') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + elif (isinstance(predicate, TEMPORAL_PREDICATES) + and type_category != 'temporal'): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='datetime') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + return issues + + +def _get_type_category(dtype_str: str) -> str: + """Categorize dtype string into broad categories.""" + dtype_lower = str(dtype_str).lower() + + if any(t in dtype_lower for t in + ['int', 'float', 'double', 'numeric', 'decimal']): + return 'numeric' + elif any(t in dtype_lower for t in + ['str', 'object', 'char', 'text', 'varchar']): + return 'string' + elif any(t in dtype_lower for t in + ['date', 'time', 'timestamp', 'datetime']): + return 'temporal' + elif 'bool' in dtype_lower: + return 'boolean' + else: + return 'unknown' + + +def validate_query(chain: Union[Chain, List], + nodes_df: Optional[pd.DataFrame] = None, + edges_df: Optional[pd.DataFrame] = None + ) -> List[ValidationIssue]: + """ + Combined syntax and schema validation. + + Args: + chain: GFQL chain or list of operations + nodes_df: Optional node dataframe for schema validation + edges_df: Optional edge dataframe for schema validation + + Returns: + List of validation issues + """ + # Always do syntax validation + issues = validate_syntax(chain) + + # If data provided, also do schema validation + if nodes_df is not None or edges_df is not None: + schema = extract_schema_from_dataframes(nodes_df, edges_df) + schema_issues = validate_schema(chain, schema) + # Merge issues, avoiding duplicates + existing_errors = {(i.error_type, i.operation_index, i.field) + for i in issues} + for issue in schema_issues: + if ((issue.error_type, issue.operation_index, issue.field) + not in existing_errors): + issues.append(issue) + + return issues + + +def extract_schema(g: "Plottable") -> Schema: + """ + Extract schema from a Plottable object. + + Args: + g: Plottable object with node/edge data + + Returns: + Schema object + """ + + nodes_df = g._nodes if hasattr(g, '_nodes') else None + edges_df = g._edges if hasattr(g, '_edges') else None + + return extract_schema_from_dataframes(nodes_df, edges_df) + + +def extract_schema_from_dataframes( + nodes_df: Optional[pd.DataFrame] = None, + edges_df: Optional[pd.DataFrame] = None) -> Schema: + """ + Extract schema from pandas DataFrames. + + Args: + nodes_df: Optional node dataframe + edges_df: Optional edge dataframe + + Returns: + Schema object with column names and types + """ + node_columns = {} + edge_columns = {} + + if nodes_df is not None and hasattr(nodes_df, 'dtypes'): + node_columns = {str(col): str(dtype) + for col, dtype in nodes_df.dtypes.items()} + + if edges_df is not None and hasattr(edges_df, 'dtypes'): + edge_columns = {str(col): str(dtype) + for col, dtype in edges_df.dtypes.items()} + + return Schema(node_columns, edge_columns) + + +def format_validation_errors(issues: List[ValidationIssue]) -> str: + """ + Format validation errors for human/LLM consumption. + + Args: + issues: List of validation issues + + Returns: + Formatted error string + """ + if not issues: + return "No validation issues found." + + lines = ["GFQL Validation Report:"] + lines.append("-" * 50) + + errors = [i for i in issues if i.level == 'error'] + warnings = [i for i in issues if i.level == 'warning'] + + if errors: + lines.append(f"\nERRORS ({len(errors)}):") + for i, error in enumerate(errors, 1): + lines.append(f"\n{i}. {error.message}") + if error.operation_index is not None: + lines.append(f" Location: Operation {error.operation_index}") + if error.field: + lines.append(f" Field: {error.field}") + if error.suggestion: + lines.append(f" ๐Ÿ’ก {error.suggestion}") + + if warnings: + lines.append(f"\nWARNINGS ({len(warnings)}):") + for i, warning in enumerate(warnings, 1): + lines.append(f"\n{i}. {warning.message}") + if warning.operation_index is not None: + lines.append( + f" Location: Operation {warning.operation_index}") + if warning.suggestion: + lines.append(f" ๐Ÿ’ก {warning.suggestion}") + + return "\n".join(lines) + + +def suggest_fixes(chain: Union[Chain, List], + issues: List[ValidationIssue]) -> List[str]: + """ + Generate fix suggestions for validation issues. + + Args: + chain: The problematic chain + issues: Validation issues found + + Returns: + List of suggested fixes + """ + suggestions = [] + + # Group by error type for consolidated suggestions + error_types: Dict[str, List[ValidationIssue]] = {} + for issue in issues: + if issue.error_type: + error_types.setdefault(issue.error_type, []).append(issue) + + # Generate type-specific suggestions + if 'COLUMN_NOT_FOUND' in error_types: + missing_cols = {issue.field for issue in + error_types['COLUMN_NOT_FOUND'] if issue.field} + suggestions.append( + f"Missing columns: {', '.join(missing_cols)}. " + f"Check column names match your data.") + + if 'TYPE_MISMATCH' in error_types: + suggestions.append( + "Type mismatches found. Use numeric predicates (gt, lt) " + "for numbers, string predicates (contains, startswith) for text.") + + if 'ORPHANED_EDGE' in error_types: + suggestions.append( + "Edge operations should connect nodes. " + "Use pattern: n() -> e() -> n()") + + # Add general suggestions from issues + for issue in issues: + if issue.suggestion and issue.suggestion not in suggestions: + suggestions.append(issue.suggestion) + + return suggestions diff --git a/graphistry/dgl_utils.py b/graphistry/dgl_utils.py index 0afd49facf..1d52408875 100644 --- a/graphistry/dgl_utils.py +++ b/graphistry/dgl_utils.py @@ -53,8 +53,8 @@ def convert_to_torch(X_enc: pd.DataFrame, y_enc: Optional[pd.DataFrame]): # type: ignore """ - Converts X, y to torch tensors compatible with ndata/edata of DGL graph - _________________________________________________________________________ + Converts X, y to torch tensors compatible with ndata/edata of DGL graph + :param X_enc: DataFrame Matrix of Values for Model Matrix :param y_enc: DataFrame Matrix of Values for Target :return: Dictionary of torch encoded arrays @@ -100,16 +100,17 @@ def get_available_devices(): def reindex_edgelist(df, src, dst): """Since DGL needs integer contiguous node labels, this relabels as pre-processing step - :eg + Example:: + df, ordered_nodes_dict = reindex_edgelist(df, 'to_node', 'from_node') - creates new columns given by config.SRC and config.DST + # creates new columns given by config.SRC and config.DST + :param df: edge dataFrame :param src: source column of dataframe :param dst: destination column of dataframe - - :returns - df, pandas DataFrame with new edges. - ordered_nodes_dict, dict ordered from most common src and dst nodes. + :return: Tuple of (df, ordered_nodes_dict) where: + - df: pandas DataFrame with new edges + - ordered_nodes_dict: dict ordered from most common src and dst nodes """ srclist = df[src] dstlist = df[dst] @@ -173,17 +174,20 @@ def pandas_to_dgl_graph( df: pd.DataFrame, src: str, dst: str, weight_col: Optional[str] = None, device: str = "cpu" ) -> Tuple["dgl.DGLGraph", "scipy.sparse.coo_matrix", Dict]: """Turns an edge DataFrame with named src and dst nodes, to DGL graph - :eg + + Example:: + g, sp_mat, ordered_nodes_dict = pandas_to_sparse_adjacency(df, 'to_node', 'from_node') + :param df: DataFrame with source and destination and optionally weight column :param src: source column of DataFrame for coo matrix :param dst: destination column of DataFrame for coo matrix :param weight_col: optional weight column when constructing coo matrix :param device: whether to put dgl graph on cpu or gpu - :return - g: dgl graph - sp_mat: sparse scipy matrix - ordered_nodes_dict: dict ordered from most common src and dst nodes + :return: Tuple of (g, sp_mat, ordered_nodes_dict) where: + - g: dgl graph + - sp_mat: sparse scipy matrix + - ordered_nodes_dict: dict ordered from most common src and dst nodes """ _, _, dgl = lazy_dgl_import() # noqa: F811 sp_mat, ordered_nodes_dict = pandas_to_sparse_adjacency(df, src, dst, weight_col) diff --git a/graphistry/plugins/spanner.py b/graphistry/plugins/spanner.py index 25c6431510..5ce903defd 100644 --- a/graphistry/plugins/spanner.py +++ b/graphistry/plugins/spanner.py @@ -212,11 +212,11 @@ def convert_spanner_json(data: List[Any]) -> List[Dict[str, Any]]: @staticmethod def add_type_from_label_to_df(df: pd.DataFrame) -> pd.DataFrame: - """Add 'type' column from 'label' for Graphistry type handling. + """Add ``type`` column from ``label`` for Graphistry type handling. - Creates a 'type' column from the 'label' column for proper visualization - in Graphistry. If a 'type' column already exists, it is renamed to 'type_' - before creating the new 'type' column. + Creates a ``type`` column from the ``label`` column for proper visualization + in Graphistry. If a ``type`` column already exists, it is renamed to ``type_`` + before creating the new ``type`` column. :param df: DataFrame containing node or edge data with 'label' column :type df: pd.DataFrame diff --git a/graphistry/tests/compute/test_exceptions.py b/graphistry/tests/compute/test_exceptions.py new file mode 100644 index 0000000000..22ed62fa38 --- /dev/null +++ b/graphistry/tests/compute/test_exceptions.py @@ -0,0 +1,135 @@ +"""Unit tests for GFQL exceptions module.""" + +import pytest +from graphistry.compute.gfql.exceptions import ( + GFQLException, GFQLValidationError, GFQLSyntaxError, + GFQLSchemaError, GFQLSemanticError, GFQLTypeError, + GFQLColumnNotFoundError +) +from graphistry.tests.common import NoAuthTestCase + + +class TestGFQLException(NoAuthTestCase): + """Test base GFQL exception.""" + + def test_init_without_context(self): + exc = GFQLException("Test error") + self.assertEqual(str(exc), "Test error") + self.assertEqual(exc.context, {}) + + def test_init_with_context(self): + context = {'field': 'test', 'value': 123} + exc = GFQLException("Test error", context) + self.assertEqual(exc.context, context) + + def test_str_with_context(self): + exc = GFQLException("Test error", {'field': 'test', 'value': 123}) + exc_str = str(exc) + self.assertIn("Test error", exc_str) + self.assertIn("field=test", exc_str) + self.assertIn("value=123", exc_str) + + +class TestGFQLValidationError(NoAuthTestCase): + """Test validation error hierarchy.""" + + def test_inheritance(self): + exc = GFQLValidationError("Validation failed") + self.assertIsInstance(exc, GFQLException) + self.assertIsInstance(exc, ValueError) + + def test_subclasses(self): + syntax_err = GFQLSyntaxError("Syntax error") + schema_err = GFQLSchemaError("Schema error") + semantic_err = GFQLSemanticError("Semantic error") + + self.assertIsInstance(syntax_err, GFQLValidationError) + self.assertIsInstance(schema_err, GFQLValidationError) + self.assertIsInstance(semantic_err, GFQLValidationError) + + +class TestGFQLTypeError(NoAuthTestCase): + """Test type error exception.""" + + def test_init(self): + exc = GFQLTypeError( + column='age', + column_type='string', + predicate='gt', + expected_type='numeric' + ) + + self.assertIsInstance(exc, GFQLSchemaError) + self.assertIn("'age'", str(exc)) + self.assertIn("'string'", str(exc)) + self.assertIn("'gt'", str(exc)) + self.assertIn("'numeric'", str(exc)) + + def test_context(self): + exc = GFQLTypeError('col', 'str', 'pred', 'num') + self.assertEqual(exc.context['column'], 'col') + self.assertEqual(exc.context['column_type'], 'str') + self.assertEqual(exc.context['predicate'], 'pred') + self.assertEqual(exc.context['expected_type'], 'num') + + +class TestGFQLColumnNotFoundError(NoAuthTestCase): + """Test column not found exception.""" + + def test_init(self): + exc = GFQLColumnNotFoundError( + column='missing_col', + table='node', + available_columns=['id', 'name', 'type'] + ) + + self.assertIsInstance(exc, GFQLSchemaError) + self.assertIn("'missing_col'", str(exc)) + self.assertIn("node", str(exc)) + + def test_context(self): + exc = GFQLColumnNotFoundError('col', 'table', ['a', 'b']) + self.assertEqual(exc.context['column'], 'col') + self.assertEqual(exc.context['table'], 'table') + self.assertEqual(exc.context['available_columns'], ['a', 'b']) + + +class TestExceptionUsage(NoAuthTestCase): + """Test exception usage patterns.""" + + def test_catching_specific_errors(self): + """Ensure exceptions can be caught at different levels.""" + + def raise_syntax_error(): + raise GFQLSyntaxError("Invalid syntax") + + # Can catch as GFQLSyntaxError + with self.assertRaises(GFQLSyntaxError): + raise_syntax_error() + + # Can catch as GFQLValidationError + with self.assertRaises(GFQLValidationError): + raise_syntax_error() + + # Can catch as GFQLException + with self.assertRaises(GFQLException): + raise_syntax_error() + + # Can catch as ValueError (for backwards compatibility) + with self.assertRaises(ValueError): + raise_syntax_error() + + def test_exception_messages(self): + """Test that exception messages are informative.""" + + exc1 = GFQLTypeError('age', 'object', 'gt', 'int64') + self.assertIn("Column 'age' has type 'object'", str(exc1)) + self.assertIn("predicate 'gt' expects 'int64'", str(exc1)) + + exc2 = GFQLColumnNotFoundError('foo', 'edge', ['bar', 'baz']) + self.assertIn("Column 'foo' not found", str(exc2)) + self.assertIn("edge data", str(exc2)) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/graphistry/tests/compute/test_gfql_validation.py b/graphistry/tests/compute/test_gfql_validation.py new file mode 100644 index 0000000000..1a93888f91 --- /dev/null +++ b/graphistry/tests/compute/test_gfql_validation.py @@ -0,0 +1,203 @@ +"""Test script for GFQL validation helpers.""" + +import pandas as pd + +from graphistry import edges, nodes +from graphistry.compute.ast import n, e_forward, e_reverse +from graphistry.compute.gfql.validate import ( + validate_syntax, validate_schema, validate_query, + extract_schema_from_dataframes, format_validation_errors, + Schema, ValidationIssue +) +from graphistry.compute.predicates.numeric import gt, lt +from graphistry.compute.predicates.str import contains + + +def test_syntax_validation(): + """Test syntax validation without data.""" + print("=== Testing Syntax Validation ===\n") + + # Valid query + valid_chain = [n({"type": "person"}), e_forward(), n()] + issues = validate_syntax(valid_chain) + print(f"Valid chain issues: {len(issues)}") + if issues: + print(format_validation_errors(issues)) + + # Invalid query - not a list + try: + issues = validate_syntax("not a list") + print(f"\nInvalid type issues: {len(issues)}") + print(format_validation_errors(issues)) + except Exception as e: + print(f"Error: {e}") + + # Empty chain + issues = validate_syntax([]) + print(f"\nEmpty chain issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Invalid operation type + issues = validate_syntax([n(), "not an operation", e_forward()]) + print(f"\nInvalid operation issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Orphaned edge warning + issues = validate_syntax([e_forward(), e_reverse()]) + print(f"\nOrphaned edge issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Unbounded hops warning + issues = validate_syntax([n(), e_forward(to_fixed_point=True), n()]) + print(f"\nUnbounded hops issues: {len(issues)}") + print(format_validation_errors(issues)) + + +def test_schema_validation(): + """Test schema validation with data.""" + print("\n\n=== Testing Schema Validation ===\n") + + # Create sample data + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'], + 'age': [25, 30, 5], + 'name': ['Alice', 'Bob', 'Corp'] + }) + + edges_df = pd.DataFrame({ + 'source': ['a', 'b', 'c'], + 'target': ['b', 'c', 'a'], + 'relationship': ['knows', 'works_at', 'employs'], + 'weight': [1.0, 2.0, 3.0] + }) + + # Extract schema + schema = extract_schema_from_dataframes(nodes_df, edges_df) + print(f"Schema: {schema}") + print(f"Node columns: {list(schema.node_columns.keys())}") + print(f"Edge columns: {list(schema.edge_columns.keys())}") + + # Valid query + valid_chain = [n({"type": "person"}), e_forward({"relationship": "knows"}), n()] + issues = validate_schema(valid_chain, schema) + print(f"\nValid query issues: {len(issues)}") + if issues: + print(format_validation_errors(issues)) + + # Column not found + invalid_chain = [n({"nonexistent": "value"})] + issues = validate_schema(invalid_chain, schema) + print(f"\nColumn not found issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Type mismatch - string predicate on numeric column + type_mismatch_chain = [n({"age": contains("25")})] + issues = validate_schema(type_mismatch_chain, schema) + print(f"\nType mismatch issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Valid numeric predicate + valid_numeric_chain = [n({"age": gt(20)})] + issues = validate_schema(valid_numeric_chain, schema) + print(f"\nValid numeric predicate issues: {len(issues)}") + if issues: + print(format_validation_errors(issues)) + + +def test_combined_validation(): + """Test combined validation.""" + print("\n\n=== Testing Combined Validation ===\n") + + nodes_df = pd.DataFrame({ + 'id': range(5), + 'value': [10, 20, 30, 40, 50] + }) + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 3], + 'target': [1, 2, 3, 4], + 'weight': [1.0, 2.0, 3.0, 4.0] + }) + + # Query with both syntax and schema issues + problematic_chain = [ + n({"missing_col": "value"}), # Schema error + e_forward(hops=-1), # Syntax error + n({"value": gt(25)}) + ] + + issues = validate_query(problematic_chain, nodes_df, edges_df) + print(f"Combined validation issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Show issue details + print("\nIssue details:") + for issue in issues: + print(f" - {issue.level}: {issue.message}") + if issue.operation_index is not None: + print(f" at operation {issue.operation_index}") + if issue.field: + print(f" field: {issue.field}") + + +def test_edge_validation(): + """Test edge-specific validation.""" + print("\n\n=== Testing Edge Validation ===\n") + + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['A', 'B', 'C'] + }) + + edges_df = pd.DataFrame({ + 'source': ['a', 'b'], + 'target': ['b', 'c'], + 'edge_type': ['follows', 'likes'] + }) + + schema = extract_schema_from_dataframes(nodes_df, edges_df) + + # Valid edge query + valid_chain = [ + n(), + e_forward( + edge_match={"edge_type": "follows"}, + source_node_match={"type": "A"}, + destination_node_match={"type": "B"} + ), + n() + ] + issues = validate_schema(valid_chain, schema) + print(f"Valid edge query issues: {len(issues)}") + if issues: + print(format_validation_errors(issues)) + + # Invalid edge column + invalid_chain = [ + n(), + e_forward({"missing_edge_col": "value"}), + n() + ] + issues = validate_schema(invalid_chain, schema) + print(f"\nInvalid edge column issues: {len(issues)}") + print(format_validation_errors(issues)) + + # Invalid node filter in edge + invalid_node_filter = [ + n(), + e_forward(source_node_match={"missing_node_col": "value"}), + n() + ] + issues = validate_schema(invalid_node_filter, schema) + print(f"\nInvalid node filter in edge issues: {len(issues)}") + print(format_validation_errors(issues)) + + +if __name__ == "__main__": + test_syntax_validation() + test_schema_validation() + test_combined_validation() + test_edge_validation() + + print("\n\n=== All tests completed! ===") diff --git a/graphistry/tests/compute/test_validate.py b/graphistry/tests/compute/test_validate.py new file mode 100644 index 0000000000..7b2c558c54 --- /dev/null +++ b/graphistry/tests/compute/test_validate.py @@ -0,0 +1,316 @@ +"""Unit tests for GFQL validation module.""" + +import pandas as pd +import pytest +from typing import List + +from graphistry.compute.gfql.validate import ( + validate_syntax, validate_schema, validate_query, + extract_schema, extract_schema_from_dataframes, + format_validation_errors, suggest_fixes, + ValidationIssue, Schema, + _format_error, _get_type_category +) +from graphistry.compute.ast import n, e_forward, e_reverse, e +from graphistry.compute.predicates.numeric import gt, lt, between +from graphistry.compute.predicates.str import contains, startswith +from graphistry.compute.chain import Chain +from graphistry.tests.common import NoAuthTestCase + + +class TestValidationIssue(NoAuthTestCase): + """Test ValidationIssue class.""" + + def test_init(self): + issue = ValidationIssue('error', 'Test message') + self.assertEqual(issue.level, 'error') + self.assertEqual(issue.message, 'Test message') + self.assertIsNone(issue.operation_index) + self.assertIsNone(issue.field) + self.assertIsNone(issue.suggestion) + self.assertIsNone(issue.error_type) + + def test_init_with_all_fields(self): + issue = ValidationIssue( + 'warning', 'Test warning', + operation_index=2, + field='test_field', + suggestion='Fix it', + error_type='TEST_ERROR' + ) + self.assertEqual(issue.level, 'warning') + self.assertEqual(issue.operation_index, 2) + self.assertEqual(issue.field, 'test_field') + self.assertEqual(issue.suggestion, 'Fix it') + self.assertEqual(issue.error_type, 'TEST_ERROR') + + def test_repr(self): + issue = ValidationIssue('error', 'Test message', suggestion='Fix it') + repr_str = repr(issue) + self.assertIn('ERROR', repr_str) + self.assertIn('Test message', repr_str) + self.assertIn('Fix it', repr_str) + + def test_to_dict(self): + issue = ValidationIssue( + 'error', 'Test', + operation_index=1, + field='field', + suggestion='Suggestion', + error_type='TYPE' + ) + d = issue.to_dict() + self.assertEqual(d['level'], 'error') + self.assertEqual(d['message'], 'Test') + self.assertEqual(d['operation_index'], 1) + self.assertEqual(d['field'], 'field') + self.assertEqual(d['suggestion'], 'Suggestion') + self.assertEqual(d['error_type'], 'TYPE') + + +class TestSchema(NoAuthTestCase): + """Test Schema class.""" + + def test_init_empty(self): + schema = Schema() + self.assertEqual(schema.node_columns, {}) + self.assertEqual(schema.edge_columns, {}) + + def test_init_with_data(self): + node_cols = {'id': 'int64', 'name': 'object'} + edge_cols = {'source': 'int64', 'target': 'int64'} + schema = Schema(node_cols, edge_cols) + self.assertEqual(schema.node_columns, node_cols) + self.assertEqual(schema.edge_columns, edge_cols) + + def test_repr(self): + schema = Schema({'id': 'int64', 'name': 'object'}, {'source': 'int64'}) + repr_str = repr(schema) + self.assertIn('nodes', repr_str) + self.assertIn('edges', repr_str) + self.assertIn('id', repr_str) + self.assertIn('name', repr_str) + + +class TestSyntaxValidation(NoAuthTestCase): + """Test syntax validation functions.""" + + def test_valid_chain(self): + chain = [n({"type": "person"}), e_forward(), n()] + issues = validate_syntax(chain) + self.assertEqual(len(issues), 0) + + def test_invalid_chain_type(self): + issues = validate_syntax("not a list") + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].level, 'error') + self.assertEqual(issues[0].error_type, 'INVALID_CHAIN_TYPE') + + def test_empty_chain(self): + issues = validate_syntax([]) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].level, 'error') + self.assertEqual(issues[0].error_type, 'EMPTY_CHAIN') + + def test_invalid_operation(self): + chain = [n(), "not an operation", e_forward()] + issues = validate_syntax(chain) + errors = [i for i in issues if i.level == 'error'] + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0].operation_index, 1) + self.assertEqual(errors[0].error_type, 'INVALID_OPERATION') + + def test_invalid_filter_key(self): + chain = [n({123: "value"})] # Non-string key + issues = validate_syntax(chain) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].error_type, 'INVALID_FILTER_KEY') + + def test_invalid_hops(self): + chain = [n(), e_forward(hops=-1), n()] + issues = validate_syntax(chain) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].error_type, 'INVALID_HOPS') + + def test_unbounded_hops_warning(self): + chain = [n(), e_forward(to_fixed_point=True), n()] + issues = validate_syntax(chain) + warnings = [i for i in issues if i.level == 'warning'] + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].error_type, 'UNBOUNDED_HOPS_WARNING') + + def test_orphaned_edge_warning(self): + chain = [e_forward(), e_reverse()] + issues = validate_syntax(chain) + warnings = [i for i in issues if i.level == 'warning'] + self.assertEqual(len(warnings), 2) + self.assertTrue(all(w.error_type == 'ORPHANED_EDGE' for w in warnings)) + + def test_chain_object(self): + chain_obj = Chain([n(), e_forward(), n()]) + issues = validate_syntax(chain_obj) + self.assertEqual(len(issues), 0) + + +class TestSchemaValidation(NoAuthTestCase): + """Test schema validation functions.""" + + def setUp(self): + super().setUp() + self.schema = Schema( + node_columns={'id': 'int64', 'name': 'object', 'age': 'int64'}, + edge_columns={'source': 'int64', 'target': 'int64', 'type': 'object', 'weight': 'float64'} + ) + + def test_valid_query(self): + chain = [n({"name": "Alice"}), e_forward({"type": "knows"}), n()] + issues = validate_schema(chain, self.schema) + self.assertEqual(len(issues), 0) + + def test_column_not_found_node(self): + chain = [n({"missing_col": "value"})] + issues = validate_schema(chain, self.schema) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].error_type, 'COLUMN_NOT_FOUND') + self.assertIn('node', issues[0].message) + + def test_column_not_found_edge(self): + chain = [n(), e_forward({"missing_edge_col": "value"}), n()] + issues = validate_schema(chain, self.schema) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].error_type, 'COLUMN_NOT_FOUND') + self.assertIn('edge', issues[0].message) + + def test_type_mismatch_numeric(self): + chain = [n({"age": contains("25")})] # String predicate on numeric + issues = validate_schema(chain, self.schema) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].error_type, 'TYPE_MISMATCH') + self.assertIn('string', issues[0].message) + + def test_type_mismatch_string(self): + chain = [n({"name": gt(10)})] # Numeric predicate on string + issues = validate_schema(chain, self.schema) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].error_type, 'TYPE_MISMATCH') + self.assertIn('numeric', issues[0].message) + + def test_edge_node_filters(self): + chain = [ + n(), + e_forward( + source_node_match={"missing": "value"}, + destination_node_match={"id": 1} + ), + n() + ] + issues = validate_schema(chain, self.schema) + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].field, 'source_node_match.missing') + + +class TestValidateQuery(NoAuthTestCase): + """Test combined validation function.""" + + def test_without_data(self): + chain = [n(), e_forward(hops=-1), n()] + issues = validate_query(chain) + # Should only do syntax validation + self.assertTrue(any(i.error_type == 'INVALID_HOPS' for i in issues)) + + def test_with_data(self): + nodes_df = pd.DataFrame({'id': [1, 2, 3], 'type': ['A', 'B', 'C']}) + edges_df = pd.DataFrame({'source': [1, 2], 'target': [2, 3]}) + + chain = [n({"missing": "value"})] + issues = validate_query(chain, nodes_df, edges_df) + # Should include schema validation + self.assertTrue(any(i.error_type == 'COLUMN_NOT_FOUND' for i in issues)) + + +class TestSchemaExtraction(NoAuthTestCase): + """Test schema extraction functions.""" + + def test_extract_from_dataframes(self): + nodes_df = pd.DataFrame({'id': [1, 2], 'name': ['A', 'B']}) + edges_df = pd.DataFrame({'source': [1], 'target': [2], 'weight': [1.0]}) + + schema = extract_schema_from_dataframes(nodes_df, edges_df) + + self.assertIn('id', schema.node_columns) + self.assertIn('name', schema.node_columns) + self.assertIn('source', schema.edge_columns) + self.assertIn('weight', schema.edge_columns) + self.assertEqual(schema.edge_columns['weight'], 'float64') + + def test_extract_none_dataframes(self): + schema = extract_schema_from_dataframes(None, None) + self.assertEqual(schema.node_columns, {}) + self.assertEqual(schema.edge_columns, {}) + + def test_extract_partial_dataframes(self): + nodes_df = pd.DataFrame({'id': [1, 2]}) + schema = extract_schema_from_dataframes(nodes_df, None) + self.assertIn('id', schema.node_columns) + self.assertEqual(schema.edge_columns, {}) + + +class TestErrorFormatting(NoAuthTestCase): + """Test error formatting functions.""" + + def test_format_no_issues(self): + result = format_validation_errors([]) + self.assertEqual(result, "No validation issues found.") + + def test_format_errors_and_warnings(self): + issues = [ + ValidationIssue('error', 'Error 1', operation_index=0, suggestion='Fix 1'), + ValidationIssue('warning', 'Warning 1', operation_index=1, suggestion='Fix 2') + ] + result = format_validation_errors(issues) + self.assertIn('ERRORS (1)', result) + self.assertIn('WARNINGS (1)', result) + self.assertIn('Error 1', result) + self.assertIn('Warning 1', result) + self.assertIn('Fix 1', result) + + def test_suggest_fixes(self): + issues = [ + ValidationIssue('error', 'Column not found', error_type='COLUMN_NOT_FOUND', field='col1'), + ValidationIssue('error', 'Column not found', error_type='COLUMN_NOT_FOUND', field='col2'), + ValidationIssue('error', 'Type mismatch', error_type='TYPE_MISMATCH') + ] + suggestions = suggest_fixes([], issues) + self.assertTrue(any('Missing columns' in s for s in suggestions)) + self.assertTrue(any('Type mismatches' in s for s in suggestions)) + + +class TestHelperFunctions(NoAuthTestCase): + """Test internal helper functions.""" + + def test_format_error(self): + message, suggestion = _format_error('EMPTY_CHAIN') + self.assertEqual(message, 'Chain is empty') + self.assertIn('Add at least one operation', suggestion) + + def test_format_error_with_kwargs(self): + message, suggestion = _format_error('COLUMN_NOT_FOUND', + column='test', + table='node', + available='id, name') + self.assertIn('test', message) + self.assertIn('node', message) + self.assertIn('id, name', suggestion) + + def test_get_type_category(self): + self.assertEqual(_get_type_category('int64'), 'numeric') + self.assertEqual(_get_type_category('float32'), 'numeric') + self.assertEqual(_get_type_category('object'), 'string') + self.assertEqual(_get_type_category('str'), 'string') + self.assertEqual(_get_type_category('datetime64'), 'temporal') + self.assertEqual(_get_type_category('bool'), 'boolean') + self.assertEqual(_get_type_category('custom_type'), 'unknown') + + +if __name__ == '__main__': + pytest.main([__file__])