diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e6b7091f9..cac98b405f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -389,11 +389,6 @@ jobs: run: | ./bin/build.sh - - name: Validate py.typed in wheel - run: | - unzip -l dist/graphistry*.whl | grep -q "graphistry/py.typed" || (echo "ERROR: py.typed marker missing from wheel - users won't get type information" && exit 1) - echo "✅ py.typed marker confirmed in wheel distribution" - test-docs: needs: [changes, python-lint-types] diff --git a/.gitignore b/.gitignore index 1c7cd046f7..f01581d693 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ docs/source/demos # AI assistant working directories AI_PROGRESS/ PLAN.md +test_env/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 917241168f..5baf1137a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,17 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Dev -## [0.41.0 - 2025-07-26] - ### Added -* Typing: Add PEP 561 type distribution support (#714) - * Add py.typed marker file to enable type checking with mypy, pyright, and PyCharm - * Configure MANIFEST.in and setup.cfg to include py.typed in source and wheel distributions - * Add CI validation to prevent regressions where py.typed might be accidentally removed - * Enables accurate type information and autocompletion for PyGraphistry APIs +* GFQL: Add comprehensive validation framework with detailed error reporting + * Built-in validation: `Chain()` constructor validates syntax automatically + * Schema validation: `validate_chain_schema()` validates queries against DataFrame schemas + * Pre-execution validation: `g.chain(ops, validate_schema=True)` catches errors before execution + * Structured error types: `GFQLValidationError`, `GFQLSyntaxError`, `GFQLTypeError`, `GFQLSchemaError` + * Error codes (E1xx syntax, E2xx type, E3xx schema) for programmatic error handling + * Collect-all mode: `validate(collect_all=True)` returns all errors instead of fail-fast + * JSON validation: `Chain.from_json()` validates during parsing for safe LLM integration + * Helpful error suggestions for common mistakes + * Example notebook: `demos/gfql/gfql_validation_fundamentals.ipynb` ### Fixed * Docs: Fix notebook validation error in hop_and_chain_graph_pattern_mining.ipynb by adding missing 'outputs' field to code cell diff --git a/MANIFEST.in b/MANIFEST.in index e7c5c401be..adf9e9ba45 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include versioneer.py include graphistry/_version.py -include graphistry/py.typed diff --git a/demos/gfql/gfql_remote.ipynb b/demos/gfql/gfql_remote.ipynb index 11440662cd..2a0a39a032 100644 --- a/demos/gfql/gfql_remote.ipynb +++ b/demos/gfql/gfql_remote.ipynb @@ -1401,6 +1401,82 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "markdown", + "id": "fs1pabrqfaj", + "source": "## Combining Let Bindings with Call Operations\n\nLet bindings in GFQL allow you to create named intermediate results and compose complex operations. When combined with call operations in remote mode, you can orchestrate sophisticated graph analyses entirely on the server, minimizing data transfer and leveraging server-side GPU acceleration.", + "metadata": {} + }, + { + "cell_type": "markdown", + "id": "bs7rghntlp", + "source": "### Example 1: PageRank Analysis with Filtering\n\nThis example demonstrates using let bindings to:\n1. Compute PageRank scores\n2. Filter high-value nodes\n3. Extract subgraphs around important nodes\n4. Return results for visualization", + "metadata": {} + }, + { + "cell_type": "code", + "id": "wurwk0xplp", + "source": "# Create a more complex graph for demonstration\ncomplex_edges = pd.DataFrame({\n 's': ['a', 'b', 'c', 'd', 'e', 'f', 'a', 'b', 'c', 'd'],\n 'd': ['b', 'c', 'd', 'e', 'f', 'a', 'c', 'd', 'e', 'f'],\n 'weight': [1, 2, 1, 3, 1, 2, 1, 2, 1, 1],\n 'type': ['follow', 'mention', 'follow', 'follow', 'mention', 'follow', 'mention', 'follow', 'follow', 'mention']\n})\n\ng_complex = graphistry.edges(complex_edges, 's', 'd').upload()\nprint(f\"Uploaded graph with {len(complex_edges)} edges\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "mwr3948llv", + "source": "%%time\n\n# Define a complex query using let bindings with call operations\npagerank_analysis_query = {\n 'let': {\n # Step 1: Compute PageRank scores\n 'with_pagerank': {\n 'call': {\n 'method': 'compute_pagerank',\n 'args': [],\n 'kwargs': {}\n }\n },\n \n # Step 2: Filter nodes with high PageRank scores\n 'important_nodes': {\n 'pipe': [\n {'var': 'with_pagerank'},\n {\n 'n': {\n 'filter': {\n 'gte': [\n {'col': 'pagerank'},\n 0.15 # Threshold for \"important\" nodes\n ]\n }\n }\n }\n ]\n },\n \n # Step 3: Get 1-hop neighborhoods of important nodes\n 'important_neighborhoods': {\n 'pipe': [\n {'var': 'important_nodes'},\n {'e_undirected': {'hops': 1}},\n {'n': {}}\n ]\n }\n },\n \n # Return the final result\n 'in': {'var': 'important_neighborhoods'}\n}\n\n# Execute the query remotely\nresult = g_complex.chain_remote([pagerank_analysis_query])\n\nprint(f\"Result has {len(result._nodes)} nodes and {len(result._edges)} edges\")\nprint(\"\\nNodes with PageRank scores:\")\nprint(result._nodes)", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "jdzoowghv2q", + "source": "### Example 2: Multi-Stage Analysis with Different Edge Types\n\nThis example shows how to use let bindings to analyze different edge types separately and combine the results:", + "metadata": {} + }, + { + "cell_type": "code", + "id": "40m2wl0yn3m", + "source": "%%time\n\n# Analyze different edge types and combine results\nedge_type_analysis = {\n 'let': {\n # Analyze follow edges\n 'follow_network': {\n 'pipe': [\n {\n 'e_undirected': {\n 'filter': {\n 'eq': [\n {'col': 'type'},\n 'follow'\n ]\n }\n }\n },\n {'n': {}}\n ]\n },\n \n # Compute centrality on follow network\n 'follow_centrality': {\n 'pipe': [\n {'var': 'follow_network'},\n {\n 'call': {\n 'method': 'compute_degree_centrality',\n 'args': [],\n 'kwargs': {}\n }\n }\n ]\n },\n \n # Find mention patterns\n 'mention_edges': {\n 'e_undirected': {\n 'filter': {\n 'eq': [\n {'col': 'type'},\n 'mention'\n ]\n }\n }\n },\n \n # Get nodes that are both highly connected and frequently mentioned\n 'influential_nodes': {\n 'pipe': [\n {'var': 'follow_centrality'},\n {\n 'n': {\n 'filter': {\n 'gte': [\n {'col': 'degree_centrality'},\n 0.5\n ]\n }\n }\n },\n {'var': 'mention_edges'},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'var': 'influential_nodes'}\n}\n\ninfluential_result = g_complex.chain_remote([edge_type_analysis])\n\nprint(f\"Found {len(influential_result._nodes)} influential nodes\")\nprint(f\"Connected by {len(influential_result._edges)} edges\")\nprint(\"\\nInfluential nodes with centrality scores:\")\nprint(influential_result._nodes)", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "5y02h8p7mkk", + "source": "### Example 3: Conditional Analysis with Let Bindings\n\nThis example demonstrates using let bindings to perform conditional analysis based on graph properties:", + "metadata": {} + }, + { + "cell_type": "code", + "id": "gmy30drvbts", + "source": "%%time\n\n# Complex analysis with multiple algorithms\ncomprehensive_analysis = {\n 'let': {\n # Base graph with all computations\n 'enriched_graph': {\n 'pipe': [\n {'call': {'method': 'compute_pagerank', 'args': [], 'kwargs': {}}},\n {'call': {'method': 'compute_degree_centrality', 'args': [], 'kwargs': {}}},\n # Add edge weight normalization\n {\n 'call': {\n 'method': 'edges',\n 'args': [],\n 'kwargs': {\n 'normalize': {\n 'weight': {\n 'min': 0,\n 'max': 1\n }\n }\n }\n }\n }\n ]\n },\n \n # Find bridge nodes (high betweenness)\n 'bridge_nodes': {\n 'pipe': [\n {'var': 'enriched_graph'},\n {\n 'n': {\n 'filter': {\n 'and': [\n {'gte': [{'col': 'pagerank'}, 0.1]},\n {'lte': [{'col': 'degree_centrality'}, 0.7]}\n ]\n }\n }\n }\n ]\n },\n \n # Find hub nodes (high degree centrality)\n 'hub_nodes': {\n 'pipe': [\n {'var': 'enriched_graph'},\n {\n 'n': {\n 'filter': {\n 'gte': [{'col': 'degree_centrality'}, 0.7]\n }\n }\n }\n ]\n },\n \n # Get connections between bridges and hubs\n 'critical_paths': {\n 'pipe': [\n {'var': 'bridge_nodes'},\n {'e_forward': {'to_nodes': {'var': 'hub_nodes'}}},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'var': 'critical_paths'}\n}\n\n# Execute remotely with GPU acceleration\ncritical_paths_result = g_complex.chain_remote([comprehensive_analysis], engine='cudf')\n\nprint(f\"Critical paths network: {len(critical_paths_result._nodes)} nodes, {len(critical_paths_result._edges)} edges\")\n\n# Check if we got results\nif len(critical_paths_result._nodes) > 0:\n print(\"\\nCritical path nodes:\")\n print(critical_paths_result._nodes)\nelse:\n print(\"\\nNo critical paths found with current thresholds\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "wbt02937wz", + "source": "### Example 4: Visualization-Ready Analysis\n\nThis example shows how to prepare data for visualization by enriching it with multiple metrics and creating a focused subgraph:", + "metadata": {} + }, + { + "cell_type": "code", + "id": "4glzmgi1u3s", + "source": "%%time\n\n# Prepare visualization-ready data with all enrichments\nviz_prep_query = {\n 'let': {\n # Compute all metrics\n 'with_metrics': {\n 'pipe': [\n {'call': {'method': 'compute_pagerank', 'args': [], 'kwargs': {}}},\n {'call': {'method': 'compute_degree_centrality', 'args': [], 'kwargs': {}}},\n # Add node colors based on PageRank\n {\n 'call': {\n 'method': 'nodes',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'node_color': {\n 'case': [\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.2]},\n 'then': 65280 # Green for high PageRank\n },\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.15]},\n 'then': 16776960 # Yellow for medium\n }\n ],\n 'else': 16711680 # Red for low\n },\n 'node_size': {\n 'mul': [\n {'col': 'degree_centrality'},\n 50 # Scale factor\n ]\n }\n }\n }\n }\n }\n ]\n },\n \n # Add edge styling based on type and weight\n 'styled_graph': {\n 'pipe': [\n {'var': 'with_metrics'},\n {\n 'call': {\n 'method': 'edges',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'edge_color': {\n 'case': [\n {\n 'when': {'eq': [{'col': 'type'}, 'follow']},\n 'then': 255 # Blue for follows\n }\n ],\n 'else': 16711935 # Magenta for mentions\n },\n 'edge_weight': {\n 'col': 'weight'\n }\n }\n }\n }\n }\n ]\n },\n \n # Focus on top nodes and their connections\n 'viz_subgraph': {\n 'pipe': [\n {'var': 'styled_graph'},\n {\n 'n': {\n 'filter': {\n 'or': [\n {'gte': [{'col': 'pagerank'}, 0.15]},\n {'gte': [{'col': 'degree_centrality'}, 0.6]}\n ]\n }\n }\n },\n {'e_undirected': {'hops': 1}},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'var': 'viz_subgraph'}\n}\n\n# Get visualization-ready data\nviz_result = g_complex.chain_remote([viz_prep_query])\n\nprint(f\"Visualization subgraph: {len(viz_result._nodes)} nodes, {len(viz_result._edges)} edges\")\nprint(\"\\nNodes with visualization attributes:\")\nprint(viz_result._nodes)\nprint(\"\\nEdges with styling:\")\nprint(viz_result._edges)\n\n# Ready to visualize\n# viz_result.plot() # Uncomment to create visualization", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "p1o68dnaemk", + "source": "### Key Benefits of Let Bindings with Remote Calls\n\n1. **Server-Side Orchestration**: All operations happen on the server, minimizing data transfer\n2. **Named Intermediate Results**: Create readable, reusable steps in complex analyses\n3. **GPU Acceleration**: Leverage server GPU for compute-intensive operations like PageRank\n4. **Composability**: Build complex workflows from simple building blocks\n5. **Efficiency**: Avoid redundant computations by reusing named results\n\nWhen working with large graphs, this approach is particularly powerful as it allows you to:\n- Perform multiple analyses without downloading intermediate results\n- Chain together different algorithms and filters\n- Prepare visualization-ready data entirely on the server\n- Return only the final, filtered results you need", + "metadata": {} } ], "metadata": { @@ -1424,4 +1500,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/demos/gfql/gfql_validation_fundamentals.ipynb b/demos/gfql/gfql_validation_fundamentals.ipynb index d59cd80bf6..9736f6d08b 100644 --- a/demos/gfql/gfql_validation_fundamentals.ipynb +++ b/demos/gfql/gfql_validation_fundamentals.ipynb @@ -139,28 +139,25 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Pre-Execution Validation\n\nYou have two options for validating queries:\n\n1. **Validate-only** (no execution): Use `validate_chain_schema()` to check compatibility without running the query\n2. **Validate-and-run**: Use `g.chain(..., validate_schema=True)` to validate before execution\n\nThis is useful for catching errors early, especially in production systems." + "source": [ + "## Pre-Execution Validation\n", + "\n", + "For better performance, you can validate queries before execution:" + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Validate AND run (stops at validation if invalid)\nprint(\"Method 1: Validate-and-run with validate_schema=True\")\ntry:\n result = g.chain(chain_to_test.chain, validate_schema=True)\n print(\"Query executed successfully\")\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (check) No graph operations were performed\")\n print(\" (check) Query was rejected before execution\")" + "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Use validate_schema parameter\ntry:\n result = g.chain(chain_to_test.chain, validate_schema=True)\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (No graph operations were performed)\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Method 2: Validate ONLY (no execution)\nprint(\"\\nMethod 2: Validate-only with validate_chain_schema()\")\nfrom graphistry.compute.validate_schema import validate_chain_schema\n\n# Check if chain is compatible with graph schema WITHOUT running it\ntry:\n validate_chain_schema(g, chain_to_test)\n print(\"Chain is valid for this graph schema\")\n print(\"Note: No query execution occurred - only validation!\")\nexcept GFQLSchemaError as e:\n print(f\"Schema incompatibility detected: {e}\")\n print(\" (check) This was validation-only - no query was executed\")\n print(\" (check) Use this method to test queries before running them\")" - }, - { - "cell_type": "code", - "source": "# Example: Demonstrating the difference\nprint(\"=== Demonstrating the difference ===\\n\")\n\n# Create a valid chain\nvalid_chain = Chain([\n n({'type': 'customer'}),\n e_forward()\n])\n\n# Validate-only: Just checks, doesn't run\nprint(\"1. Validate-only with validate_chain_schema():\")\ntry:\n validate_chain_schema(g, valid_chain)\n print(\" (check) Validation passed\")\n print(\" (check) Query NOT executed\")\n print(\" (check) No result object returned\")\nexcept GFQLSchemaError as e:\n print(f\" (x) Validation failed: {e}\")\n\n# Validate-and-run: Validates first, then executes if valid\nprint(\"\\n2. Validate-and-run with g.chain(..., validate_schema=True):\")\ntry:\n result = g.chain(valid_chain.chain, validate_schema=True)\n print(\" (check) Validation passed\")\n print(\" (check) Query WAS executed\")\n print(f\" (check) Result: {len(result._nodes)} nodes, {len(result._edges)} edges\")\nexcept GFQLSchemaError as e:\n print(f\" (x) Validation failed: {e}\")\n print(\" (x) Query NOT executed\")", - "metadata": {}, - "execution_count": null, - "outputs": [] + "source": "# Method 2: Validate chain object directly\nfrom graphistry.compute.validate_schema import validate_chain_schema\n\n# Check if chain is compatible with graph schema\ntry:\n validate_chain_schema(g, chain_to_test)\n print(\"Chain is valid for this graph schema\")\nexcept GFQLSchemaError as e:\n print(f\"Schema incompatibility: {e}\")" }, { "cell_type": "markdown", @@ -190,7 +187,58 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Comprehensive error handling example\ndef safe_chain_execution(g, operations):\n \"\"\"Execute chain with proper error handling.\"\"\"\n try:\n # Create chain\n chain = Chain(operations)\n \n # Pre-validate if desired\n # errors = chain.validate_schema(g, collect_all=True)\n # if errors:\n # print(f\"Warning: {len(errors)} schema issues found\")\n \n # Execute\n result = g.chain(operations)\n return result\n \n except GFQLSyntaxError as e:\n print(f\"Syntax Error [{e.code}]: {e.message}\")\n if e.context.get('suggestion'):\n print(f\" Try: {e.context['suggestion']}\")\n return None\n \n except GFQLTypeError as e:\n print(f\"Type Error [{e.code}]: {e.message}\")\n print(f\" Field: {e.context.get('field')}\")\n print(f\" Value: {e.context.get('value')}\")\n return None\n \n except GFQLSchemaError as e:\n print(f\"Schema Error [{e.code}]: {e.message}\")\n if e.code == ErrorCode.E301:\n print(\" Column not found in data\")\n elif e.code == ErrorCode.E302:\n print(\" Type mismatch between query and data\")\n return None\n\n# Test with valid query\nprint(\"Valid query:\")\nresult = safe_chain_execution(g, [\n n({'type': 'customer'}),\n e_forward()\n])\nif result:\n print(f\" Success! Found {len(result._nodes)} nodes\")\n\n# Test with invalid query\nprint(\"\\nInvalid query:\")\nresult = safe_chain_execution(g, [\n n({'invalid_column': 'value'})\n])" + "source": [ + "# Comprehensive error handling example\n", + "def safe_chain_execution(g, operations):\n", + " \"\"\"Execute chain with proper error handling.\"\"\"\n", + " try:\n", + " # Create chain\n", + " chain = Chain(operations)\n", + " \n", + " # Pre-validate if desired\n", + " # errors = chain.validate_schema(g, collect_all=True)\n", + " # if errors:\n", + " # print(f\"Warning: {len(errors)} schema issues found\")\n", + " \n", + " # Execute\n", + " result = g.chain(operations)\n", + " return result\n", + " \n", + " except GFQLSyntaxError as e:\n", + " print(f\"Syntax Error [{e.code}]: {e.message}\")\n", + " if e.context.get('suggestion'):\n", + " print(f\" Try: {e.context['suggestion']}\")\n", + " return None\n", + " \n", + " except GFQLTypeError as e:\n", + " print(f\"Type Error [{e.code}]: {e.message}\")\n", + " print(f\" Field: {e.context.get('field')}\")\n", + " print(f\" Value: {e.context.get('value')}\")\n", + " return None\n", + " \n", + " except GFQLSchemaError as e:\n", + " print(f\"Schema Error [{e.code}]: {e.message}\")\n", + " if e.code == ErrorCode.E301:\n", + " print(\" Column not found in data\")\n", + " elif e.code == ErrorCode.E302:\n", + " print(\" Type mismatch between query and data\")\n", + " return None\n", + "\n", + "# Test with valid query\n", + "print(\"Valid query:\")\n", + "result = safe_chain_execution(g, [\n", + " n({'type': 'customer'}),\n", + " e_forward()\n", + "])\n", + "if result:\n", + " print(f\" Success! Found {len(result._nodes)} nodes\")\n", + "\n", + "# Test with invalid query\n", + "print(\"\\nInvalid query:\")\n", + "result = safe_chain_execution(g, [\n", + " n({'invalid_column': 'value'})\n", + "])" + ] }, { "cell_type": "markdown", diff --git a/demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb b/demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb index e07f836f9d..10a3f88326 100644 --- a/demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb +++ b/demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb @@ -17,7 +17,7 @@ "This tutorial demonstrates how to use PyGraphistry's `hop()` and `gfql()` methods for graph pattern mining and traversal.\n", "\n", "**Key concepts:**\n", - "- `g.hop()`: Filter by source node \u2192 edge \u2192 destination node patterns\n", + "- `g.hop()`: Filter by source node → edge → destination node patterns\n", "- `g.gfql()`: Chain multiple node and edge filters for complex patterns\n", "- Predicates: Use comparisons, string matching, and other filters\n", "- Result labeling: Name intermediate results for analysis\n", @@ -312,7 +312,7 @@ " \n", " \n", "\n", - "

475 rows \u00d7 7 columns

\n", + "

475 rows × 7 columns

\n", "\n", "
\n", "\n", @@ -1530,13 +1530,49 @@ ] }, { - "cell_type": "code", + "cell_type": "markdown", "execution_count": null, "metadata": { "id": "w3w4RRYkWXKo" }, "outputs": [], - "source": [] + "source": "## 7. Pattern Reuse with Let Bindings\n\nThe `let` operator allows you to define named graph patterns that can be referenced multiple times in your query. This is particularly useful for:\n- Creating reusable pattern components\n- Building complex patterns from simpler building blocks\n- Avoiding repetition in pattern definitions\n\nLet's explore how to use `let` bindings for finding triangles and other complex patterns." + }, + { + "cell_type": "code", + "source": "# Finding triangles using let bindings\n# Define a reusable pattern for high-influence nodes (top 30% pagerank)\ntop_30_pr = g2._nodes.pagerank.quantile(0.7)\n\n# Find triangles of high-influence members\ng_triangles = g2.gfql([\n {\n 'let': {\n # Define a pattern for high-influence nodes\n 'influential': n({'pagerank': ge(top_30_pr)}),\n # Define a pattern for strong connections\n 'strong_edge': e_undirected({'weight': ge(0.01)})\n }\n },\n # Use the defined patterns to find triangles\n {'pattern': 'influential', 'name': 'node_a'},\n {'pattern': 'strong_edge'},\n {'pattern': 'influential', 'name': 'node_b'},\n {'pattern': 'strong_edge'},\n {'pattern': 'influential', 'name': 'node_c'},\n {'pattern': 'strong_edge'},\n {'pattern': 'influential', 'name': 'node_a'} # Close the triangle\n])\n\nprint(f\"Found {len(g_triangles._nodes)} nodes in triangles\")\nprint(f\"Found {len(g_triangles._edges)} edges in triangles\")\n\n# Visualize the triangles\ng_triangles.encode_point_color('community_infomap', as_categorical=True).plot()", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": "### Finding Community Bridge Patterns with Let\n\nLet's use `let` to define reusable patterns for finding members who bridge different communities:", + "metadata": {} + }, + { + "cell_type": "code", + "source": "# Find members who bridge communities using let bindings\ng_community_bridges = g2.gfql([\n {\n 'let': {\n # Pattern for community 0 members\n 'community_0': n({'community_infomap': 0}),\n # Pattern for community 1 members \n 'community_1': n({'community_infomap': 1}),\n # Pattern for community 2 members\n 'community_2': n({'community_infomap': 2}),\n # Pattern for any edge\n 'any_edge': e_undirected()\n }\n },\n # Find paths from community 0 to community 1 through community 2\n {'pattern': 'community_0', 'name': 'start'},\n {'pattern': 'any_edge'},\n {'pattern': 'community_2', 'name': 'bridge'},\n {'pattern': 'any_edge'},\n {'pattern': 'community_1', 'name': 'end'}\n])\n\nprint(f\"Found {len(g_community_bridges._nodes)} nodes in bridging pattern\")\nbridges = g_community_bridges._nodes[g_community_bridges._nodes.bridge]\nprint(f\"Community 2 members acting as bridges: {list(bridges.title.values)}\")\n\n# Visualize with bridge nodes highlighted\ng_community_bridges.encode_point_color(\n 'bridge',\n as_categorical=True,\n categorical_mapping={\n True: 'red',\n False: 'lightgray'\n }\n).encode_point_size('bridge', categorical_mapping={True: 80, False: 40}).plot()", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": "### Complex Pattern Composition with Let\n\nLet's create more sophisticated patterns by composing smaller patterns:", + "metadata": {} + }, + { + "cell_type": "code", + "source": "# Find star patterns around influential nodes\n# A star pattern is where one central node connects to multiple others\n\ng_star_patterns = g2.gfql([\n {\n 'let': {\n # Very influential nodes (top 10%)\n 'very_influential': n({'pagerank': ge(g2._nodes.pagerank.quantile(0.9))}),\n # Moderately influential nodes (top 50%)\n 'moderately_influential': n({'pagerank': ge(g2._nodes.pagerank.quantile(0.5))}),\n # Strong bidirectional connection\n 'strong_connection': e_undirected({'weight': ge(0.02)})\n }\n },\n # Find star patterns: very influential center connected to multiple moderately influential nodes\n {'pattern': 'very_influential', 'name': 'center'},\n {'pattern': 'strong_connection'},\n {'pattern': 'moderately_influential', 'name': 'spoke1'},\n # Return to center\n e_undirected(),\n {'pattern': 'very_influential', 'name': 'center'},\n {'pattern': 'strong_connection'},\n {'pattern': 'moderately_influential', 'name': 'spoke2'},\n # Return to center again\n e_undirected(),\n {'pattern': 'very_influential', 'name': 'center'},\n {'pattern': 'strong_connection'},\n {'pattern': 'moderately_influential', 'name': 'spoke3'}\n])\n\nprint(f\"Found {len(g_star_patterns._nodes)} nodes in star patterns\")\ncenters = g_star_patterns._nodes[g_star_patterns._nodes.center]\nprint(f\"Central nodes: {list(centers.title.unique())[:5]}...\") # Show first 5\n\n# Visualize with centers highlighted\ng_star_patterns.encode_point_color(\n 'center',\n as_categorical=True,\n categorical_mapping={\n True: 'gold',\n False: 'lightblue'\n }\n).encode_point_size(\n 'center',\n categorical_mapping={True: 100, False: 50}\n).plot()", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": "### Benefits of Let Bindings\n\nThe `let` operator provides several advantages:\n\n1. **Reusability**: Define a pattern once and use it multiple times\n2. **Readability**: Give meaningful names to complex patterns\n3. **Maintainability**: Change pattern definitions in one place\n4. **Composability**: Build complex patterns from simpler components\n\nThis makes it easier to explore and mine complex graph patterns in your data!", + "metadata": {} } ], "metadata": { diff --git a/graphistry/compute/ASTSerializable.py b/graphistry/compute/ASTSerializable.py index 0ee1ad5d11..8a36535dc0 100644 --- a/graphistry/compute/ASTSerializable.py +++ b/graphistry/compute/ASTSerializable.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import Dict, List, Optional, TYPE_CHECKING +from typing import Dict, List, Optional, Sequence, TYPE_CHECKING from graphistry.utils.json import JSONVal, serialize_to_json_val @@ -68,11 +68,11 @@ def _validate_fields(self) -> None: """ pass - def _get_child_validators(self) -> List['ASTSerializable']: + def _get_child_validators(self) -> Sequence['ASTSerializable']: """Override in subclasses to return child AST nodes that need validation. Returns: - List of child AST nodes to validate + Sequence of child AST nodes to validate """ return [] diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 2b5ffd7710..99a59117fe 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -1,4 +1,5 @@ -import numpy as np, pandas as pd +import numpy as np +import pandas as pd from typing import Any, List, Union from inspect import getmodule @@ -6,7 +7,11 @@ from graphistry.Plottable import Plottable from graphistry.util import setup_logger from .chain import chain as chain_base -from .chain_remote import chain_remote as chain_remote_base, chain_remote_shape as chain_remote_shape_base +from .gfql_unified import gfql as gfql_base +from .chain_remote import ( + chain_remote as chain_remote_base, + chain_remote_shape as chain_remote_shape_base +) from .python_remote import ( python_remote_g as python_remote_g_base, python_remote_table as python_remote_table_base, @@ -460,16 +465,76 @@ def filter_edges_by_dict(self, *args, **kwargs): filter_edges_by_dict.__doc__ = filter_edges_by_dict_base.__doc__ def chain(self, *args, **kwargs): + """ + .. deprecated:: 2.XX.X + Use :meth:`gfql` instead for a unified API that supports both chains and DAGs. + """ + import warnings + warnings.warn( + "chain() is deprecated. Use gfql() instead for a unified API.", + DeprecationWarning, + stacklevel=2 + ) return chain_base(self, *args, **kwargs) - chain.__doc__ = chain_base.__doc__ + # Preserve original docstring after deprecation notice + chain.__doc__ = (chain.__doc__ or "") + "\n\n" + (chain_base.__doc__ or "") + + # chain_let removed from public API - use gfql() instead + # (chain_let_base still available internally for gfql dispatch) + + def gfql(self, *args, **kwargs): + return gfql_base(self, *args, **kwargs) + gfql.__doc__ = gfql_base.__doc__ def chain_remote(self, *args, **kwargs) -> Plottable: + """ + .. deprecated:: 2.XX.X + Use :meth:`gfql_remote` instead for a unified API that supports both chains and DAGs. + """ + import warnings + warnings.warn( + "chain_remote() is deprecated. Use gfql_remote() instead for a unified API.", + DeprecationWarning, + stacklevel=2 + ) return chain_remote_base(self, *args, **kwargs) - chain_remote.__doc__ = chain_remote_base.__doc__ + # Preserve original docstring after deprecation notice + chain_remote.__doc__ = (chain_remote.__doc__ or "") + "\n\n" + (chain_remote_base.__doc__ or "") def chain_remote_shape(self, *args, **kwargs) -> pd.DataFrame: + """ + .. deprecated:: 2.XX.X + Use :meth:`gfql_remote_shape` instead for a unified API that supports both chains and DAGs. + """ + import warnings + warnings.warn( + "chain_remote_shape() is deprecated. Use gfql_remote_shape() instead for a unified API.", + DeprecationWarning, + stacklevel=2 + ) + return chain_remote_shape_base(self, *args, **kwargs) + # Preserve original docstring after deprecation notice + chain_remote_shape.__doc__ = (chain_remote_shape.__doc__ or "") + "\n\n" + (chain_remote_shape_base.__doc__ or "") + + def gfql_remote(self, *args, **kwargs) -> Plottable: + """Run GFQL query remotely. + + This is the remote execution version of :meth:`gfql`. It supports both simple chains + and complex DAG patterns with Let bindings. + + See :meth:`chain_remote` for detailed documentation (chain_remote is deprecated). + """ + return chain_remote_base(self, *args, **kwargs) + + def gfql_remote_shape(self, *args, **kwargs) -> pd.DataFrame: + """Get shape metadata for remote GFQL query execution. + + This is the remote shape version of :meth:`gfql`. Returns metadata about the + resulting graph without downloading the full data. + + See :meth:`chain_remote_shape` for detailed documentation (chain_remote_shape is deprecated). + """ return chain_remote_shape_base(self, *args, **kwargs) - chain_remote_shape.__doc__ = chain_remote_shape_base.__doc__ def python_remote_g(self, *args, **kwargs) -> Any: return python_remote_g_base(self, *args, **kwargs) diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 7c9f36b1d0..3cbb68ac16 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -1,6 +1,7 @@ from .ComputeMixin import ComputeMixin from .ast import ( - n, e, e_forward, e_reverse, e_undirected + n, e, e_forward, e_reverse, e_undirected, + let, remote, ref, call ) from .chain import Chain from .predicates.is_in import ( @@ -54,3 +55,39 @@ notnull, NotNull, ) from .typing import DataFrameT + +__all__ = [ + # Core classes + 'ComputeMixin', 'Chain', + # AST nodes + 'n', 'e', 'e_forward', 'e_reverse', 'e_undirected', + 'let', 'remote', 'ref', 'call', + # Predicates + 'is_in', 'IsIn', + 'duplicated', 'Duplicated', + 'is_month_start', 'IsMonthStart', + 'is_month_end', 'IsMonthEnd', + 'is_quarter_start', 'IsQuarterStart', + 'is_quarter_end', 'IsQuarterEnd', + 'is_year_start', 'IsYearStart', + 'is_year_end', 'IsYearEnd', + 'is_leap_year', 'IsLeapYear', + # Temporal + 'TemporalValue', 'DateTimeValue', 'DateValue', 'TimeValue', + 'temporal_value_from_json', + # Comparison predicates + 'gt', 'GT', 'lt', 'LT', 'ge', 'GE', 'le', 'LE', + 'eq', 'EQ', 'ne', 'NE', 'between', 'Between', + 'isna', 'IsNA', 'notna', 'NotNA', + # String predicates + 'contains', 'Contains', 'startswith', 'Startswith', + 'endswith', 'Endswith', 'match', 'Match', + 'isnumeric', 'IsNumeric', 'isalpha', 'IsAlpha', + 'isdigit', 'IsDigit', 'islower', 'IsLower', + 'isupper', 'IsUpper', 'isspace', 'IsSpace', + 'isalnum', 'IsAlnum', 'isdecimal', 'IsDecimal', + 'istitle', 'IsTitle', 'isnull', 'IsNull', + 'notnull', 'NotNull', + # Types + 'DataFrameT' +] diff --git a/graphistry/compute/ast.py b/graphistry/compute/ast.py index f58c744e49..2843d0b603 100644 --- a/graphistry/compute/ast.py +++ b/graphistry/compute/ast.py @@ -1,7 +1,10 @@ from abc import abstractmethod import logging -from typing import Any, TYPE_CHECKING, Dict, Optional, Union, cast +from typing import Any, TYPE_CHECKING, Dict, List, Optional, Sequence, Union, cast from typing_extensions import Literal + +if TYPE_CHECKING: + from graphistry.compute.exceptions import GFQLValidationError import pandas as pd from graphistry.Engine import Engine @@ -175,7 +178,7 @@ def _validate_fields(self) -> None: ErrorCode.E205, "query must be a string", field="query", value=type(self.query).__name__ ) - def _get_child_validators(self) -> list: + def _get_child_validators(self) -> Sequence['ASTSerializable']: """Return predicates that need validation.""" children = [] if self.filter_dict: @@ -372,7 +375,7 @@ def _validate_fields(self) -> None: ErrorCode.E205, f"{query_name} must be a string", field=query_name, value=type(query_value).__name__ ) - def _get_child_validators(self) -> list: + def _get_child_validators(self) -> Sequence['ASTSerializable']: """Return predicates that need validation.""" children = [] for filter_dict in [self.source_node_match, self.edge_match, self.destination_node_match]: @@ -642,9 +645,468 @@ def from_json(cls, d: dict, validate: bool = True) -> 'ASTEdge': e_undirected = ASTEdgeUndirected # noqa: E305 e = ASTEdgeUndirected # noqa: E305 + +############################################################################## + + +class ASTLet(ASTObject): + """Let-bindings for named graph operations in a DAG. + + Allows defining reusable graph operations that can reference each other, + forming a directed acyclic graph (DAG) of computations. + + :param bindings: Dictionary mapping names to graph operations + :type bindings: Dict[str, ASTObject] + + :raises GFQLTypeError: If bindings is not a dict or contains invalid keys/values + + **Example::** + + dag = ASTLet({ + 'persons': n({'type': 'person'}), + 'friends': ASTRef('persons', [e_forward({'rel': 'friend'})]) + }) + """ + def __init__(self, bindings: Dict[str, 'ASTObject']) -> None: + """Initialize Let with named bindings. + + :param bindings: Dictionary mapping names to AST operations + :type bindings: Dict[str, ASTObject] + """ + super().__init__() + self.bindings = bindings + + def _validate_fields(self) -> None: + """Validate Let fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.bindings, dict): + raise GFQLTypeError( + ErrorCode.E201, + "bindings must be a dictionary", + field="bindings", + value=type(self.bindings).__name__ + ) + + for k, v in self.bindings.items(): + if not isinstance(k, str): + raise GFQLTypeError( + ErrorCode.E102, + "binding key must be string", + field=f"bindings.{k}", + value=type(k).__name__ + ) + if not isinstance(v, ASTObject): + raise GFQLTypeError( + ErrorCode.E201, + "binding value must be ASTObject", + field=f"bindings.{k}", + value=type(v).__name__ + ) + # TODO: Check for cycles in DAG + return None + + def _get_child_validators(self) -> Sequence['ASTSerializable']: + """Return child AST nodes that need validation.""" + # ASTObject inherits from ASTSerializable, so this is safe + return list(self.bindings.values()) + + def to_json(self, validate: bool = True) -> dict: + """Convert Let to JSON representation. + + :param validate: Whether to validate before serialization + :type validate: bool + :returns: JSON-serializable dictionary + :rtype: dict + """ + if validate: + self.validate() + return { + 'type': 'Let', + 'bindings': {k: v.to_json() for k, v in self.bindings.items()} + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTLet': + """Create ASTLet from JSON representation. + + :param d: JSON dictionary with 'bindings' field + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTLet instance + :rtype: ASTLet + :raises AssertionError: If 'bindings' field is missing + """ + assert 'bindings' in d, "Let missing bindings" + bindings = {k: cast(ASTObject, from_json(v, validate=validate)) for k, v in d['bindings'].items()} + out = cls(bindings=bindings) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + # Let bindings don't use wavefronts - execute via chain_let_impl + from graphistry.compute.chain_let import chain_let_impl + from graphistry.Engine import EngineAbstract + return chain_let_impl(g, self, EngineAbstract(engine.value)) + + def reverse(self) -> 'ASTLet': + raise NotImplementedError("Let reversal not supported") + + +class ASTRemoteGraph(ASTObject): + """Load a graph from Graphistry server. + + Allows fetching previously uploaded graphs by dataset ID, + optionally with an authentication token. + + :param dataset_id: Unique identifier of the dataset on the server + :type dataset_id: str + :param token: Optional authentication token + :type token: Optional[str] + + :raises GFQLTypeError: If dataset_id is not a string or is empty + + **Example::** + + # Fetch public dataset + remote = ASTRemoteGraph('my-dataset-id') + + # Fetch private dataset with token + remote = ASTRemoteGraph('private-dataset', token='auth-token') + """ + def __init__(self, dataset_id: str, token: Optional[str] = None) -> None: + """Initialize RemoteGraph with dataset ID and optional token. + + :param dataset_id: Unique identifier of the dataset + :type dataset_id: str + :param token: Optional authentication token + :type token: Optional[str] + """ + super().__init__() + self.dataset_id = dataset_id + self.token = token + + def _validate_fields(self) -> None: + """Validate RemoteGraph fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.dataset_id, str): + raise GFQLTypeError( + ErrorCode.E201, + "dataset_id must be a string", + field="dataset_id", + value=type(self.dataset_id).__name__ + ) + + if len(self.dataset_id) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "dataset_id cannot be empty", + field="dataset_id", + value=self.dataset_id + ) + + if self.token is not None and not isinstance(self.token, str): + raise GFQLTypeError( + ErrorCode.E201, + "token must be string or None", + field="token", + value=type(self.token).__name__ + ) + + def to_json(self, validate: bool = True) -> dict: + """Convert RemoteGraph to JSON representation. + + :param validate: Whether to validate before serialization + :type validate: bool + :returns: JSON-serializable dictionary + :rtype: dict + """ + if validate: + self.validate() + result = { + 'type': 'RemoteGraph', + 'dataset_id': self.dataset_id + } + if self.token is not None: + result['token'] = self.token + return result + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTRemoteGraph': + """Create ASTRemoteGraph from JSON representation. + + :param d: JSON dictionary with 'dataset_id' field + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTRemoteGraph instance + :rtype: ASTRemoteGraph + :raises AssertionError: If 'dataset_id' field is missing + """ + assert 'dataset_id' in d, "RemoteGraph missing dataset_id" + out = cls( + dataset_id=d['dataset_id'], + token=d.get('token') + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + # Implementation in PR 1.3 + raise NotImplementedError("RemoteGraph loading will be implemented in PR 1.3") + + def reverse(self) -> 'ASTRemoteGraph': + raise NotImplementedError("RemoteGraph reversal not supported") + + +class ASTRef(ASTObject): + """Execute a chain of operations starting from a DAG binding reference. + + Allows building graph operations that start from a named binding + defined in an ASTLet (DAG) and apply additional operations. + + :param ref: Name of the binding to reference from the DAG + :type ref: str + :param chain: List of operations to apply to the referenced graph + :type chain: List[ASTObject] + + :raises GFQLTypeError: If ref is not a string or chain is not a list + + **Example::** + + # Reference 'persons' binding and find their friends + friends = ASTRef('persons', [e_forward({'rel': 'friend'})]) + """ + def __init__(self, ref: str, chain: List['ASTObject']) -> None: + """Initialize Ref with reference name and operation chain. + + :param ref: Name of the binding to reference + :type ref: str + :param chain: List of operations to apply + :type chain: List[ASTObject] + """ + super().__init__() + self.ref = ref + self.chain = chain + + def _validate_fields(self) -> None: + """Validate Ref fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.ref, str): + raise GFQLTypeError( + ErrorCode.E201, + "ref must be a string", + field="ref", + value=type(self.ref).__name__ + ) + + if len(self.ref) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "ref cannot be empty", + field="ref", + value=self.ref + ) + + if not isinstance(self.chain, list): + raise GFQLTypeError( + ErrorCode.E201, + "chain must be a list", + field="chain", + value=type(self.chain).__name__ + ) + + for i, op in enumerate(self.chain): + if not isinstance(op, ASTObject): + raise GFQLTypeError( + ErrorCode.E201, + f"chain[{i}] must be ASTObject", + field=f"chain[{i}]", + value=type(op).__name__ + ) + + def _get_child_validators(self) -> Sequence['ASTSerializable']: + """Return child AST nodes that need validation.""" + # ASTObject inherits from ASTSerializable, so this is safe + return self.chain + + def to_json(self, validate: bool = True) -> dict: + """Convert Ref to JSON representation. + + :param validate: Whether to validate before serialization + :type validate: bool + :returns: JSON-serializable dictionary + :rtype: dict + """ + if validate: + self.validate() + return { + 'type': 'Ref', + 'ref': self.ref, + 'chain': [op.to_json() for op in self.chain] + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTRef': + """Create ASTRef from JSON representation. + + :param d: JSON dictionary with 'ref' and 'chain' fields + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTRef instance + :rtype: ASTRef + :raises AssertionError: If 'ref' or 'chain' fields are missing + """ + assert 'ref' in d, "Ref missing ref" + assert 'chain' in d, "Ref missing chain" + out = cls( + ref=d['ref'], + chain=[from_json(op, validate=validate) for op in d['chain']] + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + raise NotImplementedError( + "ASTRef cannot be used directly in chain(). " + "It must be used within an ASTLet/chain_let() context." + ) + + def reverse(self) -> 'ASTRef': + # Reverse the chain operations + return ASTRef(self.ref, [op.reverse() for op in reversed(self.chain)]) + + +class ASTCall(ASTObject): + """Call a method on the current graph with validated parameters. + + Allows safe execution of Plottable methods through GFQL with parameter + validation and schema checking. + + Attributes: + function: Name of the method to call (must be in safelist) + params: Dictionary of parameters to pass to the method + """ + def __init__(self, function: str, params: Optional[Dict[str, Any]] = None) -> None: + """Initialize a Call operation. + + Args: + function: Name of the Plottable method to call + params: Optional dictionary of parameters for the method + """ + super().__init__() + self.function = function + self.params = params or {} + + def _validate_fields(self) -> None: + """Validate Call fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.function, str): + raise GFQLTypeError( + ErrorCode.E201, + "function must be a string", + field="function", + value=type(self.function).__name__ + ) + + if len(self.function) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "function name cannot be empty", + field="function", + value=self.function + ) + + if not isinstance(self.params, dict): + raise GFQLTypeError( + ErrorCode.E201, + "params must be a dictionary", + field="params", + value=type(self.params).__name__ + ) + + def to_json(self, validate: bool = True) -> dict: + """Convert Call to JSON representation. + + Args: + validate: If True, validate before serialization + + Returns: + Dictionary with type, function, and params fields + """ + if validate: + self.validate() + return { + 'type': 'Call', + 'function': self.function, + 'params': self.params + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTCall': + """Create ASTCall from JSON representation. + + :param d: JSON dictionary with 'function' field and optional 'params' + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTCall instance + :rtype: ASTCall + :raises AssertionError: If 'function' field is missing + + **Example::** + + call_json = {'type': 'Call', 'function': 'hop', 'params': {'steps': 2}} + call = ASTCall.from_json(call_json) + """ + assert 'function' in d, "Call missing function" + out = cls( + function=d['function'], + params=d.get('params', {}) + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + """Execute the method call on the graph. + + Args: + g: Graph to operate on + prev_node_wavefront: Previous node wavefront (unused) + target_wave_front: Target wavefront (unused) + engine: Execution engine (pandas/cudf) + + Returns: + New Plottable with method results + + Raises: + GFQLTypeError: If method not in safelist or parameters invalid + """ + # For chain_let, we don't use wavefronts, just execute the call + from graphistry.compute.gfql.call_executor import execute_call + return execute_call(g, self.function, self.params, engine) + + def reverse(self) -> 'ASTCall': + # Most method calls cannot be reversed + raise NotImplementedError(f"Method '{self.function}' cannot be reversed") + + ### -def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge]: +def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTRef, ASTCall]: from graphistry.compute.exceptions import ErrorCode, GFQLSyntaxError if not isinstance(o, dict): @@ -652,10 +1114,10 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge]: if 'type' not in o: raise GFQLSyntaxError( - ErrorCode.E105, "AST JSON missing required 'type' field", suggestion="Add 'type' field: 'Node' or 'Edge'" + ErrorCode.E105, "AST JSON missing required 'type' field", suggestion="Add 'type' field: 'Node', 'Edge', 'Let', 'RemoteGraph', or 'ChainRef'" ) - out: Union[ASTNode, ASTEdge] + out: Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTRef, ASTCall] if o['type'] == 'Node': out = ASTNode.from_json(o, validate=validate) elif o['type'] == 'Edge': @@ -680,12 +1142,34 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge]: "Edge missing required 'direction' field", suggestion="Add 'direction' field: 'forward', 'reverse', or 'undirected'", ) + elif o['type'] == 'Let': + out = ASTLet.from_json(o, validate=validate) + elif o['type'] == 'QueryDAG': + # For backward compatibility + out = ASTLet.from_json(o, validate=validate) + elif o['type'] == 'RemoteGraph': + out = ASTRemoteGraph.from_json(o, validate=validate) + elif o['type'] == 'ChainRef': + out = ASTRef.from_json(o, validate=validate) + elif o['type'] == 'Ref': + out = ASTRef.from_json(o, validate=validate) + elif o['type'] == 'Call': + out = ASTCall.from_json(o, validate=validate) else: raise GFQLSyntaxError( ErrorCode.E101, f"Unknown AST type: {o['type']}", field="type", value=o["type"], - suggestion="Use 'Node' or 'Edge'", + suggestion="Use 'Node', 'Edge', 'Let', 'RemoteGraph', 'ChainRef', 'Ref', or 'Call'", ) return out + + +############################################################################### +# User-friendly aliases for public API + +let = ASTLet # noqa: E305 +remote = ASTRemoteGraph # noqa: E305 +ref = ASTRef # noqa: E305 +call = ASTCall # noqa: E305 diff --git a/graphistry/compute/chain_let.py b/graphistry/compute/chain_let.py new file mode 100644 index 0000000000..a5b5b36415 --- /dev/null +++ b/graphistry/compute/chain_let.py @@ -0,0 +1,445 @@ +from typing import Dict, Set, List, Optional, Tuple, Union, cast +from typing_extensions import Literal +from graphistry.Engine import Engine, EngineAbstract, resolve_engine +from graphistry.Plottable import Plottable +from graphistry.util import setup_logger +from .ast import ASTObject, ASTLet, ASTRef, ASTRemoteGraph, ASTNode, ASTEdge, ASTCall +from .execution_context import ExecutionContext + +logger = setup_logger(__name__) + + +def extract_dependencies(ast_obj: ASTObject) -> Set[str]: + """Recursively find all ASTRef references in an AST object + + :param ast_obj: AST object to analyze + :returns: Set of referenced binding names + :rtype: Set[str] + """ + deps = set() + + if isinstance(ast_obj, ASTRef): + deps.add(ast_obj.ref) + # Also check chain operations + for op in ast_obj.chain: + deps.update(extract_dependencies(op)) + + elif isinstance(ast_obj, ASTLet): + # Nested let bindings + for binding in ast_obj.bindings.values(): + deps.update(extract_dependencies(binding)) + + # Other AST types (ASTNode, ASTEdge, ASTRemoteGraph) have no dependencies + return deps + + +def build_dependency_graph(bindings: Dict[str, ASTObject]) -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]: + """Build dependency and dependent mappings from bindings + + :param bindings: Dictionary of name -> AST object bindings + :returns: Tuple of (dependencies dict, dependents dict) + :rtype: Tuple[Dict[str, Set[str]], Dict[str, Set[str]]] + """ + dependencies: Dict[str, Set[str]] = {} + dependents: Dict[str, Set[str]] = {} + + for name, ast_obj in bindings.items(): + deps = extract_dependencies(ast_obj) + dependencies[name] = deps + + # Build reverse mapping + for dep in deps: + if dep not in dependents: + dependents[dep] = set() + dependents[dep].add(name) + + return dependencies, dependents + + +def validate_dependencies(bindings: Dict[str, ASTObject], + dependencies: Dict[str, Set[str]]) -> None: + """Check for missing references and self-cycles + + :param bindings: Dictionary of available bindings + :param dependencies: Dictionary of dependencies per binding + :raises ValueError: If missing references or self-cycles found + """ + all_names = set(bindings.keys()) + + for name, deps in dependencies.items(): + # Check self-reference + if name in deps: + raise ValueError(f"Self-reference cycle detected: '{name}' depends on itself") + + # Check missing references + missing = deps - all_names + if missing: + raise ValueError( + f"Node '{name}' references undefined nodes: {sorted(missing)}. " + f"Available nodes: {sorted(all_names)}" + ) + + +def detect_cycles(dependencies: Dict[str, Set[str]]) -> Optional[List[str]]: + """Use DFS to detect cycles and return the cycle path if found + + :param dependencies: Dictionary mapping nodes to their dependencies + :returns: List representing cycle path if found, None otherwise + :rtype: Optional[List[str]] + """ + WHITE, GRAY, BLACK = 0, 1, 2 + color = {node: WHITE for node in dependencies} + + def dfs(node: str, path: List[str]) -> Optional[List[str]]: + color[node] = GRAY + path.append(node) + + for neighbor in dependencies.get(node, set()): + if color.get(neighbor, WHITE) == GRAY: + # Found cycle - build cycle path + cycle_start = path.index(neighbor) + return path[cycle_start:] + [neighbor] + + if color.get(neighbor, WHITE) == WHITE: + cycle = dfs(neighbor, path[:]) + if cycle: + return cycle + + color[node] = BLACK + return None + + for node in dependencies: + if color[node] == WHITE: + cycle = dfs(node, []) + if cycle: + return cycle + + return None + + +def topological_sort(bindings: Dict[str, ASTObject], + dependencies: Dict[str, Set[str]], + dependents: Dict[str, Set[str]]) -> List[str]: + """Kahn's algorithm for topological sort""" + # Calculate in-degrees + in_degree = {name: len(dependencies.get(name, set())) for name in bindings} + + # Start with nodes that have no dependencies + queue = [name for name, degree in in_degree.items() if degree == 0] + result = [] + + while queue: + # Process node with no remaining dependencies + current = queue.pop(0) + result.append(current) + + # Update dependents + for dependent in dependents.get(current, set()): + in_degree[dependent] -= 1 + if in_degree[dependent] == 0: + queue.append(dependent) + + if len(result) != len(bindings): + # Cycle detected - use DFS to find it for better error + cycle = detect_cycles(dependencies) + if cycle: + raise ValueError( + f"Circular dependency detected: {' -> '.join(cycle)}. " + "Please restructure your DAG to remove cycles." + ) + else: + # Should not happen, but be defensive + raise ValueError("Failed to determine execution order (possible circular dependency)") + + return result + + +def determine_execution_order(bindings: Dict[str, ASTObject]) -> List[str]: + """Determine topological execution order for DAG bindings + + Validates dependencies and computes execution order that respects + all dependencies. Detects cycles and missing references. + + :param bindings: Dictionary of name -> AST object bindings + :returns: List of binding names in execution order + :rtype: List[str] + :raises ValueError: If cycles detected or references missing + """ + # Handle trivial cases + if not bindings: + return [] + if len(bindings) == 1: + return list(bindings.keys()) + + # Build dependency graph + dependencies, dependents = build_dependency_graph(bindings) + + # Validate all references exist + validate_dependencies(bindings, dependencies) + + # Check for cycles with detailed error + cycle = detect_cycles(dependencies) + if cycle: + raise ValueError( + f"Circular dependency detected: {' -> '.join(cycle)}. " + "Please restructure your DAG to remove cycles." + ) + + # Compute topological sort + return topological_sort(bindings, dependencies, dependents) + + +def execute_node(name: str, ast_obj: ASTObject, g: Plottable, + context: ExecutionContext, engine: Engine) -> Plottable: + """Execute a single node in the DAG + + Handles different AST object types: + - ASTLet: Recursive let execution + - ASTRef: Reference resolution and chain execution + - ASTNode: Node filtering operations + - ASTEdge: Edge traversal operations + - Others: NotImplementedError + + :param name: Binding name for this node + :param ast_obj: AST object to execute + :param g: Input graph + :param context: Execution context for storing/retrieving results + :param engine: Engine to use (pandas/cudf) + :returns: Resulting Plottable + :rtype: Plottable + :raises ValueError: If reference not found in context + :raises NotImplementedError: For unsupported AST types + """ + logger.debug("Executing node '%s' of type %s", name, type(ast_obj).__name__) + + # Handle different AST object types + if isinstance(ast_obj, ASTLet): + # Nested let execution + result = chain_let_impl(g, ast_obj, EngineAbstract(engine.value)) + elif isinstance(ast_obj, ASTRef): + # Resolve reference from context + try: + referenced_result = context.get_binding(ast_obj.ref) + except KeyError as e: + available = sorted(context.get_all_bindings().keys()) + raise ValueError( + f"Node '{name}' references '{ast_obj.ref}' which has not been executed yet. " + f"Available bindings: {available}" + ) from e + + # Execute the chain on the referenced result + if ast_obj.chain: + # Import chain function to execute the operations + from .chain import chain as chain_impl + result = chain_impl(referenced_result, ast_obj.chain, EngineAbstract(engine.value)) + else: + # Empty chain - just return the referenced result + result = referenced_result + elif isinstance(ast_obj, ASTNode): + # For chain_let, we execute nodes in a simpler way than chain() + # No wavefront propagation - just filter the graph's nodes + node_obj = cast(ASTNode, ast_obj) # Help mypy understand the type + if node_obj.filter_dict or node_obj.query: + filtered_g = g + if node_obj.filter_dict: + filtered_g = filtered_g.filter_nodes_by_dict(node_obj.filter_dict) + if node_obj.query: + filtered_g = filtered_g.nodes(lambda g: g._nodes.query(node_obj.query)) + result = filtered_g + else: + # Empty filter - return original graph + result = g + + # Add name column if specified + if node_obj._name: + result = result.nodes(result._nodes.assign(**{node_obj._name: True})) + elif isinstance(ast_obj, ASTEdge): + # For chain_let, execute edge operations using hop() + # This is simpler than the full chain() wavefront approach + result = g.hop( + nodes=None, # Start from all nodes + hops=ast_obj.hops, + to_fixed_point=ast_obj.to_fixed_point, + direction=ast_obj.direction, + source_node_match=ast_obj.source_node_match, + edge_match=ast_obj.edge_match, + destination_node_match=ast_obj.destination_node_match, + source_node_query=ast_obj.source_node_query, + edge_query=ast_obj.edge_query, + destination_node_query=ast_obj.destination_node_query, + return_as_wave_front=False # Return full graph + ) + + # Add name column to edges if specified + if ast_obj._name: + result = result.edges(result._edges.assign(**{ast_obj._name: True})) + elif isinstance(ast_obj, ASTRemoteGraph): + # Create a new plottable bound to the remote dataset_id + # This doesn't fetch the data immediately - it just creates a reference + result = g.bind(dataset_id=ast_obj.dataset_id) + + # If we need to actually fetch the data, we would use chain_remote + # For now, we'll fetch it immediately to ensure we have the data + from .chain_remote import chain_remote as chain_remote_impl + + # Fetch the remote dataset with an empty chain (no filtering) + # Convert engine to the expected type for chain_remote + chain_engine: Optional[Literal["pandas", "cudf"]] = None + if engine.value == "pandas": + chain_engine = "pandas" + elif engine.value == "cudf": + chain_engine = "cudf" + + result = chain_remote_impl( + result, + [], # Empty chain - just fetch the entire dataset + api_token=ast_obj.token, + dataset_id=ast_obj.dataset_id, + output_type="all", # Get full graph (nodes and edges) + engine=chain_engine + ) + elif isinstance(ast_obj, ASTCall): + # Execute method call with validation + from .gfql.call_executor import execute_call + result = execute_call(g, ast_obj.function, ast_obj.params, engine) + else: + # Other AST object types not yet implemented + raise NotImplementedError(f"Execution of {type(ast_obj).__name__} not yet implemented") + + # Store result in context + context.set_binding(name, result) + + return result + + +def chain_let_impl(g: Plottable, dag: ASTLet, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + output: Optional[str] = None) -> Plottable: + """Internal implementation of chain_let execution + + Validates DAG, determines execution order, and executes nodes + in topological order. + + :param g: Input graph + :param dag: Let specification with named bindings + :param engine: Engine selection (auto/pandas/cudf) + :param output: Name of binding to return (default: last executed) + :returns: Result from specified or last executed node + :rtype: Plottable + :raises TypeError: If dag is not an ASTLet + :raises RuntimeError: If node execution fails + :raises ValueError: If output binding not found + """ + if isinstance(engine, str): + engine = EngineAbstract(engine) + + # Validate the let parameter + if not isinstance(dag, ASTLet): + raise TypeError(f"dag must be an ASTLet, got {type(dag).__name__}") + + # Validate the let bindings + dag.validate() + + # Resolve engine + engine_concrete = resolve_engine(engine, g) + logger.debug('chain_let engine: %s => %s', engine, engine_concrete) + + # Materialize nodes if needed (following chain.py pattern) + g = g.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) + + # Create execution context + context = ExecutionContext() + + # Handle empty let bindings + if not dag.bindings: + return g + + # Determine execution order + order = determine_execution_order(dag.bindings) + logger.debug("DAG execution order: %s", order) + + # Execute nodes in topological order + last_result = g + for node_name in order: + ast_obj = dag.bindings[node_name] + logger.debug("Executing node '%s' in DAG", node_name) + + # Execute the node and store result in context + try: + result = execute_node(node_name, ast_obj, g, context, engine_concrete) + last_result = result + except Exception as e: + # Add context to error + raise RuntimeError( + f"Failed to execute node '{node_name}' in DAG. " + f"Error: {type(e).__name__}: {str(e)}" + ) from e + + # Return requested output or last executed result + if output is not None: + if output not in context.get_all_bindings(): + available = sorted(context.get_all_bindings().keys()) + raise ValueError( + f"Output binding '{output}' not found. " + f"Available bindings: {available}" + ) + return context.get_binding(output) + else: + return last_result + + +def chain_let(self: Plottable, dag: ASTLet, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + output: Optional[str] = None) -> Plottable: + """ + Execute a DAG of named graph operations with dependency resolution + + Chain operations can reference results from other operations by name, + enabling parallel branches and complex data flows. + + :param dag: ASTLet containing named bindings of operations + :param engine: Execution engine (auto, pandas, cudf) + :param output: Name of binding to return (default: last executed) + :returns: Plottable result from the specified or last operation + :rtype: Plottable + + **Example: Single operation (no dependencies)** + + :: + + from graphistry.compute.ast import ASTLet, n + + dag = ASTLet({ + 'people': n({'type': 'person'}) + }) + result = g.chain_let(dag) + + **Example: Linear dependencies** + + :: + + from graphistry.compute.ast import ASTLet, ASTRef, n, e + + dag = ASTLet({ + 'start': n({'type': 'person'}), + 'friends': ASTRef('start', [e(), n()]) + }) + result = g.chain_let(dag) + + **Example: Diamond pattern** + + :: + + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'transactions': n({'type': 'transaction'}), + 'branch1': ASTRef('people', [e()]), + 'branch2': ASTRef('transactions', [e()]), + 'merged': g.union(ASTRef('branch1'), ASTRef('branch2')) + }) + result = g.chain_let(dag) # Returns last executed + + # Or select specific output + people_result = g.chain_let(dag, output='people') + """ + return chain_let_impl(self, dag, engine, output) diff --git a/graphistry/compute/execution_context.py b/graphistry/compute/execution_context.py new file mode 100644 index 0000000000..0cc009cb78 --- /dev/null +++ b/graphistry/compute/execution_context.py @@ -0,0 +1,75 @@ +"""Execution context for DAG operations.""" +from typing import Any, Dict + + +class ExecutionContext: + """Manages variable bindings during DAG execution. + + Provides a namespace for storing and retrieving named graph results + during the execution of ASTLet DAGs. Each binding maps a string name + to a Plottable graph instance. + + **Example::** + + context = ExecutionContext() + context.set_binding('persons', person_graph) + friends = context.get_binding('persons') + """ + + def __init__(self) -> None: + """Initialize an empty execution context.""" + self._bindings: Dict[str, Any] = {} + + def set_binding(self, name: str, value: Any) -> None: + """Store a named result in the context. + + :param name: Name for the binding + :type name: str + :param value: Value to bind (typically a Plottable) + :type value: Any + :raises TypeError: If name is not a string + """ + if not isinstance(name, str): + raise TypeError(f"Binding name must be string, got {type(name)}") + self._bindings[name] = value + + def get_binding(self, name: str) -> Any: + """Retrieve a named result from the context. + + :param name: Name of the binding to retrieve + :type name: str + :returns: The bound value + :rtype: Any + :raises TypeError: If name is not a string + :raises KeyError: If no binding exists for the given name + """ + if not isinstance(name, str): + raise TypeError(f"Binding name must be string, got {type(name)}") + if name not in self._bindings: + raise KeyError(f"No binding found for '{name}'") + return self._bindings[name] + + def has_binding(self, name: str) -> bool: + """Check if a binding exists in the context. + + :param name: Name to check + :type name: str + :returns: True if binding exists, False otherwise + :rtype: bool + :raises TypeError: If name is not a string + """ + if not isinstance(name, str): + raise TypeError(f"Binding name must be string, got {type(name)}") + return name in self._bindings + + def clear(self) -> None: + """Clear all bindings from the context.""" + self._bindings.clear() + + def get_all_bindings(self) -> Dict[str, Any]: + """Get a copy of all bindings in the context. + + :returns: Dictionary of all current bindings + :rtype: Dict[str, Any] + """ + return self._bindings.copy() diff --git a/graphistry/compute/gfql/call_executor.py b/graphistry/compute/gfql/call_executor.py new file mode 100644 index 0000000000..c6bea4ec16 --- /dev/null +++ b/graphistry/compute/gfql/call_executor.py @@ -0,0 +1,82 @@ +"""Execute validated method calls on Plottable objects. + +This module handles the actual execution of safelisted methods +after parameter validation. +""" + +from typing import Dict, Any +from graphistry.Plottable import Plottable +from graphistry.Engine import Engine +from graphistry.compute.gfql.call_safelist import validate_call_params +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: Engine) -> Plottable: + """Execute a validated method call on a Plottable. + + Args: + g: The graph to call the method on + function: Name of the method to call + params: Parameters for the method (will be validated) + engine: Execution engine + + Returns: + Result of the method call (usually a new Plottable) + + Raises: + GFQLTypeError: If validation fails or method doesn't exist + AttributeError: If method doesn't exist on Plottable + """ + # Validate parameters against safelist + validated_params = validate_call_params(function, params) + + # Check if method exists on Plottable + if not hasattr(g, function): + raise AttributeError( + f"Plottable has no method '{function}'. " + f"This should not happen if safelist is properly configured." + ) + + # Get the method + method = getattr(g, function) + + # Special handling for methods that need the engine parameter + if function in ['materialize_nodes', 'hop']: + # These methods accept an engine parameter + if 'engine' not in validated_params: + # Add current engine if not specified + validated_params['engine'] = engine + + try: + # Execute the method with validated parameters + result = method(**validated_params) + + # Ensure result is a Plottable (most methods return self or new Plottable) + if not isinstance(result, Plottable): + raise GFQLTypeError( + ErrorCode.E201, + f"Method '{function}' returned non-Plottable result", + field="function", + value=f"{type(result).__name__}", + suggestion="Only methods that return Plottable objects are allowed" + ) + + return result + + except TypeError as e: + # Handle parameter mismatch errors + raise GFQLTypeError( + ErrorCode.E201, + f"Parameter error calling '{function}': {str(e)}", + field="params", + value=validated_params, + suggestion="Check parameter names and types" + ) from e + except Exception as e: + # Re-raise other exceptions with context + raise GFQLTypeError( + ErrorCode.E303, + f"Error executing '{function}': {str(e)}", + field="function", + value=function + ) from e diff --git a/graphistry/compute/gfql/call_safelist.py b/graphistry/compute/gfql/call_safelist.py new file mode 100644 index 0000000000..9ca6765d55 --- /dev/null +++ b/graphistry/compute/gfql/call_safelist.py @@ -0,0 +1,501 @@ +"""Safelist of allowed methods for GFQL Call operations. + +This module defines which Plottable methods can be called through GFQL +and their parameter validation rules. +""" + +from typing import Dict, Any +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +# Type validators +def is_string(v: Any) -> bool: + return isinstance(v, str) + + +def is_int(v: Any) -> bool: + return isinstance(v, int) + + +def is_bool(v: Any) -> bool: + return isinstance(v, bool) + + +def is_dict(v: Any) -> bool: + return isinstance(v, dict) + + +def is_string_or_none(v: Any) -> bool: + return v is None or isinstance(v, str) + + +def is_list_of_strings(v: Any) -> bool: + return isinstance(v, list) and all(isinstance(item, str) for item in v) + + +# Safelist configuration +# Dictionary mapping allowed Plottable method names to their validation rules. +# +# Each method entry contains: +# - allowed_params (Set[str]): Parameter names that can be passed to the method +# - required_params (Set[str]): Parameters that must be provided +# - param_validators (Dict[str, Callable]): Maps param names to validation functions +# - description (str): Human-readable description of what the method does +# - schema_effects (Dict[str, List[str]]): Describes schema changes: +# - adds_node_cols: Columns added to node DataFrame +# - adds_edge_cols: Columns added to edge DataFrame +# - requires_node_cols: Node columns that must exist before calling +# - requires_edge_cols: Edge columns that must exist before calling +# +# Example entry: +# 'hop': { +# 'allowed_params': {'steps', 'to_fixed_point', 'direction'}, +# 'required_params': set(), +# 'param_validators': { +# 'steps': is_int, +# 'to_fixed_point': is_bool, +# 'direction': lambda v: v in ['forward', 'reverse', 'undirected'] +# }, +# 'description': 'Traverse graph edges for N steps', +# 'schema_effects': {} +# } + +SAFELIST_V1: Dict[str, Dict[str, Any]] = { + 'get_degrees': { + 'allowed_params': {'col_in', 'col_out', 'col', 'engine'}, + 'required_params': set(), + 'param_validators': { + 'col_in': is_string, + 'col_out': is_string, + 'col': is_string, + 'engine': is_string + }, + 'description': 'Calculate node degrees' + }, + + 'filter_nodes_by_dict': { + 'allowed_params': {'filter_dict'}, + 'required_params': {'filter_dict'}, + 'param_validators': { + 'filter_dict': is_dict + }, + 'description': 'Filter nodes by attribute values' + }, + + 'filter_edges_by_dict': { + 'allowed_params': {'filter_dict'}, + 'required_params': {'filter_dict'}, + 'param_validators': { + 'filter_dict': is_dict + }, + 'description': 'Filter edges by attribute values' + }, + + 'materialize_nodes': { + 'allowed_params': {'engine', 'reuse'}, + 'required_params': set(), + 'param_validators': { + 'engine': is_string, + 'reuse': is_bool + }, + 'description': 'Generate node table from edges' + }, + + 'hop': { + 'allowed_params': { + 'nodes', 'hops', 'to_fixed_point', 'direction', + 'source_node_match', 'edge_match', 'destination_node_match', + 'source_node_query', 'edge_query', 'destination_node_query', + 'return_as_wave_front', 'target_wave_front', 'engine' + }, + 'required_params': set(), + 'param_validators': { + 'hops': is_int, + 'to_fixed_point': is_bool, + 'direction': lambda v: v in ['forward', 'reverse', 'undirected'], + 'source_node_match': is_dict, + 'edge_match': is_dict, + 'destination_node_match': is_dict, + 'source_node_query': is_string, + 'edge_query': is_string, + 'destination_node_query': is_string, + 'return_as_wave_front': is_bool, + 'engine': is_string + }, + 'description': 'Traverse graph by following edges' + }, + + # In/out degree methods + 'get_indegrees': { + 'allowed_params': {'col'}, + 'required_params': set(), + 'param_validators': { + 'col': is_string + }, + 'description': 'Calculate node in-degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('col', 'degree_in')], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'get_outdegrees': { + 'allowed_params': {'col'}, + 'required_params': set(), + 'param_validators': { + 'col': is_string + }, + 'description': 'Calculate node out-degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('col', 'degree_out')], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + # Graph algorithm operations + 'compute_cugraph': { + 'allowed_params': {'alg', 'out_col', 'params', 'kind', 'directed', 'G'}, + 'required_params': {'alg'}, + 'param_validators': { + 'alg': is_string, + 'out_col': is_string_or_none, + 'params': is_dict, + 'kind': is_string, + 'directed': is_bool, + 'G': lambda x: x is None # Allow None only + }, + 'description': 'Run cuGraph algorithms (pagerank, louvain, etc)', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('out_col', p['alg'])], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'compute_igraph': { + 'allowed_params': {'alg', 'out_col', 'directed', 'use_vids', 'params'}, + 'required_params': {'alg'}, + 'param_validators': { + 'alg': is_string, + 'out_col': is_string_or_none, + 'directed': is_bool, + 'use_vids': is_bool, + 'params': is_dict + }, + 'description': 'Run igraph algorithms' + }, + + # Layout operations + 'layout_cugraph': { + 'allowed_params': {'layout', 'params', 'kind', 'directed', 'G', 'bind_position', 'x_out_col', 'y_out_col', 'play'}, + 'required_params': set(), + 'param_validators': { + 'layout': is_string, + 'params': is_dict, + 'kind': is_string, + 'directed': is_bool, + 'G': lambda x: x is None, + 'bind_position': is_bool, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'play': is_int + }, + 'description': 'GPU-accelerated graph layouts' + }, + + 'layout_igraph': { + 'allowed_params': {'layout', 'directed', 'use_vids', 'bind_position', 'x_out_col', 'y_out_col', 'params', 'play'}, + 'required_params': {'layout'}, + 'param_validators': { + 'layout': is_string, + 'directed': is_bool, + 'use_vids': is_bool, + 'bind_position': is_bool, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'params': is_dict, + 'play': is_int + }, + 'description': 'igraph-based layouts' + }, + + 'layout_graphviz': { + 'allowed_params': { + 'prog', 'args', 'directed', 'strict', 'graph_attr', + 'node_attr', 'edge_attr', 'x_out_col', 'y_out_col', 'bind_position' + }, + 'required_params': set(), + 'param_validators': { + 'prog': is_string, + 'args': is_string_or_none, + 'directed': is_bool, + 'strict': is_bool, + 'graph_attr': is_dict, + 'node_attr': is_dict, + 'edge_attr': is_dict, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'bind_position': is_bool + }, + 'description': 'Graphviz layouts (dot, neato, etc)' + }, + + 'fa2_layout': { + 'allowed_params': {'fa2_params', 'circle_layout_params', 'partition_key', 'remove_self_edges', 'engine', 'featurize'}, + 'required_params': set(), + 'param_validators': { + 'fa2_params': is_dict, + 'circle_layout_params': is_dict, + 'partition_key': is_string_or_none, + 'remove_self_edges': is_bool, + 'engine': is_string, + 'featurize': is_dict + }, + 'description': 'ForceAtlas2 layout algorithm' + }, + + # Self-edge pruning + 'prune_self_edges': { + 'allowed_params': set(), + 'required_params': set(), + 'param_validators': {}, + 'description': 'Remove self-loops from graph' + }, + + # Graph transformations + 'collapse': { + 'allowed_params': {'node', 'attribute', 'column', 'self_edges', 'unwrap', 'verbose'}, + 'required_params': set(), + 'param_validators': { + 'node': is_string_or_none, + 'attribute': is_string_or_none, + 'column': is_string_or_none, + 'self_edges': is_bool, + 'unwrap': is_bool, + 'verbose': is_bool + }, + 'description': 'Collapse nodes by shared attribute values' + }, + + 'drop_nodes': { + 'allowed_params': {'nodes'}, + 'required_params': {'nodes'}, + 'param_validators': { + 'nodes': lambda v: isinstance(v, list) or is_dict(v) + }, + 'description': 'Remove specified nodes and their edges' + }, + + 'keep_nodes': { + 'allowed_params': {'nodes'}, + 'required_params': {'nodes'}, + 'param_validators': { + 'nodes': lambda v: isinstance(v, list) or is_dict(v) + }, + 'description': 'Keep only specified nodes and their edges' + }, + + # Topology analysis + 'get_topological_levels': { + 'allowed_params': {'level_col', 'allow_cycles', 'warn_cycles', 'remove_self_loops'}, + 'required_params': set(), + 'param_validators': { + 'level_col': is_string, + 'allow_cycles': is_bool, + 'warn_cycles': is_bool, + 'remove_self_loops': is_bool + }, + 'description': 'Compute topological levels for DAG analysis' + }, + + # Visual encoding methods + 'encode_point_color': { + 'allowed_params': {'column', 'palette', 'as_categorical', 'as_continuous', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'palette': lambda v: isinstance(v, list), + 'as_categorical': is_bool, + 'as_continuous': is_bool, + 'categorical_mapping': is_dict, + 'default_mapping': is_string_or_none + }, + 'description': 'Map node column values to colors' + }, + + 'encode_edge_color': { + 'allowed_params': {'column', 'palette', 'as_categorical', 'as_continuous', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'palette': lambda v: isinstance(v, list), + 'as_categorical': is_bool, + 'as_continuous': is_bool, + 'categorical_mapping': is_dict, + 'default_mapping': is_string_or_none + }, + 'description': 'Map edge column values to colors' + }, + + 'encode_point_size': { + 'allowed_params': {'column', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'categorical_mapping': is_dict, + 'default_mapping': lambda v: isinstance(v, (int, float)) + }, + 'description': 'Map node column values to sizes' + }, + + 'encode_point_icon': { + 'allowed_params': {'column', 'categorical_mapping', 'continuous_binning', 'default_mapping', 'as_text'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'categorical_mapping': is_dict, + 'continuous_binning': lambda v: isinstance(v, list), + 'default_mapping': is_string_or_none, + 'as_text': is_bool + }, + 'description': 'Map node column values to icons' + }, + + # Metadata methods + 'name': { + 'allowed_params': {'name'}, + 'required_params': {'name'}, + 'param_validators': { + 'name': is_string + }, + 'description': 'Set visualization name' + }, + + 'description': { + 'allowed_params': {'description'}, + 'required_params': {'description'}, + 'param_validators': { + 'description': is_string + }, + 'description': 'Set visualization description' + }, + + # Layout with community detection + 'group_in_a_box_layout': { + 'allowed_params': { + 'partition_alg', 'partition_params', 'layout_alg', 'layout_params', + 'x', 'y', 'w', 'h', 'encode_colors', 'colors', 'partition_key', 'engine' + }, + 'required_params': set(), + 'param_validators': { + 'partition_alg': is_string_or_none, + 'partition_params': is_dict, + 'layout_alg': lambda v: v is None or is_string(v) or callable(v), + 'layout_params': is_dict, + 'x': lambda v: isinstance(v, (int, float)), + 'y': lambda v: isinstance(v, (int, float)), + 'w': lambda v: v is None or isinstance(v, (int, float)), + 'h': lambda v: v is None or isinstance(v, (int, float)), + 'encode_colors': is_bool, + 'colors': lambda v: v is None or is_list_of_strings(v), + 'partition_key': is_string_or_none, + 'engine': lambda v: v in ['auto', 'cpu', 'gpu', 'pandas', 'cudf'] + }, + 'description': 'Group-in-a-box layout with community detection' + } +} + + +def validate_call_params(function: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Validate parameters for a GFQL Call operation against the safelist. + + Performs comprehensive validation: + 1. Checks if function is in the safelist + 2. Verifies all required parameters are present + 3. Ensures no unknown parameters are passed + 4. Validates parameter types using configured validators + 5. Returns the validated parameters unchanged + + Args: + function: Name of the Plottable method to call + params: Dictionary of parameters to validate + + Returns: + The same parameters dict if validation passes + + Raises: + GFQLTypeError: If function not in safelist (E303) + GFQLTypeError: If required parameters missing (E105) + GFQLTypeError: If unknown parameters provided (E303) + GFQLTypeError: If parameter type validation fails (E201) + + **Example::** + + # Valid call + params = validate_call_params('hop', {'steps': 2, 'direction': 'forward'}) + + # Invalid - unknown function + validate_call_params('dangerous_method', {}) # Raises E303 + + # Invalid - missing required param + validate_call_params('fa2_layout', {}) # Would raise E105 if layout was required + + # Invalid - wrong type + validate_call_params('hop', {'steps': 'two'}) # Raises E201 + """ + # Check if function is in safelist + if function not in SAFELIST_V1: + raise GFQLTypeError( + ErrorCode.E303, + f"Function '{function}' is not in the safelist", + field="function", + value=function, + suggestion=f"Available functions: {', '.join(sorted(SAFELIST_V1.keys()))}" + ) + + config = SAFELIST_V1[function] + allowed_params = config['allowed_params'] + required_params = config['required_params'] + param_validators = config['param_validators'] + + # Check for required parameters + missing_required = required_params - set(params.keys()) + if missing_required: + raise GFQLTypeError( + ErrorCode.E105, + f"Missing required parameters for '{function}'", + field="params", + value=list(missing_required), + suggestion=f"Required parameters: {', '.join(sorted(missing_required))}" + ) + + # Check for unknown parameters + unknown_params = set(params.keys()) - allowed_params + if unknown_params: + raise GFQLTypeError( + ErrorCode.E303, + f"Unknown parameters for '{function}'", + field="params", + value=list(unknown_params), + suggestion=f"Allowed parameters: {', '.join(sorted(allowed_params))}" + ) + + # Validate parameter types + for param_name, param_value in params.items(): + if param_name in param_validators: + validator = param_validators[param_name] + if not validator(param_value): + raise GFQLTypeError( + ErrorCode.E201, + f"Invalid type for parameter '{param_name}' in '{function}'", + field=f"params.{param_name}", + value=f"{type(param_value).__name__}: {param_value}", + suggestion="Check the parameter type requirements" + ) + + return params diff --git a/graphistry/compute/gfql_unified.py b/graphistry/compute/gfql_unified.py new file mode 100644 index 0000000000..04ff063666 --- /dev/null +++ b/graphistry/compute/gfql_unified.py @@ -0,0 +1,98 @@ +"""GFQL unified entrypoint for chains and DAGs""" + +from typing import List, Union, Optional +from graphistry.Plottable import Plottable +from graphistry.Engine import EngineAbstract +from graphistry.util import setup_logger +from .ast import ASTObject, ASTLet +from .chain import Chain, chain as chain_impl +from .chain_let import chain_let as chain_let_impl + +logger = setup_logger(__name__) + + +def gfql(self: Plottable, + query: Union[ASTObject, List[ASTObject], ASTLet, Chain, dict], + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + output: Optional[str] = None) -> Plottable: + """ + Execute a GFQL query - either a chain or a DAG + + Unified entrypoint that automatically detects query type and + dispatches to the appropriate execution engine. + + :param query: GFQL query - ASTObject, List[ASTObject], Chain, ASTLet, or dict + :param engine: Execution engine (auto, pandas, cudf) + :param output: For DAGs, name of binding to return (default: last executed) + :returns: Resulting Plottable + :rtype: Plottable + + **Example: Chain query** + + :: + + from graphistry.compute.ast import n, e + + # As list + result = g.gfql([n({'type': 'person'}), e(), n()]) + + # As Chain object + from graphistry.compute.chain import Chain + result = g.gfql(Chain([n({'type': 'person'}), e(), n()])) + + **Example: DAG query** + + :: + + from graphistry.compute.ast import let, ref, n, e + + result = g.gfql(let({ + 'people': n({'type': 'person'}), + 'friends': ref('people', [e({'rel': 'knows'}), n()]) + })) + + # Select specific output + friends = g.gfql(result, output='friends') + + **Example: Auto-detection** + + :: + + # List → chain execution + g.gfql([n(), e(), n()]) + + # Single ASTObject → chain execution + g.gfql(n({'type': 'person'})) + + # Dict → DAG execution (convenience) + g.gfql({'people': n({'type': 'person'})}) + """ + # Handle dict convenience first (convert to ASTLet) + if isinstance(query, dict): + query = ASTLet(query) + + # Dispatch based on type - check specific types before generic + if isinstance(query, ASTLet): + logger.debug('GFQL executing as DAG') + return chain_let_impl(self, query, engine, output) + elif isinstance(query, Chain): + logger.debug('GFQL executing as Chain') + if output is not None: + logger.warning('output parameter ignored for chain queries') + return chain_impl(self, query.chain, engine) + elif isinstance(query, ASTObject): + # Single ASTObject -> execute as single-item chain + logger.debug('GFQL executing single ASTObject as chain') + if output is not None: + logger.warning('output parameter ignored for chain queries') + return chain_impl(self, [query], engine) + elif isinstance(query, list): + logger.debug('GFQL executing list as chain') + if output is not None: + logger.warning('output parameter ignored for chain queries') + return chain_impl(self, query, engine) + else: + raise TypeError( + f"Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict. " + f"Got {type(query).__name__}" + ) diff --git a/graphistry/compute/gfql_validation/__init__.py b/graphistry/compute/gfql_validation/__init__.py new file mode 100644 index 0000000000..cd8522e0c3 --- /dev/null +++ b/graphistry/compute/gfql_validation/__init__.py @@ -0,0 +1,45 @@ +"""GFQL validation and related utilities.""" + +from graphistry.compute.gfql_validation.validate import ( + ValidationIssue, + Schema, + validate_syntax, + validate_schema, + validate_query, + extract_schema, + extract_schema_from_dataframes, + format_validation_errors, + suggest_fixes +) + +from graphistry.compute.gfql_validation.exceptions import ( + GFQLException, + GFQLValidationError, + GFQLSyntaxError, + GFQLSchemaError, + GFQLTypeError, + GFQLColumnNotFoundError +) + +__all__ = [ + # Validation classes + 'ValidationIssue', + 'Schema', + + # Validation functions + 'validate_syntax', + 'validate_schema', + 'validate_query', + 'extract_schema', + 'extract_schema_from_dataframes', + 'format_validation_errors', + 'suggest_fixes', + + # Exceptions + 'GFQLException', + 'GFQLValidationError', + 'GFQLSyntaxError', + 'GFQLSchemaError', + 'GFQLTypeError', + 'GFQLColumnNotFoundError' +] diff --git a/graphistry/compute/gfql_validation/exceptions.py b/graphistry/compute/gfql_validation/exceptions.py new file mode 100644 index 0000000000..fd44068bf6 --- /dev/null +++ b/graphistry/compute/gfql_validation/exceptions.py @@ -0,0 +1,62 @@ +"""GFQL-specific exceptions for validation and error handling.""" + +from typing import Optional, Dict, Any + + +class GFQLException(Exception): + """Base exception for all GFQL-related errors.""" + + def __init__(self, message: str, context: Optional[Dict[str, Any]] = None): + self.context = context or {} + super().__init__(message) + + def __str__(self) -> str: + if self.context: + context_str = ", ".join(f"{k}={v}" for k, v in self.context.items()) + return f"{super().__str__()} ({context_str})" + return super().__str__() + + +class GFQLValidationError(GFQLException, ValueError): + """Base validation error. Inherits from ValueError for backwards compatibility.""" + pass + + +class GFQLSyntaxError(GFQLValidationError): + """Raised when GFQL query has invalid syntax.""" + pass + + +class GFQLSchemaError(GFQLValidationError): + """Raised when GFQL query references non-existent columns or has type mismatches.""" + pass + + +class GFQLSemanticError(GFQLValidationError): + """Raised when GFQL query is syntactically valid but semantically incorrect.""" + pass + + +class GFQLTypeError(GFQLSchemaError): + """Raised when a predicate is applied to incompatible column type.""" + + def __init__(self, column: str, column_type: str, predicate: str, expected_type: str): + message = f"Column '{column}' has type '{column_type}' but predicate '{predicate}' expects '{expected_type}'" + super().__init__(message, { + 'column': column, + 'column_type': column_type, + 'predicate': predicate, + 'expected_type': expected_type + }) + + +class GFQLColumnNotFoundError(GFQLSchemaError): + """Raised when a referenced column doesn't exist in the schema.""" + + def __init__(self, column: str, table: str, available_columns: list): + message = f"Column '{column}' not found in {table} data" + super().__init__(message, { + 'column': column, + 'table': table, + 'available_columns': available_columns + }) diff --git a/graphistry/compute/gfql_validation/validate.py b/graphistry/compute/gfql_validation/validate.py new file mode 100644 index 0000000000..801aaa923d --- /dev/null +++ b/graphistry/compute/gfql_validation/validate.py @@ -0,0 +1,676 @@ +"""GFQL query validation utilities for syntax and schema checking. + +.. deprecated:: 0.34.0 + This module is deprecated. GFQL now has built-in validation. + See :doc:`/gfql/validation_migration_guide` for migration instructions. + + Instead of:: + + from graphistry.compute.gfql.validate import validate_syntax + issues = validate_syntax(query) + + Use:: + + from graphistry.compute.chain import Chain + try: + chain = Chain(query) # Automatic validation + except GFQLValidationError as e: + print(f"[{e.code}] {e.message}") +""" + +from typing import List, Optional, Dict, Union, Any, Tuple, TYPE_CHECKING +import warnings +import pandas as pd + +from graphistry.compute.chain import Chain +from graphistry.compute.ast import ASTNode, ASTEdge, ASTObject + +if TYPE_CHECKING: + from graphistry.Plottable import Plottable +from graphistry.compute.predicates.ASTPredicate import ASTPredicate +from graphistry.compute.predicates.numeric import NumericASTPredicate +from graphistry.compute.predicates.str import ( + Contains, Startswith, Endswith, Match, + IsNumeric, IsAlpha, IsDigit, IsLower, IsUpper, + IsSpace, IsAlnum, IsDecimal, IsTitle +) +from graphistry.compute.predicates.temporal import ( + IsMonthStart, IsMonthEnd, IsQuarterStart, IsQuarterEnd, + IsYearStart, IsYearEnd, IsLeapYear +) +from graphistry.util import setup_logger + +warnings.warn( + "The graphistry.compute.gfql.validate module is deprecated. " + "GFQL now has built-in validation. See the migration guide for details.", + DeprecationWarning, + stacklevel=2 +) + +logger = setup_logger(__name__) + + +class ValidationIssue: + """Represents a validation issue (error or warning).""" + + def __init__(self, + level: str, # 'error' or 'warning' + message: str, + operation_index: Optional[int] = None, + field: Optional[str] = None, + suggestion: Optional[str] = None, + error_type: Optional[str] = None): + self.level = level + self.message = message + self.operation_index = operation_index + self.field = field + self.suggestion = suggestion + self.error_type = error_type + + def __repr__(self) -> str: + parts = [f"{self.level.upper()}: {self.message}"] + if self.operation_index is not None: + parts.append(f"at operation {self.operation_index}") + if self.field: + parts.append(f"field: {self.field}") + if self.suggestion: + parts.append(f"Suggestion: {self.suggestion}") + return " | ".join(parts) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'level': self.level, + 'message': self.message, + 'operation_index': self.operation_index, + 'field': self.field, + 'suggestion': self.suggestion, + 'error_type': self.error_type + } + + +class Schema: + """Represents the schema of node and edge dataframes.""" + + def __init__(self, + node_columns: Optional[Dict[str, str]] = None, + edge_columns: Optional[Dict[str, str]] = None): + self.node_columns = node_columns or {} + self.edge_columns = edge_columns or {} + + def __repr__(self) -> str: + return (f"Schema(nodes={list(self.node_columns.keys())}, " + f"edges={list(self.edge_columns.keys())})") + + +# Error message templates +ERROR_MESSAGES = { + # Syntax errors + 'INVALID_CHAIN_TYPE': { + 'message': 'Chain must be a list of operations', + 'suggestion': 'Wrap your operations in a list: [n(), e(), n()]' + }, + 'INVALID_OPERATION': { + 'message': 'Operation at index {index} is not a valid GFQL operation', + 'suggestion': ('Use n() for nodes, ' + 'e()/e_forward()/e_reverse() for edges') + }, + 'INVALID_FILTER_KEY': { + 'message': 'Invalid filter key format: {key}', + 'suggestion': 'Filter keys must be strings' + }, + 'INVALID_HOPS': { + 'message': 'Hops must be a positive integer or None', + 'suggestion': ('Use hops=2 for specific count, ' + 'or to_fixed_point=True for unbounded') + }, + + # Schema errors + 'COLUMN_NOT_FOUND': { + 'message': 'Column "{column}" not found in {table} data', + 'suggestion': 'Available columns: {available}' + }, + 'TYPE_MISMATCH': { + 'message': ('Column "{column}" is {actual_type} but ' + 'predicate expects {expected_type}'), + 'suggestion': 'Use appropriate predicate for {actual_type} columns' + }, + 'INVALID_PREDICATE': { + 'message': ('Predicate {predicate} cannot be used with ' + '{column_type} columns'), + 'suggestion': 'Use {suggested_predicates} for {column_type} columns' + }, + + # Semantic errors + 'ORPHANED_EDGE': { + 'message': 'Edge operation at index {index} not connected to nodes', + 'suggestion': ('Add node operations before/after edge: ' + 'n() -> e() -> n()') + }, + 'UNBOUNDED_HOPS_WARNING': { + 'message': ('Unbounded hops (to_fixed_point=True) may be slow ' + 'on large graphs'), + 'suggestion': ('Consider using specific hop count for better ' + 'performance') + }, + 'EMPTY_CHAIN': { + 'message': 'Chain is empty', + 'suggestion': 'Add at least one operation: [n()]' + }, + 'INVALID_EDGE_DIRECTION': { + 'message': 'Invalid edge direction: {direction}', + 'suggestion': 'Use "forward", "reverse", or "undirected"' + } +} + + +def _format_error(error_key: str, **kwargs) -> Tuple[str, str]: + """Format error message with context.""" + template = ERROR_MESSAGES.get(error_key, { + 'message': f'Unknown error: {error_key}', + 'suggestion': 'Check documentation for valid syntax' + }) + message = template['message'].format(**kwargs) + suggestion = template['suggestion'].format(**kwargs) + return message, suggestion + + +def validate_syntax(chain: Union[Chain, List]) -> List[ValidationIssue]: + """ + Validate GFQL query syntax without requiring data. + + Args: + chain: GFQL chain or list of operations + + Returns: + List of validation issues (errors and warnings) + """ + issues = [] + + # Convert to list if Chain object + if isinstance(chain, Chain): + operations = chain.chain + elif isinstance(chain, list): + operations = chain + else: + message, suggestion = _format_error('INVALID_CHAIN_TYPE') + issues.append(ValidationIssue( + 'error', message, suggestion=suggestion, + error_type='INVALID_CHAIN_TYPE')) + return issues + + # Check empty chain + if not operations: + message, suggestion = _format_error('EMPTY_CHAIN') + issues.append(ValidationIssue( + 'error', message, suggestion=suggestion, + error_type='EMPTY_CHAIN')) + return issues + + # Validate each operation + for i, op in enumerate(operations): + # Check if valid operation type + if not isinstance(op, ASTObject): + message, suggestion = _format_error('INVALID_OPERATION', index=i) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + suggestion=suggestion, error_type='INVALID_OPERATION')) + continue + + # Validate nodes + if isinstance(op, ASTNode): + if op.filter_dict: + for key, value in op.filter_dict.items(): + if not isinstance(key, str): + message, suggestion = _format_error( + 'INVALID_FILTER_KEY', key=key) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field=str(key), suggestion=suggestion, + error_type='INVALID_FILTER_KEY')) + + # Validate edges + elif isinstance(op, ASTEdge): + # Check hops + if (op.hops is not None + and (not isinstance(op.hops, int) or op.hops < 1)): + message, suggestion = _format_error('INVALID_HOPS') + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field='hops', suggestion=suggestion, + error_type='INVALID_HOPS')) + + # Check unbounded hops warning + if op.to_fixed_point: + message, suggestion = _format_error('UNBOUNDED_HOPS_WARNING') + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='UNBOUNDED_HOPS_WARNING')) + + # Check edge filters + for filter_name, filter_dict in [ + ('edge_match', op.edge_match), + ('source_node_match', op.source_node_match), + ('destination_node_match', op.destination_node_match) + ]: + if filter_dict: + for key, value in filter_dict.items(): + if not isinstance(key, str): + message, suggestion = _format_error( + 'INVALID_FILTER_KEY', key=key) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field=f"{filter_name}.{key}", + suggestion=suggestion, + error_type='INVALID_FILTER_KEY')) + + # Check semantic issues + issues.extend(_validate_semantics(operations)) + + return issues + + +def _validate_semantics(operations: List[ASTObject]) -> List[ValidationIssue]: + """Validate semantic correctness of operation sequence.""" + issues = [] + + # Check for orphaned edges (edges not between nodes) + for i, op in enumerate(operations): + if isinstance(op, ASTEdge): + # Check if first or last operation + if i == 0 or i == len(operations) - 1: + # Edge at boundary - likely orphaned + message, suggestion = _format_error('ORPHANED_EDGE', index=i) + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='ORPHANED_EDGE')) + # Check if between two edges + elif (i > 0 and isinstance(operations[i - 1], ASTEdge) + and i < len(operations) - 1 + and isinstance(operations[i + 1], ASTEdge)): + message, suggestion = _format_error('ORPHANED_EDGE', index=i) + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='ORPHANED_EDGE')) + + return issues + + +def validate_schema(chain: Union[Chain, List], + schema: Schema) -> List[ValidationIssue]: + """ + Validate query against data schema. + + Args: + chain: GFQL chain or list of operations + schema: Schema object with column information + + Returns: + List of validation issues + """ + issues = [] + + # First do syntax validation + syntax_issues = validate_syntax(chain) + # Only keep errors from syntax validation for schema validation + issues.extend([issue for issue in syntax_issues if issue.level == 'error']) + + if any(issue.level == 'error' for issue in issues): + return issues # Don't do schema validation if syntax errors + + # Convert to list if Chain object + operations = chain.chain if isinstance(chain, Chain) else chain + + # Validate each operation against schema + for i, op in enumerate(operations): + if isinstance(op, ASTNode): + issues.extend(_validate_node_schema(op, schema, i)) + elif isinstance(op, ASTEdge): + issues.extend(_validate_edge_schema(op, schema, i)) + + return issues + + +def _validate_node_schema(node: ASTNode, schema: Schema, + op_index: int) -> List[ValidationIssue]: + """Validate node operation against schema.""" + issues = [] + + if node.filter_dict and schema.node_columns: + for col, predicate in node.filter_dict.items(): + # Check column exists + if col not in schema.node_columns: + available = list(schema.node_columns.keys())[:5] + if len(schema.node_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='node', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=col, suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + # Check predicate type compatibility + if isinstance(predicate, ASTPredicate): + col_type = schema.node_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index)) + + return issues + + +def _validate_edge_schema(edge: ASTEdge, schema: Schema, + op_index: int) -> List[ValidationIssue]: + """Validate edge operation against schema.""" + issues = [] + + # Validate edge filters + if edge.edge_match and schema.edge_columns: + for col, predicate in edge.edge_match.items(): + if col not in schema.edge_columns: + available = list(schema.edge_columns.keys())[:5] + if len(schema.edge_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='edge', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"edge_match.{col}", suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + if isinstance(predicate, ASTPredicate): + col_type = schema.edge_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index, + field_prefix="edge_match.")) + + # Validate source/dest node filters against node schema + for filter_name, filter_dict in [ + ('source_node_match', edge.source_node_match), + ('destination_node_match', edge.destination_node_match) + ]: + if filter_dict and schema.node_columns: + for col, predicate in filter_dict.items(): + if col not in schema.node_columns: + available = list(schema.node_columns.keys())[:5] + if len(schema.node_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='node', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{filter_name}.{col}", suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + if isinstance(predicate, ASTPredicate): + col_type = schema.node_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index, + field_prefix=f"{filter_name}.")) + + return issues + + +def _validate_predicate_type(predicate: ASTPredicate, column: str, + column_type: str, op_index: int, + field_prefix: str = "") -> List[ValidationIssue]: + """Validate predicate is appropriate for column type.""" + issues = [] + + # Map pandas/numpy dtypes to categories + type_category = _get_type_category(column_type) + + # Define string predicate types + STRING_PREDICATES = ( + Contains, Startswith, Endswith, Match, + IsNumeric, IsAlpha, IsDigit, IsLower, IsUpper, + IsSpace, IsAlnum, IsDecimal, IsTitle + ) + + # Define temporal predicate types + TEMPORAL_PREDICATES = ( + IsMonthStart, IsMonthEnd, IsQuarterStart, IsQuarterEnd, + IsYearStart, IsYearEnd, IsLeapYear + ) + + # Check predicate compatibility + if (isinstance(predicate, NumericASTPredicate) + and type_category not in ['numeric', 'temporal']): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='numeric') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + elif (isinstance(predicate, STRING_PREDICATES) + and type_category != 'string'): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='string') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + elif (isinstance(predicate, TEMPORAL_PREDICATES) + and type_category != 'temporal'): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='datetime') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + return issues + + +def _get_type_category(dtype_str: str) -> str: + """Categorize dtype string into broad categories.""" + dtype_lower = str(dtype_str).lower() + + if any(t in dtype_lower for t in + ['int', 'float', 'double', 'numeric', 'decimal']): + return 'numeric' + elif any(t in dtype_lower for t in + ['str', 'object', 'char', 'text', 'varchar']): + return 'string' + elif any(t in dtype_lower for t in + ['date', 'time', 'timestamp', 'datetime']): + return 'temporal' + elif 'bool' in dtype_lower: + return 'boolean' + else: + return 'unknown' + + +def validate_query(chain: Union[Chain, List], + nodes_df: Optional[pd.DataFrame] = None, + edges_df: Optional[pd.DataFrame] = None + ) -> List[ValidationIssue]: + """ + Combined syntax and schema validation. + + Args: + chain: GFQL chain or list of operations + nodes_df: Optional node dataframe for schema validation + edges_df: Optional edge dataframe for schema validation + + Returns: + List of validation issues + """ + # Always do syntax validation + issues = validate_syntax(chain) + + # If data provided, also do schema validation + if nodes_df is not None or edges_df is not None: + schema = extract_schema_from_dataframes(nodes_df, edges_df) + schema_issues = validate_schema(chain, schema) + # Merge issues, avoiding duplicates + existing_errors = {(i.error_type, i.operation_index, i.field) + for i in issues} + for issue in schema_issues: + if ((issue.error_type, issue.operation_index, issue.field) + not in existing_errors): + issues.append(issue) + + return issues + + +def extract_schema(g: "Plottable") -> Schema: + """ + Extract schema from a Plottable object. + + Args: + g: Plottable object with node/edge data + + Returns: + Schema object + """ + + nodes_df = g._nodes if hasattr(g, '_nodes') else None + edges_df = g._edges if hasattr(g, '_edges') else None + + return extract_schema_from_dataframes(nodes_df, edges_df) + + +def extract_schema_from_dataframes( + nodes_df: Optional[pd.DataFrame] = None, + edges_df: Optional[pd.DataFrame] = None) -> Schema: + """ + Extract schema from pandas DataFrames. + + Args: + nodes_df: Optional node dataframe + edges_df: Optional edge dataframe + + Returns: + Schema object with column names and types + """ + node_columns = {} + edge_columns = {} + + if nodes_df is not None and hasattr(nodes_df, 'dtypes'): + node_columns = {str(col): str(dtype) + for col, dtype in nodes_df.dtypes.items()} + + if edges_df is not None and hasattr(edges_df, 'dtypes'): + edge_columns = {str(col): str(dtype) + for col, dtype in edges_df.dtypes.items()} + + return Schema(node_columns, edge_columns) + + +def format_validation_errors(issues: List[ValidationIssue]) -> str: + """ + Format validation errors for human/LLM consumption. + + Args: + issues: List of validation issues + + Returns: + Formatted error string + """ + if not issues: + return "No validation issues found." + + lines = ["GFQL Validation Report:"] + lines.append("-" * 50) + + errors = [i for i in issues if i.level == 'error'] + warnings = [i for i in issues if i.level == 'warning'] + + if errors: + lines.append(f"\nERRORS ({len(errors)}):") + for i, error in enumerate(errors, 1): + lines.append(f"\n{i}. {error.message}") + if error.operation_index is not None: + lines.append(f" Location: Operation {error.operation_index}") + if error.field: + lines.append(f" Field: {error.field}") + if error.suggestion: + lines.append(f" 💡 {error.suggestion}") + + if warnings: + lines.append(f"\nWARNINGS ({len(warnings)}):") + for i, warning in enumerate(warnings, 1): + lines.append(f"\n{i}. {warning.message}") + if warning.operation_index is not None: + lines.append( + f" Location: Operation {warning.operation_index}") + if warning.suggestion: + lines.append(f" 💡 {warning.suggestion}") + + return "\n".join(lines) + + +def suggest_fixes(chain: Union[Chain, List], + issues: List[ValidationIssue]) -> List[str]: + """ + Generate fix suggestions for validation issues. + + Args: + chain: The problematic chain + issues: Validation issues found + + Returns: + List of suggested fixes + """ + suggestions = [] + + # Group by error type for consolidated suggestions + error_types: Dict[str, List[ValidationIssue]] = {} + for issue in issues: + if issue.error_type: + error_types.setdefault(issue.error_type, []).append(issue) + + # Generate type-specific suggestions + if 'COLUMN_NOT_FOUND' in error_types: + missing_cols = {issue.field for issue in + error_types['COLUMN_NOT_FOUND'] if issue.field} + suggestions.append( + f"Missing columns: {', '.join(missing_cols)}. " + f"Check column names match your data.") + + if 'TYPE_MISMATCH' in error_types: + suggestions.append( + "Type mismatches found. Use numeric predicates (gt, lt) " + "for numbers, string predicates (contains, startswith) for text.") + + if 'ORPHANED_EDGE' in error_types: + suggestions.append( + "Edge operations should connect nodes. " + "Use pattern: n() -> e() -> n()") + + # Add general suggestions from issues + for issue in issues: + if issue.suggestion and issue.suggestion not in suggestions: + suggestions.append(issue.suggestion) + + return suggestions diff --git a/graphistry/compute/validate_schema.py b/graphistry/compute/validate_schema.py new file mode 100644 index 0000000000..357c0c1969 --- /dev/null +++ b/graphistry/compute/validate_schema.py @@ -0,0 +1,413 @@ +"""Schema validation for GFQL chains without execution.""" + +from typing import List, Optional, Union, TYPE_CHECKING, cast +import pandas as pd +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, ASTLet, ASTRef, ASTRemoteGraph, ASTCall + +if TYPE_CHECKING: + from graphistry.compute.chain import Chain +from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError +from graphistry.compute.predicates.ASTPredicate import ASTPredicate +from graphistry.compute.predicates.numeric import NumericASTPredicate, Between +from graphistry.compute.predicates.str import Contains, Startswith, Endswith, Match + + +def validate_chain_schema( + g: Plottable, + ops: Union[List[ASTObject], 'Chain'], + collect_all: bool = False +) -> Optional[List[GFQLSchemaError]]: + """Validate chain operations against graph schema without executing. + + This performs static analysis of the chain operations to detect: + - References to non-existent columns + - Type mismatches between filters and column types + - Invalid predicate usage + + Args: + g: The graph to validate against + ops: Chain operations to validate + collect_all: If True, collect all errors. If False, raise on first error. + + Returns: + If collect_all=True: List of schema errors (empty if valid) + If collect_all=False: None if valid + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + # Handle Chain objects + from graphistry.compute.chain import Chain + if isinstance(ops, Chain): + chain_ops = ops.chain + else: + chain_ops = ops + + errors: List[GFQLSchemaError] = [] + + # Get available columns + node_columns = set(g._nodes.columns) if g._nodes is not None else set() + edge_columns = set(g._edges.columns) if g._edges is not None else set() + + for i, op in enumerate(chain_ops): + op_errors = [] + + if isinstance(op, ASTNode): + op_errors = _validate_node_op(op, node_columns, g._nodes, collect_all) + elif isinstance(op, ASTEdge): + op_errors = _validate_edge_op(op, node_columns, edge_columns, g._nodes, g._edges, collect_all) + elif isinstance(op, ASTLet): + op_errors = _validate_querydag_op(op, g, collect_all) + elif isinstance(op, ASTRef): + op_errors = _validate_chainref_op(op, g, collect_all) + elif isinstance(op, ASTRemoteGraph): + op_errors = _validate_remotegraph_op(op, collect_all) + elif isinstance(op, ASTCall): + op_errors = _validate_call_op(op, node_columns, edge_columns, collect_all) + + # Add operation index to all errors + for e in op_errors: + e.context['operation_index'] = i + + if op_errors: + if collect_all: + errors.extend(op_errors) + else: + raise op_errors[0] + + return errors if collect_all else None + + +def _validate_node_op(op: ASTNode, node_columns: set, nodes_df: Optional[pd.DataFrame], collect_all: bool) -> List[GFQLSchemaError]: + """Validate node operation against schema.""" + errors = [] + if op.filter_dict and nodes_df is not None: + errors.extend(_validate_filter_dict(op.filter_dict, node_columns, nodes_df, "node", collect_all)) + return errors + + +def _validate_edge_op( + op: ASTEdge, + node_columns: set, + edge_columns: set, + nodes_df: Optional[pd.DataFrame], + edges_df: Optional[pd.DataFrame], + collect_all: bool +) -> List[GFQLSchemaError]: + """Validate edge operation against schema.""" + errors = [] + + # Validate edge filters + if op.edge_match and edges_df is not None: + errors.extend(_validate_filter_dict(op.edge_match, edge_columns, edges_df, "edge", collect_all)) + + # Validate source node filters + if op.source_node_match and nodes_df is not None: + errors.extend(_validate_filter_dict(op.source_node_match, node_columns, nodes_df, "source node", collect_all)) + + # Validate destination node filters + if op.destination_node_match and nodes_df is not None: + errors.extend(_validate_filter_dict(op.destination_node_match, node_columns, nodes_df, "destination node", collect_all)) + + return errors + + +def _validate_querydag_op(op: ASTLet, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate Let operation against schema.""" + errors = [] + + # Validate each binding in the DAG + for binding_name, binding_value in op.bindings.items(): + try: + # Recursively validate each binding as if it's a single operation + binding_errors = validate_chain_schema(g, [binding_value], collect_all=True) + + # Add binding context to errors + if binding_errors: + for error in binding_errors: + error.context['dag_binding'] = binding_name + + if binding_errors: + if collect_all: + errors.extend(binding_errors) + else: + raise binding_errors[0] + + except GFQLSchemaError as e: + e.context['dag_binding'] = binding_name + if collect_all: + errors.append(e) + else: + raise + + return errors + + +def _validate_chainref_op(op: ASTRef, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate ChainRef operation against schema.""" + errors = [] + + # Validate the chain operations in the ChainRef + if op.chain: + try: + chain_errors = validate_chain_schema(g, op.chain, collect_all=True) + + # Add ChainRef context to errors + if chain_errors: + for error in chain_errors: + error.context['chain_ref'] = op.ref + + if chain_errors: + if collect_all: + errors.extend(chain_errors) + else: + raise chain_errors[0] + + except GFQLSchemaError as e: + e.context['chain_ref'] = op.ref + if collect_all: + errors.append(e) + else: + raise + + # Note: We don't validate that op.ref exists here since that's handled + # by the DAG dependency validation in chain_let.py + + return errors + + +def _validate_remotegraph_op(op: ASTRemoteGraph, collect_all: bool) -> List[GFQLSchemaError]: + """Validate RemoteGraph operation against schema.""" + errors = [] + + # Validate dataset_id format + if not op.dataset_id or not isinstance(op.dataset_id, str): + error = GFQLSchemaError( + ErrorCode.E303, + 'RemoteGraph dataset_id must be a non-empty string', + field='dataset_id', + value=op.dataset_id, + suggestion='Provide a valid dataset identifier string' + ) + if collect_all: + errors.append(error) + else: + raise error + + # Validate token format if provided + if op.token is not None and not isinstance(op.token, str): + error = GFQLSchemaError( + ErrorCode.E303, + 'RemoteGraph token must be a string if provided', + field='token', + value=type(op.token).__name__, + suggestion='Provide a valid token string or None' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + +def _validate_filter_dict( + filter_dict: dict, + columns: set, + df: pd.DataFrame, + context: str, + collect_all: bool = False +) -> List[GFQLSchemaError]: + """Validate filter dictionary against dataframe schema.""" + errors = [] + for col, val in filter_dict.items(): + try: + # Check column exists + if col not in columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Column "{col}" does not exist in {context} dataframe', + field=col, + value=val, + suggestion=f'Available columns: {", ".join(sorted(columns)[:10])}{"..." if len(columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + continue # Check next field + else: + raise error + + # Check type compatibility + col_dtype = df[col].dtype + + if not isinstance(val, ASTPredicate): + # Check literal value type matches + if pd.api.types.is_numeric_dtype(col_dtype) and isinstance(val, str): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: {context} column "{col}" is numeric but filter value is string', + field=col, + value=val, + column_type=str(col_dtype), + suggestion=f'Use a numeric value like {col}=123' + ) + if collect_all: + errors.append(error) + else: + raise error + elif pd.api.types.is_string_dtype(col_dtype) and isinstance(val, (int, float)): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: {context} column "{col}" is string but filter value is numeric', + field=col, + value=val, + column_type=str(col_dtype), + suggestion=f'Use a string value like {col}="value"' + ) + if collect_all: + errors.append(error) + else: + raise error + else: + # Check predicate type matches column type + if isinstance(val, (NumericASTPredicate, Between)) and not pd.api.types.is_numeric_dtype(col_dtype): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: numeric predicate used on non-numeric {context} column "{col}"', + field=col, + value=f"{val.__class__.__name__}(...)", + column_type=str(col_dtype), + suggestion='Use string predicates like contains() or startswith() for string columns' + ) + if collect_all: + errors.append(error) + else: + raise error + + if isinstance(val, (Contains, Startswith, Endswith, Match)) and not pd.api.types.is_string_dtype(col_dtype): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: string predicate used on non-string {context} column "{col}"', + field=col, + value=f"{val.__class__.__name__}(...)", + column_type=str(col_dtype), + suggestion='Use numeric predicates like gt() or lt() for numeric columns' + ) + if collect_all: + errors.append(error) + else: + raise error + + except GFQLSchemaError: + if not collect_all: + raise + + return errors + + +def _validate_call_op( + op: ASTCall, + node_columns: set, + edge_columns: set, + collect_all: bool = False +) -> List[GFQLSchemaError]: + """Validate Call operation schema requirements. + + Checks that all columns required by the called method exist in the graph. + Uses the schema_effects metadata from the safelist to determine requirements. + + Args: + op: ASTCall operation to validate + node_columns: Set of available node column names + edge_columns: Set of available edge column names + collect_all: If True, collect all errors. If False, raise on first error. + + Returns: + List of schema errors found (empty if valid) + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + errors: List[GFQLSchemaError] = [] + + # Import safelist to get schema effects + from graphistry.compute.gfql.call_safelist import SAFELIST_V1 + + # Check if method is in safelist + if op.function not in SAFELIST_V1: + # This should have been caught by parameter validation already + return errors + + method_info = SAFELIST_V1[op.function] + + # Check if method has schema effects defined + if 'schema_effects' not in method_info: + # Method doesn't define schema effects, so we can't validate + return errors + + schema_effects = method_info['schema_effects'] + + # Get required columns based on parameters + if 'requires_node_cols' in schema_effects: + if callable(schema_effects['requires_node_cols']): + required_node_cols = schema_effects['requires_node_cols'](op.params) + else: + required_node_cols = schema_effects['requires_node_cols'] + + for col in required_node_cols: + if col not in node_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires node column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available node columns: {", ".join(sorted(node_columns)[:10])}{"..." if len(node_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + if 'requires_edge_cols' in schema_effects: + if callable(schema_effects['requires_edge_cols']): + required_edge_cols = schema_effects['requires_edge_cols'](op.params) + else: + required_edge_cols = schema_effects['requires_edge_cols'] + + for col in required_edge_cols: + if col not in edge_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires edge column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available edge columns: {", ".join(sorted(edge_columns)[:10])}{"..." if len(edge_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + +# Add to Chain class +def validate_schema(self: 'Chain', g: Plottable, collect_all: bool = False) -> Optional[List[GFQLSchemaError]]: + """Validate this chain against a graph's schema without executing. + + Args: + g: Graph to validate against + collect_all: If True, collect all errors. If False, raise on first. + + Returns: + If collect_all=True: List of schema errors + If collect_all=False: None if valid + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + return validate_chain_schema(g, self, collect_all) + + +# Monkey-patching moved to chain.py to avoid circular import diff --git a/graphistry/tests/compute/README_INTEGRATION_TESTS.md b/graphistry/tests/compute/README_INTEGRATION_TESTS.md new file mode 100644 index 0000000000..0fca6c5656 --- /dev/null +++ b/graphistry/tests/compute/README_INTEGRATION_TESTS.md @@ -0,0 +1,67 @@ +# Integration Test Configuration + +This directory contains both unit tests (always run) and integration tests (opt-in). + +## Environment Variables for Integration Tests + +### GPU Tests +```bash +# Enable CUDF/GPU tests +TEST_CUDF=1 pytest test_chain_let_gpu.py +``` + +### Remote Graph Integration Tests +```bash +# Enable remote Graphistry server tests +TEST_REMOTE_INTEGRATION=1 pytest test_chain_let_remote_integration.py + +# Additional configuration for remote tests: +GRAPHISTRY_USERNAME=myuser # Username for auth +GRAPHISTRY_PASSWORD=mypass # Password for auth +GRAPHISTRY_API_KEY=key-123 # Alternative to username/password +GRAPHISTRY_SERVER=hub.graphistry.com # Server URL (optional) +GRAPHISTRY_TEST_DATASET_ID=abc123 # Known dataset for testing (optional) +``` + +## Running All Tests + +```bash +# Unit tests only (fast, no external dependencies) +pytest + +# All tests including integration +TEST_CUDF=1 TEST_REMOTE_INTEGRATION=1 pytest +``` + +## Writing New Integration Tests + +1. **Use environment variable guards:** + ```python + import os + import pytest + + REMOTE_INTEGRATION_ENABLED = os.environ.get("TEST_REMOTE_INTEGRATION") == "1" + skip_remote = pytest.mark.skipif( + not REMOTE_INTEGRATION_ENABLED, + reason="Remote integration tests need TEST_REMOTE_INTEGRATION=1" + ) + + @skip_remote + def test_my_remote_feature(): + # This only runs when TEST_REMOTE_INTEGRATION=1 + pass + ``` + +2. **Always provide mocked versions:** + - Integration tests verify real behavior + - Unit tests with mocks ensure CI/CD still validates core logic + +3. **Document requirements:** + - What env vars are needed + - What external services must be running + - Expected test data setup + +## CI/CD Configuration + +The CI/CD pipeline runs only unit tests by default. Integration tests can be enabled +in specific CI jobs by setting the appropriate environment variables. \ No newline at end of file diff --git a/graphistry/tests/compute/test_ast_errors.py b/graphistry/tests/compute/test_ast_errors.py new file mode 100644 index 0000000000..956d003f23 --- /dev/null +++ b/graphistry/tests/compute/test_ast_errors.py @@ -0,0 +1,85 @@ +"""Test error handling and messages in AST serialization""" + +import pytest +from graphistry.compute.ast import from_json +from graphistry.compute.exceptions import GFQLSyntaxError + + +class TestSerializationErrors: + """Test error handling in JSON serialization/deserialization""" + + def test_from_json_non_dict_input(self): + """Test clear error when input is not a dict""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json("not a dict") + + assert exc_info.value.code == "invalid-chain-type" + assert "AST JSON must be a dictionary" in str(exc_info.value) + + def test_from_json_none_input(self): + """Test clear error when input is None""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json(None) + + assert exc_info.value.code == "invalid-chain-type" + assert "AST JSON must be a dictionary" in str(exc_info.value) + + def test_from_json_missing_type(self): + """Test clear error when 'type' field is missing""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"no_type": "value"}) + + assert exc_info.value.code == "missing-required-field" + assert "AST JSON missing required 'type' field" in str(exc_info.value) + + def test_from_json_unknown_type(self): + """Test clear error for unknown type""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"type": "UnknownType"}) + + assert exc_info.value.code == "invalid-chain-type" + assert "Unknown AST type: UnknownType" in str(exc_info.value) + + def test_edge_missing_direction(self): + """Test clear error when Edge missing direction""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"type": "Edge"}) + + assert exc_info.value.code == "missing-required-field" + assert "Edge missing required 'direction' field" in str(exc_info.value) + + def test_edge_invalid_direction(self): + """Test clear error for invalid Edge direction""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"type": "Edge", "direction": "invalid"}) + + assert exc_info.value.code == "invalid-direction" + assert "Edge has unknown direction: invalid" in str(exc_info.value) + + def test_querydag_missing_bindings(self): + """Test clear error when Let missing bindings""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "QueryDAG"}) + + assert "Let missing bindings" in str(exc_info.value) + + def test_remotegraph_missing_dataset_id(self): + """Test clear error when RemoteGraph missing dataset_id""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "RemoteGraph"}) + + assert "RemoteGraph missing dataset_id" in str(exc_info.value) + + def test_chainref_missing_ref(self): + """Test clear error when Ref missing ref""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "Ref"}) + + assert "Ref missing ref" in str(exc_info.value) + + def test_chainref_missing_chain(self): + """Test clear error when Ref missing chain""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "Ref", "ref": "test"}) + + assert "Ref missing chain" in str(exc_info.value) diff --git a/graphistry/tests/compute/test_call_operations.py b/graphistry/tests/compute/test_call_operations.py new file mode 100644 index 0000000000..7ebb582fe8 --- /dev/null +++ b/graphistry/tests/compute/test_call_operations.py @@ -0,0 +1,485 @@ +"""Tests for GFQL Call operations.""" + +import pytest +import pandas as pd +from unittest.mock import Mock, patch, MagicMock + +from graphistry.tests.test_compute import CGFull +from graphistry.Engine import Engine, EngineAbstract +from graphistry.compute.ast import ASTCall, ASTLet, n +from graphistry.compute.chain_let import chain_let_impl +from graphistry.compute.gfql.call_safelist import validate_call_params +from graphistry.compute.gfql.call_executor import execute_call +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError, GFQLSyntaxError + + +class TestCallSafelist: + """Test method safelist validation.""" + + def test_allowed_method(self): + """Test validation of allowed methods.""" + params = validate_call_params('get_degrees', { + 'col': 'degree' + }) + assert params == {'col': 'degree'} + + def test_unknown_method(self): + """Test rejection of unknown methods.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('unknown_method', {}) + assert exc_info.value.code == ErrorCode.E303 + assert 'not in the safelist' in str(exc_info.value) + + def test_required_params(self): + """Test validation of required parameters.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('filter_nodes_by_dict', {}) + assert exc_info.value.code == ErrorCode.E105 + assert 'Missing required parameters' in str(exc_info.value) + + def test_unknown_params(self): + """Test rejection of unknown parameters.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('get_degrees', { + 'col': 'degree', + 'unknown_param': 'value' + }) + assert exc_info.value.code == ErrorCode.E303 + assert 'Unknown parameters' in str(exc_info.value) + + def test_param_type_validation(self): + """Test parameter type validation.""" + # Valid types + params = validate_call_params('hop', { + 'hops': 2, + 'direction': 'forward', + 'to_fixed_point': True + }) + assert params['hops'] == 2 + + # Invalid type + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('hop', { + 'hops': 'two' # Should be int + }) + assert exc_info.value.code == ErrorCode.E201 + assert 'Invalid type' in str(exc_info.value) + + def test_enum_validation(self): + """Test enum parameter validation.""" + # Valid enum value + params = validate_call_params('hop', { + 'direction': 'forward' + }) + assert params['direction'] == 'forward' + + # Invalid enum value + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('hop', { + 'direction': 'sideways' + }) + assert exc_info.value.code == ErrorCode.E201 + + +class TestASTCall: + """Test ASTCall node validation and serialization.""" + + def test_basic_creation(self): + """Test creating a basic ASTCall.""" + call = ASTCall('get_degrees', {'col': 'degree'}) + assert call.function == 'get_degrees' + assert call.params == {'col': 'degree'} + + def test_empty_params(self): + """Test ASTCall with no parameters.""" + call = ASTCall('prune_self_edges') + assert call.function == 'prune_self_edges' + assert call.params == {} + + def test_validation(self): + """Test ASTCall field validation.""" + # Valid call + call = ASTCall('get_degrees', {'col': 'degree'}) + call.validate() # Should not raise + + # Invalid function type + with pytest.raises(GFQLTypeError) as exc_info: + call = ASTCall(123, {}) + call.validate() + assert exc_info.value.code == ErrorCode.E201 + assert 'function must be a string' in str(exc_info.value) + + # Invalid params type + with pytest.raises(GFQLTypeError) as exc_info: + call = ASTCall('get_degrees', 'not_a_dict') + call.validate() + assert exc_info.value.code == ErrorCode.E201 + assert 'params must be a dict' in str(exc_info.value) + + def test_to_json(self): + """Test ASTCall JSON serialization.""" + call = ASTCall('get_degrees', {'col': 'degree', 'engine': 'pandas'}) + json_data = call.to_json() + + assert json_data == { + 'type': 'Call', + 'function': 'get_degrees', + 'params': {'col': 'degree', 'engine': 'pandas'} + } + + def test_from_json(self): + """Test ASTCall JSON deserialization.""" + json_data = { + 'type': 'Call', + 'function': 'filter_nodes_by_dict', + 'params': {'filter_dict': {'type': 'user'}} + } + + call = ASTCall.from_json(json_data) + assert isinstance(call, ASTCall) + assert call.function == 'filter_nodes_by_dict' + assert call.params == {'filter_dict': {'type': 'user'}} + + def test_from_json_invalid(self): + """Test ASTCall from_json with invalid data.""" + # Missing function + with pytest.raises(AssertionError) as exc_info: + ASTCall.from_json({'type': 'Call'}) + assert 'Call missing function' in str(exc_info.value) + + # Wrong type - this would be caught earlier in the AST dispatch + # so we don't test it here + + +class TestGroupInABoxExecution: + """Test actual execution of group_in_a_box_layout.""" + + @pytest.fixture + def simple_graph(self): + """Create a simple graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 3, 4, 5], + 'target': [1, 2, 0, 4, 5, 3] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3, 4, 5], + 'type': ['A', 'A', 'A', 'B', 'B', 'B'] + }) + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_group_in_a_box_basic(self, simple_graph): + """Test basic group_in_a_box_layout execution.""" + result = execute_call( + simple_graph, + 'group_in_a_box_layout', + {'engine': 'cpu'}, + Engine.PANDAS + ) + + # Should still have same number of nodes/edges + assert len(result._nodes) == len(simple_graph._nodes) + assert len(result._edges) == len(simple_graph._edges) + + # Should have position columns + assert 'x' in result._nodes.columns + assert 'y' in result._nodes.columns + + def test_group_in_a_box_with_partition_key(self, simple_graph): + """Test group_in_a_box_layout with existing partition key.""" + result = execute_call( + simple_graph, + 'group_in_a_box_layout', + { + 'partition_key': 'type', + 'engine': 'cpu', + 'encode_colors': True + }, + Engine.PANDAS + ) + + # Should have positions + assert 'x' in result._nodes.columns + assert 'y' in result._nodes.columns + + +class TestCallExecution: + """Test call execution functionality.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_execute_get_degrees(self, sample_graph): + """Test executing get_degrees method.""" + result = execute_call( + sample_graph, + 'get_degrees', + {'col': 'degree'}, + Engine.PANDAS + ) + + # Result should be a Plottable + assert hasattr(result, '_nodes') + assert 'degree' in result._nodes.columns + assert len(result._nodes) == 4 + + def test_execute_filter_nodes(self, sample_graph): + """Test executing filter_nodes_by_dict method.""" + result = execute_call( + sample_graph, + 'filter_nodes_by_dict', + {'filter_dict': {'type': 'user'}}, + Engine.PANDAS + ) + + # Should filter to only user nodes + assert len(result._nodes) == 3 + assert all(result._nodes['type'] == 'user') + + def test_execute_materialize_nodes(self): + """Test executing materialize_nodes method.""" + # Start with edges only + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0] + }) + g = CGFull().edges(edges_df).bind(source='source', destination='target') + + # No nodes initially + assert g._nodes is None + + # Execute materialize_nodes + result = execute_call( + g, + 'materialize_nodes', + {}, + Engine.PANDAS + ) + + # Should have nodes now + assert result._nodes is not None + assert len(result._nodes) == 3 + + def test_execute_with_validation_error(self, sample_graph): + """Test that validation errors are properly raised.""" + with pytest.raises(GFQLTypeError) as exc_info: + execute_call( + sample_graph, + 'hop', + {'hops': 'invalid'}, # Should be int + Engine.PANDAS + ) + assert exc_info.value.code == ErrorCode.E201 + + def test_execute_unknown_method(self, sample_graph): + """Test execution of unknown method.""" + with pytest.raises(GFQLTypeError) as exc_info: + execute_call( + sample_graph, + 'unknown_method', + {}, + Engine.PANDAS + ) + assert exc_info.value.code == ErrorCode.E303 + + +class TestCallInDAG: + """Test ASTCall execution within DAGs.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0], + 'weight': [1.0, 2.0, 1.5, 3.0, 0.5] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_call_in_dag(self, sample_graph): + """Test executing ASTCall within a DAG.""" + dag = ASTLet({ + 'filtered': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_let_impl(sample_graph, dag, EngineAbstract.PANDAS) + + # Should have degree column + assert 'degree' in result._nodes.columns + # Should still have all nodes (get_degrees doesn't filter) + assert len(result._nodes) == 4 + + def test_call_referencing_binding(self, sample_graph): + """Test ASTCall that operates on whole graph (not in chain).""" + from graphistry.compute.ast import ASTRef + + # Call operations work on the whole graph, not as part of chains + dag = ASTLet({ + 'users': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_let_impl(sample_graph, dag, EngineAbstract.PANDAS) + + # Should have degree column on all nodes + assert len(result._nodes) == 4 # All nodes + assert 'degree' in result._nodes.columns + + def test_multiple_calls(self, sample_graph): + """Test multiple call operations in sequence.""" + # First add degrees + dag1 = ASTLet({ + 'with_degrees': ASTCall('get_degrees', {'col': 'deg'}) + }) + result1 = chain_let_impl(sample_graph, dag1, EngineAbstract.PANDAS) + assert 'deg' in result1._nodes.columns + + # Then filter - use the graph that has degrees + dag2 = ASTLet({ + 'filtered': ASTCall('filter_nodes_by_dict', {'filter_dict': {'deg': 2}}) + }) + result2 = chain_let_impl(result1, dag2, EngineAbstract.PANDAS) + + # Should have nodes with degree 2 + assert len(result2._nodes) > 0 + assert all(result2._nodes['deg'] == 2) + + @patch('graphistry.compute.gfql.call_executor.getattr') + def test_call_execution_error(self, mock_getattr, sample_graph): + """Test handling of execution errors in calls.""" + # Make the method raise an error + mock_method = Mock(side_effect=RuntimeError("Method failed")) + mock_getattr.return_value = mock_method + + dag = ASTLet({ + 'failing': ASTCall('get_degrees', {}) + }) + + with pytest.raises(RuntimeError) as exc_info: + chain_let_impl(sample_graph, dag, EngineAbstract.PANDAS) + assert "Failed to execute node 'failing'" in str(exc_info.value) + + +class TestGraphAlgorithmCalls: + """Test calls to graph algorithm methods.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3] + }) + + return CGFull()\ + .edges(edges_df)\ + .bind(source='source', destination='target') + + def test_compute_cugraph_params(self): + """Test compute_cugraph parameter validation.""" + # Valid params + params = validate_call_params('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'pr_score', + 'directed': True + }) + assert params['alg'] == 'pagerank' + + # Missing required alg + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('compute_cugraph', { + 'out_col': 'pr_score' + }) + assert exc_info.value.code == ErrorCode.E105 + + def test_compute_igraph_params(self): + """Test compute_igraph parameter validation.""" + params = validate_call_params('compute_igraph', { + 'alg': 'community_louvain', + 'directed': False + }) + assert params['alg'] == 'community_louvain' + + def test_layout_methods(self): + """Test layout method parameter validation.""" + # layout_cugraph + params = validate_call_params('layout_cugraph', { + 'layout': 'force_atlas2', + 'params': {'iterations': 100} + }) + assert params['layout'] == 'force_atlas2' + + # layout_igraph + params = validate_call_params('layout_igraph', { + 'layout': 'fruchterman_reingold', + 'directed': True + }) + assert params['layout'] == 'fruchterman_reingold' + + # fa2_layout + params = validate_call_params('fa2_layout', { + 'fa2_params': {'iterations': 500} + }) + assert params['fa2_params']['iterations'] == 500 + + def test_group_in_a_box_layout_params(self): + """Test group_in_a_box_layout parameter validation.""" + # Valid params with all types + params = validate_call_params('group_in_a_box_layout', { + 'partition_alg': 'louvain', + 'partition_params': {'resolution': 1.0}, + 'layout_alg': 'force_atlas2', + 'layout_params': {'iterations': 100}, + 'x': 0, + 'y': 0, + 'w': 1000, + 'h': 1000, + 'encode_colors': True, + 'colors': ['#ff0000', '#00ff00'], + 'partition_key': 'community', + 'engine': 'cpu' + }) + assert params['partition_alg'] == 'louvain' + assert params['engine'] == 'cpu' + + # Minimal params (all optional) + params = validate_call_params('group_in_a_box_layout', {}) + assert params == {} + + # Test type validations + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('group_in_a_box_layout', { + 'x': 'not_a_number' # Should be numeric + }) + assert exc_info.value.code == ErrorCode.E201 + + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('group_in_a_box_layout', { + 'engine': 'invalid_engine' # Should be in allowed list + }) + assert exc_info.value.code == ErrorCode.E201 diff --git a/graphistry/tests/compute/test_call_operations_gpu.py b/graphistry/tests/compute/test_call_operations_gpu.py new file mode 100644 index 0000000000..f74d48ceac --- /dev/null +++ b/graphistry/tests/compute/test_call_operations_gpu.py @@ -0,0 +1,277 @@ +"""GPU tests for GFQL Call operations.""" + +import os +import pytest +import pandas as pd + +from graphistry.tests.test_compute import CGFull +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTCall, ASTLet, n +from graphistry.compute.chain_let import chain_let_impl +from graphistry.compute.gfql.call_executor import execute_call +from graphistry.compute.validate.validate_schema import validate_chain_schema +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +# Skip all tests if TEST_CUDF not set +skip_gpu = pytest.mark.skipif( + not ("TEST_CUDF" in os.environ and os.environ["TEST_CUDF"] == "1"), + reason="cudf tests need TEST_CUDF=1" +) + + +class TestCallOperationsGPU: + """Test Call operations with GPU/cudf.""" + + @skip_gpu + def test_call_with_cudf_dataframes(self): + """Test that Call operations work when starting with cudf DataFrames.""" + import cudf + + # Create cudf dataframes + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3], + 'weight': [1.0, 2.0, 3.0, 4.0] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + edges_gdf = cudf.from_pandas(edges_df) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Create graph with cudf data (may convert internally) + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Execute Call operation with CUDF engine hint + result = execute_call(g, 'get_degrees', {'col': 'degree'}, Engine.CUDF) + + # Result should have degree columns + assert 'degree' in result._nodes.columns + assert 'degree_in' in result._nodes.columns + assert 'degree_out' in result._nodes.columns + + # Verify the computation is correct + assert len(result._nodes) == 4 + # Node 2 has the highest degree (3 connections) + degrees = result._nodes['degree'].tolist() if hasattr(result._nodes['degree'], 'tolist') else list(result._nodes['degree']) + assert max(degrees) == 3 + + @skip_gpu + def test_filter_with_cudf(self): + """Test filtering operations with cudf.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'], + 'score': [0.5, 0.8, 0.9, 0.3] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Add edges to make it a valid graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 3] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Filter nodes + result = execute_call( + g, + 'filter_nodes_by_dict', + {'filter_dict': {'type': 'user'}}, + Engine.CUDF + ) + + # Should be filtered + assert len(result._nodes) == 3 + # Check the type column values + types = result._nodes['type'].to_pandas() if hasattr(result._nodes, 'to_pandas') else result._nodes['type'] + assert all(types == 'user') + + @skip_gpu + def test_compute_cugraph_call(self): + """Test compute_cugraph through Call operation.""" + import cudf + + # Create a simple graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull().edges(edges_gdf).bind(source='source', destination='target') + + # Skip if cugraph not available + try: + import cugraph + except ImportError: + pytest.skip("cugraph not installed") + + # Call compute_cugraph for pagerank + result = execute_call( + g, + 'compute_cugraph', + {'alg': 'pagerank', 'out_col': 'pr_score'}, + Engine.CUDF + ) + + # Should have pagerank scores + assert 'pr_score' in result._nodes.columns + # Verify scores are computed (all nodes should have scores) + assert len(result._nodes) == 4 # 4 unique nodes + scores = result._nodes['pr_score'].tolist() if hasattr(result._nodes['pr_score'], 'tolist') else list(result._nodes['pr_score']) + assert all(score > 0 for score in scores) + + @skip_gpu + def test_layout_cugraph_call(self): + """Test layout_cugraph through Call operation.""" + import cudf + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull().edges(edges_gdf).bind(source='source', destination='target') + + # Skip if cugraph not available + try: + import cugraph + except ImportError: + pytest.skip("cugraph not installed") + + # Call layout_cugraph + result = execute_call( + g, + 'layout_cugraph', + {'layout': 'force_atlas2'}, + Engine.CUDF + ) + + # Should have x,y coordinates + assert 'x' in result._nodes.columns + assert 'y' in result._nodes.columns + # Verify all nodes have coordinates + assert len(result._nodes) == 3 + assert result._nodes['x'].notna().all() + assert result._nodes['y'].notna().all() + + @skip_gpu + def test_chain_let_with_gpu_calls(self): + """Test DAG execution with Call operations on GPU.""" + import cudf + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0], + 'weight': [1.0, 2.0, 1.5, 3.0, 0.5] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + edges_gdf = cudf.from_pandas(edges_df) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Create DAG with Call operations + dag = ASTLet({ + 'filtered': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_let_impl(g, dag, Engine.CUDF) + + # Should have degrees column + assert 'degree' in result._nodes.columns + # Check that we have the expected number of nodes + assert len(result._nodes) == 4 # get_degrees doesn't filter + + @skip_gpu + def test_schema_validation_with_cudf(self): + """Test schema validation works with cudf DataFrames.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2], + 'type': ['A', 'B', 'C'] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull().nodes(nodes_gdf).bind(node='node') + + # Valid call - column exists + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'A'}}) + errors = validate_chain_schema(g, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid call - column doesn't exist + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing': 'X'}}) + errors = validate_chain_schema(g, [call], collect_all=True) + assert len(errors) > 0 + assert any('missing' in str(e) for e in errors) + + @skip_gpu + def test_encode_with_gpu(self): + """Test visual encoding methods with GPU data.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'category': ['A', 'B', 'A', 'C'], + 'score': [0.1, 0.5, 0.8, 0.3] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Add edges to make a valid graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 3] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Test encode_point_color + result = execute_call( + g, + 'encode_point_color', + {'column': 'category'}, + Engine.CUDF + ) + + # Should have color encoding set + assert result._point_color == 'category' + + # Test encode_point_size + result2 = execute_call( + result, + 'encode_point_size', + {'column': 'score'}, + Engine.CUDF + ) + + # Should have size encoding set + assert result2._point_size == 'score' diff --git a/graphistry/tests/compute/test_call_schema_validation.py b/graphistry/tests/compute/test_call_schema_validation.py new file mode 100644 index 0000000000..fdd666acee --- /dev/null +++ b/graphistry/tests/compute/test_call_schema_validation.py @@ -0,0 +1,131 @@ +"""Test schema validation for Call operations.""" + +import pytest +import pandas as pd +from graphistry.tests.test_compute import CGFull +from graphistry.compute.ast import ASTCall, n +from graphistry.compute.chain import Chain +from graphistry.compute.validate.validate_schema import validate_chain_schema +from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError + + +class TestCallSchemaValidation: + """Test schema validation for Call operations.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0], + 'weight': [1.0, 2.0, 3.0] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2], + 'type': ['user', 'user', 'admin'], + 'score': [0.5, 0.8, 0.9] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + @pytest.mark.skip(reason="Schema effects not yet implemented in safelist") + def test_filter_nodes_requires_columns(self, sample_graph): + """Test that filter_nodes_by_dict validates required columns.""" + # Valid: filtering by existing column + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'user'}}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: filtering by non-existent column + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing_col': 'value'}}) + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(sample_graph, [call], collect_all=False) + assert exc_info.value.code == ErrorCode.E301 + assert 'missing_col' in str(exc_info.value) + assert 'does not exist' in str(exc_info.value) + + @pytest.mark.skip(reason="Schema effects not yet implemented in safelist") + def test_filter_edges_requires_columns(self, sample_graph): + """Test that filter_edges_by_dict validates required columns.""" + # Valid: filtering by existing edge column + call = ASTCall('filter_edges_by_dict', {'filter_dict': {'weight': 2.0}}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: filtering by non-existent edge column + call = ASTCall('filter_edges_by_dict', {'filter_dict': {'edge_type': 'friend'}}) + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(sample_graph, [call], collect_all=False) + assert exc_info.value.code == ErrorCode.E301 + assert 'edge_type' in str(exc_info.value) + + def test_encode_requires_columns(self, sample_graph): + """Test that encode methods validate required columns.""" + # Valid: encoding existing column + call = ASTCall('encode_point_color', {'column': 'type'}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: encoding non-existent column + call = ASTCall('encode_point_color', {'column': 'category'}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + # Note: encode methods don't require columns to exist (they create bindings) + # so this should not produce errors + assert len(errors) == 0 + + @pytest.mark.skip(reason="Schema effects not yet implemented in safelist") + def test_chain_with_multiple_calls(self, sample_graph): + """Test validation of chains with multiple Call operations.""" + chain = Chain([ + ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'user'}}), + ASTCall('get_degrees', {'col': 'degree'}), + ASTCall('filter_nodes_by_dict', {'filter_dict': {'degree': 2}}) + ]) + + # The second filter expects 'degree' column which doesn't exist yet + # But schema validation is static and doesn't track added columns + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + # Should have error for missing 'degree' column + assert any('degree' in str(e) for e in errors) + + def test_method_without_schema_effects(self, sample_graph): + """Test that methods without schema effects don't cause errors.""" + # materialize_nodes doesn't require any columns + call = ASTCall('materialize_nodes', {}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + @pytest.mark.skip(reason="Schema effects not yet implemented in safelist") + def test_collect_all_mode(self, sample_graph): + """Test collect_all mode returns all errors.""" + chain = Chain([ + ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing1': 'a', 'missing2': 'b'}}), + ASTCall('filter_edges_by_dict', {'filter_dict': {'missing3': 'c'}}) + ]) + + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + # Should collect all 3 missing column errors + assert len(errors) >= 3 + missing_cols = {'missing1', 'missing2', 'missing3'} + error_cols = set() + for e in errors: + for col in missing_cols: + if col in str(e): + error_cols.add(col) + assert error_cols == missing_cols + + @pytest.mark.skip(reason="Schema effects not yet implemented in safelist") + def test_operation_index_in_errors(self, sample_graph): + """Test that errors include operation index.""" + chain = Chain([ + n({'type': 'user'}), # op 0 + ASTCall('filter_nodes_by_dict', {'filter_dict': {'bad_col': 1}}) # op 1 + ]) + + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + call_errors = [e for e in errors if 'bad_col' in str(e)] + assert len(call_errors) > 0 + assert call_errors[0].context['operation_index'] == 1 diff --git a/graphistry/tests/compute/test_chain_let.py b/graphistry/tests/compute/test_chain_let.py new file mode 100644 index 0000000000..4036524f6c --- /dev/null +++ b/graphistry/tests/compute/test_chain_let.py @@ -0,0 +1,1287 @@ +"""Test chain DAG functionality + +For integration tests with real remote graphs, see test_chain_let_remote_integration.py +Enable remote tests with: TEST_REMOTE_INTEGRATION=1 +""" + +import pandas as pd +import pytest +from unittest.mock import patch, MagicMock +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, ASTNode, ASTObject, n, e +from graphistry.compute.chain_let import ( + extract_dependencies, build_dependency_graph, validate_dependencies, + detect_cycles, determine_execution_order +) +from graphistry.compute.execution_context import ExecutionContext +from graphistry.compute.exceptions import GFQLTypeError +from graphistry.tests.test_compute import CGFull + + +class TestChainDagHelpers: + """Test the helper functions for DAG execution""" + + def test_extract_dependencies_no_deps(self): + """Test extracting dependencies from nodes with no dependencies""" + node = n({'type': 'person'}) + deps = extract_dependencies(node) + assert deps == set() + + remote = ASTRemoteGraph('dataset123') + deps = extract_dependencies(remote) + assert deps == set() + + def test_extract_dependencies_chain_ref(self): + """Test extracting dependencies from ASTRef""" + chain_ref = ASTRef('source', [n()]) + deps = extract_dependencies(chain_ref) + assert deps == {'source'} + + def test_extract_dependencies_nested(self): + """Test extracting dependencies from nested structures""" + # ChainRef with ChainRef in its chain + nested = ASTRef('a', [ASTRef('b', [n()])]) + deps = extract_dependencies(nested) + assert deps == {'a', 'b'} + + # Nested DAG + dag = ASTLet({ + 'inner': ASTRef('outer', [n()]) + }) + deps = extract_dependencies(dag) + assert deps == {'outer'} + + def test_build_dependency_graph(self): + """Test building dependency and dependent mappings""" + bindings = { + 'a': n(), + 'b': ASTRef('a', [n()]), + 'c': ASTRef('b', [n()]) + } + + dependencies, dependents = build_dependency_graph(bindings) + + assert dependencies == { + 'a': set(), + 'b': {'a'}, + 'c': {'b'} + } + assert dependents == { + 'a': {'b'}, + 'b': {'c'} + } + + def test_validate_dependencies_valid(self): + """Test validation passes for valid dependencies""" + bindings = { + 'a': n(), + 'b': ASTRef('a', [n()]) + } + dependencies = {'a': set(), 'b': {'a'}} + + # Should not raise + validate_dependencies(bindings, dependencies) + + def test_validate_dependencies_missing_ref(self): + """Test validation catches missing references""" + bindings = { + 'a': n() + } + dependencies = {'a': {'missing'}} + + with pytest.raises(ValueError) as exc_info: + validate_dependencies(bindings, dependencies) + + assert "references undefined nodes: ['missing']" in str(exc_info.value) + assert "Available nodes: ['a']" in str(exc_info.value) + + def test_validate_dependencies_self_ref(self): + """Test validation catches self-references""" + bindings = { + 'a': n() + } + dependencies = {'a': {'a'}} + + with pytest.raises(ValueError) as exc_info: + validate_dependencies(bindings, dependencies) + + assert "Self-reference cycle detected: 'a' depends on itself" in str(exc_info.value) + + def test_detect_cycles_no_cycle(self): + """Test cycle detection on acyclic graph""" + dependencies = { + 'a': set(), + 'b': {'a'}, + 'c': {'b'} + } + + cycle = detect_cycles(dependencies) + assert cycle is None + + def test_detect_cycles_simple_cycle(self): + """Test cycle detection on simple cycle""" + dependencies = { + 'a': {'b'}, + 'b': {'a'} + } + + cycle = detect_cycles(dependencies) + assert cycle == ['a', 'b', 'a'] or cycle == ['b', 'a', 'b'] + + def test_detect_cycles_longer_cycle(self): + """Test cycle detection on longer cycle""" + dependencies = { + 'a': {'b'}, + 'b': {'c'}, + 'c': {'a'}, + 'd': {'a'} + } + + cycle = detect_cycles(dependencies) + # Could start from any node in the cycle + assert len(cycle) == 4 # 3 nodes + repeat + assert cycle[0] == cycle[-1] # Cycle closes + + def test_determine_execution_order_empty(self): + """Test execution order for empty DAG""" + order = determine_execution_order({}) + assert order == [] + + def test_determine_execution_order_single(self): + """Test execution order for single node""" + bindings = {'only': n()} + order = determine_execution_order(bindings) + assert order == ['only'] + + def test_determine_execution_order_linear(self): + """Test execution order for linear dependencies""" + bindings = { + 'a': n(), + 'b': ASTRef('a', [n()]), + 'c': ASTRef('b', [n()]) + } + + order = determine_execution_order(bindings) + assert order == ['a', 'b', 'c'] + + def test_determine_execution_order_diamond(self): + """Test execution order for diamond pattern""" + bindings = { + 'top': n(), + 'left': ASTRef('top', [n()]), + 'right': ASTRef('top', [n()]), + 'bottom': ASTRef('left', [ASTRef('right', [n()])]) + } + + order = determine_execution_order(bindings) + # Top must come first, bottom must come last + assert order[0] == 'top' + assert order[-1] == 'bottom' + # Left and right can be in either order + assert set(order[1:3]) == {'left', 'right'} + + def test_determine_execution_order_disconnected(self): + """Test execution order for disconnected components""" + bindings = { + 'a1': n(), + 'a2': ASTRef('a1', [n()]), + 'b1': n(), + 'b2': ASTRef('b1', [n()]) + } + + order = determine_execution_order(bindings) + # Each component should be ordered correctly + assert order.index('a1') < order.index('a2') + assert order.index('b1') < order.index('b2') + + +class TestExecutionContext: + """Test ExecutionContext integration in chain_let""" + + def test_context_stores_results(self): + """Test that ExecutionContext stores node results""" + from graphistry.compute.chain_let import execute_node + + # Create a mock AST object that returns a known result + class MockNode: + def validate(self): + pass + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + mock_node = MockNode() + + # This should raise NotImplementedError but still store in context + try: + execute_node('test_node', mock_node, g, context, None) + except NotImplementedError: + pass + + # Even though execution failed, context.set_binding was called + # (we can't test this without implementing execution) + + def test_chain_ref_missing_reference(self): + """Test ASTRef with missing reference gives helpful error""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Create ASTRef that references non-existent binding + chain_ref = ASTRef('missing_ref', []) + + # Should raise ValueError with helpful message + with pytest.raises(ValueError) as exc_info: + execute_node('test', chain_ref, g, context, Engine.PANDAS) + + assert "references 'missing_ref' which has not been executed yet" in str(exc_info.value) + assert "Available bindings: []" in str(exc_info.value) + + def test_chain_ref_with_existing_reference(self): + """Test ASTRef successfully resolves existing reference""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Pre-populate context with a result + context.set_binding('previous_result', g) + + # Create ASTRef that references it (empty chain) + chain_ref = ASTRef('previous_result', []) + + # Should return the referenced result + result = execute_node('test', chain_ref, g, context, Engine.PANDAS) + assert result is g # Same object since empty chain + + # And store it under new name + assert context.get_binding('test') is g + + def test_context_passed_through_dag(self): + """Test that context is passed through DAG execution""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({}) + + # Empty DAG should work + result = g.gfql(dag) + assert result is not None + + def test_execution_order_verified(self): + """Test that execution order follows dependencies""" + # Create a DAG with known dependencies + dag = ASTLet({ + 'data': ASTRemoteGraph('dataset'), + 'filtered': ASTRef('data', []), + 'analyzed': ASTRef('filtered', []) + }) + + # Get execution order + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + + # Verify order respects dependencies + assert order == ['data', 'filtered', 'analyzed'] + + # Also test diamond pattern + dag_diamond = ASTLet({ + 'root': ASTRemoteGraph('data'), + 'left': ASTRef('root', []), + 'right': ASTRef('root', []), + 'merge': ASTRef('left', [ASTRef('right', [])]) + }) + + order_diamond = determine_execution_order(dag_diamond.bindings) + assert order_diamond[0] == 'root' + assert order_diamond[-1] == 'merge' + assert set(order_diamond[1:3]) == {'left', 'right'} + + def test_chain_ref_in_dag_execution(self): + """Test ASTRef works in DAG execution (fails on chain ops)""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Create a simple mock that can be executed + class MockExecutable(ASTObject): + def _validate_fields(self): + pass + + def _get_child_validators(self): + return [] + + def __call__(self, g, prev_node_wavefront, target_wave_front, engine): + raise NotImplementedError("Mock execution") + + def reverse(self): + return self + + # Create DAG with mock executable and chain ref + dag = ASTLet({ + 'first': MockExecutable(), + 'second': ASTRef('first', []) # Empty chain should work + }) + + # Try to execute - will fail on MockExecutable + try: + g.gfql(dag) + except RuntimeError as e: + # Should fail on first node (MockExecutable) + assert "Failed to execute node 'first'" in str(e) + assert "NotImplementedError" in str(e) + + +class TestEdgeExecution: + """Test ASTEdge execution in chain_let""" + + def test_edge_execution_basic(self): + """Test basic edge traversal in DAG""" + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c', 'd'], + 'd': ['b', 'c', 'd', 'e'], + 'type': ['knows', 'works_with', 'knows', 'manages'] + }) + g = CGFull().edges(edges_df, 's', 'd') + g = g.materialize_nodes() + + dag = ASTLet({ + 'one_hop': e() # Default forward edge + }) + + result = g.gfql(dag) + assert result is not None + # Should have traversed edges + assert len(result._nodes) > 0 + + def test_edge_with_filter(self): + """Test edge traversal with filters""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd', 'e'], + 'type': ['person', 'person', 'company', 'person', 'company'] + }) + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c', 'd'], + 'd': ['b', 'c', 'd', 'e'], + 'rel': ['knows', 'works_at', 'invests', 'works_at'] + }) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'work_edges': e(edge_match={'rel': 'works_at'}) + }) + + result = g.gfql(dag) + # Should have filtered to work relationships + assert result is not None + + def test_edge_with_direction(self): + """Test edge traversal with different directions""" + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c'], + 'd': ['b', 'c', 'd'] + }) + g = CGFull().edges(edges_df, 's', 'd') + g = g.materialize_nodes() + + # Test reverse direction + from graphistry.compute.ast import ASTEdgeReverse + dag = ASTLet({ + 'reverse': ASTEdgeReverse() + }) + + result = g.gfql(dag) + assert result is not None + + def test_edge_with_name(self): + """Test edge operation adds name column""" + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c'], + 'd': ['b', 'c', 'd'] + }) + g = CGFull().edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'tagged_edges': e(name='important') + }) + + result = g.gfql(dag) + assert 'important' in result._edges.columns + + def test_node_edge_combination(self): + """Test DAG with both node and edge operations""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c'], + 'd': ['b', 'c', 'd'] + }) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'from_people': ASTRef('people', [e()]), + 'companies': n({'type': 'company'}) + }) + + # Should execute successfully + result = g.gfql(dag) + assert result is not None + + +class TestNodeExecution: + """Test ASTNode execution in chain_let""" + + def test_node_execution_empty_filter(self): + """Test ASTNode with empty filter returns original graph""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}), 's', 'd') + g = g.materialize_nodes() # Ensure nodes exist + context = ExecutionContext() + + # Empty node filter + node = n() + result = execute_node('test', node, g, context, Engine.PANDAS) + + # Should return graph with same data + assert len(result._nodes) == len(g._nodes) + assert len(result._edges) == len(g._edges) + + def test_node_execution_with_filter(self): + """Test ASTNode with filter_dict filters nodes""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + # Create graph with node attributes + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + context = ExecutionContext() + + # Filter for person nodes + node = n({'type': 'person'}) + result = execute_node('people', node, g, context, Engine.PANDAS) + + # Should only have person nodes + assert len(result._nodes) == 2 + assert set(result._nodes['id'].tolist()) == {'a', 'b'} + assert all(result._nodes['type'] == 'person') + + def test_node_execution_with_name(self): + """Test ASTNode adds name column when specified""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g = g.materialize_nodes() + context = ExecutionContext() + + # Node with name + node = n(name='tagged') + result = execute_node('test', node, g, context, Engine.PANDAS) + + # Should have 'tagged' column + assert 'tagged' in result._nodes.columns + assert all(result._nodes['tagged']) + + def test_node_in_dag_execution(self): + """Test ASTNode works in full DAG execution""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # DAG with node filter + dag = ASTLet({ + 'people': n({'type': 'person'}) + }) + + result = g.gfql(dag) + + # Should have filtered to people only + assert len(result._nodes) == 2 + assert set(result._nodes['type'].unique()) == {'person'} + + def test_dag_with_node_and_chainref(self): + """Test DAG execution with both node and chain reference""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'], + 'active': [True, False, True, True] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'b', 'c'], 'd': ['b', 'c', 'd', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # DAG: filter people, then filter active from those + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'active_people': ASTRef('people', [n({'active': True})]) + }) + + result = g.gfql(dag) + + # Should only have active people + assert len(result._nodes) == 1 + assert result._nodes['id'].iloc[0] == 'a' + assert result._nodes['type'].iloc[0] == 'person' + assert result._nodes['active'].iloc[0] + +class TestErrorHandling: + """Test error handling and edge cases""" + + def test_invalid_dag_type(self): + """Test helpful error when dag parameter is wrong type""" + g = CGFull() + + with pytest.raises(TypeError) as exc_info: + g.gfql("not a dag") + assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) + + # When passed a dict, gfql creates an ASTLet which validates + with pytest.raises(GFQLTypeError) as exc_info: + g.gfql({'dict': 'not allowed'}) + assert exc_info.value.code == "type-mismatch" + assert "binding value must be ASTObject" in str(exc_info.value) + + def test_node_execution_error_wrapped(self): + """Test node execution errors are wrapped with context""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Create a node with invalid query syntax + dag = ASTLet({ + 'bad_query': n(query='invalid python syntax !@#') + }) + + with pytest.raises(RuntimeError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "Failed to execute node 'bad_query'" in error_msg + assert "Error:" in error_msg + + def test_cycle_detection_with_path(self): + """Test cycle detection provides the cycle path""" + dag = ASTLet({ + 'a': ASTRef('b', []), + 'b': ASTRef('c', []), + 'c': ASTRef('a', []) # Creates cycle a->b->c->a + }) + + g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + with pytest.raises(ValueError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "Circular dependency detected" in error_msg + assert "->" in error_msg # Shows the cycle path + + def test_complex_cycle_detection(self): + """Test detection of cycles in complex DAGs""" + # This DAG has no cycles, just complex dependencies + bindings = { + 'start': n(), + 'a': ASTRef('start', []), + 'b': ASTRef('a', []), + 'c': ASTRef('b', []), + 'd': ASTRef('c', []), + 'e': ASTRef('d', []), + 'f': ASTRef('b', []), # Second branch from b + 'g': ASTRef('f', []) # Note: removed nested ASTRef in chain + } + + # Test cycle detection directly + from graphistry.compute.chain_let import detect_cycles, build_dependency_graph + dependencies, _ = build_dependency_graph(bindings) + cycle = detect_cycles(dependencies) + + # Should find no cycle + assert cycle is None + + def test_missing_reference_with_suggestions(self): + """Test missing reference error includes available bindings""" + dag = ASTLet({ + 'data1': n(), + 'data2': n(), + 'result': ASTRef('data3', []) # data3 doesn't exist + }) + + g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + with pytest.raises(ValueError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "references undefined nodes: ['data3']" in error_msg + assert "Available nodes: ['data1', 'data2', 'result']" in error_msg + + +class TestExecutionMechanics: + """Test execution mechanics with granular tests""" + + def test_execute_node_stores_in_context(self): + """Test that execute_node stores results in context""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g = g.materialize_nodes() + context = ExecutionContext() + + # Execute a simple node + node = n() + result = execute_node('test_node', node, g, context, Engine.PANDAS) + + # Check result is stored in context + assert context.get_binding('test_node') is result + assert len(result._nodes) == 2 # nodes a and b + + def test_execute_node_with_different_ast_types(self): + """Test execute_node handles different AST object types""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Test ASTRemoteGraph is now implemented (will fail with missing auth) + # We'll test actual functionality with mocks in a separate test + + # Test nested ASTLet + nested_dag = ASTLet({'inner': n()}) + result = execute_node('nested', nested_dag, g, context, Engine.PANDAS) + assert result is not None + + @patch('graphistry.compute.chain_remote.chain_remote') + def test_remote_graph_execution(self, mock_chain_remote): + """Test ASTRemoteGraph executes correctly with mocked remote call""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + # Setup mock return value + mock_result = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + mock_chain_remote.return_value = mock_result + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Execute remote graph + remote = ASTRemoteGraph('dataset123', token='secret-token') + result = execute_node('remote_data', remote, g, context, Engine.PANDAS) + assert result is mock_result # Verify correct result returned + + # Verify chain_remote was called with correct params + mock_chain_remote.assert_called_once() + call_args = mock_chain_remote.call_args + assert call_args[0][1] == [] # Empty chain + assert call_args[1]['dataset_id'] == 'dataset123' + assert call_args[1]['api_token'] == 'secret-token' + assert call_args[1]['output_type'] == 'all' + + # Verify result is stored in context + assert context.get_binding('remote_data') is mock_result + + def test_chain_ref_resolution_order(self): + """Test ASTRef resolves references in correct order""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + nodes_df = pd.DataFrame({'id': ['a', 'b', 'c'], 'value': [1, 2, 3]}) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + context = ExecutionContext() + + # Store initial result + filtered = g.filter_nodes_by_dict({'value': 2}) + context.set_binding('filtered_data', filtered) + + # Create chain ref that adds more filtering + chain_ref = ASTRef('filtered_data', [n({'id': 'b'})]) + result = execute_node('final', chain_ref, g, context, Engine.PANDAS) + + # Should have only node 'b' + assert len(result._nodes) == 1 + assert result._nodes['id'].iloc[0] == 'b' + + def test_execution_context_isolation(self): + """Test that each DAG execution has isolated context""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # First DAG execution + dag1 = ASTLet({'node1': n(name='first')}) + result1 = g.gfql(dag1) + assert result1 is not None # First execution succeeds + + # Second DAG execution should not see first's context + dag2 = ASTLet({ + 'node2': n(name='second'), + 'ref_fail': ASTRef('node1', []) # Should fail - node1 not in this context + }) + + with pytest.raises(ValueError) as exc_info: + g.gfql(dag2) + assert "references undefined nodes: ['node1']" in str(exc_info.value) + + def test_execution_order_logging(self): + """Test execution order is logged correctly""" + import logging + from graphistry.compute.chain_let import logger as dag_logger + + # Capture log output + logs = [] + handler = logging.Handler() + handler.emit = lambda record: logs.append(record) + dag_logger.addHandler(handler) + dag_logger.setLevel(logging.DEBUG) + + try: + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({ + 'first': n(), + 'second': ASTRef('first', []), + 'third': ASTRef('second', []) + }) + + g.gfql(dag) + + # Check execution order was logged + order_logs = [r for r in logs if 'DAG execution order' in str(r.getMessage())] + assert len(order_logs) > 0 + assert "['first', 'second', 'third']" in str(order_logs[0].getMessage()) + + # Check individual node execution was logged + node_logs = [r for r in logs if "Executing node" in str(r.getMessage())] + assert len(node_logs) >= 3 + finally: + dag_logger.removeHandler(handler) + + +class TestDiamondPatterns: + """Test diamond and complex dependency patterns""" + + def test_diamond_pattern_execution(self): + """Test diamond pattern executes correctly""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd', 'e'], + 'type': ['source', 'middle1', 'middle2', 'target', 'other'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c', 'd'], 'd': ['b', 'd', 'd', 'e']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Diamond: top -> (left, right) -> bottom + dag = ASTLet({ + 'top': n({'type': 'source'}), + 'left': ASTRef('top', [n(name='from_left')]), + 'right': ASTRef('top', [n(name='from_right')]), + 'bottom': ASTRef('left', []) + }) + + result = g.gfql(dag) + + # Result should have source node with from_left tag + assert len(result._nodes) == 1 + assert result._nodes['type'].iloc[0] == 'source' + assert 'from_left' in result._nodes.columns + assert result._nodes['from_left'].iloc[0] + def test_multi_branch_convergence(self): + """Test multiple branches converging""" + g = CGFull().edges(pd.DataFrame({ + 's': ['a', 'b', 'c', 'd', 'e'], + 'd': ['x', 'x', 'x', 'x', 'x'] + }), 's', 'd') + g = g.materialize_nodes() + + # Multiple branches converging - test execution order + from graphistry.compute.chain_let import determine_execution_order, ExecutionContext, execute_node + from graphistry.Engine import Engine + + dag = ASTLet({ + 'branch1': n(name='b1'), + 'branch2': n(name='b2'), + 'branch3': n(name='b3'), + 'converge': n() # Gets all nodes + }) + + # Test execution order - branches can execute in any order + order = determine_execution_order(dag.bindings) + assert len(order) == 4 + assert order[-1] == 'converge' # Converge must be last + + # Execute and check final result + result = g.gfql(dag) + assert len(result._nodes) == 6 # a,b,c,d,e,x + + def test_parallel_independent_branches(self): + """Test parallel branches execute independently""" + nodes_df = pd.DataFrame({ + 'id': list('abcdefgh'), + 'branch': ['A', 'A', 'A', 'A', 'B', 'B', 'B', 'B'] + }) + edges_df = pd.DataFrame({'s': list('abcdefg'), 'd': list('bcdefgh')}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Two independent branches + dag = ASTLet({ + 'branch_a': n({'branch': 'A'}), + 'branch_b': n({'branch': 'B'}), + 'a_subset': ASTRef('branch_a', [n(query="id in ['a', 'b']")]), + 'b_subset': ASTRef('branch_b', [n(query="id in ['e', 'f']")]) + }) + + # Check execution order allows parallel execution + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + + # branch_a and branch_b can execute in any order + assert order.index('branch_a') < order.index('a_subset') + assert order.index('branch_b') < order.index('b_subset') + + # Execute DAG + result = g.gfql(dag) + + # Result should be from last executed node (b_subset) + assert len(result._nodes) == 2 + assert set(result._nodes['id'].tolist()) == {'e', 'f'} + + def test_deep_dependency_chain(self): + """Test deep linear dependency chain""" + g = CGFull().edges(pd.DataFrame({'s': list('abcdef'), 'd': list('bcdefg')}), 's', 'd') + g = g.materialize_nodes() + + # Create deep chain: n1 -> n2 -> n3 -> ... -> n10 + # Using empty chains to avoid execution issues + dag_dict = {'n1': n(name='level1')} + for i in range(2, 11): + dag_dict[f'n{i}'] = ASTRef(f'n{i - 1}', []) + + dag = ASTLet(dag_dict) + + # Test execution order is correct + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + + # Should be in sequential order + expected_order = [f'n{i}' for i in range(1, 11)] + assert order == expected_order + + # Execute DAG + result = g.gfql(dag) + + # Result should have level1 tag from n1 + assert 'level1' in result._nodes.columns + + def test_fan_out_fan_in_pattern(self): + """Test fan-out then fan-in pattern""" + g = CGFull().edges(pd.DataFrame({ + 's': ['root', 'a1', 'a2', 'b1', 'b2', 'b3'], + 'd': ['hub', 'end', 'end', 'end', 'end', 'end'] + }), 's', 'd') + g = g.materialize_nodes() + + # Test execution order for fan-out/fan-in + from graphistry.compute.chain_let import determine_execution_order + + dag = ASTLet({ + 'start': n({'id': 'root'}), + 'expand1': ASTRef('start', []), + 'expand2': ASTRef('start', []), + 'expand3': ASTRef('start', []), + 'collect': n() # Gets all nodes from original graph + }) + + # Check execution order + order = determine_execution_order(dag.bindings) + # 'start' must come before expand nodes + assert order.index('start') < order.index('expand1') + assert order.index('start') < order.index('expand2') + assert order.index('start') < order.index('expand3') + # 'collect' has no dependencies so can be anywhere + + # Execute DAG + result = g.gfql(dag) + # Result is from last executed node (one of the expand nodes) + # which references 'start' (filtered to just 'root') + assert len(result._nodes) == 1 + assert result._nodes['id'].iloc[0] == 'root' + + +class TestIntegration: + """Integration tests for complex DAG scenarios""" + + def test_empty_dag(self): + """Test empty DAG returns original graph""" + g = CGFull().edges(pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}), 's', 'd') + dag = ASTLet({}) + + result = g.gfql(dag) + + # Should return original graph + assert len(result._edges) == len(g._edges) + pd.testing.assert_frame_equal(result._edges, g._edges) + + def test_large_dag_10_nodes(self): + """Test DAG with 10+ nodes executes successfully""" + # Create a complex graph with attributes + nodes_data = [] + edges_data = [] + for i in range(20): + nodes_data.append({ + 'id': f'n{i}', + 'value': i, + 'type': 'even' if i % 2 == 0 else 'odd' + }) + for j in range(i + 1, min(i + 3, 20)): + edges_data.append({'s': f'n{i}', 'd': f'n{j}'}) + + g = CGFull().nodes(pd.DataFrame(nodes_data), 'id').edges(pd.DataFrame(edges_data), 's', 'd') + + # Create a 10+ node DAG with various patterns + dag = ASTLet({ + # Layer 1: Initial filters using filter_dict + 'high_value': n(name='high'), + 'even': n({'type': 'even'}), + 'odd': n({'type': 'odd'}), + + # Layer 2: References + 'high_even': ASTRef('even', []), + 'high_odd': ASTRef('odd', []), + + # Layer 3: More nodes + 'n1': n(name='tag1'), + 'n2': n(name='tag2'), + 'n3': n(name='tag3'), + 'n4': n(name='tag4'), + + # Layer 4: Final node + 'final': n(name='final_tag') + }) + + # Should execute without error + result = g.gfql(dag) + assert result is not None + # The DAG has 10 nodes, so it meets our 10+ node requirement + assert len(dag.bindings) == 10 + + # Verify execution order is valid + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + assert len(order) == 10 + # References come after their dependencies + assert order.index('even') < order.index('high_even') + assert order.index('odd') < order.index('high_odd') + + @patch('graphistry.compute.chain_remote.chain_remote') + def test_mock_remote_graph_placeholder(self, mock_chain_remote): + """Test DAG with mock RemoteGraph""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Setup mock to return a simple graph + mock_result = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + mock_chain_remote.return_value = mock_result + + dag = ASTLet({ + 'remote1': ASTRemoteGraph('dataset1'), + 'remote2': ASTRemoteGraph('dataset2', token='mock-token'), + 'combined': n() # Would combine results + }) + + # Should execute successfully with mocked remote calls + result = g.gfql(dag) + assert result is not None + + # Verify chain_remote was called twice (once for each RemoteGraph) + assert mock_chain_remote.call_count == 2 + + def test_memory_efficient_execution(self): + """Test that intermediate results are stored efficiently""" + + # Create a simple DAG + g = CGFull().edges(pd.DataFrame({'s': list('abc'), 'd': list('bcd')}), 's', 'd') + g = g.materialize_nodes() + + dag = ASTLet({ + 'step1': n(name='tag1'), + 'step2': n(name='tag2'), + 'step3': n(name='tag3') + }) + + # Execute and verify context usage + result = g.gfql(dag) + + # Each step should produce a result + assert result is not None + # Result has the last tag + assert 'tag3' in result._nodes.columns + + def test_error_propagation_with_context(self): + """Test errors include helpful context about which node failed""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + dag = ASTLet({ + 'good1': n(), + 'good2': n(), + 'bad': n(query='invalid syntax !@#'), + 'never_reached': n() + }) + + with pytest.raises(RuntimeError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "Failed to execute node 'bad'" in error_msg + assert "Error:" in error_msg + + +class TestCrossValidation: + """Cross-validation tests to verify implementation correctness""" + + def test_dag_vs_chain_consistency(self): + """Test that simple DAG produces same result as chain for linear flow""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Using chain + chain_result = g.chain([n({'type': 'person'})]) + + # Using DAG + dag = ASTLet({ + 'people': n({'type': 'person'}) + }) + dag_result = g.gfql(dag) + + # Should produce same nodes + assert len(chain_result._nodes) == len(dag_result._nodes) + assert set(chain_result._nodes['id'].tolist()) == set(dag_result._nodes['id'].tolist()) + + def test_execution_order_deterministic(self): + """Test that execution order is deterministic for same DAG""" + from graphistry.compute.chain_let import determine_execution_order + + dag = ASTLet({ + 'a': n(), + 'b': n(), + 'c': ASTRef('a', []), + 'd': ASTRef('b', []), + 'e': ASTRef('c', []), + 'f': ASTRef('d', []) + }) + + # Get order multiple times + orders = [] + for i in range(5): + order = determine_execution_order(dag.bindings) + orders.append(order) + + # All should be the same + for order in orders[1:]: + assert order == orders[0] + + def test_context_bindings_accessible(self): + """Test that all intermediate results are accessible in context""" + from graphistry.compute.chain_let import chain_let_impl + + g = CGFull().edges(pd.DataFrame({'s': list('abc'), 'd': list('bcd')}), 's', 'd') + g = g.materialize_nodes() + + # Create a mock context to track all bindings + bindings_tracker = {} + + class TrackingContext(ExecutionContext): + def set_binding(self, name, value): + super().set_binding(name, value) + bindings_tracker[name] = value + + # Monkey patch the execution to use our tracking context + original_chain_let_impl = chain_let_impl + + def tracking_chain_let_impl(g, dag, engine): + # Call original but capture context usage + return original_chain_let_impl(g, dag, engine) + + dag = ASTLet({ + 'step1': n(name='tag1'), + 'step2': n(name='tag2'), + 'step3': ASTRef('step1', []) + }) + + result = g.gfql(dag) + + # We can't easily intercept the context, but we can verify the result + assert result is not None + + def test_error_doesnt_corrupt_state(self): + """Test that errors don't leave DAG execution in bad state""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # First execution with error + bad_dag = ASTLet({ + 'bad': n(query='invalid syntax !!!') + }) + + try: + g.gfql(bad_dag) + except RuntimeError: + pass # Expected + + # Second execution should work fine + good_dag = ASTLet({ + 'good': n() + }) + + result = g.gfql(good_dag) + assert result is not None + + def test_node_filter_consistency(self): + """Test node filtering is consistent between chain and chain_let""" + nodes_df = pd.DataFrame({ + 'id': list('abcdef'), + 'value': [10, 20, 30, 40, 50, 60], + 'active': [True, False, True, False, True, False] + }) + edges_df = pd.DataFrame({'s': list('abcde'), 'd': list('bcdef')}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Test filter_dict + dag1 = ASTLet({'result': n({'active': True})}) + result1 = g.gfql(dag1) + assert len(result1._nodes) == 3 + assert all(result1._nodes['active']) + + # Test with name + dag2 = ASTLet({'result': n({'active': True}, name='is_active')}) + result2 = g.gfql(dag2) + assert 'is_active' in result2._nodes.columns + assert all(result2._nodes['is_active']) + + +class TestChainDagInternal: + """Test internal chain_let functionality (via gfql)""" + + def test_chain_let_via_gfql(self): + """Test that DAG execution works via gfql""" + g = CGFull() + assert hasattr(g, 'gfql') + assert callable(g.gfql) + + # chain_let should not be in public API + assert not hasattr(g, 'chain_let') + + def test_chain_let_empty(self): + """Test chain_let with empty DAG""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({}) + + # Empty DAG should return original graph + result = g.gfql(dag) + assert result is not None + + def test_chain_let_single_node_works(self): + """Test chain_let with single node now works""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g = g.materialize_nodes() + + dag = ASTLet({ + 'all_nodes': n() + }) + + # Should work now that node execution is implemented + result = g.gfql(dag) + assert result is not None + assert len(result._nodes) == 2 # nodes a and b + + @patch('graphistry.compute.chain_remote.chain_remote') + def test_chain_let_remote_not_implemented(self, mock_chain_remote): + """Test chain_let with RemoteGraph works with mocked remote""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Setup mock + mock_result = CGFull().edges(pd.DataFrame({'s': ['remote1'], 'd': ['remote2']}), 's', 'd') + mock_chain_remote.return_value = mock_result + + dag = ASTLet({ + 'remote': ASTRemoteGraph('dataset123') + }) + + # Should work now with mocked chain_remote + result = g.gfql(dag) + assert result is not None + # Result should be the mocked remote graph + assert 'remote1' in result._edges['s'].values + + def test_chain_let_multi_node_works(self): + """Test chain_let with multiple nodes now works""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({ + 'first': n(), + 'second': n() + }) + + # Should work now that node execution is implemented + result = g.gfql(dag) + assert result is not None + + # Result should be from last node ('second') + # Both nodes have empty filters so should have all data + assert len(result._nodes) == 2 # nodes a and b + + def test_chain_let_validates(self): + """Test chain_let validates the DAG""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Invalid DAG should raise during validation + with pytest.raises(TypeError) as exc_info: + g.gfql("not a dag") + + assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) + + def test_chain_let_output_selection(self): + """Test output parameter selects specific binding""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}), + 'all_nodes': n() + }) + + # Default: returns last executed + result_default = g.gfql(dag) + # Could be any of the three since they have no dependencies + assert result_default is not None + + # Select specific outputs + result_people = g.gfql(dag, output='people') + assert len(result_people._nodes) == 2 + assert all(result_people._nodes['type'] == 'person') + + result_companies = g.gfql(dag, output='companies') + assert len(result_companies._nodes) == 2 + assert all(result_companies._nodes['type'] == 'company') + + result_all = g.gfql(dag, output='all_nodes') + assert len(result_all._nodes) == 4 + + def test_chain_let_output_not_found(self): + """Test error when output binding not found""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({'node1': n()}) + + with pytest.raises(ValueError) as exc_info: + g.gfql(dag, output='missing') + + error_msg = str(exc_info.value) + assert "Output binding 'missing' not found" in error_msg + assert "Available bindings: ['node1']" in error_msg diff --git a/graphistry/tests/compute/test_chain_let_gpu.py b/graphistry/tests/compute/test_chain_let_gpu.py new file mode 100644 index 0000000000..888caedfc7 --- /dev/null +++ b/graphistry/tests/compute/test_chain_let_gpu.py @@ -0,0 +1,196 @@ +import os +import pytest +import pandas as pd + +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, n +from graphistry.compute.chain_let import chain_let_impl +from graphistry.compute.execution_context import ExecutionContext +from graphistry.tests.test_compute import CGFull + +# Skip all tests if TEST_CUDF not set +skip_gpu = pytest.mark.skipif( + not ("TEST_CUDF" in os.environ and os.environ["TEST_CUDF"] == "1"), + reason="cudf tests need TEST_CUDF=1" +) + + +class TestChainDagGPU: + """Test chain_let with GPU/cudf""" + + @skip_gpu + def test_execution_context_stores_cudf(self): + """Test that ExecutionContext can store cudf DataFrames""" + import cudf + context = ExecutionContext() + + # Create a cudf DataFrame + gdf = cudf.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']}) + + # Store it + context.set_binding('gpu_data', gdf) + + # Retrieve it + retrieved = context.get_binding('gpu_data') + + # Verify it's still a cudf DataFrame + assert isinstance(retrieved, cudf.DataFrame) + assert retrieved.equals(gdf) + + @skip_gpu + def test_chain_let_with_cudf_edges(self): + """Test chain_let with cudf edge DataFrame""" + import cudf + # Create cudf edges + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + edges_gdf = cudf.from_pandas(edges_df) + + # Create graph with cudf edges + g = CGFull().edges(edges_gdf, 's', 'd') + + # Verify edges are cudf + assert isinstance(g._edges, cudf.DataFrame) + + # Empty DAG should work + dag = ASTLet({}) + result = g.chain_let(dag) + + # Result should preserve GPU mode + assert isinstance(result._edges, cudf.DataFrame) + + @skip_gpu + def test_chain_let_engine_cudf(self): + """Test chain_let with explicit engine='cudf'""" + import cudf + # Start with pandas + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Empty DAG with cudf engine + dag = ASTLet({}) + result = g.chain_let(dag, engine='cudf') + + # Should have materialized nodes + assert result._nodes is not None + # Nodes should be cudf (due to materialize_nodes with cudf engine) + assert isinstance(result._nodes, cudf.DataFrame) + + @skip_gpu + def test_chain_let_auto_detects_gpu(self): + """Test chain_let auto-detects GPU mode from edges""" + import cudf + # Create graph with GPU edges + edges_gdf = cudf.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Create a simple DAG (will fail on execution but that's ok) + dag = ASTLet({ + 'step1': n() + }) + + # Try to execute + try: + g.chain_let(dag) # engine='auto' by default + except RuntimeError as e: + # Should fail on execution, but engine should be detected + assert "Failed to execute node 'step1'" in str(e) + + # The important part is it didn't fail on engine detection + # or materialize_nodes with GPU data + + @skip_gpu + def test_resolve_engine_with_gpu(self): + """Test that resolve_engine correctly identifies GPU mode""" + import cudf + from graphistry.Engine import resolve_engine, EngineAbstract + + # Create graph with cudf edges + edges_gdf = cudf.DataFrame({'s': ['a'], 'd': ['b']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Resolve should detect cudf + engine = resolve_engine(EngineAbstract.AUTO, g) + assert engine.value == 'cudf' + + @skip_gpu + def test_materialize_nodes_preserves_gpu(self): + """Test materialize_nodes works with GPU""" + import cudf + # Create graph with cudf edges + edges_gdf = cudf.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Materialize nodes + g2 = g.materialize_nodes() + + # Both edges and nodes should be cudf + assert isinstance(g2._edges, cudf.DataFrame) + assert isinstance(g2._nodes, cudf.DataFrame) + + # Check node content + expected_nodes = ['a', 'b', 'c', 'd'] + assert sorted(g2._nodes['id'].to_pandas().tolist()) == expected_nodes + + @skip_gpu + def test_chain_ref_with_gpu_data(self): + """Test ASTRef resolution works with GPU data""" + import cudf + from graphistry.compute.chain_let import execute_node + from graphistry.compute.execution_context import ExecutionContext + from graphistry.Engine import Engine + + # Create graph with cudf data + edges_gdf = cudf.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Create context and store GPU result + context = ExecutionContext() + context.set_binding('gpu_result', g) + + # Create chain ref to GPU data + chain_ref = ASTRef('gpu_result', []) + + # Execute should preserve GPU + result = execute_node('test', chain_ref, g, context, Engine.CUDF) + + # Result should still have GPU data + assert isinstance(result._edges, cudf.DataFrame) + assert result._edges.equals(edges_gdf) + + @skip_gpu + def test_dag_execution_preserves_gpu(self): + """Test full DAG execution preserves GPU mode""" + import cudf + + # Create graph with GPU data + edges_gdf = cudf.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Create a simple DAG + dag = ASTLet({}) # Empty DAG + + # Execute + result = g.chain_let(dag) + + # Should preserve GPU mode + assert isinstance(result._edges, cudf.DataFrame) + assert isinstance(result._nodes, cudf.DataFrame) + + @skip_gpu + def test_context_binding_with_mixed_engines(self): + """Test ExecutionContext can handle mixed pandas/cudf results""" + import cudf + from graphistry.compute.execution_context import ExecutionContext + + context = ExecutionContext() + + # Create both pandas and cudf graphs + g_pandas = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + edges_gdf = cudf.DataFrame({'s': ['x'], 'd': ['y']}) + g_cudf = CGFull().edges(edges_gdf, 's', 'd') + + # Store both + context.set_binding('pandas_graph', g_pandas) + context.set_binding('cudf_graph', g_cudf) + + # Retrieve and verify types preserved + assert isinstance(context.get_binding('pandas_graph')._edges, pd.DataFrame) + assert isinstance(context.get_binding('cudf_graph')._edges, cudf.DataFrame) diff --git a/graphistry/tests/compute/test_chain_let_remote_integration.py b/graphistry/tests/compute/test_chain_let_remote_integration.py new file mode 100644 index 0000000000..967477bb55 --- /dev/null +++ b/graphistry/tests/compute/test_chain_let_remote_integration.py @@ -0,0 +1,212 @@ +"""Integration tests for remote graph functionality in chain_let. + +These tests require a real Graphistry server and authentication. +Enable with: TEST_REMOTE_INTEGRATION=1 + +Additional optional env vars: +- GRAPHISTRY_USERNAME: Username for authentication +- GRAPHISTRY_PASSWORD: Password for authentication +- GRAPHISTRY_API_KEY: API key (alternative to username/password) +- GRAPHISTRY_SERVER: Server URL (defaults to hub.graphistry.com) +- GRAPHISTRY_TEST_DATASET_ID: Known dataset ID to test with +""" + +import os +import pytest +import pandas as pd +from unittest.mock import patch + +from graphistry import PyGraphistry +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, n +from graphistry.tests.test_compute import CGFull + + +# Check if remote integration tests are enabled +REMOTE_INTEGRATION_ENABLED = os.environ.get("TEST_REMOTE_INTEGRATION") == "1" +skip_remote = pytest.mark.skipif( + not REMOTE_INTEGRATION_ENABLED, + reason="Remote integration tests need TEST_REMOTE_INTEGRATION=1" +) + + +@skip_remote +class TestRemoteGraphIntegration: + """Integration tests that connect to a real Graphistry server.""" + + @classmethod + def setup_class(cls): + """Set up authentication for remote tests.""" + # Configure PyGraphistry with env vars if available + server = os.environ.get("GRAPHISTRY_SERVER", "hub.graphistry.com") + protocol = "https" if "443" in server or "https" in server else "http" + + if os.environ.get("GRAPHISTRY_API_KEY"): + PyGraphistry.register( + api=3, + protocol=protocol, + server=server, + api_key=os.environ["GRAPHISTRY_API_KEY"] + ) + elif os.environ.get("GRAPHISTRY_USERNAME") and os.environ.get("GRAPHISTRY_PASSWORD"): + PyGraphistry.register( + api=3, + protocol=protocol, + server=server, + username=os.environ["GRAPHISTRY_USERNAME"], + password=os.environ["GRAPHISTRY_PASSWORD"] + ) + else: + pytest.skip("Need GRAPHISTRY_API_KEY or GRAPHISTRY_USERNAME/PASSWORD for remote tests") + + def test_remote_graph_fetch_real_dataset(self): + """Test fetching a real dataset from Graphistry server.""" + # First, upload a test dataset to get a real dataset_id + test_edges = pd.DataFrame({ + 'src': ['a', 'b', 'c'], + 'dst': ['b', 'c', 'a'], + 'weight': [1.0, 2.0, 3.0] + }) + test_nodes = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'label': ['Node A', 'Node B', 'Node C'] + }) + + g = CGFull().edges(test_edges, 'src', 'dst').nodes(test_nodes, 'id') + uploaded = g.upload() + dataset_id = uploaded._dataset_id + assert dataset_id is not None + + # Now test fetching it via ASTRemoteGraph + dag = ASTLet({ + 'remote_data': ASTRemoteGraph(dataset_id) + }) + + # CGFull() creates empty graph, need one with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g2 = g_base.gfql(dag) + + # Verify we got the data back + assert len(g2._edges) == 3 + assert len(g2._nodes) == 3 + assert set(g2._edges['src'].values) == {'a', 'b', 'c'} + assert set(g2._nodes['label'].values) == {'Node A', 'Node B', 'Node C'} + + def test_remote_graph_with_token(self): + """Test using explicit token with RemoteGraph.""" + # Get current token + PyGraphistry.refresh() + token = PyGraphistry.api_token() + + if not token: + pytest.skip("No API token available") + + # Upload test data + g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + uploaded = g.upload() + dataset_id = uploaded._dataset_id + + # Fetch with explicit token + dag = ASTLet({ + 'data': ASTRemoteGraph(dataset_id, token=token) + }) + + # Need graph with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag) + assert len(result._edges) == 1 + + def test_remote_graph_in_complex_dag(self): + """Test RemoteGraph as part of a complex DAG.""" + # Upload test dataset + edges_df = pd.DataFrame({ + 'src': ['a', 'b', 'c', 'd'], + 'dst': ['b', 'c', 'd', 'a'], + 'type': ['friend', 'friend', 'enemy', 'enemy'] + }) + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'category': ['person', 'person', 'bot', 'bot'] + }) + + g = CGFull().edges(edges_df, 'src', 'dst').nodes(nodes_df, 'id') + uploaded = g.upload() + dataset_id = uploaded._dataset_id + + # Create complex DAG with remote data + dag = ASTLet({ + 'remote': ASTRemoteGraph(dataset_id), + 'persons': ASTRef('remote', [n({'category': 'person'})]), + 'friends': ASTRef('persons', [n(edge_query="type == 'friend'")]) + }) + + # Execute and verify - need graph with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag, output='friends') + + # Should only have person nodes + assert all(result._nodes['category'] == 'person') + # Should only have friend edges between persons + assert len(result._edges) > 0 + + def test_remote_graph_error_handling(self): + """Test error handling for invalid dataset IDs.""" + dag = ASTLet({ + 'bad_remote': ASTRemoteGraph('invalid-dataset-id-12345') + }) + + # Need graph with edges for materialize_nodes + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + with pytest.raises(Exception) as exc_info: + g.gfql(dag) + + # Should get some kind of HTTP error or validation error + assert 'dataset' in str(exc_info.value).lower() or 'not found' in str(exc_info.value).lower() + + @pytest.mark.skipif( + not os.environ.get("GRAPHISTRY_TEST_DATASET_ID"), + reason="Need GRAPHISTRY_TEST_DATASET_ID env var for this test" + ) + def test_remote_graph_known_dataset(self): + """Test with a known dataset ID from env var.""" + dataset_id = os.environ["GRAPHISTRY_TEST_DATASET_ID"] + + dag = ASTLet({ + 'data': ASTRemoteGraph(dataset_id) + }) + + # Need graph with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag) + + # Basic validation - should have some data + assert result._edges is not None or result._nodes is not None + if result._edges is not None: + print(f"Fetched {len(result._edges)} edges from {dataset_id}") + if result._nodes is not None: + print(f"Fetched {len(result._nodes)} nodes from {dataset_id}") + + +class TestRemoteGraphMocked: + """Tests with mocked remote calls (always run).""" + + @patch('graphistry.compute.chain_remote.chain_remote') + def test_remote_graph_execution_mocked(self, mock_chain_remote): + """Test that RemoteGraph calls chain_remote correctly.""" + # This test always runs, even without remote server + mock_result = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + mock_chain_remote.return_value = mock_result + + dag = ASTLet({ + 'remote': ASTRemoteGraph('test-dataset-123', token='test-token') + }) + + # Need a graph with edges for bind() to work + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g.gfql(dag) + assert result is not None # Verify result was returned + + # Verify chain_remote was called correctly + mock_chain_remote.assert_called_once() + call_args = mock_chain_remote.call_args + assert call_args[1]['dataset_id'] == 'test-dataset-123' + assert call_args[1]['api_token'] == 'test-token' diff --git a/graphistry/tests/compute/test_gfql.py b/graphistry/tests/compute/test_gfql.py new file mode 100644 index 0000000000..76d8b8b69c --- /dev/null +++ b/graphistry/tests/compute/test_gfql.py @@ -0,0 +1,181 @@ +import pandas as pd +import pytest +from graphistry.compute.ast import ASTLet, ASTRef, n, e +from graphistry.compute.chain import Chain +from graphistry.tests.test_compute import CGFull + + +class TestGFQLAPI: + """Test unified GFQL API and migration""" + + def test_public_api_methods(self): + """Test what methods are available on the public API""" + g = CGFull() + + # Should have gfql + assert hasattr(g, 'gfql') + assert callable(g.gfql) + + # Should still have chain (with deprecation) + assert hasattr(g, 'chain') + assert callable(g.chain) + + # Should NOT have chain_let in public API + assert not hasattr(g, 'chain_let') + + +class TestGFQL: + """Test unified GFQL entrypoint""" + + def test_gfql_exists(self): + """Test that gfql method exists on CGFull""" + g = CGFull() + assert hasattr(g, 'gfql') + assert callable(g.gfql) + + def test_gfql_with_list(self): + """Test gfql with list executes as chain""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute as chain + result = g.gfql([n({'type': 'person'})]) + + assert len(result._nodes) == 2 + assert all(result._nodes['type'] == 'person') + + def test_gfql_with_chain_object(self): + """Test gfql with Chain object""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute with Chain + chain = Chain([n({'type': 'person'}), e(), n()]) + result = g.gfql(chain) + + assert result is not None + # Result depends on graph structure + + def test_gfql_with_dag(self): + """Test gfql with ASTLet""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute as DAG + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}) + }) + + result = g.gfql(dag) + assert result is not None + + def test_gfql_with_dict_convenience(self): + """Test gfql with dict converts to DAG""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Dict should convert to DAG + result = g.gfql({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}) + }) + + assert result is not None + + def test_gfql_output_with_dag(self): + """Test gfql output parameter works with DAG""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute with output selection + result = g.gfql({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}) + }, output='people') + + assert len(result._nodes) == 2 + assert all(result._nodes['type'] == 'person') + + def test_gfql_output_ignored_for_chain(self): + """Test gfql output parameter ignored for chains""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Should work but output ignored + result = g.gfql([n()], output='ignored') + assert result is not None + + def test_gfql_with_single_ast_object(self): + """Test gfql with single ASTObject wraps in list""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Single ASTObject should work + result = g.gfql(n({'type': 'person'})) + + assert len(result._nodes) == 2 + assert all(result._nodes['type'] == 'person') + + def test_gfql_invalid_query_type(self): + """Test gfql with invalid query type""" + g = CGFull() + + with pytest.raises(TypeError) as exc_info: + g.gfql("not a valid query") + + assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) + + def test_gfql_deprecation_and_migration(self): + """Test deprecation warnings and migration path""" + import warnings + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # chain() should show deprecation warning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + chain_result = g.chain([n({'type': 'person'})]) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert "chain() is deprecated" in str(w[0].message) + assert "Use gfql()" in str(w[0].message) + + assert len(chain_result._nodes) == 2 + + # chain_let should no longer exist as public method + assert not hasattr(g, 'chain_let'), "chain_let should be removed from public API" + + # gfql should work for both patterns + gfql_chain = g.gfql([n({'type': 'person'})]) + assert len(gfql_chain._nodes) == 2 + + gfql_dag = g.gfql({'people': n({'type': 'person'})}) + assert len(gfql_dag._nodes) == 2 diff --git a/graphistry/tests/compute/test_let.py b/graphistry/tests/compute/test_let.py new file mode 100644 index 0000000000..5a26818671 --- /dev/null +++ b/graphistry/tests/compute/test_let.py @@ -0,0 +1,201 @@ +"""Tests for Let bindings and related AST nodes validation""" +import pytest +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, n, e +from graphistry.compute.execution_context import ExecutionContext +from graphistry.compute.exceptions import GFQLTypeError + + +class TestLetValidation: + """Test validation for Let bindings""" + + def test_let_valid(self): + """Valid Let should pass validation""" + dag = ASTLet({'a': n(), 'b': e()}) + dag.validate() # Should not raise + + def test_let_invalid_key_type(self): + """Let with non-string key should fail""" + # Note: This validation happens at runtime in _validate_fields + dag = ASTLet({123: n()}) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + dag.validate() + assert exc_info.value.code == "invalid-filter-key" + assert "binding key must be string" in str(exc_info.value) + + def test_let_invalid_value_type(self): + """Let with non-ASTObject value should fail""" + dag = ASTLet({'a': 'not an AST object'}) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + dag.validate() + assert exc_info.value.code == "type-mismatch" + assert "binding value must be ASTObject" in str(exc_info.value) + + def test_let_nested_validation(self): + """Let should validate nested objects""" + # This should work - nested validation of valid objects + dag = ASTLet({ + 'a': n({'type': 'person'}), + 'b': ASTRemoteGraph('dataset123') + }) + dag.validate() + + +class TestRemoteGraphValidation: + """Test validation for RemoteGraph""" + + def test_remoteGraph_valid(self): + """Valid RemoteGraph should pass validation""" + rg = ASTRemoteGraph('my-dataset') + rg.validate() # Should not raise + + rg_with_token = ASTRemoteGraph('my-dataset', token='secret') + rg_with_token.validate() # Should not raise + + def test_remoteGraph_invalid_dataset_type(self): + """RemoteGraph with non-string dataset_id should fail""" + rg = ASTRemoteGraph(123) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + rg.validate() + assert exc_info.value.code == "type-mismatch" + assert "dataset_id must be a string" in str(exc_info.value) + + def test_remoteGraph_empty_dataset(self): + """RemoteGraph with empty dataset_id should fail""" + rg = ASTRemoteGraph('') + with pytest.raises(GFQLTypeError) as exc_info: + rg.validate() + assert exc_info.value.code == "empty-chain" + assert "dataset_id cannot be empty" in str(exc_info.value) + + def test_remoteGraph_invalid_token_type(self): + """RemoteGraph with non-string token should fail""" + rg = ASTRemoteGraph('dataset', token=123) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + rg.validate() + assert exc_info.value.code == "type-mismatch" + assert "token must be string or None" in str(exc_info.value) + + +class TestChainRefValidation: + """Test validation for ChainRef""" + + def test_chainRef_valid(self): + """Valid ChainRef should pass validation""" + cr = ASTRef('myref', [n(), e()]) + cr.validate() # Should not raise + + cr_empty = ASTRef('myref', []) + cr_empty.validate() # Empty chain is valid + + def test_chainRef_invalid_ref_type(self): + """ChainRef with non-string ref should fail""" + cr = ASTRef(123, []) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + cr.validate() + assert exc_info.value.code == "type-mismatch" + assert "ref must be a string" in str(exc_info.value) + + def test_chainRef_empty_ref(self): + """ChainRef with empty ref should fail""" + cr = ASTRef('', []) + with pytest.raises(GFQLTypeError) as exc_info: + cr.validate() + assert exc_info.value.code == "empty-chain" + assert "ref cannot be empty" in str(exc_info.value) + + def test_chainRef_invalid_chain_type(self): + """ChainRef with non-list chain should fail""" + cr = ASTRef('ref', 'not a list') # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + cr.validate() + assert exc_info.value.code == "type-mismatch" + assert "chain must be a list" in str(exc_info.value) + + def test_chainRef_invalid_chain_element(self): + """ChainRef with non-ASTObject in chain should fail""" + cr = ASTRef('ref', [n(), 'not an AST object']) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: + cr.validate() + assert exc_info.value.code == "type-mismatch" + assert "must be ASTObject" in str(exc_info.value) + + def test_chainRef_nested_validation(self): + """ChainRef should validate nested operations""" + cr = ASTRef('ref', [n({'type': 'person'}), e()]) + cr.validate() # Should validate nested nodes + + +class TestExecutionContext: + """Test ExecutionContext functionality""" + + def test_context_basic_operations(self): + """Test basic get/set operations""" + ctx = ExecutionContext() + + # Set and get + ctx.set_binding('a', 'value_a') + assert ctx.get_binding('a') == 'value_a' + + # Has binding + assert ctx.has_binding('a') is True + assert ctx.has_binding('b') is False + + # Multiple bindings + ctx.set_binding('b', 123) + assert ctx.get_binding('b') == 123 + assert len(ctx.get_all_bindings()) == 2 + + def test_context_missing_binding(self): + """Test error on missing binding""" + ctx = ExecutionContext() + with pytest.raises(KeyError, match="No binding found for 'missing'"): + ctx.get_binding('missing') + + def test_context_invalid_name_type(self): + """Test error on non-string binding names""" + ctx = ExecutionContext() + + with pytest.raises(TypeError, match="Binding name must be string"): + ctx.set_binding(123, 'value') + + with pytest.raises(TypeError, match="Binding name must be string"): + ctx.get_binding(123) + + with pytest.raises(TypeError, match="Binding name must be string"): + ctx.has_binding(123) + + def test_context_clear(self): + """Test clearing all bindings""" + ctx = ExecutionContext() + ctx.set_binding('a', 1) + ctx.set_binding('b', 2) + assert len(ctx.get_all_bindings()) == 2 + + ctx.clear() + assert len(ctx.get_all_bindings()) == 0 + assert ctx.has_binding('a') is False + + def test_context_overwrite(self): + """Test overwriting existing binding""" + ctx = ExecutionContext() + ctx.set_binding('a', 'first') + assert ctx.get_binding('a') == 'first' + + ctx.set_binding('a', 'second') + assert ctx.get_binding('a') == 'second' + + +class TestChainRefReverse: + """Test reverse operation for ChainRef""" + + def test_chainRef_reverse(self): + """Test ChainRef reverse reverses operations""" + cr = ASTRef('data', [n(), e(), n()]) + reversed_cr = cr.reverse() + + assert isinstance(reversed_cr, ASTRef) + assert reversed_cr.ref == 'data' + assert len(reversed_cr.chain) == 3 + # Operations should be reversed + # Original: n, e, n + # Reversed: n, e.reverse(), n (each op is individually reversed) diff --git a/setup.cfg b/setup.cfg index f54e3fb9dc..88859d76bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,8 +8,5 @@ versionfile_build = graphistry/_version.py tag_prefix = parentdir_prefix = graphistry- -[options.package_data] -graphistry = py.typed - [aliases] test=pytest