diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb83e2b8a..5baf1137a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Dev +### Added +* GFQL: Add comprehensive validation framework with detailed error reporting + * Built-in validation: `Chain()` constructor validates syntax automatically + * Schema validation: `validate_chain_schema()` validates queries against DataFrame schemas + * Pre-execution validation: `g.chain(ops, validate_schema=True)` catches errors before execution + * Structured error types: `GFQLValidationError`, `GFQLSyntaxError`, `GFQLTypeError`, `GFQLSchemaError` + * Error codes (E1xx syntax, E2xx type, E3xx schema) for programmatic error handling + * Collect-all mode: `validate(collect_all=True)` returns all errors instead of fail-fast + * JSON validation: `Chain.from_json()` validates during parsing for safe LLM integration + * Helpful error suggestions for common mistakes + * Example notebook: `demos/gfql/gfql_validation_fundamentals.ipynb` + ### Fixed * Docs: Fix notebook validation error in hop_and_chain_graph_pattern_mining.ipynb by adding missing 'outputs' field to code cell diff --git a/demos/gfql/call_operations.ipynb b/demos/gfql/call_operations.ipynb new file mode 100644 index 0000000000..c79b8057ba --- /dev/null +++ b/demos/gfql/call_operations.ipynb @@ -0,0 +1,502 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GFQL Call Operations\n", + "\n", + "This notebook demonstrates the Call operation in GFQL, which enables:\n", + "- Invoking user-defined functions on graph data\n", + "- Custom data transformations and enrichments\n", + "- Integration with external services\n", + "- Advanced analytics within graph queries\n", + "\n", + "**Security Note**: Call operations are restricted to a safelist of allowed functions for security." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import graphistry\n", + "from graphistry import n, e_forward, e_reverse\n", + "from graphistry.compute.ast import ASTCall, ASTLet, ASTChainRef\n", + "\n", + "# For convenience, use the alias\n", + "from graphistry import call, let\n", + "\n", + "graphistry.__version__" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sample Data: Network Traffic Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create sample network traffic data\n", + "edges_df = pd.DataFrame({\n", + " 'src_ip': ['192.168.1.1', '192.168.1.1', '192.168.1.2', '10.0.0.1', '10.0.0.1', \n", + " '192.168.1.3', '192.168.1.3', '10.0.0.2', '172.16.0.1'],\n", + " 'dst_ip': ['192.168.1.2', '10.0.0.1', '192.168.1.3', '192.168.1.3', '10.0.0.2',\n", + " '10.0.0.2', '172.16.0.1', '172.16.0.1', 'external.com'],\n", + " 'protocol': ['HTTP', 'HTTPS', 'SSH', 'HTTP', 'DNS', 'HTTPS', 'SSH', 'HTTP', 'HTTPS'],\n", + " 'bytes': [1024, 2048, 512, 4096, 128, 8192, 256, 16384, 32768],\n", + " 'packets': [10, 20, 5, 40, 2, 80, 3, 160, 320],\n", + " 'timestamp': pd.to_datetime([\n", + " '2024-01-01 10:00:00', '2024-01-01 10:05:00', '2024-01-01 10:10:00',\n", + " '2024-01-01 10:15:00', '2024-01-01 10:20:00', '2024-01-01 10:25:00',\n", + " '2024-01-01 10:30:00', '2024-01-01 10:35:00', '2024-01-01 10:40:00'\n", + " ])\n", + "})\n", + "\n", + "nodes_df = pd.DataFrame({\n", + " 'ip': ['192.168.1.1', '192.168.1.2', '192.168.1.3', '10.0.0.1', \n", + " '10.0.0.2', '172.16.0.1', 'external.com'],\n", + " 'type': ['workstation', 'workstation', 'server', 'gateway', \n", + " 'dns_server', 'web_server', 'external'],\n", + " 'risk_level': [0.2, 0.3, 0.7, 0.5, 0.4, 0.8, 0.9]\n", + "})\n", + "\n", + "g = graphistry.edges(edges_df, 'src_ip', 'dst_ip').nodes(nodes_df, 'ip')\n", + "print(f\"Graph has {len(g._nodes)} nodes and {len(g._edges)} edges\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Call Operations\n", + "\n", + "Call operations allow you to invoke functions to transform or analyze data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define some analysis functions (these would be in the safelist)\n", + "def calculate_traffic_score(df):\n", + " \"\"\"Calculate a traffic anomaly score based on bytes and packets.\"\"\"\n", + " if 'bytes' in df.columns and 'packets' in df.columns:\n", + " df['traffic_score'] = (\n", + " (df['bytes'] / df['bytes'].mean()) + \n", + " (df['packets'] / df['packets'].mean())\n", + " ) / 2\n", + " return df\n", + "\n", + "def classify_protocol_risk(df):\n", + " \"\"\"Classify protocols by risk level.\"\"\"\n", + " risk_map = {\n", + " 'HTTP': 'medium',\n", + " 'HTTPS': 'low',\n", + " 'SSH': 'low',\n", + " 'DNS': 'low',\n", + " 'FTP': 'high',\n", + " 'TELNET': 'critical'\n", + " }\n", + " if 'protocol' in df.columns:\n", + " df['protocol_risk'] = df['protocol'].map(risk_map).fillna('unknown')\n", + " return df\n", + "\n", + "# Use Call to apply transformations\n", + "# Note: In production, function names must be in the safelist\n", + "enriched = g.gfql([\n", + " # Apply traffic scoring to all edges\n", + " call('calculate_traffic_score', target='edges'),\n", + " \n", + " # Apply protocol risk classification\n", + " call('classify_protocol_risk', target='edges')\n", + "])\n", + "\n", + "print(\"Enriched edges:\")\n", + "print(enriched._edges[['src_ip', 'dst_ip', 'protocol', 'traffic_score', 'protocol_risk']].head())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Call with Arguments\n", + "\n", + "Call operations can accept arguments to customize their behavior." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def filter_by_threshold(df, column, threshold, operation='gt'):\n", + " \"\"\"Filter dataframe by threshold.\"\"\"\n", + " if column in df.columns:\n", + " if operation == 'gt':\n", + " return df[df[column] > threshold]\n", + " elif operation == 'lt':\n", + " return df[df[column] < threshold]\n", + " elif operation == 'gte':\n", + " return df[df[column] >= threshold]\n", + " elif operation == 'lte':\n", + " return df[df[column] <= threshold]\n", + " return df\n", + "\n", + "# Filter high-traffic connections\n", + "high_traffic = g.gfql([\n", + " # First calculate traffic scores\n", + " call('calculate_traffic_score', target='edges'),\n", + " \n", + " # Then filter for high scores\n", + " call('filter_by_threshold', \n", + " target='edges',\n", + " args={'column': 'traffic_score', 'threshold': 1.5, 'operation': 'gt'})\n", + "])\n", + "\n", + "print(f\"High traffic connections: {len(high_traffic._edges)} edges\")\n", + "print(high_traffic._edges[['src_ip', 'dst_ip', 'bytes', 'packets', 'traffic_score']])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Combining Call with Let Bindings\n", + "\n", + "Call operations work seamlessly with Let bindings for complex analyses." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def calculate_node_centrality(g):\n", + " \"\"\"Calculate degree centrality for nodes.\"\"\"\n", + " # Count incoming and outgoing connections\n", + " in_degree = g._edges.groupby(g._destination).size().to_frame('in_degree')\n", + " out_degree = g._edges.groupby(g._source).size().to_frame('out_degree')\n", + " \n", + " # Merge with nodes\n", + " nodes = g._nodes.copy()\n", + " nodes = nodes.merge(in_degree, left_on=g._node, right_index=True, how='left')\n", + " nodes = nodes.merge(out_degree, left_on=g._node, right_index=True, how='left')\n", + " nodes['centrality'] = (nodes['in_degree'].fillna(0) + nodes['out_degree'].fillna(0)) / 2\n", + " \n", + " return g.nodes(nodes)\n", + "\n", + "# Complex analysis combining Let and Call\n", + "analysis = let({\n", + " # Find high-risk nodes\n", + " 'risky_nodes': n({'risk_level': lambda x: x > 0.6}),\n", + " \n", + " # Find their network\n", + " 'risky_network': [\n", + " ASTChainRef('risky_nodes'),\n", + " e_forward(hops=2)\n", + " ],\n", + " \n", + " # Calculate centrality for the risky network\n", + " 'analyzed_network': [\n", + " ASTChainRef('risky_network'),\n", + " call('calculate_node_centrality', target='graph')\n", + " ]\n", + "})\n", + "\n", + "result = g.gfql([analysis])\n", + "print(\"Nodes with centrality scores:\")\n", + "print(result._nodes[['ip', 'type', 'risk_level', 'centrality']].sort_values('centrality', ascending=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Call for Data Enrichment\n", + "\n", + "Call operations can enrich data with external information or complex calculations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def enrich_with_geolocation(df):\n", + " \"\"\"Simulate IP geolocation enrichment.\"\"\"\n", + " # In production, this might call an actual geolocation service\n", + " geo_data = {\n", + " '192.168.1.1': {'country': 'US', 'city': 'New York', 'lat': 40.7128, 'lon': -74.0060},\n", + " '192.168.1.2': {'country': 'US', 'city': 'New York', 'lat': 40.7128, 'lon': -74.0060},\n", + " '192.168.1.3': {'country': 'US', 'city': 'Boston', 'lat': 42.3601, 'lon': -71.0589},\n", + " '10.0.0.1': {'country': 'UK', 'city': 'London', 'lat': 51.5074, 'lon': -0.1278},\n", + " '10.0.0.2': {'country': 'UK', 'city': 'London', 'lat': 51.5074, 'lon': -0.1278},\n", + " '172.16.0.1': {'country': 'JP', 'city': 'Tokyo', 'lat': 35.6762, 'lon': 139.6503},\n", + " 'external.com': {'country': 'CN', 'city': 'Beijing', 'lat': 39.9042, 'lon': 116.4074}\n", + " }\n", + " \n", + " if 'ip' in df.columns:\n", + " for col in ['country', 'city', 'lat', 'lon']:\n", + " df[col] = df['ip'].map(lambda x: geo_data.get(x, {}).get(col))\n", + " \n", + " return df\n", + "\n", + "def calculate_geo_distance(df):\n", + " \"\"\"Calculate geographical distance for edges.\"\"\"\n", + " if all(col in df.columns for col in ['src_lat', 'src_lon', 'dst_lat', 'dst_lon']):\n", + " # Simplified distance calculation\n", + " df['geo_distance'] = np.sqrt(\n", + " (df['dst_lat'] - df['src_lat'])**2 + \n", + " (df['dst_lon'] - df['src_lon'])**2\n", + " ) * 111 # Rough conversion to km\n", + " return df\n", + "\n", + "# Enrich with geolocation data\n", + "geo_enriched = g.gfql([\n", + " # Add geolocation to nodes\n", + " call('enrich_with_geolocation', target='nodes'),\n", + " \n", + " # Calculate geographical distances for edges\n", + " call('calculate_geo_distance', target='edges')\n", + "])\n", + "\n", + "print(\"Nodes with geolocation:\")\n", + "print(geo_enriched._nodes[['ip', 'type', 'country', 'city']].head())\n", + "\n", + "# Note: Edge distance calculation would require joining node geo data to edges" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Advanced: Multi-Stage Analysis Pipeline\n", + "\n", + "Combine multiple Call operations in a complex analysis pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def detect_anomalies(df, columns, method='zscore', threshold=2):\n", + " \"\"\"Detect anomalies in specified columns.\"\"\"\n", + " for col in columns:\n", + " if col in df.columns:\n", + " if method == 'zscore':\n", + " mean = df[col].mean()\n", + " std = df[col].std()\n", + " df[f'{col}_anomaly'] = np.abs((df[col] - mean) / std) > threshold\n", + " elif method == 'iqr':\n", + " Q1 = df[col].quantile(0.25)\n", + " Q3 = df[col].quantile(0.75)\n", + " IQR = Q3 - Q1\n", + " df[f'{col}_anomaly'] = (\n", + " (df[col] < (Q1 - 1.5 * IQR)) | \n", + " (df[col] > (Q3 + 1.5 * IQR))\n", + " )\n", + " return df\n", + "\n", + "def aggregate_risk_scores(g):\n", + " \"\"\"Aggregate risk scores from edges to nodes.\"\"\"\n", + " if 'traffic_score' in g._edges.columns:\n", + " # Calculate max traffic score for each node\n", + " node_risk = g._edges.groupby(g._source)['traffic_score'].max().to_frame('max_traffic_score')\n", + " \n", + " nodes = g._nodes.copy()\n", + " nodes = nodes.merge(node_risk, left_on=g._node, right_index=True, how='left')\n", + " nodes['combined_risk'] = (\n", + " nodes['risk_level'] * 0.5 + \n", + " nodes['max_traffic_score'].fillna(0) * 0.5\n", + " )\n", + " \n", + " return g.nodes(nodes)\n", + " return g\n", + "\n", + "# Multi-stage security analysis pipeline\n", + "security_analysis = g.gfql([\n", + " # Stage 1: Calculate traffic scores\n", + " call('calculate_traffic_score', target='edges'),\n", + " \n", + " # Stage 2: Classify protocol risks\n", + " call('classify_protocol_risk', target='edges'),\n", + " \n", + " # Stage 3: Detect anomalies in traffic\n", + " call('detect_anomalies', \n", + " target='edges',\n", + " args={'columns': ['bytes', 'packets'], 'method': 'zscore', 'threshold': 1.5}),\n", + " \n", + " # Stage 4: Aggregate risks to nodes\n", + " call('aggregate_risk_scores', target='graph'),\n", + " \n", + " # Stage 5: Filter for high-risk scenarios\n", + " n({'combined_risk': lambda x: x > 0.7}),\n", + " e_forward(),\n", + " n()\n", + "])\n", + "\n", + "print(f\"Security analysis found {len(security_analysis._nodes)} nodes of interest\")\n", + "print(\"\\nHigh-risk nodes:\")\n", + "print(security_analysis._nodes[['ip', 'type', 'risk_level', 'combined_risk']].sort_values('combined_risk', ascending=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Call with Custom Return Types\n", + "\n", + "Call operations can return different types of results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def summarize_network(g):\n", + " \"\"\"Generate a summary report of the network.\"\"\"\n", + " summary = {\n", + " 'node_count': len(g._nodes),\n", + " 'edge_count': len(g._edges),\n", + " 'node_types': g._nodes['type'].value_counts().to_dict() if 'type' in g._nodes else {},\n", + " 'protocols': g._edges['protocol'].value_counts().to_dict() if 'protocol' in g._edges else {},\n", + " 'avg_bytes': g._edges['bytes'].mean() if 'bytes' in g._edges else 0,\n", + " 'total_traffic': g._edges['bytes'].sum() if 'bytes' in g._edges else 0\n", + " }\n", + " \n", + " # Add summary as graph metadata (in practice, might return separately)\n", + " g._metadata = summary\n", + " return g\n", + "\n", + "# Generate network summary\n", + "summarized = g.gfql([\n", + " call('summarize_network', target='graph')\n", + "])\n", + "\n", + "print(\"Network Summary:\")\n", + "if hasattr(summarized, '_metadata'):\n", + " for key, value in summarized._metadata.items():\n", + " print(f\" {key}: {value}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Security Considerations\n", + "\n", + "Call operations have important security features:\n", + "\n", + "1. **Safelist**: Only pre-approved functions can be called\n", + "2. **Sandboxing**: Functions run in a restricted environment\n", + "3. **Resource Limits**: Execution time and memory are bounded\n", + "4. **Input Validation**: Arguments are validated before execution" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example of safelist configuration (typically done at server level)\n", + "SAFELIST = {\n", + " 'calculate_traffic_score': {\n", + " 'module': 'network_analysis',\n", + " 'allowed_args': ['method'],\n", + " 'timeout': 30,\n", + " 'memory_limit': '1GB'\n", + " },\n", + " 'classify_protocol_risk': {\n", + " 'module': 'security_utils',\n", + " 'allowed_args': [],\n", + " 'timeout': 10\n", + " },\n", + " 'filter_by_threshold': {\n", + " 'module': 'data_filters',\n", + " 'allowed_args': ['column', 'threshold', 'operation'],\n", + " 'timeout': 20\n", + " }\n", + "}\n", + "\n", + "# Attempting to call non-safelisted function would raise an error\n", + "try:\n", + " # This would fail in production if not in safelist\n", + " result = g.gfql([\n", + " call('dangerous_function', target='graph')\n", + " ])\n", + "except Exception as e:\n", + " print(f\"Expected error: Function 'dangerous_function' not in safelist\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Call operations in GFQL provide powerful capabilities for:\n", + "\n", + "1. **Data Transformation**: Apply complex transformations to graph data\n", + "2. **Enrichment**: Add external data or calculated fields\n", + "3. **Analysis**: Run sophisticated algorithms within queries\n", + "4. **Integration**: Connect with external services and APIs\n", + "5. **Pipelines**: Build multi-stage analysis workflows\n", + "\n", + "Key concepts:\n", + "- `call(function_name, target='nodes'|'edges'|'graph', args={...})`\n", + "- Functions must be in the server's safelist\n", + "- Can be combined with Let bindings and other operations\n", + "- Support for different return types and side effects\n", + "- Security through sandboxing and resource limits" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/demos/gfql/gfql_validation_fundamentals.ipynb b/demos/gfql/gfql_validation_fundamentals.ipynb index d59cd80bf6..9736f6d08b 100644 --- a/demos/gfql/gfql_validation_fundamentals.ipynb +++ b/demos/gfql/gfql_validation_fundamentals.ipynb @@ -139,28 +139,25 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Pre-Execution Validation\n\nYou have two options for validating queries:\n\n1. **Validate-only** (no execution): Use `validate_chain_schema()` to check compatibility without running the query\n2. **Validate-and-run**: Use `g.chain(..., validate_schema=True)` to validate before execution\n\nThis is useful for catching errors early, especially in production systems." + "source": [ + "## Pre-Execution Validation\n", + "\n", + "For better performance, you can validate queries before execution:" + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Validate AND run (stops at validation if invalid)\nprint(\"Method 1: Validate-and-run with validate_schema=True\")\ntry:\n result = g.chain(chain_to_test.chain, validate_schema=True)\n print(\"Query executed successfully\")\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (check) No graph operations were performed\")\n print(\" (check) Query was rejected before execution\")" + "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Use validate_schema parameter\ntry:\n result = g.chain(chain_to_test.chain, validate_schema=True)\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (No graph operations were performed)\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Method 2: Validate ONLY (no execution)\nprint(\"\\nMethod 2: Validate-only with validate_chain_schema()\")\nfrom graphistry.compute.validate_schema import validate_chain_schema\n\n# Check if chain is compatible with graph schema WITHOUT running it\ntry:\n validate_chain_schema(g, chain_to_test)\n print(\"Chain is valid for this graph schema\")\n print(\"Note: No query execution occurred - only validation!\")\nexcept GFQLSchemaError as e:\n print(f\"Schema incompatibility detected: {e}\")\n print(\" (check) This was validation-only - no query was executed\")\n print(\" (check) Use this method to test queries before running them\")" - }, - { - "cell_type": "code", - "source": "# Example: Demonstrating the difference\nprint(\"=== Demonstrating the difference ===\\n\")\n\n# Create a valid chain\nvalid_chain = Chain([\n n({'type': 'customer'}),\n e_forward()\n])\n\n# Validate-only: Just checks, doesn't run\nprint(\"1. Validate-only with validate_chain_schema():\")\ntry:\n validate_chain_schema(g, valid_chain)\n print(\" (check) Validation passed\")\n print(\" (check) Query NOT executed\")\n print(\" (check) No result object returned\")\nexcept GFQLSchemaError as e:\n print(f\" (x) Validation failed: {e}\")\n\n# Validate-and-run: Validates first, then executes if valid\nprint(\"\\n2. Validate-and-run with g.chain(..., validate_schema=True):\")\ntry:\n result = g.chain(valid_chain.chain, validate_schema=True)\n print(\" (check) Validation passed\")\n print(\" (check) Query WAS executed\")\n print(f\" (check) Result: {len(result._nodes)} nodes, {len(result._edges)} edges\")\nexcept GFQLSchemaError as e:\n print(f\" (x) Validation failed: {e}\")\n print(\" (x) Query NOT executed\")", - "metadata": {}, - "execution_count": null, - "outputs": [] + "source": "# Method 2: Validate chain object directly\nfrom graphistry.compute.validate_schema import validate_chain_schema\n\n# Check if chain is compatible with graph schema\ntry:\n validate_chain_schema(g, chain_to_test)\n print(\"Chain is valid for this graph schema\")\nexcept GFQLSchemaError as e:\n print(f\"Schema incompatibility: {e}\")" }, { "cell_type": "markdown", @@ -190,7 +187,58 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Comprehensive error handling example\ndef safe_chain_execution(g, operations):\n \"\"\"Execute chain with proper error handling.\"\"\"\n try:\n # Create chain\n chain = Chain(operations)\n \n # Pre-validate if desired\n # errors = chain.validate_schema(g, collect_all=True)\n # if errors:\n # print(f\"Warning: {len(errors)} schema issues found\")\n \n # Execute\n result = g.chain(operations)\n return result\n \n except GFQLSyntaxError as e:\n print(f\"Syntax Error [{e.code}]: {e.message}\")\n if e.context.get('suggestion'):\n print(f\" Try: {e.context['suggestion']}\")\n return None\n \n except GFQLTypeError as e:\n print(f\"Type Error [{e.code}]: {e.message}\")\n print(f\" Field: {e.context.get('field')}\")\n print(f\" Value: {e.context.get('value')}\")\n return None\n \n except GFQLSchemaError as e:\n print(f\"Schema Error [{e.code}]: {e.message}\")\n if e.code == ErrorCode.E301:\n print(\" Column not found in data\")\n elif e.code == ErrorCode.E302:\n print(\" Type mismatch between query and data\")\n return None\n\n# Test with valid query\nprint(\"Valid query:\")\nresult = safe_chain_execution(g, [\n n({'type': 'customer'}),\n e_forward()\n])\nif result:\n print(f\" Success! Found {len(result._nodes)} nodes\")\n\n# Test with invalid query\nprint(\"\\nInvalid query:\")\nresult = safe_chain_execution(g, [\n n({'invalid_column': 'value'})\n])" + "source": [ + "# Comprehensive error handling example\n", + "def safe_chain_execution(g, operations):\n", + " \"\"\"Execute chain with proper error handling.\"\"\"\n", + " try:\n", + " # Create chain\n", + " chain = Chain(operations)\n", + " \n", + " # Pre-validate if desired\n", + " # errors = chain.validate_schema(g, collect_all=True)\n", + " # if errors:\n", + " # print(f\"Warning: {len(errors)} schema issues found\")\n", + " \n", + " # Execute\n", + " result = g.chain(operations)\n", + " return result\n", + " \n", + " except GFQLSyntaxError as e:\n", + " print(f\"Syntax Error [{e.code}]: {e.message}\")\n", + " if e.context.get('suggestion'):\n", + " print(f\" Try: {e.context['suggestion']}\")\n", + " return None\n", + " \n", + " except GFQLTypeError as e:\n", + " print(f\"Type Error [{e.code}]: {e.message}\")\n", + " print(f\" Field: {e.context.get('field')}\")\n", + " print(f\" Value: {e.context.get('value')}\")\n", + " return None\n", + " \n", + " except GFQLSchemaError as e:\n", + " print(f\"Schema Error [{e.code}]: {e.message}\")\n", + " if e.code == ErrorCode.E301:\n", + " print(\" Column not found in data\")\n", + " elif e.code == ErrorCode.E302:\n", + " print(\" Type mismatch between query and data\")\n", + " return None\n", + "\n", + "# Test with valid query\n", + "print(\"Valid query:\")\n", + "result = safe_chain_execution(g, [\n", + " n({'type': 'customer'}),\n", + " e_forward()\n", + "])\n", + "if result:\n", + " print(f\" Success! Found {len(result._nodes)} nodes\")\n", + "\n", + "# Test with invalid query\n", + "print(\"\\nInvalid query:\")\n", + "result = safe_chain_execution(g, [\n", + " n({'invalid_column': 'value'})\n", + "])" + ] }, { "cell_type": "markdown", diff --git a/docs/source/gfql/about.rst b/docs/source/gfql/about.rst index f9f633ec23..783d358c91 100644 --- a/docs/source/gfql/about.rst +++ b/docs/source/gfql/about.rst @@ -61,13 +61,13 @@ You can filter nodes based on their properties using the `n()` function. from graphistry import n - people_nodes_df = g.chain([ n({"type": "person"}) ])._nodes + people_nodes_df = g.gfql([ n({"type": "person"}) ])._nodes print('Number of person nodes:', len(people_nodes_df)) **Explanation:** - `n({"type": "person"})` filters nodes where the `type` property is `"person"`. -- `g.chain([...])` applies the chain of operations to the graph `g`. +- `g.gfql([...])` applies the chain of operations to the graph `g`. - `._nodes` retrieves the resulting nodes dataframe. 2. Find 2-Hop Edge Sequences with an Attribute @@ -81,7 +81,7 @@ Traverse multiple hops and filter edges based on attributes using `e_forward()`. from graphistry import e_forward - g_2_hops = g.chain([ e_forward({"interesting": True}, hops=2) ]) + g_2_hops = g.gfql([ e_forward({"interesting": True}, hops=2) ]) print('Number of edges in 2-hop paths:', len(g_2_hops._edges)) g_2_hops.plot() @@ -101,7 +101,7 @@ Label hops in your traversal to analyze specific relationships. from graphistry import n, e_undirected - g_2_hops = g.chain([ + g_2_hops = g.gfql([ n({g._node: "a"}), e_undirected(name="hop1"), e_undirected(name="hop2") @@ -127,7 +127,7 @@ Chain multiple traversals to find patterns between nodes. from graphistry import n, e_forward, e_reverse - g_risky = g.chain([ + g_risky = g.gfql([ n({"risk1": True}), e_forward(to_fixed_point=True), n({"type": "transaction"}, name="hit"), @@ -155,7 +155,7 @@ Use the `is_in` predicate to filter nodes or edges by multiple values. from graphistry import n, e_forward, e_reverse, is_in - g_filtered = g.chain([ + g_filtered = g.gfql([ n({"type": is_in(["person", "company"])}), e_forward({"e_type": is_in(["owns", "reviews"])}, to_fixed_point=True), n({"type": is_in(["transaction", "account"])}, name="hit"), @@ -195,7 +195,7 @@ GFQL is optimized for GPU acceleration using `cudf` and `rapids`. When using GPU g_gpu = graphistry.edges(e_gdf, 'src', 'dst').nodes(n_gdf, 'id') # Run GFQL query (executes on GPU) - g_result = g_gpu.chain([ ... ]) + g_result = g_gpu.gfql([ ... ]) print('Number of resulting edges:', len(g_result._edges)) **Explanation:** @@ -213,7 +213,7 @@ You can explicitly set the engine to ensure GPU execution. :: - g_result = g_gpu.chain([ ... ], engine='cudf') + g_result = g_gpu.gfql([ ... ], engine='cudf') **Explanation:** @@ -259,7 +259,7 @@ Use PyGraphistry's visualization capabilities to explore your graph. from graphistry import n, e # Filter nodes with high PageRank - g_high_pagerank = g_enriched.chain([ + g_high_pagerank = g_enriched.gfql([ n(query='pagerank > 0.1'), e(), n(query='pagerank > 0.1') @@ -283,7 +283,7 @@ You may want to run GFQL remotely because the data is remote or a GPU is availab from graphistry import n, e - g2 = g1.chain_remote([n(), e(), n()]) + g2 = g1.gfql_remote([n(), e(), n()]) **Example: Run GFQL remotely, and decouple the upload step** @@ -293,7 +293,7 @@ You may want to run GFQL remotely because the data is remote or a GPU is availab g2 = g1.upload() assert g2._dataset_id is not None, "Uploading sets `dataset_id` for subsequent calls" - g3 = g2.chain_remote([n(), e(), n()]) + g3 = g2.gfql_remote([n(), e(), n()]) Additional parameters enable controlling options such as the execution `engine` and what is returned @@ -306,8 +306,8 @@ Additional parameters enable controlling options such as the execution `engine` g2 = graphistry.bind(dataset_id='my-dataset-id') - nodes_df = g2.chain_remote([n()])._nodes - edges_df = g2.chain_remote([e()])._edges + nodes_df = g2.gfql_remote([n()])._nodes + edges_df = g2.gfql_remote([e()])._edges **Example: Run Python on remote GPUs over remote data** diff --git a/docs/source/gfql/combo.rst b/docs/source/gfql/combo.rst index b3fbeb92ef..4e6504c0fc 100644 --- a/docs/source/gfql/combo.rst +++ b/docs/source/gfql/combo.rst @@ -25,7 +25,7 @@ Simple Plotting .. code-block:: python - g2 = g1.chain(gfql_query) + g2 = g1.gfql(gfql_query) g2.plot() **Explanation**: diff --git a/docs/source/gfql/datetime_filtering.md b/docs/source/gfql/datetime_filtering.md index 78990da77c..4467620d47 100644 --- a/docs/source/gfql/datetime_filtering.md +++ b/docs/source/gfql/datetime_filtering.md @@ -80,12 +80,12 @@ from graphistry import n from graphistry.compute import gt, lt, between # Filter nodes created after a specific datetime -recent_nodes = g.chain([ +recent_nodes = g.gfql([ n(filter_dict={"created_at": gt(pd.Timestamp("2023-01-01 12:00:00"))}) ]) # Filter edges within a date range -date_range_edges = g.chain([ +date_range_edges = g.gfql([ n(edge_match={"timestamp": between( datetime(2023, 1, 1), datetime(2023, 12, 31) @@ -102,12 +102,12 @@ from datetime import date from graphistry.compute import eq, ge # Filter nodes by exact date -specific_date = g.chain([ +specific_date = g.gfql([ n(filter_dict={"event_date": eq(date(2023, 6, 15))}) ]) # Filter nodes on or after a date -after_date = g.chain([ +after_date = g.gfql([ n(filter_dict={"start_date": ge(date(2023, 1, 1))}) ]) ``` @@ -121,7 +121,7 @@ from datetime import time from graphistry.compute import is_in, between # Filter events at specific times -morning_events = g.chain([ +morning_events = g.gfql([ n(filter_dict={"event_time": is_in([ time(9, 0, 0), time(9, 30, 0), @@ -130,7 +130,7 @@ morning_events = g.chain([ ]) # Filter events in time range -business_hours = g.chain([ +business_hours = g.gfql([ n(filter_dict={"timestamp": between( time(9, 0, 0), time(17, 0, 0) @@ -145,7 +145,7 @@ import pytz # Timezone-aware filtering eastern = pytz.timezone('US/Eastern') -tz_aware_filter = g.chain([ +tz_aware_filter = g.gfql([ n(filter_dict={ "timestamp": gt(pd.Timestamp("2023-01-01 12:00:00", tz=eastern)) }) @@ -164,7 +164,7 @@ Combine temporal predicates with other filters: from graphistry.compute import gt, lt, eq # Complex filter with multiple conditions -complex_filter = g.chain([ +complex_filter = g.gfql([ n(filter_dict={ "created_at": gt(datetime(2023, 1, 1)), "status": eq("active"), @@ -179,7 +179,7 @@ You can pass wire protocol dictionaries directly to predicates, which is useful ```python # Pass wire protocol dictionaries directly -filter_with_dict = g.chain([ +filter_with_dict = g.gfql([ n(filter_dict={"timestamp": gt({ "type": "datetime", "value": "2023-01-01T12:00:00", @@ -188,7 +188,7 @@ filter_with_dict = g.chain([ ]) # Works with all temporal predicates -date_range_filter = g.chain([ +date_range_filter = g.gfql([ n(filter_dict={"event_date": between( {"type": "date", "value": "2023-01-01"}, {"type": "date", "value": "2023-12-31"} @@ -196,7 +196,7 @@ date_range_filter = g.chain([ ]) # And with is_in for multiple values -time_filter = g.chain([ +time_filter = g.gfql([ n(filter_dict={"event_time": is_in([ {"type": "time", "value": "09:00:00"}, {"type": "time", "value": "12:00:00"}, @@ -216,7 +216,7 @@ Use temporal filters in complex graph traversals: ```python # Find all transactions after a date, then their related accounts -recent_transactions = g.chain([ +recent_transactions = g.gfql([ n(filter_dict={"type": eq("transaction"), "date": gt(date(2023, 6, 1))}), n(edge_match={"relationship": eq("involves")}), @@ -239,7 +239,7 @@ from graphistry.compute import DateTimeValue, gt dt_value = DateTimeValue("2023-01-01T12:00:00", "US/Eastern") # Use in predicate -filter_dt = g.chain([ +filter_dt = g.gfql([ n(filter_dict={"timestamp": gt(dt_value)}) ]) ``` @@ -256,7 +256,7 @@ start = DateValue("2023-01-01") end = DateValue("2023-12-31") # Use in between predicate -year_filter = g.chain([ +year_filter = g.gfql([ n(filter_dict={"event_date": between(start, end)}) ]) ``` @@ -273,7 +273,7 @@ morning = TimeValue("09:00:00") noon = TimeValue("12:00:00") # Filter by specific times -time_filter = g.chain([ +time_filter = g.gfql([ n(filter_dict={"daily_event": is_in([morning, noon])}) ]) ``` @@ -300,12 +300,12 @@ from datetime import datetime, timedelta # Find events within last 7 days now = datetime.now() week_ago = now - timedelta(days=7) -recent_events = g.chain([ +recent_events = g.gfql([ n(filter_dict={"timestamp": gt(pd.Timestamp(week_ago))}) ]) # For recurring intervals, use multiple conditions -business_days = g.chain([ +business_days = g.gfql([ n(filter_dict={ "timestamp": between( pd.Timestamp("2023-01-01"), @@ -324,7 +324,7 @@ from datetime import datetime, timedelta # Get data from last 30 days thirty_days_ago = datetime.now() - timedelta(days=30) -recent_data = g.chain([ +recent_data = g.gfql([ n(filter_dict={"timestamp": gt(pd.Timestamp(thirty_days_ago))}) ]) ``` @@ -333,7 +333,7 @@ recent_data = g.chain([ ```python # Filter events during business hours -business_hours = g.chain([ +business_hours = g.gfql([ n(filter_dict={ "timestamp": between(time(9, 0, 0), time(17, 0, 0)) }) @@ -344,7 +344,7 @@ business_hours = g.chain([ ```python # Q1 2023 data -q1_2023 = g.chain([ +q1_2023 = g.gfql([ n(filter_dict={ "date": between( date(2023, 1, 1), diff --git a/docs/source/gfql/index.rst b/docs/source/gfql/index.rst index e8b26a10db..0b044c7a26 100644 --- a/docs/source/gfql/index.rst +++ b/docs/source/gfql/index.rst @@ -21,7 +21,6 @@ See also: quick predicates/quick datetime_filtering - wire_protocol_examples .. toctree:: :maxdepth: 2 diff --git a/docs/source/gfql/predicates/quick.rst b/docs/source/gfql/predicates/quick.rst index 7eeda5a7b2..c0a880a72b 100644 --- a/docs/source/gfql/predicates/quick.rst +++ b/docs/source/gfql/predicates/quick.rst @@ -176,7 +176,7 @@ Usage Examples from graphistry import n, gt, lt # Find nodes where age is greater than 18 and less than 30 - g_filtered = g.chain([ + g_filtered = g.gfql([ n({ "age": gt(18) }), n({ "age": lt(30) }) ]) @@ -188,7 +188,7 @@ Usage Examples from graphistry import n, is_in # Find nodes of type 'person' or 'company' - g_filtered = g.chain([ + g_filtered = g.gfql([ n({ "type": is_in(["person", "company"]) }) ]) @@ -199,7 +199,7 @@ Usage Examples from graphistry import e_forward, contains # Find edges where the relation contains 'friend' - g_filtered = g.chain([ + g_filtered = g.gfql([ e_forward({ "relation": contains("friend") }) ]) @@ -210,7 +210,7 @@ Usage Examples from graphistry import n, eq, gt # Find 'person' nodes with age greater than 18 - g_filtered = g.chain([ + g_filtered = g.gfql([ n({ "type": eq("person"), "age": gt(18) diff --git a/docs/source/gfql/remote.rst b/docs/source/gfql/remote.rst index ad3f040cd9..4311b6586a 100644 --- a/docs/source/gfql/remote.rst +++ b/docs/source/gfql/remote.rst @@ -14,13 +14,13 @@ Run chain remotely and fetch results .. code-block:: python from graphistry import n, e - g2 = g1.chain_remote([n(), e(), n()]) + g2 = g1.gfql_remote([n(), e(), n()]) assert len(g2._nodes) <= len(g1._nodes) -Method :meth:`chain_remote ` runs chain remotely and fetched the computed graph +Method :meth:`gfql_remote ` runs chain remotely and fetched the computed graph - **chain**: Sequence of graph node and edge matchers (:class:`ASTObject ` instances). -- **output_type**: Defaulting to "all", whether to return the nodes (`'nodes'`), edges (`'edges'`), or both. See :meth:`chain_remote_shape ` to return only metadata. +- **output_type**: Defaulting to "all", whether to return the nodes (`'nodes'`), edges (`'edges'`), or both. See :meth:`gfql_remote_shape ` to return only metadata. - **node_col_subset**: Optionally limit which node attributes are returned to an allowlist. - **edge_col_subset**: Optionally limit which edge attributes are returned to an allowlist. - **engine**: Optional execution engine. Engine is typically not set, defaulting to `'auto'`. Use `'cudf'` for GPU acceleration and `'pandas'` for CPU. @@ -40,7 +40,7 @@ Run on GPU remotely and fetch results .. code-block:: python from graphistry import n, e - g2 = g1.chain_remote([n(), e(), n()], engine='cudf') + g2 = g1.gfql_remote([n(), e(), n()], engine='cudf') assert len(g2._nodes) <= len(g1._nodes) CPU @@ -51,7 +51,7 @@ Run on CPU remotely and fetch results .. code-block:: python from graphistry import n, e - g2 = g1.chain_remote([n(), e(), n()], engine='pandas') + g2 = g1.gfql_remote([n(), e(), n()], engine='pandas') @@ -68,8 +68,8 @@ Explicit uploads via :meth:`upload ` g2 = g1.upload() assert g2._dataset_id is not None, "Uploading sets `dataset_id` for subsequent calls" - g3a = g2.chain_remote([n()]) - g3b = g2.chain_remote([n(), e(), n()]) + g3a = g2.gfql_remote([n()]) + g3b = g2.gfql_remote([n(), e(), n()]) assert len(g3a._nodes) >= len(g3b._nodes) @@ -86,7 +86,7 @@ If data is already uploaded and your user has access to it, such as from a previ g1 = graphistry.bind(dataset_id='abc123') assert g1._nodes is None, "Binding does not fetch data" - connected_graph_g = g1.chain_remote([n(), e()]) + connected_graph_g = g1.gfql_remote([n(), e()]) connected_nodes_df = connected_graph_g._nodes print(connected_nodes_df.shape) @@ -102,7 +102,7 @@ Return only nodes .. code-block:: python - g1.chain_remote([n(), e(), n()], output_type="nodes") + g1.gfql_remote([n(), e(), n()], output_type="nodes") Return only nodes and specific columns ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -110,7 +110,7 @@ Return only nodes and specific columns .. code-block:: python cols = [g1._node, 'time'] - g2b = g1.chain_remote( + g2b = g1.gfql_remote( [n(), e(), n()], output_type="nodes", node_col_subset=cols) @@ -122,7 +122,7 @@ Return only edges .. code-block:: python - g2a = g1.chain_remote([n(), e(), n()], output_type="edges") + g2a = g1.gfql_remote([n(), e(), n()], output_type="edges") Return only edges and specific columns ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -130,7 +130,7 @@ Return only edges and specific columns .. code-block:: python cols = [g1._source, g1._destination, 'time'] - g2b = g1.chain_remote([n(), e(), n()], + g2b = g1.gfql_remote([n(), e(), n()], output_type="edges", edge_col_subset=cols) assert len(g2b._edges.columns) == len(cols) @@ -141,7 +141,7 @@ Return metadata but not the actual graph .. code-block:: python from graphistry import n, e - shape_df = g1.chain_remote_shape([n(), e(), n()]) + shape_df = g1.gfql_remote_shape([n(), e(), n()]) assert len(shape_df) == 2 print(shape_df) @@ -170,7 +170,7 @@ Run remote python on the current graph # Upload any local graph data to the remote server g2 = g1.upload() - g3 = g2.chain_remote_python(my_remote_trim_graph_task) + g3 = g2.gfql_remote_python(my_remote_trim_graph_task) assert len(g3._nodes) == 10 assert len(g3._edges) == 10 diff --git a/docs/source/gfql/spec/language.md b/docs/source/gfql/spec/language.md index 65fae87b4b..d2903812d6 100644 --- a/docs/source/gfql/spec/language.md +++ b/docs/source/gfql/spec/language.md @@ -330,7 +330,7 @@ GFQL follows a declarative execution model similar to Neo4j's Cypher: Query execution returns filtered node and edge datasets. In the Python embedding: ```python -result = g.chain([...]) +result = g.gfql([...]) nodes_df = result._nodes # Filtered nodes edges_df = result._edges # Filtered edges ``` @@ -340,7 +340,7 @@ edges_df = result._edges # Filtered edges Operations with `name` parameter add boolean columns to mark matched entities: ```python -result = g.chain([ +result = g.gfql([ n({"type": "person"}, name="people"), e_forward(name="connections"), n({"active": True}, name="active_targets") @@ -361,6 +361,119 @@ people_nodes = result._nodes.query("people == True") This pattern is essential for extracting specific subsets from complex graph traversals. +## Call Operations and Security + +### Call Operations + +GFQL supports calling Plottable methods through the `call()` operation, providing controlled access to graph transformation and analysis capabilities: + +```python +call(function: str, params: dict) -> ASTCall +``` + +Call operations enable: +- Graph algorithms (PageRank, community detection) +- Layout computations (ForceAtlas2, Graphviz) +- Data transformations (filtering, collapsing) +- Visual encodings (color, size, icons) + +### Safelist Architecture + +For security and stability, Call operations are restricted to a predefined safelist of methods. This prevents: +- Arbitrary code execution +- Access to filesystem or network operations +- Modification of global state +- Unsafe graph operations + +#### Safelist Categories + +**Graph Analysis** +- `get_degrees`, `get_indegrees`, `get_outdegrees`: Calculate node degrees +- `compute_cugraph`: Run GPU algorithms (pagerank, louvain, etc.) +- `compute_igraph`: Run CPU algorithms +- `get_topological_levels`: Analyze DAG structure + +**Filtering & Transformation** +- `filter_nodes_by_dict`, `filter_edges_by_dict`: Filter by attributes +- `hop`: Traverse graph with conditions +- `drop_nodes`, `keep_nodes`: Node selection +- `collapse`: Merge nodes by attribute +- `prune_self_edges`: Remove self-loops +- `materialize_nodes`: Generate node table + +**Layout** +- `layout_cugraph`: GPU-accelerated layouts +- `layout_igraph`: CPU-based layouts +- `layout_graphviz`: Graphviz layouts +- `fa2_layout`: ForceAtlas2 layout + +**Visual Encoding** +- `encode_point_color`, `encode_edge_color`: Color mapping +- `encode_point_size`: Size mapping +- `encode_point_icon`: Icon mapping + +**Metadata** +- `name`: Set visualization name +- `description`: Set visualization description + +### Parameter Validation + +Call operations enforce strict parameter validation: + +1. **Type Validation**: Parameters must match expected types + ```python + call('hop', {'hops': 2}) # Valid: integer + call('hop', {'hops': 'two'}) # Error: E201 type mismatch + ``` + +2. **Required Parameters**: Missing required parameters raise errors + ```python + call('filter_nodes_by_dict', {}) # Error: E105 missing filter_dict + ``` + +3. **Unknown Parameters**: Extra parameters are rejected + ```python + call('hop', {'steps': 2}) # Error: E303 unknown parameter + ``` + +### Schema Effects + +Many Call operations modify the graph schema by adding columns: + +```python +# get_degrees adds degree columns +call('get_degrees', { + 'col': 'degree', + 'col_in': 'in_degree', + 'col_out': 'out_degree' +}) +# Schema effect: adds 3 new node columns + +# compute_cugraph adds result column +call('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'pr_score' +}) +# Schema effect: adds 'pr_score' node column +``` + +### Security Considerations + +1. **No Direct Code Execution**: Call operations cannot execute arbitrary Python code +2. **No System Access**: Methods cannot access filesystem, network, or system resources +3. **Validated Parameters**: All inputs are type-checked and validated +4. **Deterministic Effects**: Operations have predictable, documented effects +5. **No State Mutation**: Operations return new graphs without modifying originals + +### Error Handling + +Call operations use GFQL's standard error codes: +- **E303**: Function not in safelist +- **E105**: Missing required parameter +- **E201**: Parameter type mismatch +- **E303**: Unknown parameter +- **E301**: Required column not found (runtime) + ## Best Practices 1. **Use specific filters early**: Filter nodes before traversing edges diff --git a/docs/source/gfql/spec/python_embedding.md b/docs/source/gfql/spec/python_embedding.md index cf20db1741..7984061a4d 100644 --- a/docs/source/gfql/spec/python_embedding.md +++ b/docs/source/gfql/spec/python_embedding.md @@ -41,8 +41,8 @@ Graph edges can be accessed similarly: ```python from graphistry import n, e_forward -# Execute a chain -result = g.chain([ +# Execute a GFQL query +result = g.gfql([ n({"type": "person"}), e_forward(), n() @@ -53,6 +53,268 @@ nodes_df = result._nodes # Filtered nodes DataFrame edges_df = result._edges # Filtered edges DataFrame ``` +## DAG Patterns with Let Bindings + +GFQL supports directed acyclic graph (DAG) patterns using Let bindings, which allow you to define named graph operations that can reference each other. + +### Let Bindings + +```python +from graphistry import let, ref, n, e_forward + +# Define DAG patterns with named bindings +result = g.gfql(let({ + 'persons': n({'type': 'person'}), + 'adults': ref('persons', [n({'age': ge(18)})]), + 'connections': ref('adults', [ + e_forward({'type': 'knows'}), + ref('adults') # Find connections between adults + ]) +})) + +# Access individual binding results +persons_df = result._nodes[result._nodes['persons']] +adults_df = result._nodes[result._nodes['adults']] +connection_edges = result._edges[result._edges['connections']] +``` + +### ChainRef (Reference to Named Bindings) + +The `ref()` function creates references to named bindings within a Let: + +```python +# Basic reference - just the binding result +result = g.gfql(let({ + 'base': n({'status': 'active'}), + 'extended': ref('base') # Just references 'base' +})) + +# Reference with additional operations +result = g.gfql(let({ + 'suspects': n({'risk_score': gt(80)}), + 'lateral_movement': ref('suspects', [ + e_forward({'type': 'ssh', 'failed_attempts': gt(5)}), + n({'type': 'server'}) + ]) +})) +``` + +### Complex DAG Patterns + +```python +# Multi-level analysis pattern +result = g.gfql(let({ + # Find high-value accounts + 'high_value': n({'balance': gt(100000)}), + + # Find transactions from high-value accounts + 'high_value_txns': ref('high_value', [ + e_forward({'type': 'transaction', 'amount': gt(10000)}) + ]), + + # Find recipients of high-value transactions + 'recipients': ref('high_value_txns', [n()]), + + # Find second-hop connections + 'network': ref('recipients', [ + e_forward({'type': 'transaction'}, hops=2) + ]) +})) +``` + +### RemoteGraph (Load Remote Datasets) + +```python +from graphistry import remote_dataset + +# Load a public dataset +remote_g = remote_dataset('public-dataset-id') +result = remote_g.gfql([n({'type': 'user'})]) + +# Load a private dataset with authentication +remote_g = remote_dataset('private-dataset-id', token='auth-token') + +# Use remote dataset in Let bindings +result = g.gfql(let({ + 'remote_data': remote_dataset('dataset-123'), + 'filtered': ref('remote_data', [n({'active': True})]) +})) +``` + +## Call Operations + +GFQL supports calling Plottable methods through the `call()` function, providing a safe way to invoke graph transformations and analysis operations. + +### Basic Call Usage + +```python +from graphistry import call + +# Calculate node degrees +result = g.gfql([ + n({'type': 'person'}), + call('get_degrees', { + 'col': 'centrality', + 'col_in': 'in_centrality', + 'col_out': 'out_centrality' + }) +]) + +# Access degree columns +degree_df = result._nodes[['centrality', 'in_centrality', 'out_centrality']] +``` + +### Graph Analysis Operations + +```python +# PageRank computation +result = g.gfql([ + call('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'pagerank_score', + 'params': {'alpha': 0.85} + }) +]) + +# Community detection +result = g.gfql([ + call('compute_cugraph', { + 'alg': 'louvain', + 'out_col': 'community' + }) +]) + +# Topological analysis +result = g.gfql([ + call('get_topological_levels', { + 'level_col': 'topo_level', + 'allow_cycles': True + }) +]) +``` + +### Layout Operations + +```python +# GPU-accelerated layout +result = g.gfql([ + call('layout_cugraph', { + 'layout': 'force_atlas2', + 'params': { + 'iterations': 500, + 'outbound_attraction_distribution': True + } + }) +]) + +# Graphviz layouts +result = g.gfql([ + call('layout_graphviz', { + 'prog': 'dot', + 'directed': True + }) +]) +``` + +### Filtering and Transformation + +```python +# Complex filtering +result = g.gfql([ + call('filter_nodes_by_dict', { + 'filter_dict': {'type': 'server', 'critical': True} + }), + call('hop', { + 'hops': 2, + 'direction': 'forward', + 'edge_match': {'port': 22} + }) +]) + +# Node transformations +result = g.gfql([ + call('collapse', { + 'column': 'department', + 'self_edges': False + }) +]) +``` + +### Visual Encoding + +```python +# Encode visual properties +result = g.gfql([ + call('encode_point_color', { + 'column': 'risk_score', + 'palette': ['green', 'yellow', 'red'], + 'as_continuous': True + }), + call('encode_point_size', { + 'column': 'importance', + 'categorical_mapping': { + 'low': 10, + 'medium': 20, + 'high': 30 + } + }) +]) +``` + +### Call with Let Bindings + +```python +from graphistry import let, ref, call + +# Combine Let bindings with Call operations +result = g.gfql(let({ + 'high_risk': n({'risk_score': gt(80)}), + 'connected': ref('high_risk', [ + e_forward({'type': 'communicates'}) + ]), + 'analyzed': call('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'influence' + }) +})) +``` + +### Available Call Methods + +Call operations are restricted to a safelist of Plottable methods: + +- **Graph Analysis**: `get_degrees`, `get_indegrees`, `get_outdegrees`, `compute_cugraph`, `compute_igraph`, `get_topological_levels` +- **Filtering**: `filter_nodes_by_dict`, `filter_edges_by_dict`, `hop`, `drop_nodes`, `keep_nodes` +- **Transformation**: `collapse`, `prune_self_edges`, `materialize_nodes` +- **Layout**: `layout_cugraph`, `layout_igraph`, `layout_graphviz`, `fa2_layout` +- **Visual Encoding**: `encode_point_color`, `encode_edge_color`, `encode_point_size`, `encode_point_icon` +- **Metadata**: `name`, `description` + +### Call Validation + +Call operations are validated at multiple levels: + +1. **Function validation**: Only safelist methods allowed +2. **Parameter validation**: Type checking for all parameters +3. **Schema validation**: Ensures required columns exist + +```python +try: + result = g.gfql([ + call('dangerous_method', {}) # Raises E303: not in safelist + ]) +except GFQLTypeError as e: + print(f"Error: {e.message}") + +# Parameter type validation +try: + result = g.gfql([ + call('hop', {'hops': 'two'}) # Raises E201: wrong type + ]) +except GFQLTypeError as e: + print(f"Error: {e.message}") +``` + ## Engine Selection GFQL supports multiple execution engines: @@ -63,9 +325,9 @@ GFQL supports multiple execution engines: ```python # Force specific engine -g.chain([...], engine='cudf') # GPU execution -g.chain([...], engine='pandas') # CPU execution -g.chain([...], engine='auto') # Auto-select +g.gfql([...], engine='cudf') # GPU execution +g.gfql([...], engine='pandas') # CPU execution +g.gfql([...], engine='auto') # Auto-select ``` ## Python-Specific Values @@ -108,51 +370,31 @@ GFQL provides comprehensive validation to catch errors early: Operations are automatically validated during construction: ```python -from graphistry.compute.chain import Chain from graphistry.compute.ast import n, e_forward # Automatic validation on construction -chain = Chain([ +from graphistry import ASTNode, ASTEdge +operations = [ n({'type': 'person'}), e_forward({'hops': -1}) # Raises GFQLTypeError: hops must be positive -]) +] +# Validation happens when operations are created ``` ### Schema Validation -You have two options for validating queries against your data schema: - -1. **Validate-only** (no execution): Use `validate_chain_schema()` to check compatibility without running the query -2. **Validate-and-run**: Use `g.chain(..., validate_schema=True)` to validate before execution +Schema validation happens during execution or can be done pre-emptively: ```python -# Method 1: Validate-only (no execution) -from graphistry.compute.validate_schema import validate_chain_schema - -chain = Chain([n({'missing_column': 'value'})]) -try: - validate_chain_schema(g, chain) # Only validates, doesn't execute - print("Chain is valid for this graph") -except GFQLSchemaError as e: - print(f"Schema incompatibility: {e}") - print("No query was executed") - -# Method 2: Runtime validation (automatic) -try: - result = g.chain([ - n({'missing_column': 'value'}) - ]) # Validates during execution, raises GFQLSchemaError -except GFQLSchemaError as e: - print(f"Runtime validation error: {e}") +# Runtime validation (automatic) +result = g.gfql([ + n({'missing_column': 'value'}) # Raises GFQLSchemaError during execution +]) -# Method 3: Validate-and-run (pre-execution validation) -try: - result = g.chain([ - n({'missing_column': 'value'}) - ], validate_schema=True) # Validates first, only executes if valid -except GFQLSchemaError as e: - print(f"Pre-execution validation failed: {e}") - print("Query was not executed") +# Pre-execution validation (optional) +result = g.gfql([ + n({'missing_column': 'value'}) +], validate_schema=True) # Raises GFQLSchemaError before execution ``` ### Error Types @@ -160,7 +402,7 @@ except GFQLSchemaError as e: GFQL uses structured exceptions with error codes: - **GFQLSyntaxError** (E1xx): Structural issues - - E101: Invalid type (e.g., chain not a list) + - E101: Invalid type (e.g., operations not a list) - E103: Invalid parameter value (e.g., negative hops) - E104: Invalid direction - E105: Missing required field @@ -178,20 +420,26 @@ GFQL uses structured exceptions with error codes: ```python # Fail-fast mode (default) - raises on first error -chain.validate() +for op in operations: + op.validate() # Collect-all mode - returns list of all errors -errors = chain.validate(collect_all=True) +errors = [] +for op in operations: + try: + op.validate() + except Exception as e: + errors.append(e) for error in errors: print(f"[{error.code}] {error.message}") if error.suggestion: print(f" Suggestion: {error.suggestion}") # Pre-validate schema without execution -from graphistry.compute.validate_schema import validate_chain_schema +from graphistry.compute.validate_schema import validate_gfql_schema # Check schema compatibility -errors = validate_chain_schema(g, chain, collect_all=True) +errors = validate_gfql_schema(g, operations, collect_all=True) ``` ### Example: Handling Validation Errors @@ -200,7 +448,7 @@ errors = validate_chain_schema(g, chain, collect_all=True) from graphistry.compute.exceptions import GFQLValidationError, GFQLSchemaError try: - result = g.chain([ + result = g.gfql([ n({'age': 'twenty-five'}) # Type mismatch ]) except GFQLSchemaError as e: @@ -238,27 +486,27 @@ n({"created": gt(pd.Timestamp("2024-01-01"))}) print(g._nodes.columns) # ['id', 'type', 'name'] # Wrong - Column doesn't exist -g.chain([n({"username": "Alice"})]) # KeyError +g.gfql([n({"username": "Alice"})]) # KeyError # Correct - Use existing column -g.chain([n({"name": "Alice"})]) +g.gfql([n({"name": "Alice"})]) ``` ### Unsupported Operations ```python -# Wrong - Can't aggregate in chain -# g.chain([n(), e(), count()]) +# Wrong - Can't aggregate in GFQL query +# g.gfql([n(), e(), count()]) -# Correct - Aggregate after chain -result = g.chain([n(), e()]) +# Correct - Aggregate after GFQL query +result = g.gfql([n(), e()]) count = len(result._edges) # Wrong - OPTIONAL MATCH not supported # No direct GFQL equivalent # Correct - Handle optionality in post-processing -result = g.chain([n(), e_forward()]) +result = g.gfql([n(), e_forward()]) # Check for nodes without edges nodes_with_edges = result._nodes[result._nodes[g._node].isin(result._edges[g._source])] ``` @@ -271,16 +519,16 @@ nodes_with_edges = result._nodes[result._nodes[g._node].isin(result._edges[g._so node_filters = {"type": "User"} if min_age: node_filters["age"] = gt(min_age) -g.chain([n(node_filters)]) +g.gfql([n(node_filters)]) # Avoid: Hardcoded query strings -g.chain([n(query=f"type == 'User' and age > {min_age}")]) # SQL injection risk +g.gfql([n(query=f"type == 'User' and age > {min_age}")]) # SQL injection risk ``` ### Memory Efficiency ```python # Good: Filter early and use named results -result = g.chain([ +result = g.gfql([ n({"active": True}, name="active_users"), # Filter first e_forward({"recent": True}) ]) diff --git a/docs/source/gfql/spec/wire_protocol.md b/docs/source/gfql/spec/wire_protocol.md index beacfdf20f..d82cc88aef 100644 --- a/docs/source/gfql/spec/wire_protocol.md +++ b/docs/source/gfql/spec/wire_protocol.md @@ -33,6 +33,10 @@ All GFQL wire protocol messages are JSON objects with a `type` field: - `Chain`: Complete query chain - `Node`: Node matcher operation - `Edge`: Edge traversal operation +- `Let`: DAG bindings for named graph patterns +- `ChainRef`: Reference to a named binding in Let +- `RemoteGraph`: Load graph from remote dataset +- `Call`: Method invocation with validated parameters - Predicates: `GT`, `LT`, `EQ`, `IsIn`, `Between`, etc. - Temporal values: `datetime`, `date`, `time` @@ -43,7 +47,7 @@ All GFQL wire protocol messages are JSON objects with a `type` field that identi ### Type Identification Each object includes a `type` field: -- Operations: `"Node"`, `"Edge"`, `"Chain"` +- Operations: `"Node"`, `"Edge"`, `"Chain"`, `"Let"`, `"ChainRef"`, `"RemoteGraph"`, `"Call"` - Predicates: `"GT"`, `"LT"`, `"IsIn"`, etc. - Temporal values: `"datetime"`, `"date"`, `"time"` @@ -135,6 +139,233 @@ chain([ } ``` +### Let Bindings (DAG Patterns) + +**Python**: +```python +ASTLet({ + 'persons': n({'type': 'Person'}), + 'adults': ASTChainRef('persons', [n({'age': ge(18)})]), + 'connections': ASTChainRef('adults', [ + e_forward({'type': 'knows'}), + ASTChainRef('adults') + ]) +}) +``` + +**Wire Format**: +```json +{ + "type": "Let", + "bindings": { + "persons": { + "type": "Node", + "filter_dict": {"type": "Person"} + }, + "adults": { + "type": "ChainRef", + "ref": "persons", + "chain": [{ + "type": "Node", + "filter_dict": { + "age": {"type": "GE", "val": 18} + } + }] + }, + "connections": { + "type": "ChainRef", + "ref": "adults", + "chain": [ + { + "type": "Edge", + "direction": "forward", + "edge_match": {"type": "knows"} + }, + { + "type": "ChainRef", + "ref": "adults", + "chain": [] + } + ] + } + } +} +``` + +### ChainRef (Reference to Named Binding) + +**Python**: +```python +ASTChainRef('base_pattern', [ + e_forward({'status': 'active'}), + n({'verified': True}) +]) +``` + +**Wire Format**: +```json +{ + "type": "ChainRef", + "ref": "base_pattern", + "chain": [ + { + "type": "Edge", + "direction": "forward", + "edge_match": {"status": "active"} + }, + { + "type": "Node", + "filter_dict": {"verified": true} + } + ] +} +``` + +### RemoteGraph (Load Remote Dataset) + +**Python**: +```python +ASTRemoteGraph('dataset-123', token='auth-token') +``` + +**Wire Format**: +```json +{ + "type": "RemoteGraph", + "dataset_id": "dataset-123", + "token": "auth-token" +} +``` + +Without token (public dataset): +```json +{ + "type": "RemoteGraph", + "dataset_id": "public-dataset-456" +} +``` + +### Call Operation + +**Python**: +```python +ASTCall('get_degrees', { + 'col': 'centrality', + 'col_in': 'in_centrality', + 'col_out': 'out_centrality' +}) +``` + +**Wire Format**: +```json +{ + "type": "Call", + "function": "get_degrees", + "params": { + "col": "centrality", + "col_in": "in_centrality", + "col_out": "out_centrality" + } +} +``` + +#### Call Operation Examples + +**PageRank computation**: +```json +{ + "type": "Call", + "function": "compute_cugraph", + "params": { + "alg": "pagerank", + "out_col": "pagerank_score", + "params": {"alpha": 0.85} + } +} +``` + +**Graph layout**: +```json +{ + "type": "Call", + "function": "layout_cugraph", + "params": { + "layout": "force_atlas2", + "params": { + "iterations": 500, + "outbound_attraction_distribution": true, + "edge_weight_influence": 1.0 + } + } +} +``` + +**Node filtering**: +```json +{ + "type": "Call", + "function": "filter_nodes_by_dict", + "params": { + "filter_dict": { + "type": "person", + "active": true + } + } +} +``` + +**Complex hop traversal**: +```json +{ + "type": "Call", + "function": "hop", + "params": { + "hops": 3, + "direction": "forward", + "edge_match": {"type": "transfer"}, + "destination_node_match": {"account_type": "checking"} + } +} +``` + +#### Available Call Methods + +The following Plottable methods are available through Call operations: + +**Graph Analysis**: +- `get_degrees`: Calculate node degrees +- `get_indegrees`: Calculate in-degrees only +- `get_outdegrees`: Calculate out-degrees only +- `compute_cugraph`: Run GPU algorithms (pagerank, louvain, etc.) +- `compute_igraph`: Run CPU algorithms +- `get_topological_levels`: Analyze DAG structure + +**Filtering & Transformation**: +- `filter_nodes_by_dict`: Filter nodes by attributes +- `filter_edges_by_dict`: Filter edges by attributes +- `hop`: Traverse graph with complex conditions +- `drop_nodes`: Remove specified nodes +- `keep_nodes`: Keep only specified nodes +- `collapse`: Merge nodes by attribute +- `prune_self_edges`: Remove self-loops +- `materialize_nodes`: Generate node table from edges + +**Layout**: +- `layout_cugraph`: GPU-accelerated layouts +- `layout_igraph`: CPU-based layouts +- `layout_graphviz`: Graphviz layouts (dot, neato, etc.) +- `fa2_layout`: ForceAtlas2 layout + +**Visual Encoding**: +- `encode_point_color`: Map node values to colors +- `encode_edge_color`: Map edge values to colors +- `encode_point_size`: Map node values to sizes +- `encode_point_icon`: Map node values to icons + +**Metadata**: +- `name`: Set visualization name +- `description`: Set visualization description + ## Predicate Serialization ### Comparison Predicates @@ -235,6 +466,235 @@ null // null **Note**: The `timezone` field is optional for DateTime values and defaults to "UTC" if omitted. This ensures consistent behavior across systems while allowing explicit timezone specification when needed. +### Temporal Predicate Examples + +The wire protocol uses tagged dictionaries to preserve type information during JSON serialization. This enables cross-language compatibility and configuration-driven predicate creation. + +#### DateTime Comparisons + +**Python**: +```python +import pandas as pd +from datetime import datetime +from graphistry import n +from graphistry.compute import gt, between + +# Using pandas Timestamp +filter1 = n(filter_dict={ + "created_at": gt(pd.Timestamp("2023-01-01 12:00:00")) +}) + +# Using Python datetime +filter2 = n(edge_match={ + "timestamp": between( + datetime(2023, 1, 1), + datetime(2023, 12, 31, 23, 59, 59) + ) +}) +``` + +**Wire Format**: +```json +// GT with datetime +{ + "type": "Node", + "filter_dict": { + "created_at": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + } + } + } +} + +// Between with datetime range +{ + "type": "Node", + "edge_match": { + "timestamp": { + "type": "Between", + "lower": { + "type": "datetime", + "value": "2023-01-01T00:00:00", + "timezone": "UTC" + }, + "upper": { + "type": "datetime", + "value": "2023-12-31T23:59:59", + "timezone": "UTC" + }, + "inclusive": true + } + } +} +``` + +#### Date Comparisons + +**Python**: +```python +from datetime import date +from graphistry.compute import eq, ge + +# Date equality +filter1 = n(filter_dict={ + "event_date": eq(date(2023, 6, 15)) +}) + +# Date range check +filter2 = n(filter_dict={ + "start_date": ge(date(2023, 1, 1)) +}) +``` + +**Wire Format**: +```json +// Date equality +{ + "type": "Node", + "filter_dict": { + "event_date": { + "type": "EQ", + "val": { + "type": "date", + "value": "2023-06-15" + } + } + } +} +``` + +#### Time Comparisons + +**Python**: +```python +from datetime import time +from graphistry.compute import is_in, between + +# Specific times +filter1 = n(filter_dict={ + "event_time": is_in([ + time(9, 0, 0), + time(12, 0, 0), + time(17, 0, 0) + ]) +}) +``` + +**Wire Format**: +```json +// IsIn with times +{ + "type": "Node", + "filter_dict": { + "event_time": { + "type": "IsIn", + "options": [ + {"type": "time", "value": "09:00:00"}, + {"type": "time", "value": "12:00:00"}, + {"type": "time", "value": "17:00:00"} + ] + } + } +} +``` + +#### Timezone-Aware DateTime + +**Python**: +```python +import pytz + +# Using timezone-aware timestamp +eastern = pytz.timezone('US/Eastern') +filter1 = n(filter_dict={ + "timestamp": gt( + pd.Timestamp("2023-01-01 09:00:00", tz=eastern) + ) +}) +``` + +**Wire Format**: +```json +{ + "type": "Node", + "filter_dict": { + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-01-01T09:00:00", + "timezone": "US/Eastern" + } + } + } +} +``` + +#### Round-Trip Serialization + +```python +# Create predicate +from graphistry.compute import gt +pred = gt(pd.Timestamp("2023-01-01 12:00:00")) + +# Serialize to JSON +json_data = pred.to_json() +# Output: { +# 'type': 'GT', +# 'val': { +# 'type': 'datetime', +# 'value': '2023-01-01T12:00:00', +# 'timezone': 'UTC' +# } +# } + +# Deserialize from JSON +from graphistry.compute.predicates.numeric import GT +pred2 = GT.from_json(json_data) +``` + +#### TypeScript Interfaces + +For frontend developers, temporal values follow these interfaces: + +```typescript +// DateTime with timezone +interface DateTimeWire { + type: "datetime"; + value: string; // ISO 8601 format + timezone?: string; // IANA timezone (default: "UTC") +} + +// Date only +interface DateWire { + type: "date"; + value: string; // YYYY-MM-DD format +} + +// Time only +interface TimeWire { + type: "time"; + value: string; // HH:MM:SS[.ffffff] format +} + +// Temporal predicates +interface TemporalPredicate { + type: "GT" | "LT" | "GE" | "LE" | "EQ" | "NE"; + val: DateTimeWire | DateWire | TimeWire; +} + +interface BetweenPredicate { + type: "Between"; + lower: DateTimeWire | DateWire | TimeWire; + upper: DateTimeWire | DateWire | TimeWire; + inclusive: boolean; +} +``` + ## Examples ### User 360 Query @@ -325,6 +785,122 @@ g.chain([ } ``` +### Complex DAG Pattern + +**Python**: +```python +g.gfql(ASTLet({ + 'suspicious_ips': n({'risk_score': gt(80)}), + 'lateral_movement': ASTChainRef('suspicious_ips', [ + e_forward({'type': 'ssh', 'failed_attempts': gt(5)}), + n({'type': 'server'}) + ]), + 'escalation': ASTChainRef('lateral_movement', [ + e_forward({'type': 'privilege_change'}), + n({'admin': True}) + ]) +})) +``` + +**Wire Format**: +```json +{ + "type": "Let", + "bindings": { + "suspicious_ips": { + "type": "Node", + "filter_dict": { + "risk_score": {"type": "GT", "val": 80} + } + }, + "lateral_movement": { + "type": "ChainRef", + "ref": "suspicious_ips", + "chain": [ + { + "type": "Edge", + "direction": "forward", + "edge_match": { + "type": "ssh", + "failed_attempts": {"type": "GT", "val": 5} + } + }, + { + "type": "Node", + "filter_dict": {"type": "server"} + } + ] + }, + "escalation": { + "type": "ChainRef", + "ref": "lateral_movement", + "chain": [ + { + "type": "Edge", + "direction": "forward", + "edge_match": {"type": "privilege_change"} + }, + { + "type": "Node", + "filter_dict": {"admin": true} + } + ] + } + } +} +``` + +### DAG Pattern with Call Operations + +**Python**: +```python +g.gfql(ASTLet({ + 'high_value': n({'amount': gt(100000)}), + 'connected': ASTChainRef('high_value', [ + e_forward({'type': 'transfer'}, hops=2) + ]), + 'analyzed': ASTCall('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'influence_score' + }) +})) +``` + +**Wire Format**: +```json +{ + "type": "Let", + "bindings": { + "high_value": { + "type": "Node", + "filter_dict": { + "amount": {"type": "GT", "val": 100000} + } + }, + "connected": { + "type": "ChainRef", + "ref": "high_value", + "chain": [ + { + "type": "Edge", + "direction": "forward", + "edge_match": {"type": "transfer"}, + "hops": 2 + } + ] + }, + "analyzed": { + "type": "Call", + "function": "compute_cugraph", + "params": { + "alg": "pagerank", + "out_col": "influence_score" + } + } + } +} +``` + ## Best Practices @@ -333,6 +909,12 @@ g.chain([ 3. **Handle timezones consistently**: Include timezone for datetime values when precision matters (defaults to UTC) 4. **Validate before sending**: Use JSON Schema validation 5. **Handle unknown fields**: Ignore unrecognized fields for compatibility +6. **Let bindings**: Define bindings in dependency order (referenced names must be defined first) +7. **ChainRef validation**: Ensure referenced names exist in the Let binding scope +8. **RemoteGraph security**: Protect authentication tokens in transit and storage +9. **Call operations**: Only use function names from the safelist +10. **Parameter validation**: Ensure Call parameters match expected types +11. **Error handling**: Call operations may fail if schema requirements aren't met ## See Also diff --git a/docs/source/gfql/validation/fundamentals.rst b/docs/source/gfql/validation/fundamentals.rst index 6de638e1f3..97dfea9557 100644 --- a/docs/source/gfql/validation/fundamentals.rst +++ b/docs/source/gfql/validation/fundamentals.rst @@ -51,7 +51,7 @@ Built-in Validation GFQL validates automatically - no separate validation calls needed: * **Syntax validation**: Happens during chain construction -* **Schema validation**: Happens by default during ``g.chain()`` execution +* **Schema validation**: Happens by default during ``g.gfql()`` execution * **Structured errors**: Error codes (E1xx, E2xx, E3xx) for programmatic handling Error Types @@ -85,13 +85,13 @@ Missing Columns # Wrong - column doesn't exist try: - result = g.chain([n({'category': 'VIP'})]) + result = g.gfql([n({'category': 'VIP'})]) except GFQLSchemaError as e: print(f"Error: {e.message}") # Column "category" does not exist print(f"Suggestion: {e.context.get('suggestion')}") # Correct - use existing columns - result = g.chain([n({'type': 'customer'})]) + result = g.gfql([n({'type': 'customer'})]) Type Mismatches ^^^^^^^^^^^^^^^ @@ -100,13 +100,13 @@ Type Mismatches # Wrong - string value on numeric column try: - result = g.chain([n({'score': 'high'})]) + result = g.gfql([n({'score': 'high'})]) except GFQLSchemaError as e: print(f"Error: {e.message}") # Type mismatch # Correct - use numeric predicate from graphistry.compute.predicates.numeric import gt - result = g.chain([n({'score': gt(80)})]) + result = g.gfql([n({'score': gt(80)})]) Temporal Comparisons ^^^^^^^^^^^^^^^^^^^^ @@ -117,12 +117,12 @@ Temporal Comparisons from graphistry.compute.predicates.numeric import gt, lt # Compare datetime columns - result = g.chain([ + result = g.gfql([ n({'created_at': gt(pd.Timestamp('2024-01-01'))}) ]) # Find recent activity (last 7 days) - result = g.chain([ + result = g.gfql([ e_forward({ 'timestamp': gt(pd.Timestamp.now() - pd.Timedelta(days=7)) }) @@ -139,39 +139,28 @@ GFQL validates automatically - just write your queries and run them: .. code-block:: python # Validation happens automatically - result = g.chain([n({'type': 'customer'})]) + result = g.gfql([n({'type': 'customer'})]) # Errors are caught and reported clearly try: - result = g.chain([n({'invalid_column': 'value'})]) + result = g.gfql([n({'invalid_column': 'value'})]) except GFQLSchemaError as e: print(f"Error: {e.message}") -Pre-Execution Validation Options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Advanced: Manual Validation +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You have two options for validating queries: - -1. **Validate-only** (no execution): Use ``validate_chain_schema()`` to check compatibility without running the query -2. **Validate-and-run**: Use ``g.gfql(..., validate_schema=True)`` to validate before execution +For advanced users who need to validate before execution: .. code-block:: python - # Method 1: Validate-only (no execution) - from graphistry.compute.validate_schema import validate_chain_schema - - try: - validate_chain_schema(g, chain) # Only validates, doesn't execute - print("Chain is valid for this graph schema") - except GFQLSchemaError as e: - print(f"Schema incompatibility: {e}") + # Validate syntax without running + chain = Chain([n(), e_forward()]) + errors = chain.validate(collect_all=True) - # Method 2: Validate-and-run - try: - result = g.gfql(chain.chain, validate_schema=True) # Validates, then executes if valid - print(f"Query executed: {len(result._nodes)} nodes") - except GFQLSchemaError as e: - print(f"Validation failed, query not executed: {e}") + # Pre-validate against schema (rarely needed) + from graphistry.compute.validate_schema import validate_chain_schema + schema_errors = validate_chain_schema(g, chain, collect_all=True) Error Collection ^^^^^^^^^^^^^^^^ diff --git a/docs/source/gfql/validation/production.rst b/docs/source/gfql/validation/production.rst index d78c2ba462..bdda405977 100644 --- a/docs/source/gfql/validation/production.rst +++ b/docs/source/gfql/validation/production.rst @@ -132,7 +132,7 @@ pytest Fixtures chain = Chain(operations) # Should not raise # Test schema validation - result = sample_plottable.chain(operations) # Should not raise + result = sample_plottable.gfql(operations) # Should not raise assert len(result._nodes) > 0 def test_invalid_query_syntax(sample_plottable): @@ -144,7 +144,7 @@ pytest Fixtures operations = [n({'missing_column': eq('value')})] with pytest.raises(GFQLValidationError) as exc_info: - result = sample_plottable.chain(operations) # Schema validation fails + result = sample_plottable.gfql(operations) # Schema validation fails assert exc_info.value.code == 'E301' # Column not found API Integration @@ -238,7 +238,7 @@ Instead of generating Python code directly, generate JSON and use GFQL's validat .. code-block:: python # DON'T: Generate Python code (security risk) - # query = f"g.chain([n({{'user_id': '{user_input}'}})])" + # query = f"g.gfql([n({{'user_id': '{user_input}'}})])" # eval(query) # NEVER DO THIS # DO: Generate JSON and validate @@ -253,7 +253,7 @@ Instead of generating Python code directly, generate JSON and use GFQL's validat # Safe parsing with validation from graphistry.compute.chain import Chain chain = Chain.from_json(query_json, validate=True) - result = g.chain(chain.chain) + result = g.gfql(chain.chain) **Key Security Features** diff --git a/docs/source/gfql/wire_protocol_examples.md b/docs/source/gfql/wire_protocol_examples.md deleted file mode 100644 index 40f507b097..0000000000 --- a/docs/source/gfql/wire_protocol_examples.md +++ /dev/null @@ -1,516 +0,0 @@ -# Temporal Predicates Wire Protocol Reference - -This document provides a comprehensive reference for how temporal predicates serialize to JSON in the GFQL wire protocol. The wire protocol enables interoperability between Python and other systems. - -## Overview - -The wire protocol uses tagged dictionaries to preserve type information during JSON serialization. This enables: -- Cross-language compatibility -- Configuration-driven predicate creation -- Network transport of queries -- Storage of predicate definitions - -**Key Concept**: Wire protocol dictionaries can be used directly in the Python API: - -```python -# These are equivalent: -pred1 = gt(100) -pred2 = gt(pd.Timestamp("2023-01-01")) -pred3 = gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"}) -``` - -## 1. DateTime Comparisons - -### Python API -```python -import pandas as pd -from datetime import datetime -from graphistry import n -from graphistry.compute import gt, between - -# Using pandas Timestamp -filter1 = n(filter_dict={ - "created_at": gt(pd.Timestamp("2023-01-01 12:00:00")) -}) - -# Using Python datetime -filter2 = n(edge_match={ - "timestamp": between( - datetime(2023, 1, 1), - datetime(2023, 12, 31, 23, 59, 59) - ) -}) -``` - -### Wire Protocol (JSON) -```json -// GT with datetime -{ - "type": "ASTNode", - "filter_dict": { - "created_at": { - "type": "GT", - "val": { - "type": "datetime", - "value": "2023-01-01T12:00:00", - "timezone": "UTC" - } - } - } -} - -// Between with datetime range -{ - "type": "ASTNode", - "edge_match": { - "timestamp": { - "type": "Between", - "lower": { - "type": "datetime", - "value": "2023-01-01T00:00:00", - "timezone": "UTC" - }, - "upper": { - "type": "datetime", - "value": "2023-12-31T23:59:59", - "timezone": "UTC" - }, - "inclusive": true - } - } -} -``` - -### Round-trip Example -```python -# Create predicate -from graphistry.compute import gt -pred = gt(pd.Timestamp("2023-01-01 12:00:00")) - -# Serialize to JSON -json_data = pred.to_json() -print(json_data) -# Output: { -# 'type': 'GT', -# 'val': { -# 'type': 'datetime', -# 'value': '2023-01-01T12:00:00', -# 'timezone': 'UTC' -# } -# } - -# Deserialize from JSON -from graphistry.compute.predicates.numeric import GT -pred2 = GT.from_json(json_data) -# pred2 is functionally equivalent to pred -``` - -## 2. Date-Only Comparisons - -### Python API -```python -from datetime import date -from graphistry.compute import eq, ge - -# Date equality -filter1 = n(filter_dict={ - "event_date": eq(date(2023, 6, 15)) -}) - -# Date range check -filter2 = n(filter_dict={ - "start_date": ge(date(2023, 1, 1)) -}) -``` - -### Wire Protocol (JSON) -```json -// Date equality -{ - "type": "ASTNode", - "filter_dict": { - "event_date": { - "type": "EQ", - "val": { - "type": "date", - "value": "2023-06-15" - } - } - } -} - -// Date greater than or equal -{ - "type": "ASTNode", - "filter_dict": { - "start_date": { - "type": "GE", - "val": { - "type": "date", - "value": "2023-01-01" - } - } - } -} -``` - -## 3. Time-Only Comparisons - -### Python API -```python -from datetime import time -from graphistry.compute import is_in, between - -# Specific times -filter1 = n(filter_dict={ - "event_time": is_in([ - time(9, 0, 0), - time(12, 0, 0), - time(17, 0, 0) - ]) -}) - -# Time range -filter2 = n(edge_match={ - "daily_schedule": between( - time(9, 0, 0), - time(17, 30, 0) - ) -}) -``` - -### Wire Protocol (JSON) -```json -// IsIn with times -{ - "type": "ASTNode", - "filter_dict": { - "event_time": { - "type": "IsIn", - "options": [ - {"type": "time", "value": "09:00:00"}, - {"type": "time", "value": "12:00:00"}, - {"type": "time", "value": "17:00:00"} - ] - } - } -} - -// Time range -{ - "type": "ASTNode", - "edge_match": { - "daily_schedule": { - "type": "Between", - "lower": {"type": "time", "value": "09:00:00"}, - "upper": {"type": "time", "value": "17:30:00"}, - "inclusive": true - } - } -} -``` - -## 4. Timezone-Aware DateTime - -### Python API -```python -import pytz -from graphistry.compute import DateTimeValue, gt - -# Using timezone-aware timestamp -eastern = pytz.timezone('US/Eastern') -filter1 = n(filter_dict={ - "timestamp": gt( - pd.Timestamp("2023-01-01 09:00:00", tz=eastern) - ) -}) - -# Using DateTimeValue with explicit timezone -dt_val = DateTimeValue("2023-01-01T09:00:00", "US/Eastern") -filter2 = n(filter_dict={ - "timestamp": gt(dt_val) -}) -``` - -### Wire Protocol (JSON) -```json -// Timezone-aware datetime -{ - "type": "ASTNode", - "filter_dict": { - "timestamp": { - "type": "GT", - "val": { - "type": "datetime", - "value": "2023-01-01T09:00:00", - "timezone": "US/Eastern" - } - } - } -} -``` - -## 5. Complex Chain with Temporal Predicates - -### Python API -```python -from graphistry import n, e_forward -from graphistry.compute import gt, eq, between -from datetime import datetime, timedelta - -# Multi-hop query with temporal filters -chain = g.chain([ - # Recent transactions - n(edge_match={ - "timestamp": gt(datetime.now() - timedelta(days=7)), - "amount": gt(1000) - }), - # To active accounts - n(filter_dict={ - "status": eq("active"), - "last_login": between( - datetime.now() - timedelta(days=30), - datetime.now() - ) - }), - # Outgoing transfers - e_forward(edge_match={ - "type": eq("transfer"), - "timestamp": gt(datetime.now() - timedelta(days=1)) - }) -]) -``` - -### Wire Protocol (JSON) -```json -{ - "type": "Chain", - "queries": [ - { - "type": "ASTNode", - "edge_match": { - "timestamp": { - "type": "GT", - "val": { - "type": "datetime", - "value": "2023-12-18T10:30:00", - "timezone": "UTC" - } - }, - "amount": { - "type": "GT", - "val": 1000 - } - } - }, - { - "type": "ASTNode", - "filter_dict": { - "status": { - "type": "EQ", - "val": "active" - }, - "last_login": { - "type": "Between", - "lower": { - "type": "datetime", - "value": "2023-11-25T10:30:00", - "timezone": "UTC" - }, - "upper": { - "type": "datetime", - "value": "2023-12-25T10:30:00", - "timezone": "UTC" - }, - "inclusive": true - } - } - }, - { - "type": "ASTEdge", - "direction": "forward", - "edge_match": { - "type": { - "type": "EQ", - "val": "transfer" - }, - "timestamp": { - "type": "GT", - "val": { - "type": "datetime", - "value": "2023-12-24T10:30:00", - "timezone": "UTC" - } - } - } - } - ] -} -``` - -## 6. Temporal Value Classes Direct Usage - -### Python API -```python -from graphistry.compute import ( - DateTimeValue, DateValue, TimeValue, - temporal_value_from_json, gt -) - -# Create temporal values -dt_val = DateTimeValue("2023-06-15T14:30:00", "Europe/London") -date_val = DateValue("2023-06-15") -time_val = TimeValue("14:30:00") - -# Use in predicates -filter1 = n(filter_dict={"timestamp": gt(dt_val)}) -filter2 = n(filter_dict={"event_date": eq(date_val)}) -filter3 = n(filter_dict={"daily_time": eq(time_val)}) - -# Create from JSON -json_dt = { - "type": "datetime", - "value": "2023-06-15T14:30:00", - "timezone": "Europe/London" -} -dt_from_json = temporal_value_from_json(json_dt) -``` - -### Wire Protocol (JSON) -```json -// DateTimeValue serialization -{ - "type": "datetime", - "value": "2023-06-15T14:30:00", - "timezone": "Europe/London" -} - -// DateValue serialization -{ - "type": "date", - "value": "2023-06-15" -} - -// TimeValue serialization -{ - "type": "time", - "value": "14:30:00" -} -``` - -## 7. Full Round-Trip Example - -```python -# 1. Create a complex query with temporal predicates -from graphistry import n, Chain -from graphistry.compute import gt, between, is_in -from datetime import datetime, date, time -import pandas as pd - -query = Chain([ - n(filter_dict={ - "created": gt(pd.Timestamp("2023-01-01")), - "event_date": between(date(2023, 6, 1), date(2023, 6, 30)), - "event_time": is_in([time(9, 0), time(12, 0), time(17, 0)]) - }) -]) - -# 2. Serialize to JSON -json_query = query.to_json() -print(json_query) - -# 3. Send over wire (simulated) -import json -wire_data = json.dumps(json_query) -received_data = json.loads(wire_data) - -# 4. Deserialize on receiving end -from graphistry.compute.ast import Chain -reconstructed_query = Chain.from_json(received_data) - -# 5. Apply to graph data -result = g.chain(reconstructed_query.queries) -``` - -## Wire Protocol Structure - -### Temporal Value Types - -All temporal values in the wire protocol follow this pattern: - -```typescript -// DateTime with timezone -interface DateTimeWire { - type: "datetime"; - value: string; // ISO 8601 format - timezone?: string; // IANA timezone (default: "UTC") -} - -// Date only -interface DateWire { - type: "date"; - value: string; // YYYY-MM-DD format -} - -// Time only -interface TimeWire { - type: "time"; - value: string; // HH:MM:SS[.ffffff] format -} -``` - -### Predicate Structure - -Predicates containing temporal values serialize as: - -```typescript -interface TemporalPredicate { - type: "GT" | "LT" | "GE" | "LE" | "EQ" | "NE"; - val: DateTimeWire | DateWire | TimeWire; -} - -interface BetweenPredicate { - type: "Between"; - lower: DateTimeWire | DateWire | TimeWire; - upper: DateTimeWire | DateWire | TimeWire; - inclusive: boolean; -} - -interface IsInPredicate { - type: "IsIn"; - options: Array; -} -``` - -## Key Points - -1. **Type Safety**: Raw strings are rejected in the Python API to avoid ambiguity -2. **Automatic Conversion**: Python datetime objects are automatically converted to appropriate temporal values -3. **Timezone Preservation**: Timezone information is preserved through serialization -4. **Tagged Format**: JSON uses tagged dictionaries to preserve type information -5. **Direct Usage**: Wire protocol dicts can be passed directly to Python predicates - -## Error Handling - -```python -# Raw strings raise ValueError -try: - filter_raw = n(filter_dict={"date": gt("2023-01-01")}) -except ValueError as e: - print(e) - # Output: Raw string '2023-01-01' is ambiguous. Use: - # - gt(pd.Timestamp('2023-01-01')) for datetime - # - gt({'type': 'datetime', 'value': '2023-01-01T00:00:00'}) for explicit type - -# Valid approaches -filter1 = n(filter_dict={"date": gt(pd.Timestamp("2023-01-01"))}) -filter2 = n(filter_dict={"date": gt(datetime(2023, 1, 1))}) -filter3 = n(filter_dict={"date": gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"})}) -``` - -## Performance Considerations - -- Temporal predicates leverage pandas' optimized datetime operations -- Timezone conversions are handled efficiently -- For large datasets, ensure datetime columns are properly typed (not object dtype) -- Use `pd.Timestamp` for best performance when creating many predicates programmatically \ No newline at end of file diff --git a/docs/source/notebooks/gfql.rst b/docs/source/notebooks/gfql.rst index a967eae076..c363aa1b70 100644 --- a/docs/source/notebooks/gfql.rst +++ b/docs/source/notebooks/gfql.rst @@ -8,6 +8,7 @@ GFQL Graph queries Intro to graph queries with hop and chain <../demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb> GFQL Validation Fundamentals <../demos/gfql/gfql_validation_fundamentals.ipynb> + Call Operations <../demos/gfql/call_operations.ipynb> DateTime Filtering Examples <../demos/gfql/temporal_predicates.ipynb> GPU Benchmarking <../demos/gfql/benchmark_hops_cpu_gpu.ipynb> GFQL Remote mode <../demos/gfql/gfql_remote.ipynb> diff --git a/graphistry/compute/ASTSerializable.py b/graphistry/compute/ASTSerializable.py index 0ee1ad5d11..8a36535dc0 100644 --- a/graphistry/compute/ASTSerializable.py +++ b/graphistry/compute/ASTSerializable.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import Dict, List, Optional, TYPE_CHECKING +from typing import Dict, List, Optional, Sequence, TYPE_CHECKING from graphistry.utils.json import JSONVal, serialize_to_json_val @@ -68,11 +68,11 @@ def _validate_fields(self) -> None: """ pass - def _get_child_validators(self) -> List['ASTSerializable']: + def _get_child_validators(self) -> Sequence['ASTSerializable']: """Override in subclasses to return child AST nodes that need validation. Returns: - List of child AST nodes to validate + Sequence of child AST nodes to validate """ return [] diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 0736b21538..e4d14abad5 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -7,7 +7,7 @@ from graphistry.Plottable import Plottable from graphistry.util import setup_logger from .chain import chain as chain_base -from .gfql import gfql as gfql_base +from .gfql_unified import gfql as gfql_base from .chain_remote import ( chain_remote as chain_remote_base, chain_remote_shape as chain_remote_shape_base @@ -487,12 +487,54 @@ def gfql(self, *args, **kwargs): gfql.__doc__ = gfql_base.__doc__ def chain_remote(self, *args, **kwargs) -> Plottable: + """ + .. deprecated:: 2.XX.X + Use :meth:`gfql_remote` instead for a unified API that supports both chains and DAGs. + """ + import warnings + warnings.warn( + "chain_remote() is deprecated. Use gfql_remote() instead for a unified API.", + DeprecationWarning, + stacklevel=2 + ) return chain_remote_base(self, *args, **kwargs) - chain_remote.__doc__ = chain_remote_base.__doc__ + # Preserve original docstring after deprecation notice + chain_remote.__doc__ = (chain_remote.__doc__ or "") + "\n\n" + (chain_remote_base.__doc__ or "") def chain_remote_shape(self, *args, **kwargs) -> pd.DataFrame: + """ + .. deprecated:: 2.XX.X + Use :meth:`gfql_remote_shape` instead for a unified API that supports both chains and DAGs. + """ + import warnings + warnings.warn( + "chain_remote_shape() is deprecated. Use gfql_remote_shape() instead for a unified API.", + DeprecationWarning, + stacklevel=2 + ) + return chain_remote_shape_base(self, *args, **kwargs) + # Preserve original docstring after deprecation notice + chain_remote_shape.__doc__ = (chain_remote_shape.__doc__ or "") + "\n\n" + (chain_remote_shape_base.__doc__ or "") + + def gfql_remote(self, *args, **kwargs) -> Plottable: + """Run GFQL query remotely. + + This is the remote execution version of :meth:`gfql`. It supports both simple chains + and complex DAG patterns with Let bindings. + + See :meth:`chain_remote` for detailed documentation (chain_remote is deprecated). + """ + return chain_remote_base(self, *args, **kwargs) + + def gfql_remote_shape(self, *args, **kwargs) -> pd.DataFrame: + """Get shape metadata for remote GFQL query execution. + + This is the remote shape version of :meth:`gfql`. Returns metadata about the + resulting graph without downloading the full data. + + See :meth:`chain_remote_shape` for detailed documentation (chain_remote_shape is deprecated). + """ return chain_remote_shape_base(self, *args, **kwargs) - chain_remote_shape.__doc__ = chain_remote_shape_base.__doc__ def python_remote_g(self, *args, **kwargs) -> Any: return python_remote_g_base(self, *args, **kwargs) diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 726b1c0706..3cbb68ac16 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -1,7 +1,7 @@ from .ComputeMixin import ComputeMixin from .ast import ( n, e, e_forward, e_reverse, e_undirected, - let, remote, ref + let, remote, ref, call ) from .chain import Chain from .predicates.is_in import ( @@ -61,7 +61,7 @@ 'ComputeMixin', 'Chain', # AST nodes 'n', 'e', 'e_forward', 'e_reverse', 'e_undirected', - 'let', 'remote', 'ref', + 'let', 'remote', 'ref', 'call', # Predicates 'is_in', 'IsIn', 'duplicated', 'Duplicated', diff --git a/graphistry/compute/ast.py b/graphistry/compute/ast.py index 629cfa67c9..1ce9b7f258 100644 --- a/graphistry/compute/ast.py +++ b/graphistry/compute/ast.py @@ -1,6 +1,6 @@ from abc import abstractmethod import logging -from typing import TYPE_CHECKING, Dict, List, Optional, Union, cast +from typing import Any, TYPE_CHECKING, Dict, List, Optional, Sequence, Union, cast from typing_extensions import Literal if TYPE_CHECKING: @@ -133,7 +133,7 @@ def _validate_fields(self) -> None: ErrorCode.E205, "query must be a string", field="query", value=type(self.query).__name__ ) - def _get_child_validators(self) -> list: + def _get_child_validators(self) -> Sequence['ASTSerializable']: """Return predicates that need validation.""" children = [] if self.filter_dict: @@ -330,7 +330,7 @@ def _validate_fields(self) -> None: ErrorCode.E205, f"{query_name} must be a string", field=query_name, value=type(query_value).__name__ ) - def _get_child_validators(self) -> list: + def _get_child_validators(self) -> Sequence['ASTSerializable']: """Return predicates that need validation.""" children = [] for filter_dict in [self.source_node_match, self.edge_match, self.destination_node_match]: @@ -605,21 +605,75 @@ def from_json(cls, d: dict, validate: bool = True) -> 'ASTEdge': class ASTLet(ASTObject): - """Let bindings for named graph operations""" - def __init__(self, bindings: Dict[str, ASTObject]): + """Let-bindings for named graph operations in a DAG. + + Allows defining reusable graph operations that can reference each other, + forming a directed acyclic graph (DAG) of computations. + + :param bindings: Dictionary mapping names to graph operations + :type bindings: Dict[str, ASTObject] + + :raises GFQLTypeError: If bindings is not a dict or contains invalid keys/values + + **Example::** + + dag = ASTLet({ + 'persons': n({'type': 'person'}), + 'friends': ASTRef('persons', [e_forward({'rel': 'friend'})]) + }) + """ + def __init__(self, bindings: Dict[str, 'ASTObject']) -> None: + """Initialize Let with named bindings. + + :param bindings: Dictionary mapping names to AST operations + :type bindings: Dict[str, ASTObject] + """ super().__init__() self.bindings = bindings - def validate(self, collect_all: bool = False) -> Optional[List['GFQLValidationError']]: - assert isinstance(self.bindings, dict), "bindings must be a dictionary" + def _validate_fields(self) -> None: + """Validate Let fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.bindings, dict): + raise GFQLTypeError( + ErrorCode.E201, + "bindings must be a dictionary", + field="bindings", + value=type(self.bindings).__name__ + ) + for k, v in self.bindings.items(): - assert isinstance(k, str), f"binding key must be string, got {type(k)}" - assert isinstance(v, ASTObject), f"binding value must be ASTObject, got {type(v)}" - v.validate() + if not isinstance(k, str): + raise GFQLTypeError( + ErrorCode.E102, + "binding key must be string", + field=f"bindings.{k}", + value=type(k).__name__ + ) + if not isinstance(v, ASTObject): + raise GFQLTypeError( + ErrorCode.E201, + "binding value must be ASTObject", + field=f"bindings.{k}", + value=type(v).__name__ + ) # TODO: Check for cycles in DAG return None - def to_json(self, validate=True) -> dict: + def _get_child_validators(self) -> Sequence['ASTSerializable']: + """Return child AST nodes that need validation.""" + # ASTObject inherits from ASTSerializable, so this is safe + return list(self.bindings.values()) + + def to_json(self, validate: bool = True) -> dict: + """Convert Let to JSON representation. + + :param validate: Whether to validate before serialization + :type validate: bool + :returns: JSON-serializable dictionary + :rtype: dict + """ if validate: self.validate() return { @@ -629,6 +683,16 @@ def to_json(self, validate=True) -> dict: @classmethod def from_json(cls, d: dict, validate: bool = True) -> 'ASTLet': + """Create ASTLet from JSON representation. + + :param d: JSON dictionary with 'bindings' field + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTLet instance + :rtype: ASTLet + :raises AssertionError: If 'bindings' field is missing + """ assert 'bindings' in d, "Let missing bindings" bindings = {k: cast(ASTObject, from_json(v, validate=validate)) for k, v in d['bindings'].items()} out = cls(bindings=bindings) @@ -648,19 +712,74 @@ def reverse(self) -> 'ASTLet': class ASTRemoteGraph(ASTObject): - """Load a graph from Graphistry server""" - def __init__(self, dataset_id: str, token: Optional[str] = None): + """Load a graph from Graphistry server. + + Allows fetching previously uploaded graphs by dataset ID, + optionally with an authentication token. + + :param dataset_id: Unique identifier of the dataset on the server + :type dataset_id: str + :param token: Optional authentication token + :type token: Optional[str] + + :raises GFQLTypeError: If dataset_id is not a string or is empty + + **Example::** + + # Fetch public dataset + remote = ASTRemoteGraph('my-dataset-id') + + # Fetch private dataset with token + remote = ASTRemoteGraph('private-dataset', token='auth-token') + """ + def __init__(self, dataset_id: str, token: Optional[str] = None) -> None: + """Initialize RemoteGraph with dataset ID and optional token. + + :param dataset_id: Unique identifier of the dataset + :type dataset_id: str + :param token: Optional authentication token + :type token: Optional[str] + """ super().__init__() self.dataset_id = dataset_id self.token = token - def validate(self, collect_all: bool = False) -> Optional[List['GFQLValidationError']]: - assert isinstance(self.dataset_id, str), "dataset_id must be a string" - assert len(self.dataset_id) > 0, "dataset_id cannot be empty" - assert self.token is None or isinstance(self.token, str), "token must be string or None" - return None + def _validate_fields(self) -> None: + """Validate RemoteGraph fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.dataset_id, str): + raise GFQLTypeError( + ErrorCode.E201, + "dataset_id must be a string", + field="dataset_id", + value=type(self.dataset_id).__name__ + ) + + if len(self.dataset_id) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "dataset_id cannot be empty", + field="dataset_id", + value=self.dataset_id + ) + + if self.token is not None and not isinstance(self.token, str): + raise GFQLTypeError( + ErrorCode.E201, + "token must be string or None", + field="token", + value=type(self.token).__name__ + ) - def to_json(self, validate=True) -> dict: + def to_json(self, validate: bool = True) -> dict: + """Convert RemoteGraph to JSON representation. + + :param validate: Whether to validate before serialization + :type validate: bool + :returns: JSON-serializable dictionary + :rtype: dict + """ if validate: self.validate() result = { @@ -673,6 +792,16 @@ def to_json(self, validate=True) -> dict: @classmethod def from_json(cls, d: dict, validate: bool = True) -> 'ASTRemoteGraph': + """Create ASTRemoteGraph from JSON representation. + + :param d: JSON dictionary with 'dataset_id' field + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTRemoteGraph instance + :rtype: ASTRemoteGraph + :raises AssertionError: If 'dataset_id' field is missing + """ assert 'dataset_id' in d, "RemoteGraph missing dataset_id" out = cls( dataset_id=d['dataset_id'], @@ -691,35 +820,108 @@ def reverse(self) -> 'ASTRemoteGraph': raise NotImplementedError("RemoteGraph reversal not supported") -class ASTChainRef(ASTObject): - """Execute a chain with reference to a DAG binding""" - def __init__(self, ref: str, chain: List[ASTObject]): +class ASTRef(ASTObject): + """Execute a chain of operations starting from a DAG binding reference. + + Allows building graph operations that start from a named binding + defined in an ASTLet (DAG) and apply additional operations. + + :param ref: Name of the binding to reference from the DAG + :type ref: str + :param chain: List of operations to apply to the referenced graph + :type chain: List[ASTObject] + + :raises GFQLTypeError: If ref is not a string or chain is not a list + + **Example::** + + # Reference 'persons' binding and find their friends + friends = ASTRef('persons', [e_forward({'rel': 'friend'})]) + """ + def __init__(self, ref: str, chain: List['ASTObject']) -> None: + """Initialize Ref with reference name and operation chain. + + :param ref: Name of the binding to reference + :type ref: str + :param chain: List of operations to apply + :type chain: List[ASTObject] + """ super().__init__() self.ref = ref self.chain = chain - def validate(self, collect_all: bool = False) -> Optional[List['GFQLValidationError']]: - assert isinstance(self.ref, str), "ref must be a string" - assert len(self.ref) > 0, "ref cannot be empty" - assert isinstance(self.chain, list), "chain must be a list" + def _validate_fields(self) -> None: + """Validate Ref fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.ref, str): + raise GFQLTypeError( + ErrorCode.E201, + "ref must be a string", + field="ref", + value=type(self.ref).__name__ + ) + + if len(self.ref) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "ref cannot be empty", + field="ref", + value=self.ref + ) + + if not isinstance(self.chain, list): + raise GFQLTypeError( + ErrorCode.E201, + "chain must be a list", + field="chain", + value=type(self.chain).__name__ + ) + for i, op in enumerate(self.chain): - assert isinstance(op, ASTObject), f"chain[{i}] must be ASTObject, got {type(op)}" - op.validate() - return None + if not isinstance(op, ASTObject): + raise GFQLTypeError( + ErrorCode.E201, + f"chain[{i}] must be ASTObject", + field=f"chain[{i}]", + value=type(op).__name__ + ) - def to_json(self, validate=True) -> dict: + def _get_child_validators(self) -> Sequence['ASTSerializable']: + """Return child AST nodes that need validation.""" + # ASTObject inherits from ASTSerializable, so this is safe + return self.chain + + def to_json(self, validate: bool = True) -> dict: + """Convert Ref to JSON representation. + + :param validate: Whether to validate before serialization + :type validate: bool + :returns: JSON-serializable dictionary + :rtype: dict + """ if validate: self.validate() return { - 'type': 'ChainRef', + 'type': 'Ref', 'ref': self.ref, 'chain': [op.to_json() for op in self.chain] } @classmethod - def from_json(cls, d: dict, validate: bool = True) -> 'ASTChainRef': - assert 'ref' in d, "ChainRef missing ref" - assert 'chain' in d, "ChainRef missing chain" + def from_json(cls, d: dict, validate: bool = True) -> 'ASTRef': + """Create ASTRef from JSON representation. + + :param d: JSON dictionary with 'ref' and 'chain' fields + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTRef instance + :rtype: ASTRef + :raises AssertionError: If 'ref' or 'chain' fields are missing + """ + assert 'ref' in d, "Ref missing ref" + assert 'chain' in d, "Ref missing chain" out = cls( ref=d['ref'], chain=[from_json(op, validate=validate) for op in d['chain']] @@ -730,17 +932,144 @@ def from_json(cls, d: dict, validate: bool = True) -> 'ASTChainRef': def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: - # Implementation in PR 1.2 - raise NotImplementedError("ChainRef execution will be implemented in PR 1.2") + raise NotImplementedError( + "ASTRef cannot be used directly in chain(). " + "It must be used within an ASTLet/chain_dag() context." + ) - def reverse(self) -> 'ASTChainRef': + def reverse(self) -> 'ASTRef': # Reverse the chain operations - return ASTChainRef(self.ref, [op.reverse() for op in reversed(self.chain)]) + return ASTRef(self.ref, [op.reverse() for op in reversed(self.chain)]) + + +class ASTCall(ASTObject): + """Call a method on the current graph with validated parameters. + + Allows safe execution of Plottable methods through GFQL with parameter + validation and schema checking. + + Attributes: + function: Name of the method to call (must be in safelist) + params: Dictionary of parameters to pass to the method + """ + def __init__(self, function: str, params: Optional[Dict[str, Any]] = None) -> None: + """Initialize a Call operation. + + Args: + function: Name of the Plottable method to call + params: Optional dictionary of parameters for the method + """ + super().__init__() + self.function = function + self.params = params or {} + + def _validate_fields(self) -> None: + """Validate Call fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.function, str): + raise GFQLTypeError( + ErrorCode.E201, + "function must be a string", + field="function", + value=type(self.function).__name__ + ) + + if len(self.function) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "function name cannot be empty", + field="function", + value=self.function + ) + + if not isinstance(self.params, dict): + raise GFQLTypeError( + ErrorCode.E201, + "params must be a dictionary", + field="params", + value=type(self.params).__name__ + ) + + def to_json(self, validate: bool = True) -> dict: + """Convert Call to JSON representation. + + Args: + validate: If True, validate before serialization + + Returns: + Dictionary with type, function, and params fields + """ + if validate: + self.validate() + return { + 'type': 'Call', + 'function': self.function, + 'params': self.params + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTCall': + """Create ASTCall from JSON representation. + + :param d: JSON dictionary with 'function' field and optional 'params' + :type d: dict + :param validate: Whether to validate after creation + :type validate: bool + :returns: New ASTCall instance + :rtype: ASTCall + :raises AssertionError: If 'function' field is missing + + **Example::** + + call_json = {'type': 'Call', 'function': 'hop', 'params': {'steps': 2}} + call = ASTCall.from_json(call_json) + """ + assert 'function' in d, "Call missing function" + out = cls( + function=d['function'], + params=d.get('params', {}) + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + """Execute the method call on the graph. + + Args: + g: Graph to operate on + prev_node_wavefront: Previous node wavefront (unused) + target_wave_front: Target wavefront (unused) + engine: Execution engine (pandas/cudf) + + Returns: + New Plottable with method results + + Raises: + GFQLTypeError: If method not in safelist or parameters invalid + """ + # For chain_dag, we don't use wavefronts, just execute the call + from graphistry.compute.call_executor import execute_call + return execute_call(g, self.function, self.params, engine) + + def reverse(self) -> 'ASTCall': + """Reverse is not supported for Call operations. + + Most Plottable methods are not reversible as they perform + transformations that cannot be undone. + + Raises: + NotImplementedError: Always raised as calls cannot be reversed + """ + # Most method calls cannot be reversed + raise NotImplementedError(f"Method '{self.function}' cannot be reversed") ### -def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTChainRef]: +def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTRef, ASTCall]: from graphistry.compute.exceptions import ErrorCode, GFQLSyntaxError if not isinstance(o, dict): @@ -748,10 +1077,10 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTL if 'type' not in o: raise GFQLSyntaxError( - ErrorCode.E105, "AST JSON missing required 'type' field", suggestion="Add 'type' field: 'Node', 'Edge', 'QueryDAG', 'RemoteGraph', or 'ChainRef'" + ErrorCode.E105, "AST JSON missing required 'type' field", suggestion="Add 'type' field: 'Node', 'Edge', 'Let', 'RemoteGraph', or 'ChainRef'" ) - out: Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTChainRef] + out: Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTRef, ASTCall] if o['type'] == 'Node': out = ASTNode.from_json(o, validate=validate) elif o['type'] == 'Edge': @@ -776,20 +1105,26 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTL "Edge missing required 'direction' field", suggestion="Add 'direction' field: 'forward', 'reverse', or 'undirected'", ) - elif o['type'] == 'QueryDAG' or o['type'] == 'Let': - # Support both types for backward compatibility + elif o['type'] == 'Let': + out = ASTLet.from_json(o, validate=validate) + elif o['type'] == 'QueryDAG': + # For backward compatibility out = ASTLet.from_json(o, validate=validate) elif o['type'] == 'RemoteGraph': out = ASTRemoteGraph.from_json(o, validate=validate) elif o['type'] == 'ChainRef': - out = ASTChainRef.from_json(o, validate=validate) + out = ASTRef.from_json(o, validate=validate) + elif o['type'] == 'Ref': + out = ASTRef.from_json(o, validate=validate) + elif o['type'] == 'Call': + out = ASTCall.from_json(o, validate=validate) else: raise GFQLSyntaxError( ErrorCode.E101, f"Unknown AST type: {o['type']}", field="type", value=o["type"], - suggestion="Use 'Node', 'Edge', 'Let', 'RemoteGraph', or 'ChainRef'", + suggestion="Use 'Node', 'Edge', 'Let', 'RemoteGraph', 'ChainRef', 'Ref', or 'Call'", ) return out @@ -799,4 +1134,5 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTL let = ASTLet # noqa: E305 remote = ASTRemoteGraph # noqa: E305 -ref = ASTChainRef # noqa: E305 +ref = ASTRef # noqa: E305 +call = ASTCall # noqa: E305 diff --git a/graphistry/compute/chain_dag.py b/graphistry/compute/chain_dag.py index ad551a5f1c..866143073e 100644 --- a/graphistry/compute/chain_dag.py +++ b/graphistry/compute/chain_dag.py @@ -3,14 +3,14 @@ from graphistry.Engine import Engine, EngineAbstract, resolve_engine from graphistry.Plottable import Plottable from graphistry.util import setup_logger -from .ast import ASTObject, ASTLet, ASTChainRef, ASTRemoteGraph, ASTNode, ASTEdge +from .ast import ASTObject, ASTLet, ASTRef, ASTRemoteGraph, ASTNode, ASTEdge, ASTCall from .execution_context import ExecutionContext logger = setup_logger(__name__) def extract_dependencies(ast_obj: ASTObject) -> Set[str]: - """Recursively find all ASTChainRef references in an AST object + """Recursively find all ASTRef references in an AST object :param ast_obj: AST object to analyze :returns: Set of referenced binding names @@ -18,7 +18,7 @@ def extract_dependencies(ast_obj: ASTObject) -> Set[str]: """ deps = set() - if isinstance(ast_obj, ASTChainRef): + if isinstance(ast_obj, ASTRef): deps.add(ast_obj.ref) # Also check chain operations for op in ast_obj.chain: @@ -194,8 +194,12 @@ def execute_node(name: str, ast_obj: ASTObject, g: Plottable, """Execute a single node in the DAG Handles different AST object types: +<<<<<<< HEAD - ASTLet: Recursive let execution - - ASTChainRef: Reference resolution and chain execution +======= + - ASTLet: Recursive DAG execution +>>>>>>> refactor: rename ASTQueryDAG to ASTLet throughout codebase + - ASTRef: Reference resolution and chain execution - ASTNode: Node filtering operations - ASTEdge: Edge traversal operations - Others: NotImplementedError @@ -215,8 +219,8 @@ def execute_node(name: str, ast_obj: ASTObject, g: Plottable, # Handle different AST object types if isinstance(ast_obj, ASTLet): # Nested let execution - result = chain_dag_impl(g, ast_obj, engine.value) - elif isinstance(ast_obj, ASTChainRef): + result = chain_dag_impl(g, ast_obj, EngineAbstract(engine.value)) + elif isinstance(ast_obj, ASTRef): # Resolve reference from context try: referenced_result = context.get_binding(ast_obj.ref) @@ -231,14 +235,14 @@ def execute_node(name: str, ast_obj: ASTObject, g: Plottable, if ast_obj.chain: # Import chain function to execute the operations from .chain import chain as chain_impl - result = chain_impl(referenced_result, ast_obj.chain, engine.value) + result = chain_impl(referenced_result, ast_obj.chain, EngineAbstract(engine.value)) else: # Empty chain - just return the referenced result result = referenced_result elif isinstance(ast_obj, ASTNode): # For chain_dag, we execute nodes in a simpler way than chain() # No wavefront propagation - just filter the graph's nodes - node_obj = cast(ASTNode, ast_obj) + node_obj = cast(ASTNode, ast_obj) # Help mypy understand the type if node_obj.filter_dict or node_obj.query: filtered_g = g if node_obj.filter_dict: @@ -283,14 +287,25 @@ def execute_node(name: str, ast_obj: ASTObject, g: Plottable, from .chain_remote import chain_remote as chain_remote_impl # Fetch the remote dataset with an empty chain (no filtering) + # Convert engine to the expected type for chain_remote + chain_engine: Optional[Literal["pandas", "cudf"]] = None + if engine.value == "pandas": + chain_engine = "pandas" + elif engine.value == "cudf": + chain_engine = "cudf" + result = chain_remote_impl( result, [], # Empty chain - just fetch the entire dataset api_token=ast_obj.token, dataset_id=ast_obj.dataset_id, output_type="all", # Get full graph (nodes and edges) - engine=cast(Literal["pandas", "cudf"], engine.value) + engine=chain_engine ) + elif isinstance(ast_obj, ASTCall): + # Execute method call with validation + from .gfql.call_executor import execute_call + result = execute_call(g, ast_obj.function, ast_obj.params, engine) else: # Other AST object types not yet implemented raise NotImplementedError(f"Execution of {type(ast_obj).__name__} not yet implemented") @@ -315,7 +330,7 @@ def chain_dag_impl(g: Plottable, dag: ASTLet, :param output: Name of binding to return (default: last executed) :returns: Result from specified or last executed node :rtype: Plottable - :raises TypeError: If dag is not an ASTQueryDAG + :raises TypeError: If dag is not an ASTLet :raises RuntimeError: If node execution fails :raises ValueError: If output binding not found """ @@ -407,11 +422,11 @@ def chain_dag(self: Plottable, dag: ASTLet, :: - from graphistry.compute.ast import ASTLet, ASTChainRef, n, e + from graphistry.compute.ast import ASTLet, ASTRef, n, e dag = ASTLet({ 'start': n({'type': 'person'}), - 'friends': ASTChainRef('start', [e(), n()]) + 'friends': ASTRef('start', [e(), n()]) }) result = g.chain_dag(dag) @@ -422,9 +437,9 @@ def chain_dag(self: Plottable, dag: ASTLet, dag = ASTLet({ 'people': n({'type': 'person'}), 'transactions': n({'type': 'transaction'}), - 'branch1': ASTChainRef('people', [e()]), - 'branch2': ASTChainRef('transactions', [e()]), - 'merged': g.union(ASTChainRef('branch1'), ASTChainRef('branch2')) + 'branch1': ASTRef('people', [e()]), + 'branch2': ASTRef('transactions', [e()]), + 'merged': g.union(ASTRef('branch1'), ASTRef('branch2')) }) result = g.chain_dag(dag) # Returns last executed diff --git a/graphistry/compute/execution_context.py b/graphistry/compute/execution_context.py index 3b8e034b8b..0cc009cb78 100644 --- a/graphistry/compute/execution_context.py +++ b/graphistry/compute/execution_context.py @@ -1,21 +1,48 @@ -"""Execution context for DAG operations""" +"""Execution context for DAG operations.""" from typing import Any, Dict class ExecutionContext: - """Manages variable bindings during DAG execution""" + """Manages variable bindings during DAG execution. - def __init__(self): + Provides a namespace for storing and retrieving named graph results + during the execution of ASTLet DAGs. Each binding maps a string name + to a Plottable graph instance. + + **Example::** + + context = ExecutionContext() + context.set_binding('persons', person_graph) + friends = context.get_binding('persons') + """ + + def __init__(self) -> None: + """Initialize an empty execution context.""" self._bindings: Dict[str, Any] = {} def set_binding(self, name: str, value: Any) -> None: - """Store a named result""" + """Store a named result in the context. + + :param name: Name for the binding + :type name: str + :param value: Value to bind (typically a Plottable) + :type value: Any + :raises TypeError: If name is not a string + """ if not isinstance(name, str): raise TypeError(f"Binding name must be string, got {type(name)}") self._bindings[name] = value def get_binding(self, name: str) -> Any: - """Retrieve a named result""" + """Retrieve a named result from the context. + + :param name: Name of the binding to retrieve + :type name: str + :returns: The bound value + :rtype: Any + :raises TypeError: If name is not a string + :raises KeyError: If no binding exists for the given name + """ if not isinstance(name, str): raise TypeError(f"Binding name must be string, got {type(name)}") if name not in self._bindings: @@ -23,15 +50,26 @@ def get_binding(self, name: str) -> Any: return self._bindings[name] def has_binding(self, name: str) -> bool: - """Check if binding exists""" + """Check if a binding exists in the context. + + :param name: Name to check + :type name: str + :returns: True if binding exists, False otherwise + :rtype: bool + :raises TypeError: If name is not a string + """ if not isinstance(name, str): raise TypeError(f"Binding name must be string, got {type(name)}") return name in self._bindings def clear(self) -> None: - """Clear all bindings""" + """Clear all bindings from the context.""" self._bindings.clear() def get_all_bindings(self) -> Dict[str, Any]: - """Get a copy of all bindings""" + """Get a copy of all bindings in the context. + + :returns: Dictionary of all current bindings + :rtype: Dict[str, Any] + """ return self._bindings.copy() diff --git a/graphistry/compute/gfql/__init__.py b/graphistry/compute/gfql/__init__.py new file mode 100644 index 0000000000..f1b014164c --- /dev/null +++ b/graphistry/compute/gfql/__init__.py @@ -0,0 +1,6 @@ +"""GFQL module - re-export validation functionality.""" + +# Re-export all validation functionality +from graphistry.compute.gfql.validate import * # noqa: F403, F401 + +# Note: The gfql function is in ../gfql.py, not in this package diff --git a/graphistry/compute/gfql/call_executor.py b/graphistry/compute/gfql/call_executor.py new file mode 100644 index 0000000000..221a33a050 --- /dev/null +++ b/graphistry/compute/gfql/call_executor.py @@ -0,0 +1,79 @@ +"""Execute validated method calls on Plottable objects. + +This module provides the execution layer for GFQL Call operations after +parameter validation has been performed by the safelist module. +""" + +from typing import Dict, Any +from graphistry.Plottable import Plottable +from graphistry.Engine import Engine +from graphistry.compute.gfql.call_safelist import validate_call_params +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: Engine) -> Plottable: + """Execute a validated method call on a Plottable. + + Args: + g: The graph to call the method on + function: Name of the method to call + params: Parameters for the method (will be validated) + engine: Execution engine + + Returns: + Result of the method call (usually a new Plottable) + + Raises: + GFQLTypeError: If validation fails or method doesn't exist + AttributeError: If method doesn't exist on Plottable + """ + # Validate parameters against safelist + validated_params = validate_call_params(function, params) + # Check if method exists on Plottable + if not hasattr(g, function): + raise AttributeError( + f"Plottable has no method '{function}'. " + f"This should not happen if safelist is properly configured." + ) + + # Get the method + method = getattr(g, function) + + # Special handling for methods that need the engine parameter + if function in ['materialize_nodes', 'hop']: + # These methods accept an engine parameter + if 'engine' not in validated_params: + # Add current engine if not specified + validated_params['engine'] = engine + + try: + # Execute the method with validated parameters + result = method(**validated_params) + # Ensure result is a Plottable (most methods return self or new Plottable) + if not isinstance(result, Plottable): + raise GFQLTypeError( + ErrorCode.E201, + f"Method '{function}' returned non-Plottable result", + field="function", + value=f"{type(result).__name__}", + suggestion="Only methods that return Plottable objects are allowed" + ) + + return result + except TypeError as e: + # Handle parameter mismatch errors + raise GFQLTypeError( + ErrorCode.E201, + f"Parameter error calling '{function}': {str(e)}", + field="params", + value=validated_params, + suggestion="Check parameter names and types" + ) from e + except Exception as e: + # Re-raise other exceptions with context + raise GFQLTypeError( + ErrorCode.E303, + f"Error executing '{function}': {str(e)}", + field="function", + value=function + ) from e diff --git a/graphistry/compute/gfql/call_safelist.py b/graphistry/compute/gfql/call_safelist.py new file mode 100644 index 0000000000..4fed1736e7 --- /dev/null +++ b/graphistry/compute/gfql/call_safelist.py @@ -0,0 +1,552 @@ +"""Safelist of allowed methods for GFQL Call operations. + +This module defines which Plottable methods can be called through GFQL +and their parameter validation rules. +""" + +from typing import Dict, Any +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +# Type validators +def is_string(v: Any) -> bool: + """Check if value is a string. + + Args: + v: Value to check + + Returns: + True if v is a string, False otherwise + """ + return isinstance(v, str) + + +def is_int(v: Any) -> bool: + """Check if value is an integer. + + Args: + v: Value to check + + Returns: + True if v is an integer, False otherwise + """ + return isinstance(v, int) + + +def is_bool(v: Any) -> bool: + """Check if value is a boolean. + + Args: + v: Value to check + + Returns: + True if v is a boolean, False otherwise + """ + return isinstance(v, bool) + + +def is_dict(v: Any) -> bool: + """Check if value is a dictionary. + + Args: + v: Value to check + + Returns: + True if v is a dictionary, False otherwise + """ + return isinstance(v, dict) + + +def is_string_or_none(v: Any) -> bool: + """Check if value is a string or None. + + Args: + v: Value to check + + Returns: + True if v is a string or None, False otherwise + """ + return v is None or isinstance(v, str) + + +def is_list_of_strings(v: Any) -> bool: + """Check if value is a list of strings. + + Args: + v: Value to check + + Returns: + True if v is a list containing only strings, False otherwise + """ + return isinstance(v, list) and all(isinstance(item, str) for item in v) + + +# Safelist configuration +# Dictionary mapping allowed Plottable method names to their validation rules. +# +# Each method entry contains: +# - allowed_params (Set[str]): Parameter names that can be passed to the method +# - required_params (Set[str]): Parameters that must be provided +# - param_validators (Dict[str, Callable]): Maps param names to validation functions +# - description (str): Human-readable description of what the method does +# - schema_effects (Dict[str, List[str]]): Describes schema changes: +# - adds_node_cols: Columns added to node DataFrame +# - adds_edge_cols: Columns added to edge DataFrame +# - requires_node_cols: Node columns that must exist before calling +# - requires_edge_cols: Edge columns that must exist before calling +# +# Example entry: +# 'hop': { +# 'allowed_params': {'steps', 'to_fixed_point', 'direction'}, +# 'required_params': set(), +# 'param_validators': { +# 'steps': is_int, +# 'to_fixed_point': is_bool, +# 'direction': lambda v: v in ['forward', 'reverse', 'undirected'] +# }, +# 'description': 'Traverse graph edges for N steps', +# 'schema_effects': {} +# } + +SAFELIST_V1: Dict[str, Dict[str, Any]] = { + 'get_degrees': { + 'allowed_params': {'col_in', 'col_out', 'col'}, + 'required_params': set(), + 'param_validators': { + 'col_in': is_string, + 'col_out': is_string, + 'col': is_string + }, + 'description': 'Calculate node degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [ + p.get('col', 'degree'), + p.get('col_in', 'degree_in'), + p.get('col_out', 'degree_out') + ], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'filter_nodes_by_dict': { + 'allowed_params': {'filter_dict'}, + 'required_params': {'filter_dict'}, + 'param_validators': { + 'filter_dict': is_dict + }, + 'description': 'Filter nodes by attribute values', + 'schema_effects': { + 'adds_node_cols': [], + 'adds_edge_cols': [], + 'requires_node_cols': lambda p: list(p.get('filter_dict', {}).keys()), + 'requires_edge_cols': [] + } + }, + + 'filter_edges_by_dict': { + 'allowed_params': {'filter_dict'}, + 'required_params': {'filter_dict'}, + 'param_validators': { + 'filter_dict': is_dict + }, + 'description': 'Filter edges by attribute values', + 'schema_effects': { + 'adds_node_cols': [], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': lambda p: list(p.get('filter_dict', {}).keys()) + } + }, + + 'materialize_nodes': { + 'allowed_params': {'engine', 'reuse'}, + 'required_params': set(), + 'param_validators': { + 'engine': is_string, + 'reuse': is_bool + }, + 'description': 'Generate node table from edges', + 'schema_effects': { + 'adds_node_cols': ['node'], # Creates node column + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'hop': { + 'allowed_params': { + 'nodes', 'hops', 'to_fixed_point', 'direction', + 'source_node_match', 'edge_match', 'destination_node_match', + 'source_node_query', 'edge_query', 'destination_node_query', + 'return_as_wave_front', 'target_wave_front', 'engine' + }, + 'required_params': set(), + 'param_validators': { + 'hops': is_int, + 'to_fixed_point': is_bool, + 'direction': lambda v: v in ['forward', 'reverse', 'undirected'], + 'source_node_match': is_dict, + 'edge_match': is_dict, + 'destination_node_match': is_dict, + 'source_node_query': is_string, + 'edge_query': is_string, + 'destination_node_query': is_string, + 'return_as_wave_front': is_bool, + 'engine': is_string + }, + 'description': 'Traverse graph by following edges' + }, + + # In/out degree methods + 'get_indegrees': { + 'allowed_params': {'col'}, + 'required_params': set(), + 'param_validators': { + 'col': is_string + }, + 'description': 'Calculate node in-degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('col', 'degree_in')], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'get_outdegrees': { + 'allowed_params': {'col'}, + 'required_params': set(), + 'param_validators': { + 'col': is_string + }, + 'description': 'Calculate node out-degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('col', 'degree_out')], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + # Graph algorithm operations + 'compute_cugraph': { + 'allowed_params': {'alg', 'out_col', 'params', 'kind', 'directed', 'G'}, + 'required_params': {'alg'}, + 'param_validators': { + 'alg': is_string, + 'out_col': is_string_or_none, + 'params': is_dict, + 'kind': is_string, + 'directed': is_bool, + 'G': lambda x: x is None # Allow None only + }, + 'description': 'Run cuGraph algorithms (pagerank, louvain, etc)', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('out_col', p['alg'])], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'compute_igraph': { + 'allowed_params': {'alg', 'out_col', 'directed', 'use_vids', 'params'}, + 'required_params': {'alg'}, + 'param_validators': { + 'alg': is_string, + 'out_col': is_string_or_none, + 'directed': is_bool, + 'use_vids': is_bool, + 'params': is_dict + }, + 'description': 'Run igraph algorithms' + }, + + # Layout operations + 'layout_cugraph': { + 'allowed_params': {'layout', 'params', 'kind', 'directed', 'G', 'bind_position', 'x_out_col', 'y_out_col', 'play'}, + 'required_params': set(), + 'param_validators': { + 'layout': is_string, + 'params': is_dict, + 'kind': is_string, + 'directed': is_bool, + 'G': lambda x: x is None, + 'bind_position': is_bool, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'play': is_int + }, + 'description': 'GPU-accelerated graph layouts' + }, + + 'layout_igraph': { + 'allowed_params': {'layout', 'directed', 'use_vids', 'bind_position', 'x_out_col', 'y_out_col', 'params', 'play'}, + 'required_params': {'layout'}, + 'param_validators': { + 'layout': is_string, + 'directed': is_bool, + 'use_vids': is_bool, + 'bind_position': is_bool, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'params': is_dict, + 'play': is_int + }, + 'description': 'igraph-based layouts' + }, + + 'layout_graphviz': { + 'allowed_params': { + 'prog', 'args', 'directed', 'strict', 'graph_attr', + 'node_attr', 'edge_attr', 'x_out_col', 'y_out_col', 'bind_position' + }, + 'required_params': set(), + 'param_validators': { + 'prog': is_string, + 'args': is_string_or_none, + 'directed': is_bool, + 'strict': is_bool, + 'graph_attr': is_dict, + 'node_attr': is_dict, + 'edge_attr': is_dict, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'bind_position': is_bool + }, + 'description': 'Graphviz layouts (dot, neato, etc)' + }, + + 'fa2_layout': { + 'allowed_params': {'fa2_params', 'circle_layout_params', 'partition_key', 'remove_self_edges', 'engine', 'featurize'}, + 'required_params': set(), + 'param_validators': { + 'fa2_params': is_dict, + 'circle_layout_params': is_dict, + 'partition_key': is_string_or_none, + 'remove_self_edges': is_bool, + 'engine': is_string, + 'featurize': is_dict + }, + 'description': 'ForceAtlas2 layout algorithm' + }, + + # Self-edge pruning + 'prune_self_edges': { + 'allowed_params': set(), + 'required_params': set(), + 'param_validators': {}, + 'description': 'Remove self-loops from graph' + }, + + # Graph transformations + 'collapse': { + 'allowed_params': {'node', 'attribute', 'column', 'self_edges', 'unwrap', 'verbose'}, + 'required_params': set(), + 'param_validators': { + 'node': is_string_or_none, + 'attribute': is_string_or_none, + 'column': is_string_or_none, + 'self_edges': is_bool, + 'unwrap': is_bool, + 'verbose': is_bool + }, + 'description': 'Collapse nodes by shared attribute values' + }, + + 'drop_nodes': { + 'allowed_params': {'nodes'}, + 'required_params': {'nodes'}, + 'param_validators': { + 'nodes': lambda v: isinstance(v, list) or is_dict(v) + }, + 'description': 'Remove specified nodes and their edges' + }, + + 'keep_nodes': { + 'allowed_params': {'nodes'}, + 'required_params': {'nodes'}, + 'param_validators': { + 'nodes': lambda v: isinstance(v, list) or is_dict(v) + }, + 'description': 'Keep only specified nodes and their edges' + }, + + # Topology analysis + 'get_topological_levels': { + 'allowed_params': {'level_col', 'allow_cycles', 'warn_cycles', 'remove_self_loops'}, + 'required_params': set(), + 'param_validators': { + 'level_col': is_string, + 'allow_cycles': is_bool, + 'warn_cycles': is_bool, + 'remove_self_loops': is_bool + }, + 'description': 'Compute topological levels for DAG analysis' + }, + + # Visual encoding methods + 'encode_point_color': { + 'allowed_params': {'column', 'palette', 'as_categorical', 'as_continuous', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'palette': lambda v: isinstance(v, list), + 'as_categorical': is_bool, + 'as_continuous': is_bool, + 'categorical_mapping': is_dict, + 'default_mapping': is_string_or_none + }, + 'description': 'Map node column values to colors' + }, + + 'encode_edge_color': { + 'allowed_params': {'column', 'palette', 'as_categorical', 'as_continuous', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'palette': lambda v: isinstance(v, list), + 'as_categorical': is_bool, + 'as_continuous': is_bool, + 'categorical_mapping': is_dict, + 'default_mapping': is_string_or_none + }, + 'description': 'Map edge column values to colors' + }, + + 'encode_point_size': { + 'allowed_params': {'column', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'categorical_mapping': is_dict, + 'default_mapping': lambda v: isinstance(v, (int, float)) + }, + 'description': 'Map node column values to sizes' + }, + + 'encode_point_icon': { + 'allowed_params': {'column', 'categorical_mapping', 'continuous_binning', 'default_mapping', 'as_text'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'categorical_mapping': is_dict, + 'continuous_binning': lambda v: isinstance(v, list), + 'default_mapping': is_string_or_none, + 'as_text': is_bool + }, + 'description': 'Map node column values to icons' + }, + + # Metadata methods + 'name': { + 'allowed_params': {'name'}, + 'required_params': {'name'}, + 'param_validators': { + 'name': is_string + }, + 'description': 'Set visualization name' + }, + + 'description': { + 'allowed_params': {'description'}, + 'required_params': {'description'}, + 'param_validators': { + 'description': is_string + }, + 'description': 'Set visualization description' + } +} + + +def validate_call_params(function: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Validate parameters for a GFQL Call operation against the safelist. + + Performs comprehensive validation: + 1. Checks if function is in the safelist + 2. Verifies all required parameters are present + 3. Ensures no unknown parameters are passed + 4. Validates parameter types using configured validators + 5. Returns the validated parameters unchanged + + Args: + function: Name of the Plottable method to call + params: Dictionary of parameters to validate + + Returns: + The same parameters dict if validation passes + + Raises: + GFQLTypeError: If function not in safelist (E303) + GFQLTypeError: If required parameters missing (E105) + GFQLTypeError: If unknown parameters provided (E303) + GFQLTypeError: If parameter type validation fails (E201) + + **Example::** + + # Valid call + params = validate_call_params('hop', {'steps': 2, 'direction': 'forward'}) + + # Invalid - unknown function + validate_call_params('dangerous_method', {}) # Raises E303 + + # Invalid - missing required param + validate_call_params('fa2_layout', {}) # Would raise E105 if layout was required + + # Invalid - wrong type + validate_call_params('hop', {'steps': 'two'}) # Raises E201 + """ + # Check if function is in safelist + if function not in SAFELIST_V1: + raise GFQLTypeError( + ErrorCode.E303, + f"Function '{function}' is not in the safelist", + field="function", + value=function, + suggestion=f"Available functions: {', '.join(sorted(SAFELIST_V1.keys()))}" + ) + + config = SAFELIST_V1[function] + allowed_params = config['allowed_params'] + required_params = config['required_params'] + param_validators = config['param_validators'] + + # Check for required parameters + missing_required = required_params - set(params.keys()) + if missing_required: + raise GFQLTypeError( + ErrorCode.E105, + f"Missing required parameters for '{function}'", + field="params", + value=list(missing_required), + suggestion=f"Required parameters: {', '.join(sorted(missing_required))}" + ) + + # Check for unknown parameters + unknown_params = set(params.keys()) - allowed_params + if unknown_params: + raise GFQLTypeError( + ErrorCode.E303, + f"Unknown parameters for '{function}'", + field="params", + value=list(unknown_params), + suggestion=f"Allowed parameters: {', '.join(sorted(allowed_params))}" + ) + + # Validate parameter types + for param_name, param_value in params.items(): + if param_name in param_validators: + validator = param_validators[param_name] + if not validator(param_value): + raise GFQLTypeError( + ErrorCode.E201, + f"Invalid type for parameter '{param_name}' in '{function}'", + field=f"params.{param_name}", + value=f"{type(param_value).__name__}: {param_value}", + suggestion="Check the parameter type requirements" + ) + + return params diff --git a/graphistry/compute/gfql/validate/__init__.py b/graphistry/compute/gfql/validate/__init__.py new file mode 100644 index 0000000000..dcb8484930 --- /dev/null +++ b/graphistry/compute/gfql/validate/__init__.py @@ -0,0 +1,45 @@ +"""GFQL validation and related utilities.""" + +from graphistry.compute.gfql.validate.validate import ( + ValidationIssue, + Schema, + validate_syntax, + validate_schema, + validate_query, + extract_schema, + extract_schema_from_dataframes, + format_validation_errors, + suggest_fixes +) + +from graphistry.compute.gfql.validate.exceptions import ( + GFQLException, + GFQLValidationError, + GFQLSyntaxError, + GFQLSchemaError, + GFQLTypeError, + GFQLColumnNotFoundError +) + +__all__ = [ + # Validation classes + 'ValidationIssue', + 'Schema', + + # Validation functions + 'validate_syntax', + 'validate_schema', + 'validate_query', + 'extract_schema', + 'extract_schema_from_dataframes', + 'format_validation_errors', + 'suggest_fixes', + + # Exceptions + 'GFQLException', + 'GFQLValidationError', + 'GFQLSyntaxError', + 'GFQLSchemaError', + 'GFQLTypeError', + 'GFQLColumnNotFoundError' +] diff --git a/graphistry/compute/gfql/validate/exceptions.py b/graphistry/compute/gfql/validate/exceptions.py new file mode 100644 index 0000000000..fd44068bf6 --- /dev/null +++ b/graphistry/compute/gfql/validate/exceptions.py @@ -0,0 +1,62 @@ +"""GFQL-specific exceptions for validation and error handling.""" + +from typing import Optional, Dict, Any + + +class GFQLException(Exception): + """Base exception for all GFQL-related errors.""" + + def __init__(self, message: str, context: Optional[Dict[str, Any]] = None): + self.context = context or {} + super().__init__(message) + + def __str__(self) -> str: + if self.context: + context_str = ", ".join(f"{k}={v}" for k, v in self.context.items()) + return f"{super().__str__()} ({context_str})" + return super().__str__() + + +class GFQLValidationError(GFQLException, ValueError): + """Base validation error. Inherits from ValueError for backwards compatibility.""" + pass + + +class GFQLSyntaxError(GFQLValidationError): + """Raised when GFQL query has invalid syntax.""" + pass + + +class GFQLSchemaError(GFQLValidationError): + """Raised when GFQL query references non-existent columns or has type mismatches.""" + pass + + +class GFQLSemanticError(GFQLValidationError): + """Raised when GFQL query is syntactically valid but semantically incorrect.""" + pass + + +class GFQLTypeError(GFQLSchemaError): + """Raised when a predicate is applied to incompatible column type.""" + + def __init__(self, column: str, column_type: str, predicate: str, expected_type: str): + message = f"Column '{column}' has type '{column_type}' but predicate '{predicate}' expects '{expected_type}'" + super().__init__(message, { + 'column': column, + 'column_type': column_type, + 'predicate': predicate, + 'expected_type': expected_type + }) + + +class GFQLColumnNotFoundError(GFQLSchemaError): + """Raised when a referenced column doesn't exist in the schema.""" + + def __init__(self, column: str, table: str, available_columns: list): + message = f"Column '{column}' not found in {table} data" + super().__init__(message, { + 'column': column, + 'table': table, + 'available_columns': available_columns + }) diff --git a/graphistry/compute/gfql/validate/validate.py b/graphistry/compute/gfql/validate/validate.py new file mode 100644 index 0000000000..801aaa923d --- /dev/null +++ b/graphistry/compute/gfql/validate/validate.py @@ -0,0 +1,676 @@ +"""GFQL query validation utilities for syntax and schema checking. + +.. deprecated:: 0.34.0 + This module is deprecated. GFQL now has built-in validation. + See :doc:`/gfql/validation_migration_guide` for migration instructions. + + Instead of:: + + from graphistry.compute.gfql.validate import validate_syntax + issues = validate_syntax(query) + + Use:: + + from graphistry.compute.chain import Chain + try: + chain = Chain(query) # Automatic validation + except GFQLValidationError as e: + print(f"[{e.code}] {e.message}") +""" + +from typing import List, Optional, Dict, Union, Any, Tuple, TYPE_CHECKING +import warnings +import pandas as pd + +from graphistry.compute.chain import Chain +from graphistry.compute.ast import ASTNode, ASTEdge, ASTObject + +if TYPE_CHECKING: + from graphistry.Plottable import Plottable +from graphistry.compute.predicates.ASTPredicate import ASTPredicate +from graphistry.compute.predicates.numeric import NumericASTPredicate +from graphistry.compute.predicates.str import ( + Contains, Startswith, Endswith, Match, + IsNumeric, IsAlpha, IsDigit, IsLower, IsUpper, + IsSpace, IsAlnum, IsDecimal, IsTitle +) +from graphistry.compute.predicates.temporal import ( + IsMonthStart, IsMonthEnd, IsQuarterStart, IsQuarterEnd, + IsYearStart, IsYearEnd, IsLeapYear +) +from graphistry.util import setup_logger + +warnings.warn( + "The graphistry.compute.gfql.validate module is deprecated. " + "GFQL now has built-in validation. See the migration guide for details.", + DeprecationWarning, + stacklevel=2 +) + +logger = setup_logger(__name__) + + +class ValidationIssue: + """Represents a validation issue (error or warning).""" + + def __init__(self, + level: str, # 'error' or 'warning' + message: str, + operation_index: Optional[int] = None, + field: Optional[str] = None, + suggestion: Optional[str] = None, + error_type: Optional[str] = None): + self.level = level + self.message = message + self.operation_index = operation_index + self.field = field + self.suggestion = suggestion + self.error_type = error_type + + def __repr__(self) -> str: + parts = [f"{self.level.upper()}: {self.message}"] + if self.operation_index is not None: + parts.append(f"at operation {self.operation_index}") + if self.field: + parts.append(f"field: {self.field}") + if self.suggestion: + parts.append(f"Suggestion: {self.suggestion}") + return " | ".join(parts) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'level': self.level, + 'message': self.message, + 'operation_index': self.operation_index, + 'field': self.field, + 'suggestion': self.suggestion, + 'error_type': self.error_type + } + + +class Schema: + """Represents the schema of node and edge dataframes.""" + + def __init__(self, + node_columns: Optional[Dict[str, str]] = None, + edge_columns: Optional[Dict[str, str]] = None): + self.node_columns = node_columns or {} + self.edge_columns = edge_columns or {} + + def __repr__(self) -> str: + return (f"Schema(nodes={list(self.node_columns.keys())}, " + f"edges={list(self.edge_columns.keys())})") + + +# Error message templates +ERROR_MESSAGES = { + # Syntax errors + 'INVALID_CHAIN_TYPE': { + 'message': 'Chain must be a list of operations', + 'suggestion': 'Wrap your operations in a list: [n(), e(), n()]' + }, + 'INVALID_OPERATION': { + 'message': 'Operation at index {index} is not a valid GFQL operation', + 'suggestion': ('Use n() for nodes, ' + 'e()/e_forward()/e_reverse() for edges') + }, + 'INVALID_FILTER_KEY': { + 'message': 'Invalid filter key format: {key}', + 'suggestion': 'Filter keys must be strings' + }, + 'INVALID_HOPS': { + 'message': 'Hops must be a positive integer or None', + 'suggestion': ('Use hops=2 for specific count, ' + 'or to_fixed_point=True for unbounded') + }, + + # Schema errors + 'COLUMN_NOT_FOUND': { + 'message': 'Column "{column}" not found in {table} data', + 'suggestion': 'Available columns: {available}' + }, + 'TYPE_MISMATCH': { + 'message': ('Column "{column}" is {actual_type} but ' + 'predicate expects {expected_type}'), + 'suggestion': 'Use appropriate predicate for {actual_type} columns' + }, + 'INVALID_PREDICATE': { + 'message': ('Predicate {predicate} cannot be used with ' + '{column_type} columns'), + 'suggestion': 'Use {suggested_predicates} for {column_type} columns' + }, + + # Semantic errors + 'ORPHANED_EDGE': { + 'message': 'Edge operation at index {index} not connected to nodes', + 'suggestion': ('Add node operations before/after edge: ' + 'n() -> e() -> n()') + }, + 'UNBOUNDED_HOPS_WARNING': { + 'message': ('Unbounded hops (to_fixed_point=True) may be slow ' + 'on large graphs'), + 'suggestion': ('Consider using specific hop count for better ' + 'performance') + }, + 'EMPTY_CHAIN': { + 'message': 'Chain is empty', + 'suggestion': 'Add at least one operation: [n()]' + }, + 'INVALID_EDGE_DIRECTION': { + 'message': 'Invalid edge direction: {direction}', + 'suggestion': 'Use "forward", "reverse", or "undirected"' + } +} + + +def _format_error(error_key: str, **kwargs) -> Tuple[str, str]: + """Format error message with context.""" + template = ERROR_MESSAGES.get(error_key, { + 'message': f'Unknown error: {error_key}', + 'suggestion': 'Check documentation for valid syntax' + }) + message = template['message'].format(**kwargs) + suggestion = template['suggestion'].format(**kwargs) + return message, suggestion + + +def validate_syntax(chain: Union[Chain, List]) -> List[ValidationIssue]: + """ + Validate GFQL query syntax without requiring data. + + Args: + chain: GFQL chain or list of operations + + Returns: + List of validation issues (errors and warnings) + """ + issues = [] + + # Convert to list if Chain object + if isinstance(chain, Chain): + operations = chain.chain + elif isinstance(chain, list): + operations = chain + else: + message, suggestion = _format_error('INVALID_CHAIN_TYPE') + issues.append(ValidationIssue( + 'error', message, suggestion=suggestion, + error_type='INVALID_CHAIN_TYPE')) + return issues + + # Check empty chain + if not operations: + message, suggestion = _format_error('EMPTY_CHAIN') + issues.append(ValidationIssue( + 'error', message, suggestion=suggestion, + error_type='EMPTY_CHAIN')) + return issues + + # Validate each operation + for i, op in enumerate(operations): + # Check if valid operation type + if not isinstance(op, ASTObject): + message, suggestion = _format_error('INVALID_OPERATION', index=i) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + suggestion=suggestion, error_type='INVALID_OPERATION')) + continue + + # Validate nodes + if isinstance(op, ASTNode): + if op.filter_dict: + for key, value in op.filter_dict.items(): + if not isinstance(key, str): + message, suggestion = _format_error( + 'INVALID_FILTER_KEY', key=key) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field=str(key), suggestion=suggestion, + error_type='INVALID_FILTER_KEY')) + + # Validate edges + elif isinstance(op, ASTEdge): + # Check hops + if (op.hops is not None + and (not isinstance(op.hops, int) or op.hops < 1)): + message, suggestion = _format_error('INVALID_HOPS') + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field='hops', suggestion=suggestion, + error_type='INVALID_HOPS')) + + # Check unbounded hops warning + if op.to_fixed_point: + message, suggestion = _format_error('UNBOUNDED_HOPS_WARNING') + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='UNBOUNDED_HOPS_WARNING')) + + # Check edge filters + for filter_name, filter_dict in [ + ('edge_match', op.edge_match), + ('source_node_match', op.source_node_match), + ('destination_node_match', op.destination_node_match) + ]: + if filter_dict: + for key, value in filter_dict.items(): + if not isinstance(key, str): + message, suggestion = _format_error( + 'INVALID_FILTER_KEY', key=key) + issues.append(ValidationIssue( + 'error', message, operation_index=i, + field=f"{filter_name}.{key}", + suggestion=suggestion, + error_type='INVALID_FILTER_KEY')) + + # Check semantic issues + issues.extend(_validate_semantics(operations)) + + return issues + + +def _validate_semantics(operations: List[ASTObject]) -> List[ValidationIssue]: + """Validate semantic correctness of operation sequence.""" + issues = [] + + # Check for orphaned edges (edges not between nodes) + for i, op in enumerate(operations): + if isinstance(op, ASTEdge): + # Check if first or last operation + if i == 0 or i == len(operations) - 1: + # Edge at boundary - likely orphaned + message, suggestion = _format_error('ORPHANED_EDGE', index=i) + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='ORPHANED_EDGE')) + # Check if between two edges + elif (i > 0 and isinstance(operations[i - 1], ASTEdge) + and i < len(operations) - 1 + and isinstance(operations[i + 1], ASTEdge)): + message, suggestion = _format_error('ORPHANED_EDGE', index=i) + issues.append(ValidationIssue( + 'warning', message, operation_index=i, + suggestion=suggestion, + error_type='ORPHANED_EDGE')) + + return issues + + +def validate_schema(chain: Union[Chain, List], + schema: Schema) -> List[ValidationIssue]: + """ + Validate query against data schema. + + Args: + chain: GFQL chain or list of operations + schema: Schema object with column information + + Returns: + List of validation issues + """ + issues = [] + + # First do syntax validation + syntax_issues = validate_syntax(chain) + # Only keep errors from syntax validation for schema validation + issues.extend([issue for issue in syntax_issues if issue.level == 'error']) + + if any(issue.level == 'error' for issue in issues): + return issues # Don't do schema validation if syntax errors + + # Convert to list if Chain object + operations = chain.chain if isinstance(chain, Chain) else chain + + # Validate each operation against schema + for i, op in enumerate(operations): + if isinstance(op, ASTNode): + issues.extend(_validate_node_schema(op, schema, i)) + elif isinstance(op, ASTEdge): + issues.extend(_validate_edge_schema(op, schema, i)) + + return issues + + +def _validate_node_schema(node: ASTNode, schema: Schema, + op_index: int) -> List[ValidationIssue]: + """Validate node operation against schema.""" + issues = [] + + if node.filter_dict and schema.node_columns: + for col, predicate in node.filter_dict.items(): + # Check column exists + if col not in schema.node_columns: + available = list(schema.node_columns.keys())[:5] + if len(schema.node_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='node', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=col, suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + # Check predicate type compatibility + if isinstance(predicate, ASTPredicate): + col_type = schema.node_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index)) + + return issues + + +def _validate_edge_schema(edge: ASTEdge, schema: Schema, + op_index: int) -> List[ValidationIssue]: + """Validate edge operation against schema.""" + issues = [] + + # Validate edge filters + if edge.edge_match and schema.edge_columns: + for col, predicate in edge.edge_match.items(): + if col not in schema.edge_columns: + available = list(schema.edge_columns.keys())[:5] + if len(schema.edge_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='edge', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"edge_match.{col}", suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + if isinstance(predicate, ASTPredicate): + col_type = schema.edge_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index, + field_prefix="edge_match.")) + + # Validate source/dest node filters against node schema + for filter_name, filter_dict in [ + ('source_node_match', edge.source_node_match), + ('destination_node_match', edge.destination_node_match) + ]: + if filter_dict and schema.node_columns: + for col, predicate in filter_dict.items(): + if col not in schema.node_columns: + available = list(schema.node_columns.keys())[:5] + if len(schema.node_columns) > 5: + available.append('...') + message, suggestion = _format_error( + 'COLUMN_NOT_FOUND', + column=col, + table='node', + available=', '.join(available)) + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{filter_name}.{col}", suggestion=suggestion, + error_type='COLUMN_NOT_FOUND')) + continue + + if isinstance(predicate, ASTPredicate): + col_type = schema.node_columns[col] + issues.extend(_validate_predicate_type( + predicate, col, col_type, op_index, + field_prefix=f"{filter_name}.")) + + return issues + + +def _validate_predicate_type(predicate: ASTPredicate, column: str, + column_type: str, op_index: int, + field_prefix: str = "") -> List[ValidationIssue]: + """Validate predicate is appropriate for column type.""" + issues = [] + + # Map pandas/numpy dtypes to categories + type_category = _get_type_category(column_type) + + # Define string predicate types + STRING_PREDICATES = ( + Contains, Startswith, Endswith, Match, + IsNumeric, IsAlpha, IsDigit, IsLower, IsUpper, + IsSpace, IsAlnum, IsDecimal, IsTitle + ) + + # Define temporal predicate types + TEMPORAL_PREDICATES = ( + IsMonthStart, IsMonthEnd, IsQuarterStart, IsQuarterEnd, + IsYearStart, IsYearEnd, IsLeapYear + ) + + # Check predicate compatibility + if (isinstance(predicate, NumericASTPredicate) + and type_category not in ['numeric', 'temporal']): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='numeric') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + elif (isinstance(predicate, STRING_PREDICATES) + and type_category != 'string'): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='string') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + elif (isinstance(predicate, TEMPORAL_PREDICATES) + and type_category != 'temporal'): + message, suggestion = _format_error( + 'TYPE_MISMATCH', + column=column, + actual_type=column_type, + expected_type='datetime') + issues.append(ValidationIssue( + 'error', message, operation_index=op_index, + field=f"{field_prefix}{column}", + suggestion=suggestion, + error_type='TYPE_MISMATCH')) + + return issues + + +def _get_type_category(dtype_str: str) -> str: + """Categorize dtype string into broad categories.""" + dtype_lower = str(dtype_str).lower() + + if any(t in dtype_lower for t in + ['int', 'float', 'double', 'numeric', 'decimal']): + return 'numeric' + elif any(t in dtype_lower for t in + ['str', 'object', 'char', 'text', 'varchar']): + return 'string' + elif any(t in dtype_lower for t in + ['date', 'time', 'timestamp', 'datetime']): + return 'temporal' + elif 'bool' in dtype_lower: + return 'boolean' + else: + return 'unknown' + + +def validate_query(chain: Union[Chain, List], + nodes_df: Optional[pd.DataFrame] = None, + edges_df: Optional[pd.DataFrame] = None + ) -> List[ValidationIssue]: + """ + Combined syntax and schema validation. + + Args: + chain: GFQL chain or list of operations + nodes_df: Optional node dataframe for schema validation + edges_df: Optional edge dataframe for schema validation + + Returns: + List of validation issues + """ + # Always do syntax validation + issues = validate_syntax(chain) + + # If data provided, also do schema validation + if nodes_df is not None or edges_df is not None: + schema = extract_schema_from_dataframes(nodes_df, edges_df) + schema_issues = validate_schema(chain, schema) + # Merge issues, avoiding duplicates + existing_errors = {(i.error_type, i.operation_index, i.field) + for i in issues} + for issue in schema_issues: + if ((issue.error_type, issue.operation_index, issue.field) + not in existing_errors): + issues.append(issue) + + return issues + + +def extract_schema(g: "Plottable") -> Schema: + """ + Extract schema from a Plottable object. + + Args: + g: Plottable object with node/edge data + + Returns: + Schema object + """ + + nodes_df = g._nodes if hasattr(g, '_nodes') else None + edges_df = g._edges if hasattr(g, '_edges') else None + + return extract_schema_from_dataframes(nodes_df, edges_df) + + +def extract_schema_from_dataframes( + nodes_df: Optional[pd.DataFrame] = None, + edges_df: Optional[pd.DataFrame] = None) -> Schema: + """ + Extract schema from pandas DataFrames. + + Args: + nodes_df: Optional node dataframe + edges_df: Optional edge dataframe + + Returns: + Schema object with column names and types + """ + node_columns = {} + edge_columns = {} + + if nodes_df is not None and hasattr(nodes_df, 'dtypes'): + node_columns = {str(col): str(dtype) + for col, dtype in nodes_df.dtypes.items()} + + if edges_df is not None and hasattr(edges_df, 'dtypes'): + edge_columns = {str(col): str(dtype) + for col, dtype in edges_df.dtypes.items()} + + return Schema(node_columns, edge_columns) + + +def format_validation_errors(issues: List[ValidationIssue]) -> str: + """ + Format validation errors for human/LLM consumption. + + Args: + issues: List of validation issues + + Returns: + Formatted error string + """ + if not issues: + return "No validation issues found." + + lines = ["GFQL Validation Report:"] + lines.append("-" * 50) + + errors = [i for i in issues if i.level == 'error'] + warnings = [i for i in issues if i.level == 'warning'] + + if errors: + lines.append(f"\nERRORS ({len(errors)}):") + for i, error in enumerate(errors, 1): + lines.append(f"\n{i}. {error.message}") + if error.operation_index is not None: + lines.append(f" Location: Operation {error.operation_index}") + if error.field: + lines.append(f" Field: {error.field}") + if error.suggestion: + lines.append(f" 💡 {error.suggestion}") + + if warnings: + lines.append(f"\nWARNINGS ({len(warnings)}):") + for i, warning in enumerate(warnings, 1): + lines.append(f"\n{i}. {warning.message}") + if warning.operation_index is not None: + lines.append( + f" Location: Operation {warning.operation_index}") + if warning.suggestion: + lines.append(f" 💡 {warning.suggestion}") + + return "\n".join(lines) + + +def suggest_fixes(chain: Union[Chain, List], + issues: List[ValidationIssue]) -> List[str]: + """ + Generate fix suggestions for validation issues. + + Args: + chain: The problematic chain + issues: Validation issues found + + Returns: + List of suggested fixes + """ + suggestions = [] + + # Group by error type for consolidated suggestions + error_types: Dict[str, List[ValidationIssue]] = {} + for issue in issues: + if issue.error_type: + error_types.setdefault(issue.error_type, []).append(issue) + + # Generate type-specific suggestions + if 'COLUMN_NOT_FOUND' in error_types: + missing_cols = {issue.field for issue in + error_types['COLUMN_NOT_FOUND'] if issue.field} + suggestions.append( + f"Missing columns: {', '.join(missing_cols)}. " + f"Check column names match your data.") + + if 'TYPE_MISMATCH' in error_types: + suggestions.append( + "Type mismatches found. Use numeric predicates (gt, lt) " + "for numbers, string predicates (contains, startswith) for text.") + + if 'ORPHANED_EDGE' in error_types: + suggestions.append( + "Edge operations should connect nodes. " + "Use pattern: n() -> e() -> n()") + + # Add general suggestions from issues + for issue in issues: + if issue.suggestion and issue.suggestion not in suggestions: + suggestions.append(issue.suggestion) + + return suggestions diff --git a/graphistry/compute/gfql.py b/graphistry/compute/gfql_unified.py similarity index 100% rename from graphistry/compute/gfql.py rename to graphistry/compute/gfql_unified.py diff --git a/graphistry/compute/validate/validate_schema.py b/graphistry/compute/validate/validate_schema.py index cd4217f870..228bc4c937 100644 --- a/graphistry/compute/validate/validate_schema.py +++ b/graphistry/compute/validate/validate_schema.py @@ -3,11 +3,10 @@ from typing import List, Optional, Union, TYPE_CHECKING, cast import pandas as pd from graphistry.Plottable import Plottable -from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, ASTLet, ASTChainRef, ASTRemoteGraph +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, ASTLet, ASTRef, ASTRemoteGraph, ASTCall if TYPE_CHECKING: from graphistry.compute.chain import Chain - from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError from graphistry.compute.predicates.ASTPredicate import ASTPredicate from graphistry.compute.predicates.numeric import NumericASTPredicate, Between @@ -60,10 +59,12 @@ def validate_chain_schema( op_errors = _validate_edge_op(op, node_columns, edge_columns, g._nodes, g._edges, collect_all) elif isinstance(op, ASTLet): op_errors = _validate_querydag_op(op, g, collect_all) - elif isinstance(op, ASTChainRef): - op_errors = _validate_chainref_op(op, g, collect_all) + elif isinstance(op, ASTRef): + op_errors = _validate_ref_op(op, g, collect_all) elif isinstance(op, ASTRemoteGraph): op_errors = _validate_remotegraph_op(op, collect_all) + elif isinstance(op, ASTCall): + op_errors = _validate_call_op(op, node_columns, edge_columns, collect_all) # Add operation index to all errors for e in op_errors: @@ -126,8 +127,6 @@ def _validate_querydag_op(op: ASTLet, g: Plottable, collect_all: bool) -> List[G if binding_errors: for error in binding_errors: error.context['dag_binding'] = binding_name - - if binding_errors: if collect_all: errors.extend(binding_errors) else: @@ -143,28 +142,26 @@ def _validate_querydag_op(op: ASTLet, g: Plottable, collect_all: bool) -> List[G return errors -def _validate_chainref_op(op: ASTChainRef, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: - """Validate ChainRef operation against schema.""" +def _validate_ref_op(op: ASTRef, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate Ref operation against schema.""" errors = [] - # Validate the chain operations in the ChainRef + # Validate the chain operations in the Ref if op.chain: try: chain_errors = validate_chain_schema(g, op.chain, collect_all=True) - # Add ChainRef context to errors + # Add Ref context to errors if chain_errors: for error in chain_errors: - error.context['chain_ref'] = op.ref - - if chain_errors: + error.context['ref'] = op.ref if collect_all: errors.extend(chain_errors) else: raise chain_errors[0] except GFQLSchemaError as e: - e.context['chain_ref'] = op.ref + e.context['ref'] = op.ref if collect_all: errors.append(e) else: @@ -305,6 +302,92 @@ def _validate_filter_dict( return errors +def _validate_call_op( + op: ASTCall, + node_columns: set, + edge_columns: set, + collect_all: bool = False +) -> List[GFQLSchemaError]: + """Validate Call operation schema requirements. + + Checks that all columns required by the called method exist in the graph. + Uses the schema_effects metadata from the safelist to determine requirements. + + Args: + op: ASTCall operation to validate + node_columns: Set of available node column names + edge_columns: Set of available edge column names + collect_all: If True, collect all errors. If False, raise on first error. + + Returns: + List of schema errors found (empty if valid) + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + errors: List[GFQLSchemaError] = [] + + # Import safelist to get schema effects + from graphistry.compute.call_safelist import SAFELIST_V1 + + # Check if method is in safelist + if op.function not in SAFELIST_V1: + # This should have been caught by parameter validation already + return errors + + method_info = SAFELIST_V1[op.function] + + # Check if method has schema effects defined + if 'schema_effects' not in method_info: + # Method doesn't define schema effects, so we can't validate + return errors + + schema_effects = method_info['schema_effects'] + + # Get required columns based on parameters + if 'requires_node_cols' in schema_effects: + if callable(schema_effects['requires_node_cols']): + required_node_cols = schema_effects['requires_node_cols'](op.params) + else: + required_node_cols = schema_effects['requires_node_cols'] + + for col in required_node_cols: + if col not in node_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires node column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available node columns: {", ".join(sorted(node_columns)[:10])}{"..." if len(node_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + if 'requires_edge_cols' in schema_effects: + if callable(schema_effects['requires_edge_cols']): + required_edge_cols = schema_effects['requires_edge_cols'](op.params) + else: + required_edge_cols = schema_effects['requires_edge_cols'] + + for col in required_edge_cols: + if col not in edge_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires edge column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available edge columns: {", ".join(sorted(edge_columns)[:10])}{"..." if len(edge_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + # Add to Chain class def validate_schema(self: 'Chain', g: Plottable, collect_all: bool = False) -> Optional[List[GFQLSchemaError]]: """Validate this chain against a graph's schema without executing. diff --git a/graphistry/compute/validate_schema.py b/graphistry/compute/validate_schema.py new file mode 100644 index 0000000000..76e1092943 --- /dev/null +++ b/graphistry/compute/validate_schema.py @@ -0,0 +1,413 @@ +"""Schema validation for GFQL chains without execution.""" + +from typing import List, Optional, Union, TYPE_CHECKING, cast +import pandas as pd +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, ASTLet, ASTRef, ASTRemoteGraph, ASTCall + +if TYPE_CHECKING: + from graphistry.compute.chain import Chain +from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError +from graphistry.compute.predicates.ASTPredicate import ASTPredicate +from graphistry.compute.predicates.numeric import NumericASTPredicate, Between +from graphistry.compute.predicates.str import Contains, Startswith, Endswith, Match + + +def validate_chain_schema( + g: Plottable, + ops: Union[List[ASTObject], 'Chain'], + collect_all: bool = False +) -> Optional[List[GFQLSchemaError]]: + """Validate chain operations against graph schema without executing. + + This performs static analysis of the chain operations to detect: + - References to non-existent columns + - Type mismatches between filters and column types + - Invalid predicate usage + + Args: + g: The graph to validate against + ops: Chain operations to validate + collect_all: If True, collect all errors. If False, raise on first error. + + Returns: + If collect_all=True: List of schema errors (empty if valid) + If collect_all=False: None if valid + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + # Handle Chain objects + from graphistry.compute.chain import Chain + if isinstance(ops, Chain): + chain_ops = ops.chain + else: + chain_ops = ops + + errors: List[GFQLSchemaError] = [] + + # Get available columns + node_columns = set(g._nodes.columns) if g._nodes is not None else set() + edge_columns = set(g._edges.columns) if g._edges is not None else set() + + for i, op in enumerate(chain_ops): + op_errors = [] + + if isinstance(op, ASTNode): + op_errors = _validate_node_op(op, node_columns, g._nodes, collect_all) + elif isinstance(op, ASTEdge): + op_errors = _validate_edge_op(op, node_columns, edge_columns, g._nodes, g._edges, collect_all) + elif isinstance(op, ASTLet): + op_errors = _validate_querydag_op(op, g, collect_all) + elif isinstance(op, ASTRef): + op_errors = _validate_chainref_op(op, g, collect_all) + elif isinstance(op, ASTRemoteGraph): + op_errors = _validate_remotegraph_op(op, collect_all) + elif isinstance(op, ASTCall): + op_errors = _validate_call_op(op, node_columns, edge_columns, collect_all) + + # Add operation index to all errors + for e in op_errors: + e.context['operation_index'] = i + + if op_errors: + if collect_all: + errors.extend(op_errors) + else: + raise op_errors[0] + + return errors if collect_all else None + + +def _validate_node_op(op: ASTNode, node_columns: set, nodes_df: Optional[pd.DataFrame], collect_all: bool) -> List[GFQLSchemaError]: + """Validate node operation against schema.""" + errors = [] + if op.filter_dict and nodes_df is not None: + errors.extend(_validate_filter_dict(op.filter_dict, node_columns, nodes_df, "node", collect_all)) + return errors + + +def _validate_edge_op( + op: ASTEdge, + node_columns: set, + edge_columns: set, + nodes_df: Optional[pd.DataFrame], + edges_df: Optional[pd.DataFrame], + collect_all: bool +) -> List[GFQLSchemaError]: + """Validate edge operation against schema.""" + errors = [] + + # Validate edge filters + if op.edge_match and edges_df is not None: + errors.extend(_validate_filter_dict(op.edge_match, edge_columns, edges_df, "edge", collect_all)) + + # Validate source node filters + if op.source_node_match and nodes_df is not None: + errors.extend(_validate_filter_dict(op.source_node_match, node_columns, nodes_df, "source node", collect_all)) + + # Validate destination node filters + if op.destination_node_match and nodes_df is not None: + errors.extend(_validate_filter_dict(op.destination_node_match, node_columns, nodes_df, "destination node", collect_all)) + + return errors + + +def _validate_querydag_op(op: ASTLet, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate Let operation against schema.""" + errors = [] + + # Validate each binding in the DAG + for binding_name, binding_value in op.bindings.items(): + try: + # Recursively validate each binding as if it's a single operation + binding_errors = validate_chain_schema(g, [binding_value], collect_all=True) + + # Add binding context to errors + if binding_errors: + for error in binding_errors: + error.context['dag_binding'] = binding_name + + if binding_errors: + if collect_all: + errors.extend(binding_errors) + else: + raise binding_errors[0] + + except GFQLSchemaError as e: + e.context['dag_binding'] = binding_name + if collect_all: + errors.append(e) + else: + raise + + return errors + + +def _validate_chainref_op(op: ASTRef, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate ChainRef operation against schema.""" + errors = [] + + # Validate the chain operations in the ChainRef + if op.chain: + try: + chain_errors = validate_chain_schema(g, op.chain, collect_all=True) + + # Add ChainRef context to errors + if chain_errors: + for error in chain_errors: + error.context['chain_ref'] = op.ref + + if chain_errors: + if collect_all: + errors.extend(chain_errors) + else: + raise chain_errors[0] + + except GFQLSchemaError as e: + e.context['chain_ref'] = op.ref + if collect_all: + errors.append(e) + else: + raise + + # Note: We don't validate that op.ref exists here since that's handled + # by the DAG dependency validation in chain_dag.py + + return errors + + +def _validate_remotegraph_op(op: ASTRemoteGraph, collect_all: bool) -> List[GFQLSchemaError]: + """Validate RemoteGraph operation against schema.""" + errors = [] + + # Validate dataset_id format + if not op.dataset_id or not isinstance(op.dataset_id, str): + error = GFQLSchemaError( + ErrorCode.E303, + 'RemoteGraph dataset_id must be a non-empty string', + field='dataset_id', + value=op.dataset_id, + suggestion='Provide a valid dataset identifier string' + ) + if collect_all: + errors.append(error) + else: + raise error + + # Validate token format if provided + if op.token is not None and not isinstance(op.token, str): + error = GFQLSchemaError( + ErrorCode.E303, + 'RemoteGraph token must be a string if provided', + field='token', + value=type(op.token).__name__, + suggestion='Provide a valid token string or None' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + +def _validate_filter_dict( + filter_dict: dict, + columns: set, + df: pd.DataFrame, + context: str, + collect_all: bool = False +) -> List[GFQLSchemaError]: + """Validate filter dictionary against dataframe schema.""" + errors = [] + for col, val in filter_dict.items(): + try: + # Check column exists + if col not in columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Column "{col}" does not exist in {context} dataframe', + field=col, + value=val, + suggestion=f'Available columns: {", ".join(sorted(columns)[:10])}{"..." if len(columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + continue # Check next field + else: + raise error + + # Check type compatibility + col_dtype = df[col].dtype + + if not isinstance(val, ASTPredicate): + # Check literal value type matches + if pd.api.types.is_numeric_dtype(col_dtype) and isinstance(val, str): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: {context} column "{col}" is numeric but filter value is string', + field=col, + value=val, + column_type=str(col_dtype), + suggestion=f'Use a numeric value like {col}=123' + ) + if collect_all: + errors.append(error) + else: + raise error + elif pd.api.types.is_string_dtype(col_dtype) and isinstance(val, (int, float)): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: {context} column "{col}" is string but filter value is numeric', + field=col, + value=val, + column_type=str(col_dtype), + suggestion=f'Use a string value like {col}="value"' + ) + if collect_all: + errors.append(error) + else: + raise error + else: + # Check predicate type matches column type + if isinstance(val, (NumericASTPredicate, Between)) and not pd.api.types.is_numeric_dtype(col_dtype): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: numeric predicate used on non-numeric {context} column "{col}"', + field=col, + value=f"{val.__class__.__name__}(...)", + column_type=str(col_dtype), + suggestion='Use string predicates like contains() or startswith() for string columns' + ) + if collect_all: + errors.append(error) + else: + raise error + + if isinstance(val, (Contains, Startswith, Endswith, Match)) and not pd.api.types.is_string_dtype(col_dtype): + error = GFQLSchemaError( + ErrorCode.E302, + f'Type mismatch: string predicate used on non-string {context} column "{col}"', + field=col, + value=f"{val.__class__.__name__}(...)", + column_type=str(col_dtype), + suggestion='Use numeric predicates like gt() or lt() for numeric columns' + ) + if collect_all: + errors.append(error) + else: + raise error + + except GFQLSchemaError: + if not collect_all: + raise + + return errors + + +def _validate_call_op( + op: ASTCall, + node_columns: set, + edge_columns: set, + collect_all: bool = False +) -> List[GFQLSchemaError]: + """Validate Call operation schema requirements. + + Checks that all columns required by the called method exist in the graph. + Uses the schema_effects metadata from the safelist to determine requirements. + + Args: + op: ASTCall operation to validate + node_columns: Set of available node column names + edge_columns: Set of available edge column names + collect_all: If True, collect all errors. If False, raise on first error. + + Returns: + List of schema errors found (empty if valid) + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + errors: List[GFQLSchemaError] = [] + + # Import safelist to get schema effects + from graphistry.compute.gfql.call_safelist import SAFELIST_V1 + + # Check if method is in safelist + if op.function not in SAFELIST_V1: + # This should have been caught by parameter validation already + return errors + + method_info = SAFELIST_V1[op.function] + + # Check if method has schema effects defined + if 'schema_effects' not in method_info: + # Method doesn't define schema effects, so we can't validate + return errors + + schema_effects = method_info['schema_effects'] + + # Get required columns based on parameters + if 'requires_node_cols' in schema_effects: + if callable(schema_effects['requires_node_cols']): + required_node_cols = schema_effects['requires_node_cols'](op.params) + else: + required_node_cols = schema_effects['requires_node_cols'] + + for col in required_node_cols: + if col not in node_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires node column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available node columns: {", ".join(sorted(node_columns)[:10])}{"..." if len(node_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + if 'requires_edge_cols' in schema_effects: + if callable(schema_effects['requires_edge_cols']): + required_edge_cols = schema_effects['requires_edge_cols'](op.params) + else: + required_edge_cols = schema_effects['requires_edge_cols'] + + for col in required_edge_cols: + if col not in edge_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires edge column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available edge columns: {", ".join(sorted(edge_columns)[:10])}{"..." if len(edge_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + +# Add to Chain class +def validate_schema(self: 'Chain', g: Plottable, collect_all: bool = False) -> Optional[List[GFQLSchemaError]]: + """Validate this chain against a graph's schema without executing. + + Args: + g: Graph to validate against + collect_all: If True, collect all errors. If False, raise on first. + + Returns: + If collect_all=True: List of schema errors + If collect_all=False: None if valid + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + return validate_chain_schema(g, self, collect_all) + + +# Monkey-patching moved to chain.py to avoid circular import diff --git a/graphistry/tests/compute/test_ast.py b/graphistry/tests/compute/test_ast.py index 29e6aa476e..9785587797 100644 --- a/graphistry/tests/compute/test_ast.py +++ b/graphistry/tests/compute/test_ast.py @@ -1,5 +1,5 @@ from graphistry.compute.ast import ( - from_json, ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTChainRef, + from_json, ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTRef, n, e, e_forward, e_reverse, e_undirected ) @@ -94,24 +94,24 @@ def test_serialization_remoteGraph_with_token(): def test_serialization_chainRef_empty(): """Test ChainRef with empty chain""" - cr = ASTChainRef('mydata', []) + cr = ASTRef('mydata', []) o = cr.to_json() - assert o == {'type': 'ChainRef', 'ref': 'mydata', 'chain': []} + assert o == {'type': 'Ref', 'ref': 'mydata', 'chain': []} cr2 = from_json(o) - assert isinstance(cr2, ASTChainRef) + assert isinstance(cr2, ASTRef) assert cr2.ref == 'mydata' assert cr2.chain == [] def test_serialization_chainRef_with_ops(): """Test ChainRef with operations""" - cr = ASTChainRef('data1', [n({'type': 'person'}), e_forward()]) + cr = ASTRef('data1', [n({'type': 'person'}), e_forward()]) o = cr.to_json() - assert o['type'] == 'ChainRef' + assert o['type'] == 'Ref' assert o['ref'] == 'data1' assert len(o['chain']) == 2 cr2 = from_json(o) - assert isinstance(cr2, ASTChainRef) + assert isinstance(cr2, ASTRef) assert cr2.ref == 'data1' assert len(cr2.chain) == 2 assert isinstance(cr2.chain[0], ASTNode) diff --git a/graphistry/tests/compute/test_ast_errors.py b/graphistry/tests/compute/test_ast_errors.py index fcbe12ee3a..956d003f23 100644 --- a/graphistry/tests/compute/test_ast_errors.py +++ b/graphistry/tests/compute/test_ast_errors.py @@ -13,6 +13,7 @@ def test_from_json_non_dict_input(self): with pytest.raises(GFQLSyntaxError) as exc_info: from_json("not a dict") + assert exc_info.value.code == "invalid-chain-type" assert "AST JSON must be a dictionary" in str(exc_info.value) def test_from_json_none_input(self): @@ -20,6 +21,7 @@ def test_from_json_none_input(self): with pytest.raises(GFQLSyntaxError) as exc_info: from_json(None) + assert exc_info.value.code == "invalid-chain-type" assert "AST JSON must be a dictionary" in str(exc_info.value) def test_from_json_missing_type(self): @@ -27,6 +29,7 @@ def test_from_json_missing_type(self): with pytest.raises(GFQLSyntaxError) as exc_info: from_json({"no_type": "value"}) + assert exc_info.value.code == "missing-required-field" assert "AST JSON missing required 'type' field" in str(exc_info.value) def test_from_json_unknown_type(self): @@ -34,6 +37,7 @@ def test_from_json_unknown_type(self): with pytest.raises(GFQLSyntaxError) as exc_info: from_json({"type": "UnknownType"}) + assert exc_info.value.code == "invalid-chain-type" assert "Unknown AST type: UnknownType" in str(exc_info.value) def test_edge_missing_direction(self): @@ -41,6 +45,7 @@ def test_edge_missing_direction(self): with pytest.raises(GFQLSyntaxError) as exc_info: from_json({"type": "Edge"}) + assert exc_info.value.code == "missing-required-field" assert "Edge missing required 'direction' field" in str(exc_info.value) def test_edge_invalid_direction(self): @@ -48,10 +53,11 @@ def test_edge_invalid_direction(self): with pytest.raises(GFQLSyntaxError) as exc_info: from_json({"type": "Edge", "direction": "invalid"}) + assert exc_info.value.code == "invalid-direction" assert "Edge has unknown direction: invalid" in str(exc_info.value) def test_querydag_missing_bindings(self): - """Test clear error when QueryDAG missing bindings""" + """Test clear error when Let missing bindings""" with pytest.raises(AssertionError) as exc_info: from_json({"type": "QueryDAG"}) @@ -65,15 +71,15 @@ def test_remotegraph_missing_dataset_id(self): assert "RemoteGraph missing dataset_id" in str(exc_info.value) def test_chainref_missing_ref(self): - """Test clear error when ChainRef missing ref""" + """Test clear error when Ref missing ref""" with pytest.raises(AssertionError) as exc_info: - from_json({"type": "ChainRef"}) + from_json({"type": "Ref"}) - assert "ChainRef missing ref" in str(exc_info.value) + assert "Ref missing ref" in str(exc_info.value) def test_chainref_missing_chain(self): - """Test clear error when ChainRef missing chain""" + """Test clear error when Ref missing chain""" with pytest.raises(AssertionError) as exc_info: - from_json({"type": "ChainRef", "ref": "test"}) + from_json({"type": "Ref", "ref": "test"}) - assert "ChainRef missing chain" in str(exc_info.value) + assert "Ref missing chain" in str(exc_info.value) diff --git a/graphistry/tests/compute/test_call_operations.py b/graphistry/tests/compute/test_call_operations.py new file mode 100644 index 0000000000..d47ad8b95d --- /dev/null +++ b/graphistry/tests/compute/test_call_operations.py @@ -0,0 +1,394 @@ +"""Tests for GFQL Call operations.""" + +import pytest +import pandas as pd +from unittest.mock import Mock, patch, MagicMock + +from graphistry.tests.test_compute import CGFull +from graphistry.Engine import Engine, EngineAbstract +from graphistry.compute.ast import ASTCall, ASTLet, n +from graphistry.compute.chain_dag import chain_dag_impl +from graphistry.compute.gfql.call_safelist import validate_call_params +from graphistry.compute.gfql.call_executor import execute_call +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError, GFQLSyntaxError + + +class TestCallSafelist: + """Test method safelist validation.""" + + def test_allowed_method(self): + """Test validation of allowed methods.""" + params = validate_call_params('get_degrees', { + 'col': 'degree' + }) + assert params == {'col': 'degree'} + + def test_unknown_method(self): + """Test rejection of unknown methods.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('unknown_method', {}) + assert exc_info.value.code == ErrorCode.E303 + assert 'not in the safelist' in str(exc_info.value) + + def test_required_params(self): + """Test validation of required parameters.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('filter_nodes_by_dict', {}) + assert exc_info.value.code == ErrorCode.E105 + assert 'Missing required parameters' in str(exc_info.value) + + def test_unknown_params(self): + """Test rejection of unknown parameters.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('get_degrees', { + 'col': 'degree', + 'unknown_param': 'value' + }) + assert exc_info.value.code == ErrorCode.E303 + assert 'Unknown parameters' in str(exc_info.value) + + def test_param_type_validation(self): + """Test parameter type validation.""" + # Valid types + params = validate_call_params('hop', { + 'hops': 2, + 'direction': 'forward', + 'to_fixed_point': True + }) + assert params['hops'] == 2 + + # Invalid type + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('hop', { + 'hops': 'two' # Should be int + }) + assert exc_info.value.code == ErrorCode.E201 + assert 'Invalid type' in str(exc_info.value) + + def test_enum_validation(self): + """Test enum parameter validation.""" + # Valid enum value + params = validate_call_params('hop', { + 'direction': 'forward' + }) + assert params['direction'] == 'forward' + + # Invalid enum value + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('hop', { + 'direction': 'sideways' + }) + assert exc_info.value.code == ErrorCode.E201 + + +class TestASTCall: + """Test ASTCall node validation and serialization.""" + + def test_basic_creation(self): + """Test creating a basic ASTCall.""" + call = ASTCall('get_degrees', {'col': 'degree'}) + assert call.function == 'get_degrees' + assert call.params == {'col': 'degree'} + + def test_empty_params(self): + """Test ASTCall with no parameters.""" + call = ASTCall('prune_self_edges') + assert call.function == 'prune_self_edges' + assert call.params == {} + + def test_validation(self): + """Test ASTCall field validation.""" + # Valid call + call = ASTCall('get_degrees', {'col': 'degree'}) + call.validate() # Should not raise + + # Invalid function type + with pytest.raises(GFQLTypeError) as exc_info: + call = ASTCall(123, {}) + call.validate() + assert exc_info.value.code == ErrorCode.E201 + assert 'function must be a string' in str(exc_info.value) + + # Invalid params type + with pytest.raises(GFQLTypeError) as exc_info: + call = ASTCall('get_degrees', 'not_a_dict') + call.validate() + assert exc_info.value.code == ErrorCode.E201 + assert 'params must be a dict' in str(exc_info.value) + + def test_to_json(self): + """Test ASTCall JSON serialization.""" + call = ASTCall('get_degrees', {'col': 'degree', 'engine': 'pandas'}) + json_data = call.to_json() + + assert json_data == { + 'type': 'Call', + 'function': 'get_degrees', + 'params': {'col': 'degree', 'engine': 'pandas'} + } + + def test_from_json(self): + """Test ASTCall JSON deserialization.""" + json_data = { + 'type': 'Call', + 'function': 'filter_nodes_by_dict', + 'params': {'filter_dict': {'type': 'user'}} + } + + call = ASTCall.from_json(json_data) + assert isinstance(call, ASTCall) + assert call.function == 'filter_nodes_by_dict' + assert call.params == {'filter_dict': {'type': 'user'}} + + def test_from_json_invalid(self): + """Test ASTCall from_json with invalid data.""" + # Missing function + with pytest.raises(AssertionError) as exc_info: + ASTCall.from_json({'type': 'Call'}) + assert 'Call missing function' in str(exc_info.value) + + # Wrong type - this would be caught earlier in the AST dispatch + # so we don't test it here + + +class TestCallExecution: + """Test call execution functionality.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_execute_get_degrees(self, sample_graph): + """Test executing get_degrees method.""" + result = execute_call( + sample_graph, + 'get_degrees', + {'col': 'degree'}, + Engine.PANDAS + ) + + # Result should be a Plottable + assert hasattr(result, '_nodes') + assert 'degree' in result._nodes.columns + assert len(result._nodes) == 4 + + def test_execute_filter_nodes(self, sample_graph): + """Test executing filter_nodes_by_dict method.""" + result = execute_call( + sample_graph, + 'filter_nodes_by_dict', + {'filter_dict': {'type': 'user'}}, + Engine.PANDAS + ) + + # Should filter to only user nodes + assert len(result._nodes) == 3 + assert all(result._nodes['type'] == 'user') + + def test_execute_materialize_nodes(self): + """Test executing materialize_nodes method.""" + # Start with edges only + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0] + }) + g = CGFull().edges(edges_df).bind(source='source', destination='target') + + # No nodes initially + assert g._nodes is None + + # Execute materialize_nodes + result = execute_call( + g, + 'materialize_nodes', + {}, + Engine.PANDAS + ) + + # Should have nodes now + assert result._nodes is not None + assert len(result._nodes) == 3 + + def test_execute_with_validation_error(self, sample_graph): + """Test that validation errors are properly raised.""" + with pytest.raises(GFQLTypeError) as exc_info: + execute_call( + sample_graph, + 'hop', + {'hops': 'invalid'}, # Should be int + Engine.PANDAS + ) + assert exc_info.value.code == ErrorCode.E201 + + def test_execute_unknown_method(self, sample_graph): + """Test execution of unknown method.""" + with pytest.raises(GFQLTypeError) as exc_info: + execute_call( + sample_graph, + 'unknown_method', + {}, + Engine.PANDAS + ) + assert exc_info.value.code == ErrorCode.E303 + + +class TestCallInDAG: + """Test ASTCall execution within DAGs.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0], + 'weight': [1.0, 2.0, 1.5, 3.0, 0.5] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_call_in_dag(self, sample_graph): + """Test executing ASTCall within a DAG.""" + dag = ASTLet({ + 'filtered': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_dag_impl(sample_graph, dag, EngineAbstract.PANDAS) + + # Should have degree column + assert 'degree' in result._nodes.columns + # Should still have all nodes (get_degrees doesn't filter) + assert len(result._nodes) == 4 + + def test_call_referencing_binding(self, sample_graph): + """Test ASTCall that operates on whole graph (not in chain).""" + from graphistry.compute.ast import ASTChainRef + + # Call operations work on the whole graph, not as part of chains + dag = ASTLet({ + 'users': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_dag_impl(sample_graph, dag, EngineAbstract.PANDAS) + + # Should have degree column on all nodes + assert len(result._nodes) == 4 # All nodes + assert 'degree' in result._nodes.columns + + def test_multiple_calls(self, sample_graph): + """Test multiple call operations in sequence.""" + # First add degrees + dag1 = ASTLet({ + 'with_degrees': ASTCall('get_degrees', {'col': 'deg'}) + }) + result1 = chain_dag_impl(sample_graph, dag1, EngineAbstract.PANDAS) + assert 'deg' in result1._nodes.columns + + # Then filter - use the graph that has degrees + dag2 = ASTLet({ + 'filtered': ASTCall('filter_nodes_by_dict', {'filter_dict': {'deg': 2}}) + }) + result2 = chain_dag_impl(result1, dag2, EngineAbstract.PANDAS) + + # Should have nodes with degree 2 + assert len(result2._nodes) > 0 + assert all(result2._nodes['deg'] == 2) + + @patch('graphistry.compute.gfql.call_executor.getattr') + def test_call_execution_error(self, mock_getattr, sample_graph): + """Test handling of execution errors in calls.""" + # Make the method raise an error + mock_method = Mock(side_effect=RuntimeError("Method failed")) + mock_getattr.return_value = mock_method + + dag = ASTLet({ + 'failing': ASTCall('get_degrees', {}) + }) + + with pytest.raises(RuntimeError) as exc_info: + chain_dag_impl(sample_graph, dag, EngineAbstract.PANDAS) + assert "Failed to execute node 'failing'" in str(exc_info.value) + + +class TestGraphAlgorithmCalls: + """Test calls to graph algorithm methods.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3] + }) + + return CGFull()\ + .edges(edges_df)\ + .bind(source='source', destination='target') + + def test_compute_cugraph_params(self): + """Test compute_cugraph parameter validation.""" + # Valid params + params = validate_call_params('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'pr_score', + 'directed': True + }) + assert params['alg'] == 'pagerank' + + # Missing required alg + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('compute_cugraph', { + 'out_col': 'pr_score' + }) + assert exc_info.value.code == ErrorCode.E105 + + def test_compute_igraph_params(self): + """Test compute_igraph parameter validation.""" + params = validate_call_params('compute_igraph', { + 'alg': 'community_louvain', + 'directed': False + }) + assert params['alg'] == 'community_louvain' + + def test_layout_methods(self): + """Test layout method parameter validation.""" + # layout_cugraph + params = validate_call_params('layout_cugraph', { + 'layout': 'force_atlas2', + 'params': {'iterations': 100} + }) + assert params['layout'] == 'force_atlas2' + + # layout_igraph + params = validate_call_params('layout_igraph', { + 'layout': 'fruchterman_reingold', + 'directed': True + }) + assert params['layout'] == 'fruchterman_reingold' + + # fa2_layout + params = validate_call_params('fa2_layout', { + 'fa2_params': {'iterations': 500} + }) + assert params['fa2_params']['iterations'] == 500 diff --git a/graphistry/tests/compute/test_call_operations_gpu.py b/graphistry/tests/compute/test_call_operations_gpu.py new file mode 100644 index 0000000000..0a7902286d --- /dev/null +++ b/graphistry/tests/compute/test_call_operations_gpu.py @@ -0,0 +1,277 @@ +"""GPU tests for GFQL Call operations.""" + +import os +import pytest +import pandas as pd + +from graphistry.tests.test_compute import CGFull +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTCall, ASTLet, n +from graphistry.compute.chain_dag import chain_dag_impl +from graphistry.compute.gfql.call_executor import execute_call +from graphistry.compute.validate.validate_schema import validate_chain_schema +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +# Skip all tests if TEST_CUDF not set +skip_gpu = pytest.mark.skipif( + not ("TEST_CUDF" in os.environ and os.environ["TEST_CUDF"] == "1"), + reason="cudf tests need TEST_CUDF=1" +) + + +class TestCallOperationsGPU: + """Test Call operations with GPU/cudf.""" + + @skip_gpu + def test_call_with_cudf_dataframes(self): + """Test that Call operations work when starting with cudf DataFrames.""" + import cudf + + # Create cudf dataframes + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3], + 'weight': [1.0, 2.0, 3.0, 4.0] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + edges_gdf = cudf.from_pandas(edges_df) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Create graph with cudf data (may convert internally) + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Execute Call operation with CUDF engine hint + result = execute_call(g, 'get_degrees', {'col': 'degree'}, Engine.CUDF) + + # Result should have degree columns + assert 'degree' in result._nodes.columns + assert 'degree_in' in result._nodes.columns + assert 'degree_out' in result._nodes.columns + + # Verify the computation is correct + assert len(result._nodes) == 4 + # Node 2 has the highest degree (3 connections) + degrees = result._nodes['degree'].tolist() if hasattr(result._nodes['degree'], 'tolist') else list(result._nodes['degree']) + assert max(degrees) == 3 + + @skip_gpu + def test_filter_with_cudf(self): + """Test filtering operations with cudf.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'], + 'score': [0.5, 0.8, 0.9, 0.3] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Add edges to make it a valid graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 3] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Filter nodes + result = execute_call( + g, + 'filter_nodes_by_dict', + {'filter_dict': {'type': 'user'}}, + Engine.CUDF + ) + + # Should be filtered + assert len(result._nodes) == 3 + # Check the type column values + types = result._nodes['type'].to_pandas() if hasattr(result._nodes, 'to_pandas') else result._nodes['type'] + assert all(types == 'user') + + @skip_gpu + def test_compute_cugraph_call(self): + """Test compute_cugraph through Call operation.""" + import cudf + + # Create a simple graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull().edges(edges_gdf).bind(source='source', destination='target') + + # Skip if cugraph not available + try: + import cugraph + except ImportError: + pytest.skip("cugraph not installed") + + # Call compute_cugraph for pagerank + result = execute_call( + g, + 'compute_cugraph', + {'alg': 'pagerank', 'out_col': 'pr_score'}, + Engine.CUDF + ) + + # Should have pagerank scores + assert 'pr_score' in result._nodes.columns + # Verify scores are computed (all nodes should have scores) + assert len(result._nodes) == 4 # 4 unique nodes + scores = result._nodes['pr_score'].tolist() if hasattr(result._nodes['pr_score'], 'tolist') else list(result._nodes['pr_score']) + assert all(score > 0 for score in scores) + + @skip_gpu + def test_layout_cugraph_call(self): + """Test layout_cugraph through Call operation.""" + import cudf + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull().edges(edges_gdf).bind(source='source', destination='target') + + # Skip if cugraph not available + try: + import cugraph + except ImportError: + pytest.skip("cugraph not installed") + + # Call layout_cugraph + result = execute_call( + g, + 'layout_cugraph', + {'layout': 'force_atlas2'}, + Engine.CUDF + ) + + # Should have x,y coordinates + assert 'x' in result._nodes.columns + assert 'y' in result._nodes.columns + # Verify all nodes have coordinates + assert len(result._nodes) == 3 + assert result._nodes['x'].notna().all() + assert result._nodes['y'].notna().all() + + @skip_gpu + def test_chain_dag_with_gpu_calls(self): + """Test DAG execution with Call operations on GPU.""" + import cudf + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0], + 'weight': [1.0, 2.0, 1.5, 3.0, 0.5] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + edges_gdf = cudf.from_pandas(edges_df) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Create DAG with Call operations + dag = ASTLet({ + 'filtered': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_dag_impl(g, dag, Engine.CUDF) + + # Should have degrees column + assert 'degree' in result._nodes.columns + # Check that we have the expected number of nodes + assert len(result._nodes) == 4 # get_degrees doesn't filter + + @skip_gpu + def test_schema_validation_with_cudf(self): + """Test schema validation works with cudf DataFrames.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2], + 'type': ['A', 'B', 'C'] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull().nodes(nodes_gdf).bind(node='node') + + # Valid call - column exists + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'A'}}) + errors = validate_chain_schema(g, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid call - column doesn't exist + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing': 'X'}}) + errors = validate_chain_schema(g, [call], collect_all=True) + assert len(errors) > 0 + assert any('missing' in str(e) for e in errors) + + @skip_gpu + def test_encode_with_gpu(self): + """Test visual encoding methods with GPU data.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'category': ['A', 'B', 'A', 'C'], + 'score': [0.1, 0.5, 0.8, 0.3] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Add edges to make a valid graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 3] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Test encode_point_color + result = execute_call( + g, + 'encode_point_color', + {'column': 'category'}, + Engine.CUDF + ) + + # Should have color encoding set + assert result._point_color == 'category' + + # Test encode_point_size + result2 = execute_call( + result, + 'encode_point_size', + {'column': 'score'}, + Engine.CUDF + ) + + # Should have size encoding set + assert result2._point_size == 'score' diff --git a/graphistry/tests/compute/test_call_schema_validation.py b/graphistry/tests/compute/test_call_schema_validation.py new file mode 100644 index 0000000000..3dd2d6f372 --- /dev/null +++ b/graphistry/tests/compute/test_call_schema_validation.py @@ -0,0 +1,126 @@ +"""Test schema validation for Call operations.""" + +import pytest +import pandas as pd +from graphistry.tests.test_compute import CGFull +from graphistry.compute.ast import ASTCall, n +from graphistry.compute.chain import Chain +from graphistry.compute.validate.validate_schema import validate_chain_schema +from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError + + +class TestCallSchemaValidation: + """Test schema validation for Call operations.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0], + 'weight': [1.0, 2.0, 3.0] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2], + 'type': ['user', 'user', 'admin'], + 'score': [0.5, 0.8, 0.9] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_filter_nodes_requires_columns(self, sample_graph): + """Test that filter_nodes_by_dict validates required columns.""" + # Valid: filtering by existing column + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'user'}}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: filtering by non-existent column + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing_col': 'value'}}) + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(sample_graph, [call], collect_all=False) + assert exc_info.value.code == ErrorCode.E301 + assert 'missing_col' in str(exc_info.value) + assert 'does not exist' in str(exc_info.value) + + def test_filter_edges_requires_columns(self, sample_graph): + """Test that filter_edges_by_dict validates required columns.""" + # Valid: filtering by existing edge column + call = ASTCall('filter_edges_by_dict', {'filter_dict': {'weight': 2.0}}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: filtering by non-existent edge column + call = ASTCall('filter_edges_by_dict', {'filter_dict': {'edge_type': 'friend'}}) + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(sample_graph, [call], collect_all=False) + assert exc_info.value.code == ErrorCode.E301 + assert 'edge_type' in str(exc_info.value) + + def test_encode_requires_columns(self, sample_graph): + """Test that encode methods validate required columns.""" + # Valid: encoding existing column + call = ASTCall('encode_point_color', {'column': 'type'}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: encoding non-existent column + call = ASTCall('encode_point_color', {'column': 'category'}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + # Note: encode methods don't require columns to exist (they create bindings) + # so this should not produce errors + assert len(errors) == 0 + + def test_chain_with_multiple_calls(self, sample_graph): + """Test validation of chains with multiple Call operations.""" + chain = Chain([ + ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'user'}}), + ASTCall('get_degrees', {'col': 'degree'}), + ASTCall('filter_nodes_by_dict', {'filter_dict': {'degree': 2}}) + ]) + + # The second filter expects 'degree' column which doesn't exist yet + # But schema validation is static and doesn't track added columns + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + # Should have error for missing 'degree' column + assert any('degree' in str(e) for e in errors) + + def test_method_without_schema_effects(self, sample_graph): + """Test that methods without schema effects don't cause errors.""" + # materialize_nodes doesn't require any columns + call = ASTCall('materialize_nodes', {}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + def test_collect_all_mode(self, sample_graph): + """Test collect_all mode returns all errors.""" + chain = Chain([ + ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing1': 'a', 'missing2': 'b'}}), + ASTCall('filter_edges_by_dict', {'filter_dict': {'missing3': 'c'}}) + ]) + + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + # Should collect all 3 missing column errors + assert len(errors) >= 3 + missing_cols = {'missing1', 'missing2', 'missing3'} + error_cols = set() + for e in errors: + for col in missing_cols: + if col in str(e): + error_cols.add(col) + assert error_cols == missing_cols + + def test_operation_index_in_errors(self, sample_graph): + """Test that errors include operation index.""" + chain = Chain([ + n({'type': 'user'}), # op 0 + ASTCall('filter_nodes_by_dict', {'filter_dict': {'bad_col': 1}}) # op 1 + ]) + + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + call_errors = [e for e in errors if 'bad_col' in str(e)] + assert len(call_errors) > 0 + assert call_errors[0].context['operation_index'] == 1 diff --git a/graphistry/tests/compute/test_chain_dag.py b/graphistry/tests/compute/test_chain_dag.py index 00e592cf59..dc62851d80 100644 --- a/graphistry/tests/compute/test_chain_dag.py +++ b/graphistry/tests/compute/test_chain_dag.py @@ -7,12 +7,13 @@ import pandas as pd import pytest from unittest.mock import patch, MagicMock -from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, ASTNode, ASTObject, n, e +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, ASTNode, ASTObject, n, e from graphistry.compute.chain_dag import ( extract_dependencies, build_dependency_graph, validate_dependencies, detect_cycles, determine_execution_order ) from graphistry.compute.execution_context import ExecutionContext +from graphistry.compute.exceptions import GFQLTypeError from graphistry.tests.test_compute import CGFull @@ -30,21 +31,21 @@ def test_extract_dependencies_no_deps(self): assert deps == set() def test_extract_dependencies_chain_ref(self): - """Test extracting dependencies from ASTChainRef""" - chain_ref = ASTChainRef('source', [n()]) + """Test extracting dependencies from ASTRef""" + chain_ref = ASTRef('source', [n()]) deps = extract_dependencies(chain_ref) assert deps == {'source'} def test_extract_dependencies_nested(self): """Test extracting dependencies from nested structures""" # ChainRef with ChainRef in its chain - nested = ASTChainRef('a', [ASTChainRef('b', [n()])]) + nested = ASTRef('a', [ASTRef('b', [n()])]) deps = extract_dependencies(nested) assert deps == {'a', 'b'} # Nested DAG dag = ASTLet({ - 'inner': ASTChainRef('outer', [n()]) + 'inner': ASTRef('outer', [n()]) }) deps = extract_dependencies(dag) assert deps == {'outer'} @@ -53,8 +54,8 @@ def test_build_dependency_graph(self): """Test building dependency and dependent mappings""" bindings = { 'a': n(), - 'b': ASTChainRef('a', [n()]), - 'c': ASTChainRef('b', [n()]) + 'b': ASTRef('a', [n()]), + 'c': ASTRef('b', [n()]) } dependencies, dependents = build_dependency_graph(bindings) @@ -73,7 +74,7 @@ def test_validate_dependencies_valid(self): """Test validation passes for valid dependencies""" bindings = { 'a': n(), - 'b': ASTChainRef('a', [n()]) + 'b': ASTRef('a', [n()]) } dependencies = {'a': set(), 'b': {'a'}} @@ -155,8 +156,8 @@ def test_determine_execution_order_linear(self): """Test execution order for linear dependencies""" bindings = { 'a': n(), - 'b': ASTChainRef('a', [n()]), - 'c': ASTChainRef('b', [n()]) + 'b': ASTRef('a', [n()]), + 'c': ASTRef('b', [n()]) } order = determine_execution_order(bindings) @@ -166,9 +167,9 @@ def test_determine_execution_order_diamond(self): """Test execution order for diamond pattern""" bindings = { 'top': n(), - 'left': ASTChainRef('top', [n()]), - 'right': ASTChainRef('top', [n()]), - 'bottom': ASTChainRef('left', [ASTChainRef('right', [n()])]) + 'left': ASTRef('top', [n()]), + 'right': ASTRef('top', [n()]), + 'bottom': ASTRef('left', [ASTRef('right', [n()])]) } order = determine_execution_order(bindings) @@ -182,9 +183,9 @@ def test_determine_execution_order_disconnected(self): """Test execution order for disconnected components""" bindings = { 'a1': n(), - 'a2': ASTChainRef('a1', [n()]), + 'a2': ASTRef('a1', [n()]), 'b1': n(), - 'b2': ASTChainRef('b1', [n()]) + 'b2': ASTRef('b1', [n()]) } order = determine_execution_order(bindings) @@ -219,15 +220,15 @@ def validate(self): # (we can't test this without implementing execution) def test_chain_ref_missing_reference(self): - """Test ASTChainRef with missing reference gives helpful error""" + """Test ASTRef with missing reference gives helpful error""" from graphistry.compute.chain_dag import execute_node from graphistry.Engine import Engine g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') context = ExecutionContext() - # Create ASTChainRef that references non-existent binding - chain_ref = ASTChainRef('missing_ref', []) + # Create ASTRef that references non-existent binding + chain_ref = ASTRef('missing_ref', []) # Should raise ValueError with helpful message with pytest.raises(ValueError) as exc_info: @@ -237,7 +238,7 @@ def test_chain_ref_missing_reference(self): assert "Available bindings: []" in str(exc_info.value) def test_chain_ref_with_existing_reference(self): - """Test ASTChainRef successfully resolves existing reference""" + """Test ASTRef successfully resolves existing reference""" from graphistry.compute.chain_dag import execute_node from graphistry.Engine import Engine @@ -247,8 +248,8 @@ def test_chain_ref_with_existing_reference(self): # Pre-populate context with a result context.set_binding('previous_result', g) - # Create ASTChainRef that references it (empty chain) - chain_ref = ASTChainRef('previous_result', []) + # Create ASTRef that references it (empty chain) + chain_ref = ASTRef('previous_result', []) # Should return the referenced result result = execute_node('test', chain_ref, g, context, Engine.PANDAS) @@ -271,8 +272,8 @@ def test_execution_order_verified(self): # Create a DAG with known dependencies dag = ASTLet({ 'data': ASTRemoteGraph('dataset'), - 'filtered': ASTChainRef('data', []), - 'analyzed': ASTChainRef('filtered', []) + 'filtered': ASTRef('data', []), + 'analyzed': ASTRef('filtered', []) }) # Get execution order @@ -285,9 +286,9 @@ def test_execution_order_verified(self): # Also test diamond pattern dag_diamond = ASTLet({ 'root': ASTRemoteGraph('data'), - 'left': ASTChainRef('root', []), - 'right': ASTChainRef('root', []), - 'merge': ASTChainRef('left', [ASTChainRef('right', [])]) + 'left': ASTRef('root', []), + 'right': ASTRef('root', []), + 'merge': ASTRef('left', [ASTRef('right', [])]) }) order_diamond = determine_execution_order(dag_diamond.bindings) @@ -296,13 +297,16 @@ def test_execution_order_verified(self): assert set(order_diamond[1:3]) == {'left', 'right'} def test_chain_ref_in_dag_execution(self): - """Test ASTChainRef works in DAG execution (fails on chain ops)""" + """Test ASTRef works in DAG execution (fails on chain ops)""" g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') # Create a simple mock that can be executed class MockExecutable(ASTObject): - def validate(self): + def _validate_fields(self): pass + + def _get_child_validators(self): + return [] def __call__(self, g, prev_node_wavefront, target_wave_front, engine): raise NotImplementedError("Mock execution") @@ -313,7 +317,7 @@ def reverse(self): # Create DAG with mock executable and chain ref dag = ASTLet({ 'first': MockExecutable(), - 'second': ASTChainRef('first', []) # Empty chain should work + 'second': ASTRef('first', []) # Empty chain should work }) # Try to execute - will fail on MockExecutable @@ -415,7 +419,7 @@ def test_node_edge_combination(self): dag = ASTLet({ 'people': n({'type': 'person'}), - 'from_people': ASTChainRef('people', [e()]), + 'from_people': ASTRef('people', [e()]), 'companies': n({'type': 'company'}) }) @@ -517,7 +521,7 @@ def test_dag_with_node_and_chainref(self): # DAG: filter people, then filter active from those dag = ASTLet({ 'people': n({'type': 'person'}), - 'active_people': ASTChainRef('people', [n({'active': True})]) + 'active_people': ASTRef('people', [n({'active': True})]) }) result = g.gfql(dag) @@ -526,8 +530,7 @@ def test_dag_with_node_and_chainref(self): assert len(result._nodes) == 1 assert result._nodes['id'].iloc[0] == 'a' assert result._nodes['type'].iloc[0] == 'person' - assert result._nodes['active'].iloc[0] # Just check truthiness - + assert result._nodes['active'].iloc[0] class TestErrorHandling: """Test error handling and edge cases""" @@ -540,8 +543,10 @@ def test_invalid_dag_type(self): g.gfql("not a dag") assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) - with pytest.raises(AssertionError) as exc_info: + # When passed a dict, gfql creates an ASTLet which validates + with pytest.raises(GFQLTypeError) as exc_info: g.gfql({'dict': 'not allowed'}) + assert exc_info.value.code == "type-mismatch" assert "binding value must be ASTObject" in str(exc_info.value) def test_node_execution_error_wrapped(self): @@ -563,9 +568,9 @@ def test_node_execution_error_wrapped(self): def test_cycle_detection_with_path(self): """Test cycle detection provides the cycle path""" dag = ASTLet({ - 'a': ASTChainRef('b', []), - 'b': ASTChainRef('c', []), - 'c': ASTChainRef('a', []) # Creates cycle a->b->c->a + 'a': ASTRef('b', []), + 'b': ASTRef('c', []), + 'c': ASTRef('a', []) # Creates cycle a->b->c->a }) g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') @@ -581,13 +586,13 @@ def test_complex_cycle_detection(self): # This DAG has no cycles, just complex dependencies bindings = { 'start': n(), - 'a': ASTChainRef('start', []), - 'b': ASTChainRef('a', []), - 'c': ASTChainRef('b', []), - 'd': ASTChainRef('c', []), - 'e': ASTChainRef('d', []), - 'f': ASTChainRef('b', []), # Second branch from b - 'g': ASTChainRef('f', []) # Note: removed nested ASTChainRef in chain + 'a': ASTRef('start', []), + 'b': ASTRef('a', []), + 'c': ASTRef('b', []), + 'd': ASTRef('c', []), + 'e': ASTRef('d', []), + 'f': ASTRef('b', []), # Second branch from b + 'g': ASTRef('f', []) # Note: removed nested ASTRef in chain } # Test cycle detection directly @@ -603,7 +608,7 @@ def test_missing_reference_with_suggestions(self): dag = ASTLet({ 'data1': n(), 'data2': n(), - 'result': ASTChainRef('data3', []) # data3 doesn't exist + 'result': ASTRef('data3', []) # data3 doesn't exist }) g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') @@ -681,7 +686,7 @@ def test_remote_graph_execution(self, mock_chain_remote): assert context.get_binding('remote_data') is mock_result def test_chain_ref_resolution_order(self): - """Test ASTChainRef resolves references in correct order""" + """Test ASTRef resolves references in correct order""" from graphistry.compute.chain_dag import execute_node from graphistry.Engine import Engine @@ -695,7 +700,7 @@ def test_chain_ref_resolution_order(self): context.set_binding('filtered_data', filtered) # Create chain ref that adds more filtering - chain_ref = ASTChainRef('filtered_data', [n({'id': 'b'})]) + chain_ref = ASTRef('filtered_data', [n({'id': 'b'})]) result = execute_node('final', chain_ref, g, context, Engine.PANDAS) # Should have only node 'b' @@ -714,7 +719,7 @@ def test_execution_context_isolation(self): # Second DAG execution should not see first's context dag2 = ASTLet({ 'node2': n(name='second'), - 'ref_fail': ASTChainRef('node1', []) # Should fail - node1 not in this context + 'ref_fail': ASTRef('node1', []) # Should fail - node1 not in this context }) with pytest.raises(ValueError) as exc_info: @@ -737,8 +742,8 @@ def test_execution_order_logging(self): g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') dag = ASTLet({ 'first': n(), - 'second': ASTChainRef('first', []), - 'third': ASTChainRef('second', []) + 'second': ASTRef('first', []), + 'third': ASTRef('second', []) }) g.gfql(dag) @@ -770,9 +775,9 @@ def test_diamond_pattern_execution(self): # Diamond: top -> (left, right) -> bottom dag = ASTLet({ 'top': n({'type': 'source'}), - 'left': ASTChainRef('top', [n(name='from_left')]), - 'right': ASTChainRef('top', [n(name='from_right')]), - 'bottom': ASTChainRef('left', []) + 'left': ASTRef('top', [n(name='from_left')]), + 'right': ASTRef('top', [n(name='from_right')]), + 'bottom': ASTRef('left', []) }) result = g.gfql(dag) @@ -781,8 +786,7 @@ def test_diamond_pattern_execution(self): assert len(result._nodes) == 1 assert result._nodes['type'].iloc[0] == 'source' assert 'from_left' in result._nodes.columns - assert result._nodes['from_left'].iloc[0] # Check truthiness - + assert result._nodes['from_left'].iloc[0] def test_multi_branch_convergence(self): """Test multiple branches converging""" g = CGFull().edges(pd.DataFrame({ @@ -824,8 +828,8 @@ def test_parallel_independent_branches(self): dag = ASTLet({ 'branch_a': n({'branch': 'A'}), 'branch_b': n({'branch': 'B'}), - 'a_subset': ASTChainRef('branch_a', [n(query="id in ['a', 'b']")]), - 'b_subset': ASTChainRef('branch_b', [n(query="id in ['e', 'f']")]) + 'a_subset': ASTRef('branch_a', [n(query="id in ['a', 'b']")]), + 'b_subset': ASTRef('branch_b', [n(query="id in ['e', 'f']")]) }) # Check execution order allows parallel execution @@ -852,7 +856,7 @@ def test_deep_dependency_chain(self): # Using empty chains to avoid execution issues dag_dict = {'n1': n(name='level1')} for i in range(2, 11): - dag_dict[f'n{i}'] = ASTChainRef(f'n{i - 1}', []) + dag_dict[f'n{i}'] = ASTRef(f'n{i - 1}', []) dag = ASTLet(dag_dict) @@ -883,9 +887,9 @@ def test_fan_out_fan_in_pattern(self): dag = ASTLet({ 'start': n({'id': 'root'}), - 'expand1': ASTChainRef('start', []), - 'expand2': ASTChainRef('start', []), - 'expand3': ASTChainRef('start', []), + 'expand1': ASTRef('start', []), + 'expand2': ASTRef('start', []), + 'expand3': ASTRef('start', []), 'collect': n() # Gets all nodes from original graph }) @@ -943,8 +947,8 @@ def test_large_dag_10_nodes(self): 'odd': n({'type': 'odd'}), # Layer 2: References - 'high_even': ASTChainRef('even', []), - 'high_odd': ASTChainRef('odd', []), + 'high_even': ASTRef('even', []), + 'high_odd': ASTRef('odd', []), # Layer 3: More nodes 'n1': n(name='tag1'), @@ -970,21 +974,27 @@ def test_large_dag_10_nodes(self): assert order.index('even') < order.index('high_even') assert order.index('odd') < order.index('high_odd') - def test_mock_remote_graph_placeholder(self): - """Test DAG with RemoteGraph requires authentication""" + @patch('graphistry.compute.chain_remote.chain_remote') + def test_mock_remote_graph_placeholder(self, mock_chain_remote): + """Test DAG with mock RemoteGraph""" g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + # Setup mock to return a simple graph + mock_result = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + mock_chain_remote.return_value = mock_result + dag = ASTLet({ 'remote1': ASTRemoteGraph('dataset1'), 'remote2': ASTRemoteGraph('dataset2', token='mock-token'), 'combined': n() # Would combine results }) - # Should raise error due to missing authentication - with pytest.raises(RuntimeError) as exc_info: - g.gfql(dag) + # Should execute successfully with mocked remote calls + result = g.gfql(dag) + assert result is not None - assert "Must call login() first" in str(exc_info.value) + # Verify chain_remote was called twice (once for each RemoteGraph) + assert mock_chain_remote.call_count == 2 def test_memory_efficient_execution(self): """Test that intermediate results are stored efficiently""" @@ -1058,10 +1068,10 @@ def test_execution_order_deterministic(self): dag = ASTLet({ 'a': n(), 'b': n(), - 'c': ASTChainRef('a', []), - 'd': ASTChainRef('b', []), - 'e': ASTChainRef('c', []), - 'f': ASTChainRef('d', []) + 'c': ASTRef('a', []), + 'd': ASTRef('b', []), + 'e': ASTRef('c', []), + 'f': ASTRef('d', []) }) # Get order multiple times @@ -1099,7 +1109,7 @@ def tracking_chain_dag_impl(g, dag, engine): dag = ASTLet({ 'step1': n(name='tag1'), 'step2': n(name='tag2'), - 'step3': ASTChainRef('step1', []) + 'step3': ASTRef('step1', []) }) result = g.gfql(dag) @@ -1187,19 +1197,24 @@ def test_chain_dag_single_node_works(self): assert result is not None assert len(result._nodes) == 2 # nodes a and b - def test_chain_dag_remote_not_implemented(self): - """Test chain_dag with RemoteGraph requires authentication""" + @patch('graphistry.compute.chain_remote.chain_remote') + def test_chain_dag_remote_not_implemented(self, mock_chain_remote): + """Test chain_dag with RemoteGraph works with mocked remote""" g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Setup mock + mock_result = CGFull().edges(pd.DataFrame({'s': ['remote1'], 'd': ['remote2']}), 's', 'd') + mock_chain_remote.return_value = mock_result + dag = ASTLet({ 'remote': ASTRemoteGraph('dataset123') }) - # Should raise RuntimeError due to missing authentication - with pytest.raises(RuntimeError) as exc_info: - g.gfql(dag) - - assert "Failed to execute node 'remote' in DAG" in str(exc_info.value) - assert "Must call login() first" in str(exc_info.value) + # Should work now with mocked chain_remote + result = g.gfql(dag) + assert result is not None + # Result should be the mocked remote graph + assert 'remote1' in result._edges['s'].values def test_chain_dag_multi_node_works(self): """Test chain_dag with multiple nodes now works""" diff --git a/graphistry/tests/compute/test_chain_dag_gpu.py b/graphistry/tests/compute/test_chain_dag_gpu.py index d397e54f83..c7db8592d1 100644 --- a/graphistry/tests/compute/test_chain_dag_gpu.py +++ b/graphistry/tests/compute/test_chain_dag_gpu.py @@ -2,7 +2,7 @@ import pytest import pandas as pd -from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, n +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, n from graphistry.compute.chain_dag import chain_dag_impl from graphistry.compute.execution_context import ExecutionContext from graphistry.tests.test_compute import CGFull @@ -131,7 +131,7 @@ def test_materialize_nodes_preserves_gpu(self): @skip_gpu def test_chain_ref_with_gpu_data(self): - """Test ASTChainRef resolution works with GPU data""" + """Test ASTRef resolution works with GPU data""" import cudf from graphistry.compute.chain_dag import execute_node from graphistry.compute.execution_context import ExecutionContext @@ -146,7 +146,7 @@ def test_chain_ref_with_gpu_data(self): context.set_binding('gpu_result', g) # Create chain ref to GPU data - chain_ref = ASTChainRef('gpu_result', []) + chain_ref = ASTRef('gpu_result', []) # Execute should preserve GPU result = execute_node('test', chain_ref, g, context, Engine.CUDF) diff --git a/graphistry/tests/compute/test_chain_dag_remote_integration.py b/graphistry/tests/compute/test_chain_dag_remote_integration.py index 09f1931642..3c862ece98 100644 --- a/graphistry/tests/compute/test_chain_dag_remote_integration.py +++ b/graphistry/tests/compute/test_chain_dag_remote_integration.py @@ -17,7 +17,7 @@ from unittest.mock import patch from graphistry import PyGraphistry -from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, n +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, n from graphistry.tests.test_compute import CGFull @@ -135,8 +135,8 @@ def test_remote_graph_in_complex_dag(self): # Create complex DAG with remote data dag = ASTLet({ 'remote': ASTRemoteGraph(dataset_id), - 'persons': ASTChainRef('remote', [n({'category': 'person'})]), - 'friends': ASTChainRef('persons', [n(edge_query="type == 'friend'")]) + 'persons': ASTRef('remote', [n({'category': 'person'})]), + 'friends': ASTRef('persons', [n(edge_query="type == 'friend'")]) }) # Execute and verify - need graph with edges for materialize_nodes @@ -200,9 +200,9 @@ def test_remote_graph_execution_mocked(self, mock_chain_remote): 'remote': ASTRemoteGraph('test-dataset-123', token='test-token') }) - # Mock result should be used, but we still need edges for materialize_nodes - g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') - result = g_base.gfql(dag) + # Need a graph with edges for bind() to work + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g.gfql(dag) assert result is not None # Verify result was returned # Verify chain_remote was called correctly diff --git a/graphistry/tests/compute/test_chain_schema_validation.py b/graphistry/tests/compute/test_chain_schema_validation.py index 0494d1ba15..caf58514ec 100644 --- a/graphistry/tests/compute/test_chain_schema_validation.py +++ b/graphistry/tests/compute/test_chain_schema_validation.py @@ -4,7 +4,7 @@ import pandas as pd from graphistry import edges, nodes from graphistry.compute.chain import Chain -from graphistry.compute.ast import n, e_forward, ASTLet, ASTChainRef, ASTRemoteGraph +from graphistry.compute.ast import n, e_forward, ASTLet, ASTRef, ASTRemoteGraph from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError from graphistry.compute.validate.validate_schema import validate_chain_schema @@ -193,7 +193,7 @@ def test_let_collect_all_errors(self): def test_chainref_valid_schema(self): """Valid ChainRef passes schema validation.""" - chain_ref = ASTChainRef('other_data', [ + chain_ref = ASTRef('other_data', [ n({'type': 'person'}), e_forward({'edge_type': 'friend'}) ]) @@ -204,7 +204,7 @@ def test_chainref_valid_schema(self): def test_chainref_invalid_chain_operation(self): """ChainRef with invalid chain operation fails.""" - chain_ref = ASTChainRef('other_data', [ + chain_ref = ASTRef('other_data', [ n({'missing_column': 'value'}) ]) @@ -217,7 +217,7 @@ def test_chainref_invalid_chain_operation(self): def test_chainref_empty_chain(self): """ChainRef with empty chain passes validation.""" - chain_ref = ASTChainRef('other_data', []) + chain_ref = ASTRef('other_data', []) # Should not raise errors = validate_chain_schema(self.g, [chain_ref], collect_all=True) diff --git a/graphistry/tests/compute/test_gfql.py b/graphistry/tests/compute/test_gfql.py index 9c689ef521..4a68c18755 100644 --- a/graphistry/tests/compute/test_gfql.py +++ b/graphistry/tests/compute/test_gfql.py @@ -1,6 +1,6 @@ import pandas as pd import pytest -from graphistry.compute.ast import ASTLet, ASTChainRef, n, e +from graphistry.compute.ast import ASTLet, ASTRef, n, e from graphistry.compute.chain import Chain from graphistry.tests.test_compute import CGFull diff --git a/graphistry/tests/compute/test_gfql_validation.py b/graphistry/tests/compute/test_gfql_validation.py index 982c546bf9..d04b5cd705 100644 --- a/graphistry/tests/compute/test_gfql_validation.py +++ b/graphistry/tests/compute/test_gfql_validation.py @@ -4,7 +4,7 @@ from graphistry import edges, nodes from graphistry.compute.ast import n, e_forward, e_reverse -from graphistry.compute.gfql_validation.validate import ( +from graphistry.compute.gfql import ( validate_syntax, validate_schema, validate_query, extract_schema_from_dataframes, format_validation_errors, Schema, ValidationIssue diff --git a/graphistry/tests/compute/test_let.py b/graphistry/tests/compute/test_let.py index 4acbb3ef20..5a26818671 100644 --- a/graphistry/tests/compute/test_let.py +++ b/graphistry/tests/compute/test_let.py @@ -1,7 +1,8 @@ """Tests for Let bindings and related AST nodes validation""" import pytest -from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, n, e +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTRef, n, e from graphistry.compute.execution_context import ExecutionContext +from graphistry.compute.exceptions import GFQLTypeError class TestLetValidation: @@ -14,15 +15,20 @@ def test_let_valid(self): def test_let_invalid_key_type(self): """Let with non-string key should fail""" - with pytest.raises(AssertionError, match="binding key must be string"): - dag = ASTLet({123: n()}) + # Note: This validation happens at runtime in _validate_fields + dag = ASTLet({123: n()}) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: dag.validate() + assert exc_info.value.code == "invalid-filter-key" + assert "binding key must be string" in str(exc_info.value) def test_let_invalid_value_type(self): """Let with non-ASTObject value should fail""" - with pytest.raises(AssertionError, match="binding value must be ASTObject"): - dag = ASTLet({'a': 'not an AST object'}) + dag = ASTLet({'a': 'not an AST object'}) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: dag.validate() + assert exc_info.value.code == "type-mismatch" + assert "binding value must be ASTObject" in str(exc_info.value) def test_let_nested_validation(self): """Let should validate nested objects""" @@ -47,21 +53,27 @@ def test_remoteGraph_valid(self): def test_remoteGraph_invalid_dataset_type(self): """RemoteGraph with non-string dataset_id should fail""" - with pytest.raises(AssertionError, match="dataset_id must be a string"): - rg = ASTRemoteGraph(123) + rg = ASTRemoteGraph(123) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: rg.validate() + assert exc_info.value.code == "type-mismatch" + assert "dataset_id must be a string" in str(exc_info.value) def test_remoteGraph_empty_dataset(self): """RemoteGraph with empty dataset_id should fail""" - with pytest.raises(AssertionError, match="dataset_id cannot be empty"): - rg = ASTRemoteGraph('') + rg = ASTRemoteGraph('') + with pytest.raises(GFQLTypeError) as exc_info: rg.validate() + assert exc_info.value.code == "empty-chain" + assert "dataset_id cannot be empty" in str(exc_info.value) def test_remoteGraph_invalid_token_type(self): """RemoteGraph with non-string token should fail""" - with pytest.raises(AssertionError, match="token must be string or None"): - rg = ASTRemoteGraph('dataset', token=123) + rg = ASTRemoteGraph('dataset', token=123) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: rg.validate() + assert exc_info.value.code == "type-mismatch" + assert "token must be string or None" in str(exc_info.value) class TestChainRefValidation: @@ -69,39 +81,47 @@ class TestChainRefValidation: def test_chainRef_valid(self): """Valid ChainRef should pass validation""" - cr = ASTChainRef('myref', [n(), e()]) + cr = ASTRef('myref', [n(), e()]) cr.validate() # Should not raise - cr_empty = ASTChainRef('myref', []) + cr_empty = ASTRef('myref', []) cr_empty.validate() # Empty chain is valid def test_chainRef_invalid_ref_type(self): """ChainRef with non-string ref should fail""" - with pytest.raises(AssertionError, match="ref must be a string"): - cr = ASTChainRef(123, []) + cr = ASTRef(123, []) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: cr.validate() + assert exc_info.value.code == "type-mismatch" + assert "ref must be a string" in str(exc_info.value) def test_chainRef_empty_ref(self): """ChainRef with empty ref should fail""" - with pytest.raises(AssertionError, match="ref cannot be empty"): - cr = ASTChainRef('', []) + cr = ASTRef('', []) + with pytest.raises(GFQLTypeError) as exc_info: cr.validate() + assert exc_info.value.code == "empty-chain" + assert "ref cannot be empty" in str(exc_info.value) def test_chainRef_invalid_chain_type(self): """ChainRef with non-list chain should fail""" - with pytest.raises(AssertionError, match="chain must be a list"): - cr = ASTChainRef('ref', 'not a list') + cr = ASTRef('ref', 'not a list') # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: cr.validate() + assert exc_info.value.code == "type-mismatch" + assert "chain must be a list" in str(exc_info.value) def test_chainRef_invalid_chain_element(self): """ChainRef with non-ASTObject in chain should fail""" - with pytest.raises(AssertionError, match="must be ASTObject"): - cr = ASTChainRef('ref', [n(), 'not an AST object']) + cr = ASTRef('ref', [n(), 'not an AST object']) # type: ignore + with pytest.raises(GFQLTypeError) as exc_info: cr.validate() + assert exc_info.value.code == "type-mismatch" + assert "must be ASTObject" in str(exc_info.value) def test_chainRef_nested_validation(self): """ChainRef should validate nested operations""" - cr = ASTChainRef('ref', [n({'type': 'person'}), e()]) + cr = ASTRef('ref', [n({'type': 'person'}), e()]) cr.validate() # Should validate nested nodes @@ -170,10 +190,10 @@ class TestChainRefReverse: def test_chainRef_reverse(self): """Test ChainRef reverse reverses operations""" - cr = ASTChainRef('data', [n(), e(), n()]) + cr = ASTRef('data', [n(), e(), n()]) reversed_cr = cr.reverse() - assert isinstance(reversed_cr, ASTChainRef) + assert isinstance(reversed_cr, ASTRef) assert reversed_cr.ref == 'data' assert len(reversed_cr.chain) == 3 # Operations should be reversed diff --git a/graphistry/tests/compute/test_validate.py b/graphistry/tests/compute/test_validate.py index a54c211597..073b5c6153 100644 --- a/graphistry/tests/compute/test_validate.py +++ b/graphistry/tests/compute/test_validate.py @@ -4,13 +4,14 @@ import pytest from typing import List -from graphistry.compute.gfql_validation.validate import ( +from graphistry.compute.gfql import ( validate_syntax, validate_schema, validate_query, extract_schema, extract_schema_from_dataframes, format_validation_errors, suggest_fixes, - ValidationIssue, Schema, - _format_error, _get_type_category + ValidationIssue, Schema ) +# Import internals directly from the module +from graphistry.compute.gfql.validate.validate import _format_error, _get_type_category from graphistry.compute.ast import n, e_forward, e_reverse, e from graphistry.compute.predicates.numeric import gt, lt, between from graphistry.compute.predicates.str import contains, startswith diff --git a/test_env/bin/Activate.ps1 b/test_env/bin/Activate.ps1 new file mode 100644 index 0000000000..2fb3852c3c --- /dev/null +++ b/test_env/bin/Activate.ps1 @@ -0,0 +1,241 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/test_env/bin/activate b/test_env/bin/activate new file mode 100644 index 0000000000..aa874798a2 --- /dev/null +++ b/test_env/bin/activate @@ -0,0 +1,76 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/home/lmeyerov/Work/pygraphistry/test_env" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + if [ "x(test_env) " != x ] ; then + PS1="(test_env) ${PS1:-}" + else + if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then + # special case for Aspen magic directories + # see https://aspen.io/ + PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" + else + PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" + fi + fi + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r +fi diff --git a/test_env/bin/activate.csh b/test_env/bin/activate.csh new file mode 100644 index 0000000000..bf0337ce66 --- /dev/null +++ b/test_env/bin/activate.csh @@ -0,0 +1,37 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/home/lmeyerov/Work/pygraphistry/test_env" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + if ("test_env" != "") then + set env_name = "test_env" + else + if (`basename "VIRTUAL_ENV"` == "__") then + # special case for Aspen magic directories + # see https://aspen.io/ + set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` + else + set env_name = `basename "$VIRTUAL_ENV"` + endif + endif + set prompt = "[$env_name] $prompt" + unset env_name +endif + +alias pydoc python -m pydoc + +rehash diff --git a/test_env/bin/activate.fish b/test_env/bin/activate.fish new file mode 100644 index 0000000000..4664a4f344 --- /dev/null +++ b/test_env/bin/activate.fish @@ -0,0 +1,75 @@ +# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) +# you cannot run it directly + +function deactivate -d "Exit virtualenv and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self destruct! + functions -e deactivate + end +end + +# unset irrelevant variables +deactivate nondestructive + +set -gx VIRTUAL_ENV "/home/lmeyerov/Work/pygraphistry/test_env" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# unset PYTHONHOME if set +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # save the current fish_prompt function as the function _old_fish_prompt + functions -c fish_prompt _old_fish_prompt + + # with the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command + set -l old_status $status + + # Prompt override? + if test -n "(test_env) " + printf "%s%s" "(test_env) " (set_color normal) + else + # ...Otherwise, prepend env + set -l _checkbase (basename "$VIRTUAL_ENV") + if test $_checkbase = "__" + # special case for Aspen magic directories + # see https://aspen.io/ + printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) + else + printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) + end + end + + # Restore the return status of the previous command. + echo "exit $old_status" | . + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/test_env/bin/dmypy b/test_env/bin/dmypy new file mode 100755 index 0000000000..f33c67e53f --- /dev/null +++ b/test_env/bin/dmypy @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from mypy.dmypy.client import console_entry +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(console_entry()) diff --git a/test_env/bin/f2py b/test_env/bin/f2py new file mode 100755 index 0000000000..804d1b479f --- /dev/null +++ b/test_env/bin/f2py @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/f2py3 b/test_env/bin/f2py3 new file mode 100755 index 0000000000..804d1b479f --- /dev/null +++ b/test_env/bin/f2py3 @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/f2py3.8 b/test_env/bin/f2py3.8 new file mode 100755 index 0000000000..804d1b479f --- /dev/null +++ b/test_env/bin/f2py3.8 @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/flake8 b/test_env/bin/flake8 new file mode 100755 index 0000000000..4ca64b8410 --- /dev/null +++ b/test_env/bin/flake8 @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from flake8.main.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/ipython b/test_env/bin/ipython new file mode 100755 index 0000000000..edb496de32 --- /dev/null +++ b/test_env/bin/ipython @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from IPython import start_ipython +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(start_ipython()) diff --git a/test_env/bin/ipython3 b/test_env/bin/ipython3 new file mode 100755 index 0000000000..edb496de32 --- /dev/null +++ b/test_env/bin/ipython3 @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from IPython import start_ipython +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(start_ipython()) diff --git a/test_env/bin/mypy b/test_env/bin/mypy new file mode 100755 index 0000000000..5c81dc519b --- /dev/null +++ b/test_env/bin/mypy @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from mypy.__main__ import console_entry +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(console_entry()) diff --git a/test_env/bin/mypyc b/test_env/bin/mypyc new file mode 100755 index 0000000000..85a7169dbb --- /dev/null +++ b/test_env/bin/mypyc @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from mypyc.__main__ import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/normalizer b/test_env/bin/normalizer new file mode 100755 index 0000000000..97621f7136 --- /dev/null +++ b/test_env/bin/normalizer @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from charset_normalizer import cli +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli.cli_detect()) diff --git a/test_env/bin/pip b/test_env/bin/pip new file mode 100755 index 0000000000..75f5f0bf68 --- /dev/null +++ b/test_env/bin/pip @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/pip3 b/test_env/bin/pip3 new file mode 100755 index 0000000000..75f5f0bf68 --- /dev/null +++ b/test_env/bin/pip3 @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/pip3.8 b/test_env/bin/pip3.8 new file mode 100755 index 0000000000..75f5f0bf68 --- /dev/null +++ b/test_env/bin/pip3.8 @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/py.test b/test_env/bin/py.test new file mode 100755 index 0000000000..82b67974b6 --- /dev/null +++ b/test_env/bin/py.test @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pytest import console_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(console_main()) diff --git a/test_env/bin/pycodestyle b/test_env/bin/pycodestyle new file mode 100755 index 0000000000..8bcdd0a2cc --- /dev/null +++ b/test_env/bin/pycodestyle @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pycodestyle import _main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(_main()) diff --git a/test_env/bin/pyflakes b/test_env/bin/pyflakes new file mode 100755 index 0000000000..3c733cf4a2 --- /dev/null +++ b/test_env/bin/pyflakes @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pyflakes.api import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/pygmentize b/test_env/bin/pygmentize new file mode 100755 index 0000000000..99e7324397 --- /dev/null +++ b/test_env/bin/pygmentize @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pygments.cmdline import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/pytest b/test_env/bin/pytest new file mode 100755 index 0000000000..82b67974b6 --- /dev/null +++ b/test_env/bin/pytest @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pytest import console_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(console_main()) diff --git a/test_env/bin/python b/test_env/bin/python new file mode 120000 index 0000000000..6d82f132f2 --- /dev/null +++ b/test_env/bin/python @@ -0,0 +1 @@ +/home/lmeyerov/miniconda3/bin/python \ No newline at end of file diff --git a/test_env/bin/python3 b/test_env/bin/python3 new file mode 120000 index 0000000000..d8654aa0e2 --- /dev/null +++ b/test_env/bin/python3 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/test_env/bin/stubgen b/test_env/bin/stubgen new file mode 100755 index 0000000000..75bf9d63e5 --- /dev/null +++ b/test_env/bin/stubgen @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from mypy.stubgen import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/bin/stubtest b/test_env/bin/stubtest new file mode 100755 index 0000000000..ee238bc746 --- /dev/null +++ b/test_env/bin/stubtest @@ -0,0 +1,8 @@ +#!/home/lmeyerov/Work/pygraphistry/test_env/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from mypy.stubtest import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/test_env/lib64 b/test_env/lib64 new file mode 120000 index 0000000000..7951405f85 --- /dev/null +++ b/test_env/lib64 @@ -0,0 +1 @@ +lib \ No newline at end of file diff --git a/test_env/pyvenv.cfg b/test_env/pyvenv.cfg new file mode 100644 index 0000000000..13d481180b --- /dev/null +++ b/test_env/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /home/lmeyerov/miniconda3/bin +include-system-site-packages = false +version = 3.8.20 diff --git a/test_env/share/man/man1/ipython.1 b/test_env/share/man/man1/ipython.1 new file mode 100644 index 0000000000..0f4a191f3f --- /dev/null +++ b/test_env/share/man/man1/ipython.1 @@ -0,0 +1,60 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH IPYTHON 1 "July 15, 2011" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) and groff_man(7) +.\" .SH section heading +.\" .SS secondary section heading +.\" +.\" +.\" To preview this page as plain text: nroff -man ipython.1 +.\" +.SH NAME +ipython \- Tools for Interactive Computing in Python. +.SH SYNOPSIS +.B ipython +.RI [ options ] " files" ... + +.B ipython subcommand +.RI [ options ] ... + +.SH DESCRIPTION +An interactive Python shell with automatic history (input and output), dynamic +object introspection, easier configuration, command completion, access to the +system shell, integration with numerical and scientific computing tools, +web notebook, Qt console, and more. + +For more information on how to use IPython, see 'ipython \-\-help', +or 'ipython \-\-help\-all' for all available command\(hyline options. + +.SH "ENVIRONMENT VARIABLES" +.sp +.PP +\fIIPYTHONDIR\fR +.RS 4 +This is the location where IPython stores all its configuration files. The default +is $HOME/.ipython if IPYTHONDIR is not defined. + +You can see the computed value of IPYTHONDIR with `ipython locate`. + +.SH FILES + +IPython uses various configuration files stored in profiles within IPYTHONDIR. +To generate the default configuration files and start configuring IPython, +do 'ipython profile create', and edit '*_config.py' files located in +IPYTHONDIR/profile_default. + +.SH AUTHORS +IPython is written by the IPython Development Team .