diff --git a/.gitignore b/.gitignore index 1c7cd046f7..f01581d693 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ docs/source/demos # AI assistant working directories AI_PROGRESS/ PLAN.md +test_env/ diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py index 2b5ffd7710..a31b5fae50 100644 --- a/graphistry/compute/ComputeMixin.py +++ b/graphistry/compute/ComputeMixin.py @@ -6,6 +6,8 @@ from graphistry.Plottable import Plottable from graphistry.util import setup_logger from .chain import chain as chain_base +from .chain_let import chain_let as chain_let_base +from .gfql import gfql as gfql_base from .chain_remote import chain_remote as chain_remote_base, chain_remote_shape as chain_remote_shape_base from .python_remote import ( python_remote_g as python_remote_g_base, @@ -460,8 +462,26 @@ def filter_edges_by_dict(self, *args, **kwargs): filter_edges_by_dict.__doc__ = filter_edges_by_dict_base.__doc__ def chain(self, *args, **kwargs): + """ + .. deprecated:: 2.XX.X + Use :meth:`gfql` instead for a unified API that supports both chains and DAGs. + """ + import warnings + warnings.warn( + "chain() is deprecated. Use gfql() instead for a unified API.", + DeprecationWarning, + stacklevel=2 + ) return chain_base(self, *args, **kwargs) - chain.__doc__ = chain_base.__doc__ + # Preserve original docstring after deprecation notice + chain.__doc__ = (chain.__doc__ or "") + "\n\n" + (chain_base.__doc__ or "") + + # chain_let removed from public API - use gfql() instead + # (chain_let_base still available internally for gfql dispatch) + + def gfql(self, *args, **kwargs): + return gfql_base(self, *args, **kwargs) + gfql.__doc__ = gfql_base.__doc__ def chain_remote(self, *args, **kwargs) -> Plottable: return chain_remote_base(self, *args, **kwargs) diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 7c9f36b1d0..34da105be5 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -1,6 +1,7 @@ from .ComputeMixin import ComputeMixin from .ast import ( - n, e, e_forward, e_reverse, e_undirected + n, e, e_forward, e_reverse, e_undirected, + let, remote, ref, call ) from .chain import Chain from .predicates.is_in import ( diff --git a/graphistry/compute/ast.py b/graphistry/compute/ast.py index f58c744e49..438082d91c 100644 --- a/graphistry/compute/ast.py +++ b/graphistry/compute/ast.py @@ -1,7 +1,10 @@ from abc import abstractmethod import logging -from typing import Any, TYPE_CHECKING, Dict, Optional, Union, cast +from typing import Any, TYPE_CHECKING, Dict, List, Optional, Union, cast from typing_extensions import Literal + +if TYPE_CHECKING: + from graphistry.compute.exceptions import GFQLValidationError import pandas as pd from graphistry.Engine import Engine @@ -642,9 +645,300 @@ def from_json(cls, d: dict, validate: bool = True) -> 'ASTEdge': e_undirected = ASTEdgeUndirected # noqa: E305 e = ASTEdgeUndirected # noqa: E305 + +############################################################################## + + +class ASTLet(ASTObject): + """Let bindings for named graph operations""" + def __init__(self, bindings: Dict[str, ASTObject]): + super().__init__() + self.bindings = bindings + + def validate(self, collect_all: bool = False) -> Optional[List['GFQLValidationError']]: + assert isinstance(self.bindings, dict), "bindings must be a dictionary" + for k, v in self.bindings.items(): + assert isinstance(k, str), f"binding key must be string, got {type(k)}" + assert isinstance(v, ASTObject), f"binding value must be ASTObject, got {type(v)}" + v.validate() + # TODO: Check for cycles in DAG + return None + + def to_json(self, validate=True) -> dict: + if validate: + self.validate() + return { + 'type': 'Let', + 'bindings': {k: v.to_json() for k, v in self.bindings.items()} + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTLet': + assert 'bindings' in d, "Let missing bindings" + bindings = {k: cast(ASTObject, from_json(v, validate=validate)) for k, v in d['bindings'].items()} + out = cls(bindings=bindings) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + # Implementation in PR 1.2 + raise NotImplementedError("Let execution will be implemented in PR 1.2") + + def reverse(self) -> 'ASTLet': + raise NotImplementedError("Let reversal not supported") + + +class ASTRemoteGraph(ASTObject): + """Load a graph from Graphistry server""" + def __init__(self, dataset_id: str, token: Optional[str] = None): + super().__init__() + self.dataset_id = dataset_id + self.token = token + + def validate(self, collect_all: bool = False) -> Optional[List['GFQLValidationError']]: + assert isinstance(self.dataset_id, str), "dataset_id must be a string" + assert len(self.dataset_id) > 0, "dataset_id cannot be empty" + assert self.token is None or isinstance(self.token, str), "token must be string or None" + return None + + def to_json(self, validate=True) -> dict: + if validate: + self.validate() + result = { + 'type': 'RemoteGraph', + 'dataset_id': self.dataset_id + } + if self.token is not None: + result['token'] = self.token + return result + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTRemoteGraph': + assert 'dataset_id' in d, "RemoteGraph missing dataset_id" + out = cls( + dataset_id=d['dataset_id'], + token=d.get('token') + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + """Load remote graph dataset. + + This operation loads a graph from the Graphistry server by dataset_id. + The loaded graph replaces the current graph context. + + Args: + g: Current graph (ignored - remote graph becomes new context) + prev_node_wavefront: Previous node wavefront (unused) + target_wave_front: Target wavefront (unused) + engine: Execution engine (preserved for loaded graph) + + Returns: + New Plottable with remote graph data + + Raises: + NotImplementedError: Remote graph loading not yet implemented + """ + # TODO: Implement remote graph loading via Graphistry API + # This would involve: + # 1. Authenticate using self.token if provided + # 2. Fetch graph data from server using self.dataset_id + # 3. Return new Plottable with loaded edges/nodes + raise NotImplementedError( + "RemoteGraph loading is not yet implemented. " + "This requires server-side dataset loading functionality." + ) + + def reverse(self) -> 'ASTRemoteGraph': + raise NotImplementedError("RemoteGraph reversal not supported") + + +class ASTChainRef(ASTObject): + """Execute a chain with reference to a DAG binding""" + def __init__(self, ref: str, chain: List[ASTObject]): + super().__init__() + self.ref = ref + self.chain = chain + + def validate(self, collect_all: bool = False) -> Optional[List['GFQLValidationError']]: + assert isinstance(self.ref, str), "ref must be a string" + assert len(self.ref) > 0, "ref cannot be empty" + assert isinstance(self.chain, list), "chain must be a list" + for i, op in enumerate(self.chain): + assert isinstance(op, ASTObject), f"chain[{i}] must be ASTObject, got {type(op)}" + op.validate() + return None + + def to_json(self, validate=True) -> dict: + if validate: + self.validate() + return { + 'type': 'ChainRef', + 'ref': self.ref, + 'chain': [op.to_json() for op in self.chain] + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTChainRef': + assert 'ref' in d, "ChainRef missing ref" + assert 'chain' in d, "ChainRef missing chain" + out = cls( + ref=d['ref'], + chain=[from_json(op, validate=validate) for op in d['chain']] + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + """Execute chain operations with reference to a DAG binding. + + ASTChainRef cannot be executed directly as it requires access to the + let bindings context to resolve the reference. This method should only + be called from within chain_let_impl where the binding context is available. + + Args: + g: Graph to operate on + prev_node_wavefront: Previous node wavefront + target_wave_front: Target wavefront + engine: Execution engine + + Returns: + New Plottable after executing chain operations + + Raises: + NotImplementedError: ChainRef requires let bindings context + """ + raise NotImplementedError( + "ASTChainRef cannot be executed directly. " + "It must be used within an ASTLet/chain_let() context where " + "the binding reference can be resolved." + ) + + def reverse(self) -> 'ASTChainRef': + # Reverse the chain operations + return ASTChainRef(self.ref, [op.reverse() for op in reversed(self.chain)]) + + +class ASTCall(ASTObject): + """Call a method on the current graph with validated parameters. + + Allows safe execution of Plottable methods through GFQL with parameter + validation and schema checking. + + Attributes: + function: Name of the method to call (must be in safelist) + params: Dictionary of parameters to pass to the method + """ + def __init__(self, function: str, params: Optional[Dict[str, Any]] = None): + """Initialize a Call operation. + + Args: + function: Name of the Plottable method to call + params: Optional dictionary of parameters for the method + """ + super().__init__() + self.function = function + self.params = params or {} + + def _validate_fields(self) -> None: + """Validate Call fields.""" + from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + if not isinstance(self.function, str): + raise GFQLTypeError( + ErrorCode.E201, + "function must be a string", + field="function", + value=type(self.function).__name__ + ) + + if len(self.function) == 0: + raise GFQLTypeError( + ErrorCode.E106, + "function name cannot be empty", + field="function", + value=self.function + ) + + if not isinstance(self.params, dict): + raise GFQLTypeError( + ErrorCode.E201, + "params must be a dictionary", + field="params", + value=type(self.params).__name__ + ) + + def to_json(self, validate=True) -> dict: + """Convert Call to JSON representation. + + Args: + validate: If True, validate before serialization + + Returns: + Dictionary with type, function, and params fields + """ + if validate: + self.validate() + return { + 'type': 'Call', + 'function': self.function, + 'params': self.params + } + + @classmethod + def from_json(cls, d: dict, validate: bool = True) -> 'ASTCall': + assert 'function' in d, "Call missing function" + out = cls( + function=d['function'], + params=d.get('params', {}) + ) + if validate: + out.validate() + return out + + def __call__(self, g: Plottable, prev_node_wavefront: Optional[DataFrameT], + target_wave_front: Optional[DataFrameT], engine: Engine) -> Plottable: + """Execute the method call on the graph. + + Args: + g: Graph to operate on + prev_node_wavefront: Previous node wavefront (unused) + target_wave_front: Target wavefront (unused) + engine: Execution engine (pandas/cudf) + + Returns: + New Plottable with method results + + Raises: + GFQLTypeError: If method not in safelist or parameters invalid + """ + # For let bindings, we don't use wavefronts, just execute the call + from graphistry.compute.call_executor import execute_call + return execute_call(g, self.function, self.params, engine) + + def reverse(self) -> 'ASTCall': + """Reverse is not supported for Call operations. + + Most Plottable methods are not reversible as they perform + transformations that cannot be undone. + + Raises: + NotImplementedError: Always raised as calls cannot be reversed + """ + # Most method calls cannot be reversed + raise NotImplementedError(f"Method '{self.function}' cannot be reversed") + + ### -def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge]: +def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTChainRef, ASTCall]: from graphistry.compute.exceptions import ErrorCode, GFQLSyntaxError if not isinstance(o, dict): @@ -652,10 +946,10 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge]: if 'type' not in o: raise GFQLSyntaxError( - ErrorCode.E105, "AST JSON missing required 'type' field", suggestion="Add 'type' field: 'Node' or 'Edge'" + ErrorCode.E105, "AST JSON missing required 'type' field", suggestion="Add 'type' field: 'Node', 'Edge', 'QueryDAG', 'RemoteGraph', or 'ChainRef'" ) - out: Union[ASTNode, ASTEdge] + out: Union[ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTChainRef, ASTCall] if o['type'] == 'Node': out = ASTNode.from_json(o, validate=validate) elif o['type'] == 'Edge': @@ -680,12 +974,30 @@ def from_json(o: JSONVal, validate: bool = True) -> Union[ASTNode, ASTEdge]: "Edge missing required 'direction' field", suggestion="Add 'direction' field: 'forward', 'reverse', or 'undirected'", ) + elif o['type'] == 'QueryDAG' or o['type'] == 'Let': + # Support both types for backward compatibility + out = ASTLet.from_json(o, validate=validate) + elif o['type'] == 'RemoteGraph': + out = ASTRemoteGraph.from_json(o, validate=validate) + elif o['type'] == 'ChainRef': + out = ASTChainRef.from_json(o, validate=validate) + elif o['type'] == 'Call': + out = ASTCall.from_json(o, validate=validate) else: raise GFQLSyntaxError( ErrorCode.E101, f"Unknown AST type: {o['type']}", field="type", value=o["type"], - suggestion="Use 'Node' or 'Edge'", + suggestion="Use 'Node', 'Edge', 'Let', 'RemoteGraph', 'ChainRef', or 'Call'", ) return out + + +############################################################################### +# User-friendly aliases for public API + +let = ASTLet # noqa: E305 +remote = ASTRemoteGraph # noqa: E305 +ref = ASTChainRef # noqa: E305 +call = ASTCall # noqa: E305 diff --git a/graphistry/compute/call_executor.py b/graphistry/compute/call_executor.py new file mode 100644 index 0000000000..67ac04e031 --- /dev/null +++ b/graphistry/compute/call_executor.py @@ -0,0 +1,83 @@ +"""Execute validated method calls on Plottable objects. + +This module provides the execution layer for GFQL Call operations after +parameter validation has been performed by the safelist module. +""" + +from typing import Dict, Any + +from graphistry.Plottable import Plottable +from graphistry.Engine import Engine +from graphistry.compute.call_safelist import validate_call_params +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +def execute_call(g: Plottable, function: str, params: Dict[str, Any], engine: Engine) -> Plottable: + """Execute a validated method call on a Plottable. + + Args: + g: The graph to call the method on + function: Name of the method to call + params: Parameters for the method (will be validated) + engine: Execution engine + + Returns: + Result of the method call (usually a new Plottable) + + Raises: + GFQLTypeError: If validation fails or method doesn't exist + AttributeError: If method doesn't exist on Plottable + """ + # Validate parameters against safelist + validated_params = validate_call_params(function, params) + + # Check if method exists on Plottable + if not hasattr(g, function): + raise AttributeError( + f"Plottable has no method '{function}'. " + f"This should not happen if safelist is properly configured." + ) + + # Get the method + method = getattr(g, function) + + # Special handling for methods that need the engine parameter + if function in ['materialize_nodes', 'hop']: + # These methods accept an engine parameter + if 'engine' not in validated_params: + # Add current engine if not specified + validated_params['engine'] = engine + + try: + # Execute the method with validated parameters + result = method(**validated_params) + + # Ensure result is a Plottable (most methods return self or new Plottable) + if not isinstance(result, Plottable): + raise GFQLTypeError( + ErrorCode.E201, + f"Method '{function}' returned non-Plottable result", + field="function", + value=f"{type(result).__name__}", + suggestion="Only methods that return Plottable objects are allowed" + ) + + return result + + except TypeError as e: + # Handle parameter mismatch errors + raise GFQLTypeError( + ErrorCode.E201, + f"Parameter error calling '{function}': {str(e)}", + field="params", + value=validated_params, + suggestion="Check parameter names and types" + ) from e + except Exception as e: + # Re-raise other exceptions with context + raise GFQLTypeError( + ErrorCode.E303, + f"Error executing '{function}': {str(e)}", + field="function", + value=function + ) from e diff --git a/graphistry/compute/call_safelist.py b/graphistry/compute/call_safelist.py new file mode 100644 index 0000000000..3b3ae3d304 --- /dev/null +++ b/graphistry/compute/call_safelist.py @@ -0,0 +1,519 @@ +"""Safelist of allowed methods for GFQL Call operations. + +This module defines which Plottable methods can be called through GFQL +and their parameter validation rules. +""" + +from typing import Dict, Any, Set, Optional, Union, Type +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +# Type validators +def is_string(v: Any) -> bool: + """Check if value is a string. + + Args: + v: Value to check + + Returns: + True if v is a string, False otherwise + """ + return isinstance(v, str) + + +def is_int(v: Any) -> bool: + """Check if value is an integer. + + Args: + v: Value to check + + Returns: + True if v is an integer, False otherwise + """ + return isinstance(v, int) + + +def is_bool(v: Any) -> bool: + """Check if value is a boolean. + + Args: + v: Value to check + + Returns: + True if v is a boolean, False otherwise + """ + return isinstance(v, bool) + + +def is_dict(v: Any) -> bool: + """Check if value is a dictionary. + + Args: + v: Value to check + + Returns: + True if v is a dictionary, False otherwise + """ + return isinstance(v, dict) + + +def is_string_or_none(v: Any) -> bool: + """Check if value is a string or None. + + Args: + v: Value to check + + Returns: + True if v is a string or None, False otherwise + """ + return v is None or isinstance(v, str) + + +def is_list_of_strings(v: Any) -> bool: + """Check if value is a list of strings. + + Args: + v: Value to check + + Returns: + True if v is a list containing only strings, False otherwise + """ + return isinstance(v, list) and all(isinstance(item, str) for item in v) + + +# Safelist configuration +# Each entry defines: +# - allowed_params: Set of parameter names that can be passed +# - required_params: Set of parameters that must be provided +# - param_validators: Dict of param_name -> validator function +# - description: Human-readable description of what the method does +# - schema_effects: Dict describing columns added/removed/required +# - adds_node_cols: List of columns added to nodes +# - adds_edge_cols: List of columns added to edges +# - requires_node_cols: List of node columns that must exist +# - requires_edge_cols: List of edge columns that must exist + +SAFELIST_V1: Dict[str, Dict[str, Any]] = { + 'get_degrees': { + 'allowed_params': {'col_in', 'col_out', 'col'}, + 'required_params': set(), + 'param_validators': { + 'col_in': is_string, + 'col_out': is_string, + 'col': is_string + }, + 'description': 'Calculate node degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [ + p.get('col', 'degree'), + p.get('col_in', 'degree_in'), + p.get('col_out', 'degree_out') + ], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'filter_nodes_by_dict': { + 'allowed_params': {'filter_dict'}, + 'required_params': {'filter_dict'}, + 'param_validators': { + 'filter_dict': is_dict + }, + 'description': 'Filter nodes by attribute values', + 'schema_effects': { + 'adds_node_cols': [], + 'adds_edge_cols': [], + 'requires_node_cols': lambda p: list(p.get('filter_dict', {}).keys()), + 'requires_edge_cols': [] + } + }, + + 'filter_edges_by_dict': { + 'allowed_params': {'filter_dict'}, + 'required_params': {'filter_dict'}, + 'param_validators': { + 'filter_dict': is_dict + }, + 'description': 'Filter edges by attribute values', + 'schema_effects': { + 'adds_node_cols': [], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': lambda p: list(p.get('filter_dict', {}).keys()) + } + }, + + 'materialize_nodes': { + 'allowed_params': {'engine', 'reuse'}, + 'required_params': set(), + 'param_validators': { + 'engine': is_string, + 'reuse': is_bool + }, + 'description': 'Generate node table from edges', + 'schema_effects': { + 'adds_node_cols': ['node'], # Creates node column + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'hop': { + 'allowed_params': { + 'nodes', 'hops', 'to_fixed_point', 'direction', + 'source_node_match', 'edge_match', 'destination_node_match', + 'source_node_query', 'edge_query', 'destination_node_query', + 'return_as_wave_front', 'target_wave_front', 'engine' + }, + 'required_params': set(), + 'param_validators': { + 'hops': is_int, + 'to_fixed_point': is_bool, + 'direction': lambda v: v in ['forward', 'reverse', 'undirected'], + 'source_node_match': is_dict, + 'edge_match': is_dict, + 'destination_node_match': is_dict, + 'source_node_query': is_string, + 'edge_query': is_string, + 'destination_node_query': is_string, + 'return_as_wave_front': is_bool, + 'engine': is_string + }, + 'description': 'Traverse graph by following edges', + 'schema_effects': { + 'adds_node_cols': [], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + # In/out degree methods + 'get_indegrees': { + 'allowed_params': {'col'}, + 'required_params': set(), + 'param_validators': { + 'col': is_string + }, + 'description': 'Calculate node in-degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('col', 'degree_in')], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'get_outdegrees': { + 'allowed_params': {'col'}, + 'required_params': set(), + 'param_validators': { + 'col': is_string + }, + 'description': 'Calculate node out-degrees', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('col', 'degree_out')], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + # Graph algorithm operations + 'compute_cugraph': { + 'allowed_params': {'alg', 'out_col', 'params', 'kind', 'directed', 'G'}, + 'required_params': {'alg'}, + 'param_validators': { + 'alg': is_string, + 'out_col': is_string_or_none, + 'params': is_dict, + 'kind': is_string, + 'directed': is_bool, + 'G': lambda x: x is None # Allow None only + }, + 'description': 'Run cuGraph algorithms (pagerank, louvain, etc)', + 'schema_effects': { + 'adds_node_cols': lambda p: [p.get('out_col', p['alg'])], + 'adds_edge_cols': [], + 'requires_node_cols': [], + 'requires_edge_cols': [] + } + }, + + 'compute_igraph': { + 'allowed_params': {'alg', 'out_col', 'directed', 'use_vids', 'params'}, + 'required_params': {'alg'}, + 'param_validators': { + 'alg': is_string, + 'out_col': is_string_or_none, + 'directed': is_bool, + 'use_vids': is_bool, + 'params': is_dict + }, + 'description': 'Run igraph algorithms' + }, + + # Layout operations + 'layout_cugraph': { + 'allowed_params': {'layout', 'params', 'kind', 'directed', 'G', 'bind_position', 'x_out_col', 'y_out_col', 'play'}, + 'required_params': set(), + 'param_validators': { + 'layout': is_string, + 'params': is_dict, + 'kind': is_string, + 'directed': is_bool, + 'G': lambda x: x is None, + 'bind_position': is_bool, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'play': is_int + }, + 'description': 'GPU-accelerated graph layouts' + }, + + 'layout_igraph': { + 'allowed_params': {'layout', 'directed', 'use_vids', 'bind_position', 'x_out_col', 'y_out_col', 'params', 'play'}, + 'required_params': {'layout'}, + 'param_validators': { + 'layout': is_string, + 'directed': is_bool, + 'use_vids': is_bool, + 'bind_position': is_bool, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'params': is_dict, + 'play': is_int + }, + 'description': 'igraph-based layouts' + }, + + 'layout_graphviz': { + 'allowed_params': { + 'prog', 'args', 'directed', 'strict', 'graph_attr', + 'node_attr', 'edge_attr', 'x_out_col', 'y_out_col', 'bind_position' + }, + 'required_params': set(), + 'param_validators': { + 'prog': is_string, + 'args': is_string_or_none, + 'directed': is_bool, + 'strict': is_bool, + 'graph_attr': is_dict, + 'node_attr': is_dict, + 'edge_attr': is_dict, + 'x_out_col': is_string, + 'y_out_col': is_string, + 'bind_position': is_bool + }, + 'description': 'Graphviz layouts (dot, neato, etc)' + }, + + 'fa2_layout': { + 'allowed_params': {'fa2_params', 'circle_layout_params', 'partition_key', 'remove_self_edges', 'engine', 'featurize'}, + 'required_params': set(), + 'param_validators': { + 'fa2_params': is_dict, + 'circle_layout_params': is_dict, + 'partition_key': is_string_or_none, + 'remove_self_edges': is_bool, + 'engine': is_string, + 'featurize': is_dict + }, + 'description': 'ForceAtlas2 layout algorithm' + }, + + # Self-edge pruning + 'prune_self_edges': { + 'allowed_params': set(), + 'required_params': set(), + 'param_validators': {}, + 'description': 'Remove self-loops from graph' + }, + + # Graph transformations + 'collapse': { + 'allowed_params': {'node', 'attribute', 'column', 'self_edges', 'unwrap', 'verbose'}, + 'required_params': set(), + 'param_validators': { + 'node': is_string_or_none, + 'attribute': is_string_or_none, + 'column': is_string_or_none, + 'self_edges': is_bool, + 'unwrap': is_bool, + 'verbose': is_bool + }, + 'description': 'Collapse nodes by shared attribute values' + }, + + 'drop_nodes': { + 'allowed_params': {'nodes'}, + 'required_params': {'nodes'}, + 'param_validators': { + 'nodes': lambda v: isinstance(v, list) or is_dict(v) + }, + 'description': 'Remove specified nodes and their edges' + }, + + 'keep_nodes': { + 'allowed_params': {'nodes'}, + 'required_params': {'nodes'}, + 'param_validators': { + 'nodes': lambda v: isinstance(v, list) or is_dict(v) + }, + 'description': 'Keep only specified nodes and their edges' + }, + + # Topology analysis + 'get_topological_levels': { + 'allowed_params': {'level_col', 'allow_cycles', 'warn_cycles', 'remove_self_loops'}, + 'required_params': set(), + 'param_validators': { + 'level_col': is_string, + 'allow_cycles': is_bool, + 'warn_cycles': is_bool, + 'remove_self_loops': is_bool + }, + 'description': 'Compute topological levels for DAG analysis' + }, + + # Visual encoding methods + 'encode_point_color': { + 'allowed_params': {'column', 'palette', 'as_categorical', 'as_continuous', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'palette': lambda v: isinstance(v, list), + 'as_categorical': is_bool, + 'as_continuous': is_bool, + 'categorical_mapping': is_dict, + 'default_mapping': is_string_or_none + }, + 'description': 'Map node column values to colors' + }, + + 'encode_edge_color': { + 'allowed_params': {'column', 'palette', 'as_categorical', 'as_continuous', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'palette': lambda v: isinstance(v, list), + 'as_categorical': is_bool, + 'as_continuous': is_bool, + 'categorical_mapping': is_dict, + 'default_mapping': is_string_or_none + }, + 'description': 'Map edge column values to colors' + }, + + 'encode_point_size': { + 'allowed_params': {'column', 'categorical_mapping', 'default_mapping'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'categorical_mapping': is_dict, + 'default_mapping': lambda v: isinstance(v, (int, float)) + }, + 'description': 'Map node column values to sizes' + }, + + 'encode_point_icon': { + 'allowed_params': {'column', 'categorical_mapping', 'continuous_binning', 'default_mapping', 'as_text'}, + 'required_params': {'column'}, + 'param_validators': { + 'column': is_string, + 'categorical_mapping': is_dict, + 'continuous_binning': lambda v: isinstance(v, list), + 'default_mapping': is_string_or_none, + 'as_text': is_bool + }, + 'description': 'Map node column values to icons' + }, + + # Metadata methods + 'name': { + 'allowed_params': {'name'}, + 'required_params': {'name'}, + 'param_validators': { + 'name': is_string + }, + 'description': 'Set visualization name' + }, + + 'description': { + 'allowed_params': {'description'}, + 'required_params': {'description'}, + 'param_validators': { + 'description': is_string + }, + 'description': 'Set visualization description' + } +} + + +def validate_call_params(function: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Validate parameters for a function call. + + Args: + function: Name of the function to call + params: Parameters to validate + + Returns: + Validated parameters (may be modified, e.g., defaults added) + + Raises: + GFQLTypeError: If validation fails + """ + # Check if function is in safelist + if function not in SAFELIST_V1: + raise GFQLTypeError( + ErrorCode.E303, + f"Function '{function}' is not in the safelist", + field="function", + value=function, + suggestion=f"Available functions: {', '.join(sorted(SAFELIST_V1.keys()))}" + ) + + config = SAFELIST_V1[function] + allowed_params = config['allowed_params'] + required_params = config['required_params'] + param_validators = config['param_validators'] + + # Check for required parameters + missing_required = required_params - set(params.keys()) + if missing_required: + raise GFQLTypeError( + ErrorCode.E105, + f"Missing required parameters for '{function}'", + field="params", + value=list(missing_required), + suggestion=f"Required parameters: {', '.join(sorted(missing_required))}" + ) + + # Check for unknown parameters + unknown_params = set(params.keys()) - allowed_params + if unknown_params: + raise GFQLTypeError( + ErrorCode.E303, + f"Unknown parameters for '{function}'", + field="params", + value=list(unknown_params), + suggestion=f"Allowed parameters: {', '.join(sorted(allowed_params))}" + ) + + # Validate parameter types + for param_name, param_value in params.items(): + if param_name in param_validators: + validator = param_validators[param_name] + if not validator(param_value): + raise GFQLTypeError( + ErrorCode.E201, + f"Invalid type for parameter '{param_name}' in '{function}'", + field=f"params.{param_name}", + value=f"{type(param_value).__name__}: {param_value}", + suggestion="Check the parameter type requirements" + ) + + return params diff --git a/graphistry/compute/chain_let.py b/graphistry/compute/chain_let.py new file mode 100644 index 0000000000..9319590f5a --- /dev/null +++ b/graphistry/compute/chain_let.py @@ -0,0 +1,439 @@ +import logging +from typing import Dict, Set, List, Optional, Tuple, Union, cast +from typing_extensions import Literal +from graphistry.Engine import Engine, EngineAbstract, resolve_engine +from graphistry.Plottable import Plottable +from graphistry.util import setup_logger +from .ast import ASTObject, ASTLet, ASTChainRef, ASTRemoteGraph, ASTNode, ASTEdge, ASTCall +from .execution_context import ExecutionContext +from .typing import DataFrameT + +logger = setup_logger(__name__) + + +def extract_dependencies(ast_obj: ASTObject) -> Set[str]: + """Recursively find all ASTChainRef references in an AST object + + :param ast_obj: AST object to analyze + :returns: Set of referenced binding names + :rtype: Set[str] + """ + deps = set() + + if isinstance(ast_obj, ASTChainRef): + deps.add(ast_obj.ref) + # Also check chain operations + for op in ast_obj.chain: + deps.update(extract_dependencies(op)) + + elif isinstance(ast_obj, ASTLet): + # Nested let bindings + for binding in ast_obj.bindings.values(): + deps.update(extract_dependencies(binding)) + + # Other AST types (ASTNode, ASTEdge, ASTRemoteGraph) have no dependencies + return deps + + +def build_dependency_graph(bindings: Dict[str, ASTObject]) -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]: + """Build dependency and dependent mappings from bindings + + :param bindings: Dictionary of name -> AST object bindings + :returns: Tuple of (dependencies dict, dependents dict) + :rtype: Tuple[Dict[str, Set[str]], Dict[str, Set[str]]] + """ + dependencies: Dict[str, Set[str]] = {} + dependents: Dict[str, Set[str]] = {} + + for name, ast_obj in bindings.items(): + deps = extract_dependencies(ast_obj) + dependencies[name] = deps + + # Build reverse mapping + for dep in deps: + if dep not in dependents: + dependents[dep] = set() + dependents[dep].add(name) + + return dependencies, dependents + + +def validate_dependencies(bindings: Dict[str, ASTObject], + dependencies: Dict[str, Set[str]]) -> None: + """Check for missing references and self-cycles + + :param bindings: Dictionary of available bindings + :param dependencies: Dictionary of dependencies per binding + :raises ValueError: If missing references or self-cycles found + """ + all_names = set(bindings.keys()) + + for name, deps in dependencies.items(): + # Check self-reference + if name in deps: + raise ValueError(f"Self-reference cycle detected: '{name}' depends on itself") + + # Check missing references + missing = deps - all_names + if missing: + raise ValueError( + f"Node '{name}' references undefined nodes: {sorted(missing)}. " + f"Available nodes: {sorted(all_names)}" + ) + + +def detect_cycles(dependencies: Dict[str, Set[str]]) -> Optional[List[str]]: + """Use DFS to detect cycles and return the cycle path if found + + :param dependencies: Dictionary mapping nodes to their dependencies + :returns: List representing cycle path if found, None otherwise + :rtype: Optional[List[str]] + """ + WHITE, GRAY, BLACK = 0, 1, 2 + color = {node: WHITE for node in dependencies} + + def dfs(node: str, path: List[str]) -> Optional[List[str]]: + color[node] = GRAY + path.append(node) + + for neighbor in dependencies.get(node, set()): + if color.get(neighbor, WHITE) == GRAY: + # Found cycle - build cycle path + cycle_start = path.index(neighbor) + return path[cycle_start:] + [neighbor] + + if color.get(neighbor, WHITE) == WHITE: + cycle = dfs(neighbor, path[:]) + if cycle: + return cycle + + color[node] = BLACK + return None + + for node in dependencies: + if color[node] == WHITE: + cycle = dfs(node, []) + if cycle: + return cycle + + return None + + +def topological_sort(bindings: Dict[str, ASTObject], + dependencies: Dict[str, Set[str]], + dependents: Dict[str, Set[str]]) -> List[str]: + """Kahn's algorithm for topological sort""" + # Calculate in-degrees + in_degree = {name: len(dependencies.get(name, set())) for name in bindings} + + # Start with nodes that have no dependencies + queue = [name for name, degree in in_degree.items() if degree == 0] + result = [] + + while queue: + # Process node with no remaining dependencies + current = queue.pop(0) + result.append(current) + + # Update dependents + for dependent in dependents.get(current, set()): + in_degree[dependent] -= 1 + if in_degree[dependent] == 0: + queue.append(dependent) + + if len(result) != len(bindings): + # Cycle detected - use DFS to find it for better error + cycle = detect_cycles(dependencies) + if cycle: + raise ValueError( + f"Circular dependency detected: {' -> '.join(cycle)}. " + "Please restructure your DAG to remove cycles." + ) + else: + # Should not happen, but be defensive + raise ValueError("Failed to determine execution order (possible circular dependency)") + + return result + + +def determine_execution_order(bindings: Dict[str, ASTObject]) -> List[str]: + """Determine topological execution order for DAG bindings + + Validates dependencies and computes execution order that respects + all dependencies. Detects cycles and missing references. + + :param bindings: Dictionary of name -> AST object bindings + :returns: List of binding names in execution order + :rtype: List[str] + :raises ValueError: If cycles detected or references missing + """ + # Handle trivial cases + if not bindings: + return [] + if len(bindings) == 1: + return list(bindings.keys()) + + # Build dependency graph + dependencies, dependents = build_dependency_graph(bindings) + + # Validate all references exist + validate_dependencies(bindings, dependencies) + + # Check for cycles with detailed error + cycle = detect_cycles(dependencies) + if cycle: + raise ValueError( + f"Circular dependency detected: {' -> '.join(cycle)}. " + "Please restructure your DAG to remove cycles." + ) + + # Compute topological sort + return topological_sort(bindings, dependencies, dependents) + + +def execute_node(name: str, ast_obj: ASTObject, g: Plottable, + context: ExecutionContext, engine: Engine) -> Plottable: + """Execute a single node in the DAG + + Handles different AST object types: + - ASTLet: Recursive let execution + - ASTChainRef: Reference resolution and chain execution + - ASTNode: Node filtering operations + - ASTEdge: Edge traversal operations + - Others: NotImplementedError + + :param name: Binding name for this node + :param ast_obj: AST object to execute + :param g: Input graph + :param context: Execution context for storing/retrieving results + :param engine: Engine to use (pandas/cudf) + :returns: Resulting Plottable + :rtype: Plottable + :raises ValueError: If reference not found in context + :raises NotImplementedError: For unsupported AST types + """ + logger.debug("Executing node '%s' of type %s", name, type(ast_obj).__name__) + + # Handle different AST object types + if isinstance(ast_obj, ASTLet): + # Nested let execution + result = chain_let_impl(g, ast_obj, engine.value) + elif isinstance(ast_obj, ASTChainRef): + # Resolve reference from context + try: + referenced_result = context.get_binding(ast_obj.ref) + except KeyError as e: + available = sorted(context.get_all_bindings().keys()) + raise ValueError( + f"Node '{name}' references '{ast_obj.ref}' which has not been executed yet. " + f"Available bindings: {available}" + ) from e + + # Execute the chain on the referenced result + if ast_obj.chain: + # Import chain function to execute the operations + from .chain import chain as chain_impl + result = chain_impl(referenced_result, ast_obj.chain, engine.value) + else: + # Empty chain - just return the referenced result + result = referenced_result + elif isinstance(ast_obj, ASTNode): + # For let bindings, we execute nodes in a simpler way than chain() + # No wavefront propagation - just filter the graph's nodes + if ast_obj.filter_dict or ast_obj.query: + filtered_g = g + if ast_obj.filter_dict: + filtered_g = filtered_g.filter_nodes_by_dict(ast_obj.filter_dict) + if ast_obj.query: + filtered_g = filtered_g.nodes(lambda g: g._nodes.query(ast_obj.query)) + result = filtered_g + else: + # Empty filter - return original graph + result = g + + # Add name column if specified + if ast_obj._name: + result = result.nodes(result._nodes.assign(**{ast_obj._name: True})) + elif isinstance(ast_obj, ASTEdge): + # For let bindings, execute edge operations using hop() + # This is simpler than the full chain() wavefront approach + result = g.hop( + nodes=None, # Start from all nodes + hops=ast_obj.hops, + to_fixed_point=ast_obj.to_fixed_point, + direction=ast_obj.direction, + source_node_match=ast_obj.source_node_match, + edge_match=ast_obj.edge_match, + destination_node_match=ast_obj.destination_node_match, + source_node_query=ast_obj.source_node_query, + edge_query=ast_obj.edge_query, + destination_node_query=ast_obj.destination_node_query, + return_as_wave_front=False # Return full graph + ) + + # Add name column to edges if specified + if ast_obj._name: + result = result.edges(result._edges.assign(**{ast_obj._name: True})) + elif isinstance(ast_obj, ASTRemoteGraph): + # Create a new plottable bound to the remote dataset_id + # This doesn't fetch the data immediately - it just creates a reference + result = g.bind(dataset_id=ast_obj.dataset_id) + + # If we need to actually fetch the data, we would use chain_remote + # For now, we'll fetch it immediately to ensure we have the data + from .chain_remote import chain_remote as chain_remote_impl + + # Fetch the remote dataset with an empty chain (no filtering) + result = chain_remote_impl( + result, + [], # Empty chain - just fetch the entire dataset + api_token=ast_obj.token, + dataset_id=ast_obj.dataset_id, + output_type="all", # Get full graph (nodes and edges) + engine=cast(Literal["pandas", "cudf"], engine.value) + ) + elif isinstance(ast_obj, ASTCall): + # Execute method call with validation + from .call_executor import execute_call + result = execute_call(g, ast_obj.function, ast_obj.params, engine) + else: + # Other AST object types not yet implemented + raise NotImplementedError(f"Execution of {type(ast_obj).__name__} not yet implemented") + + # Store result in context + context.set_binding(name, result) + + return result + + +def chain_let_impl(g: Plottable, let_bindings: ASTLet, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + output: Optional[str] = None) -> Plottable: + """Internal implementation of chain_let execution + + Validates let bindings, determines execution order, and executes nodes + in topological order. + + :param g: Input graph + :param let_bindings: Let specification with named bindings + :param engine: Engine selection (auto/pandas/cudf) + :param output: Name of binding to return (default: last executed) + :returns: Result from specified or last executed node + :rtype: Plottable + :raises TypeError: If let_bindings is not an ASTLet + :raises RuntimeError: If node execution fails + :raises ValueError: If output binding not found + """ + if isinstance(engine, str): + engine = EngineAbstract(engine) + + # Validate the let parameter + if not isinstance(let_bindings, ASTLet): + raise TypeError(f"let_bindings must be an ASTLet, got {type(let_bindings).__name__}") + + # Validate the let bindings + let_bindings.validate() + + # Resolve engine + engine_concrete = resolve_engine(engine, g) + logger.debug('chain_let engine: %s => %s', engine, engine_concrete) + + # Materialize nodes if needed (following chain.py pattern) + g = g.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) + + # Create execution context + context = ExecutionContext() + + # Handle empty let bindings + if not let_bindings.bindings: + return g + + # Determine execution order + order = determine_execution_order(let_bindings.bindings) + logger.debug("Let execution order: %s", order) + + # Execute nodes in topological order + last_result = g + for node_name in order: + ast_obj = let_bindings.bindings[node_name] + logger.debug("Executing node '%s' in DAG", node_name) + + # Execute the node and store result in context + try: + result = execute_node(node_name, ast_obj, g, context, engine_concrete) + last_result = result + except Exception as e: + # Add context to error + raise RuntimeError( + f"Failed to execute node '{node_name}' in let bindings. " + f"Error: {type(e).__name__}: {str(e)}" + ) from e + + # Return requested output or last executed result + if output is not None: + if output not in context.get_all_bindings(): + available = sorted(context.get_all_bindings().keys()) + raise ValueError( + f"Output binding '{output}' not found. " + f"Available bindings: {available}" + ) + return context.get_binding(output) + else: + return last_result + + +def chain_let(self: Plottable, let_bindings: ASTLet, + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + output: Optional[str] = None) -> Plottable: + """ + Execute named graph operations with dependency resolution + + Let operations can reference results from other operations by name, + enabling parallel branches and complex data flows. + + :param let_bindings: ASTLet containing named bindings of operations + :param engine: Execution engine (auto, pandas, cudf) + :param output: Name of binding to return (default: last executed) + :returns: Plottable result from the specified or last operation + :rtype: Plottable + + **Example: Single operation (no dependencies)** + + :: + + from graphistry.compute.ast import ASTLet, n + + let_bindings = ASTLet({ + 'people': n({'type': 'person'}) + }) + result = g.chain_let(let_bindings) + + **Example: Linear dependencies** + + :: + + from graphistry.compute.ast import ASTLet, ASTChainRef, n, e + + let_bindings = ASTLet({ + 'start': n({'type': 'person'}), + 'friends': ASTChainRef('start', [e(), n()]) + }) + result = g.chain_let(let_bindings) + + **Example: Diamond pattern** + + :: + + let_bindings = ASTLet({ + 'people': n({'type': 'person'}), + 'transactions': n({'type': 'transaction'}), + 'branch1': ASTChainRef('people', [e()]), + 'branch2': ASTChainRef('transactions', [e()]), + 'merged': g.union(ASTChainRef('branch1'), ASTChainRef('branch2')) + }) + result = g.chain_let(let_bindings) # Returns last executed + + # Or select specific output + people_result = g.chain_let(let_bindings, output='people') + """ + return chain_let_impl(self, let_bindings, engine, output) diff --git a/graphistry/compute/execution_context.py b/graphistry/compute/execution_context.py new file mode 100644 index 0000000000..3b8e034b8b --- /dev/null +++ b/graphistry/compute/execution_context.py @@ -0,0 +1,37 @@ +"""Execution context for DAG operations""" +from typing import Any, Dict + + +class ExecutionContext: + """Manages variable bindings during DAG execution""" + + def __init__(self): + self._bindings: Dict[str, Any] = {} + + def set_binding(self, name: str, value: Any) -> None: + """Store a named result""" + if not isinstance(name, str): + raise TypeError(f"Binding name must be string, got {type(name)}") + self._bindings[name] = value + + def get_binding(self, name: str) -> Any: + """Retrieve a named result""" + if not isinstance(name, str): + raise TypeError(f"Binding name must be string, got {type(name)}") + if name not in self._bindings: + raise KeyError(f"No binding found for '{name}'") + return self._bindings[name] + + def has_binding(self, name: str) -> bool: + """Check if binding exists""" + if not isinstance(name, str): + raise TypeError(f"Binding name must be string, got {type(name)}") + return name in self._bindings + + def clear(self) -> None: + """Clear all bindings""" + self._bindings.clear() + + def get_all_bindings(self) -> Dict[str, Any]: + """Get a copy of all bindings""" + return self._bindings.copy() diff --git a/graphistry/compute/gfql.py b/graphistry/compute/gfql.py new file mode 100644 index 0000000000..19f903a8f0 --- /dev/null +++ b/graphistry/compute/gfql.py @@ -0,0 +1,99 @@ +"""GFQL unified entrypoint for chains and DAGs""" + +import logging +from typing import List, Union, Optional, Dict +from graphistry.Plottable import Plottable +from graphistry.Engine import EngineAbstract +from graphistry.util import setup_logger +from .ast import ASTObject, ASTLet +from .chain import Chain, chain as chain_impl +from .chain_let import chain_let as chain_let_impl + +logger = setup_logger(__name__) + + +def gfql(self: Plottable, + query: Union[ASTObject, List[ASTObject], ASTLet, Chain, dict], + engine: Union[EngineAbstract, str] = EngineAbstract.AUTO, + output: Optional[str] = None) -> Plottable: + """ + Execute a GFQL query - either a chain or a DAG + + Unified entrypoint that automatically detects query type and + dispatches to the appropriate execution engine. + + :param query: GFQL query - ASTObject, List[ASTObject], Chain, ASTLet, or dict + :param engine: Execution engine (auto, pandas, cudf) + :param output: For DAGs, name of binding to return (default: last executed) + :returns: Resulting Plottable + :rtype: Plottable + + **Example: Chain query** + + :: + + from graphistry.compute.ast import n, e + + # As list + result = g.gfql([n({'type': 'person'}), e(), n()]) + + # As Chain object + from graphistry.compute.chain import Chain + result = g.gfql(Chain([n({'type': 'person'}), e(), n()])) + + **Example: DAG query** + + :: + + from graphistry.compute.ast import let, ref, n, e + + result = g.gfql(let({ + 'people': n({'type': 'person'}), + 'friends': ref('people', [e({'rel': 'knows'}), n()]) + })) + + # Select specific output + friends = g.gfql(result, output='friends') + + **Example: Auto-detection** + + :: + + # List → chain execution + g.gfql([n(), e(), n()]) + + # Single ASTObject → chain execution + g.gfql(n({'type': 'person'})) + + # Dict → DAG execution (convenience) + g.gfql({'people': n({'type': 'person'})}) + """ + # Handle dict convenience first (convert to ASTLet) + if isinstance(query, dict): + query = ASTLet(query) + + # Dispatch based on type - check specific types before generic + if isinstance(query, ASTLet): + logger.debug('GFQL executing as DAG') + return chain_let_impl(self, query, engine, output) + elif isinstance(query, Chain): + logger.debug('GFQL executing as Chain') + if output is not None: + logger.warning('output parameter ignored for chain queries') + return chain_impl(self, query.chain, engine) + elif isinstance(query, ASTObject): + # Single ASTObject -> execute as single-item chain + logger.debug('GFQL executing single ASTObject as chain') + if output is not None: + logger.warning('output parameter ignored for chain queries') + return chain_impl(self, [query], engine) + elif isinstance(query, list): + logger.debug('GFQL executing list as chain') + if output is not None: + logger.warning('output parameter ignored for chain queries') + return chain_impl(self, query, engine) + else: + raise TypeError( + f"Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict. " + f"Got {type(query).__name__}" + ) diff --git a/graphistry/compute/gfql/__init__.py b/graphistry/compute/gfql_validation/__init__.py similarity index 87% rename from graphistry/compute/gfql/__init__.py rename to graphistry/compute/gfql_validation/__init__.py index 1df331d90f..cd8522e0c3 100644 --- a/graphistry/compute/gfql/__init__.py +++ b/graphistry/compute/gfql_validation/__init__.py @@ -1,6 +1,6 @@ """GFQL validation and related utilities.""" -from graphistry.compute.gfql.validate import ( +from graphistry.compute.gfql_validation.validate import ( ValidationIssue, Schema, validate_syntax, @@ -12,7 +12,7 @@ suggest_fixes ) -from graphistry.compute.gfql.exceptions import ( +from graphistry.compute.gfql_validation.exceptions import ( GFQLException, GFQLValidationError, GFQLSyntaxError, diff --git a/graphistry/compute/gfql/exceptions.py b/graphistry/compute/gfql_validation/exceptions.py similarity index 100% rename from graphistry/compute/gfql/exceptions.py rename to graphistry/compute/gfql_validation/exceptions.py diff --git a/graphistry/compute/gfql/validate.py b/graphistry/compute/gfql_validation/validate.py similarity index 100% rename from graphistry/compute/gfql/validate.py rename to graphistry/compute/gfql_validation/validate.py diff --git a/graphistry/compute/validate/validate_schema.py b/graphistry/compute/validate/validate_schema.py index 8f6597fe01..b994e8e6ed 100644 --- a/graphistry/compute/validate/validate_schema.py +++ b/graphistry/compute/validate/validate_schema.py @@ -3,11 +3,10 @@ from typing import List, Optional, Union, TYPE_CHECKING, cast import pandas as pd from graphistry.Plottable import Plottable -from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, ASTLet, ASTChainRef, ASTRemoteGraph, ASTCall if TYPE_CHECKING: from graphistry.compute.chain import Chain - from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError from graphistry.compute.predicates.ASTPredicate import ASTPredicate from graphistry.compute.predicates.numeric import NumericASTPredicate, Between @@ -57,6 +56,14 @@ def validate_chain_schema( op_errors = _validate_node_op(op, node_columns, g._nodes, collect_all) elif isinstance(op, ASTEdge): op_errors = _validate_edge_op(op, node_columns, edge_columns, g._nodes, g._edges, collect_all) + elif isinstance(op, ASTLet): + op_errors = _validate_querydag_op(op, g, collect_all) + elif isinstance(op, ASTChainRef): + op_errors = _validate_chainref_op(op, g, collect_all) + elif isinstance(op, ASTRemoteGraph): + op_errors = _validate_remotegraph_op(op, collect_all) + elif isinstance(op, ASTCall): + op_errors = _validate_call_op(op, node_columns, edge_columns, collect_all) # Add operation index to all errors for e in op_errors: @@ -105,6 +112,101 @@ def _validate_edge_op( return errors +def _validate_querydag_op(op: ASTLet, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate Let operation against schema.""" + errors = [] + + # Validate each binding in the DAG + for binding_name, binding_value in op.bindings.items(): + try: + # Recursively validate each binding as if it's a single operation + binding_errors = validate_chain_schema(g, [binding_value], collect_all=True) + + # Add binding context to errors + if binding_errors: + for error in binding_errors: + error.context['dag_binding'] = binding_name + if collect_all: + errors.extend(binding_errors) + else: + raise binding_errors[0] + + except GFQLSchemaError as e: + e.context['dag_binding'] = binding_name + if collect_all: + errors.append(e) + else: + raise + + return errors + + +def _validate_chainref_op(op: ASTChainRef, g: Plottable, collect_all: bool) -> List[GFQLSchemaError]: + """Validate ChainRef operation against schema.""" + errors = [] + + # Validate the chain operations in the ChainRef + if op.chain: + try: + chain_errors = validate_chain_schema(g, op.chain, collect_all=True) + + # Add ChainRef context to errors + if chain_errors: + for error in chain_errors: + error.context['chain_ref'] = op.ref + if collect_all: + errors.extend(chain_errors) + else: + raise chain_errors[0] + + except GFQLSchemaError as e: + e.context['chain_ref'] = op.ref + if collect_all: + errors.append(e) + else: + raise + + # Note: We don't validate that op.ref exists here since that's handled + # by the let bindings dependency validation in chain_let.py + + return errors + + +def _validate_remotegraph_op(op: ASTRemoteGraph, collect_all: bool) -> List[GFQLSchemaError]: + """Validate RemoteGraph operation against schema.""" + errors = [] + + # Validate dataset_id format + if not op.dataset_id or not isinstance(op.dataset_id, str): + error = GFQLSchemaError( + ErrorCode.E303, + 'RemoteGraph dataset_id must be a non-empty string', + field='dataset_id', + value=op.dataset_id, + suggestion='Provide a valid dataset identifier string' + ) + if collect_all: + errors.append(error) + else: + raise error + + # Validate token format if provided + if op.token is not None and not isinstance(op.token, str): + error = GFQLSchemaError( + ErrorCode.E303, + 'RemoteGraph token must be a string if provided', + field='token', + value=type(op.token).__name__, + suggestion='Provide a valid token string or None' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + def _validate_filter_dict( filter_dict: dict, columns: set, @@ -199,6 +301,92 @@ def _validate_filter_dict( return errors +def _validate_call_op( + op: ASTCall, + node_columns: set, + edge_columns: set, + collect_all: bool = False +) -> List[GFQLSchemaError]: + """Validate Call operation schema requirements. + + Checks that all columns required by the called method exist in the graph. + Uses the schema_effects metadata from the safelist to determine requirements. + + Args: + op: ASTCall operation to validate + node_columns: Set of available node column names + edge_columns: Set of available edge column names + collect_all: If True, collect all errors. If False, raise on first error. + + Returns: + List of schema errors found (empty if valid) + + Raises: + GFQLSchemaError: If collect_all=False and validation fails + """ + errors: List[GFQLSchemaError] = [] + + # Import safelist to get schema effects + from graphistry.compute.call_safelist import SAFELIST_V1 + + # Check if method is in safelist + if op.function not in SAFELIST_V1: + # This should have been caught by parameter validation already + return errors + + method_info = SAFELIST_V1[op.function] + + # Check if method has schema effects defined + if 'schema_effects' not in method_info: + # Method doesn't define schema effects, so we can't validate + return errors + + schema_effects = method_info['schema_effects'] + + # Get required columns based on parameters + if 'requires_node_cols' in schema_effects: + if callable(schema_effects['requires_node_cols']): + required_node_cols = schema_effects['requires_node_cols'](op.params) + else: + required_node_cols = schema_effects['requires_node_cols'] + + for col in required_node_cols: + if col not in node_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires node column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available node columns: {", ".join(sorted(node_columns)[:10])}{"..." if len(node_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + if 'requires_edge_cols' in schema_effects: + if callable(schema_effects['requires_edge_cols']): + required_edge_cols = schema_effects['requires_edge_cols'](op.params) + else: + required_edge_cols = schema_effects['requires_edge_cols'] + + for col in required_edge_cols: + if col not in edge_columns: + error = GFQLSchemaError( + ErrorCode.E301, + f'Call operation "{op.function}" requires edge column "{col}" which does not exist', + field=f'{op.function}.{col}', + value=col, + suggestion=f'Available edge columns: {", ".join(sorted(edge_columns)[:10])}{"..." if len(edge_columns) > 10 else ""}' + ) + if collect_all: + errors.append(error) + else: + raise error + + return errors + + # Add to Chain class def validate_schema(self: 'Chain', g: Plottable, collect_all: bool = False) -> Optional[List[GFQLSchemaError]]: """Validate this chain against a graph's schema without executing. diff --git a/graphistry/tests/compute/README_INTEGRATION_TESTS.md b/graphistry/tests/compute/README_INTEGRATION_TESTS.md new file mode 100644 index 0000000000..f60bde7d9e --- /dev/null +++ b/graphistry/tests/compute/README_INTEGRATION_TESTS.md @@ -0,0 +1,67 @@ +# Integration Test Configuration + +This directory contains both unit tests (always run) and integration tests (opt-in). + +## Environment Variables for Integration Tests + +### GPU Tests +```bash +# Enable CUDF/GPU tests +TEST_CUDF=1 pytest test_chain_dag_gpu.py +``` + +### Remote Graph Integration Tests +```bash +# Enable remote Graphistry server tests +TEST_REMOTE_INTEGRATION=1 pytest test_chain_dag_remote_integration.py + +# Additional configuration for remote tests: +GRAPHISTRY_USERNAME=myuser # Username for auth +GRAPHISTRY_PASSWORD=mypass # Password for auth +GRAPHISTRY_API_KEY=key-123 # Alternative to username/password +GRAPHISTRY_SERVER=hub.graphistry.com # Server URL (optional) +GRAPHISTRY_TEST_DATASET_ID=abc123 # Known dataset for testing (optional) +``` + +## Running All Tests + +```bash +# Unit tests only (fast, no external dependencies) +pytest + +# All tests including integration +TEST_CUDF=1 TEST_REMOTE_INTEGRATION=1 pytest +``` + +## Writing New Integration Tests + +1. **Use environment variable guards:** + ```python + import os + import pytest + + REMOTE_INTEGRATION_ENABLED = os.environ.get("TEST_REMOTE_INTEGRATION") == "1" + skip_remote = pytest.mark.skipif( + not REMOTE_INTEGRATION_ENABLED, + reason="Remote integration tests need TEST_REMOTE_INTEGRATION=1" + ) + + @skip_remote + def test_my_remote_feature(): + # This only runs when TEST_REMOTE_INTEGRATION=1 + pass + ``` + +2. **Always provide mocked versions:** + - Integration tests verify real behavior + - Unit tests with mocks ensure CI/CD still validates core logic + +3. **Document requirements:** + - What env vars are needed + - What external services must be running + - Expected test data setup + +## CI/CD Configuration + +The CI/CD pipeline runs only unit tests by default. Integration tests can be enabled +in specific CI jobs by setting the appropriate environment variables. \ No newline at end of file diff --git a/graphistry/tests/compute/test_ast.py b/graphistry/tests/compute/test_ast.py index 61f082d21c..29e6aa476e 100644 --- a/graphistry/tests/compute/test_ast.py +++ b/graphistry/tests/compute/test_ast.py @@ -1,4 +1,7 @@ -from graphistry.compute.ast import from_json, ASTNode, ASTEdge, n, e, e_forward, e_reverse, e_undirected +from graphistry.compute.ast import ( + from_json, ASTNode, ASTEdge, ASTLet, ASTRemoteGraph, ASTChainRef, + n, e, e_forward, e_reverse, e_undirected +) def test_serialization_node(): @@ -21,3 +24,95 @@ def test_serialization_edge(): assert edge2._name == 'abc' o2 = edge2.to_json() assert o == o2 + + +def test_serialization_let_empty(): + """Test Let with empty bindings""" + dag = ASTLet({}) + o = dag.to_json() + assert o == {'type': 'Let', 'bindings': {}} + dag2 = from_json(o) + assert isinstance(dag2, ASTLet) + assert dag2.bindings == {} + o2 = dag2.to_json() + assert o == o2 + + +def test_serialization_let_single(): + """Test Let with single binding""" + dag = ASTLet({'a': n()}) + o = dag.to_json() + assert o['type'] == 'Let' + assert 'a' in o['bindings'] + dag2 = from_json(o) + assert isinstance(dag2, ASTLet) + assert 'a' in dag2.bindings + assert isinstance(dag2.bindings['a'], ASTNode) + + +def test_serialization_let_multi(): + """Test Let with multiple bindings""" + dag = ASTLet({ + 'nodes': n({'type': 'person'}), + 'edges': e_forward(), + 'remote': ASTRemoteGraph('dataset123') + }) + o = dag.to_json() + dag2 = from_json(o) + assert isinstance(dag2, ASTLet) + assert len(dag2.bindings) == 3 + assert isinstance(dag2.bindings['nodes'], ASTNode) + assert isinstance(dag2.bindings['edges'], ASTEdge) + assert isinstance(dag2.bindings['remote'], ASTRemoteGraph) + + +def test_serialization_remoteGraph(): + """Test RemoteGraph serialization""" + rg = ASTRemoteGraph('my-dataset-id') + o = rg.to_json() + assert o == {'type': 'RemoteGraph', 'dataset_id': 'my-dataset-id'} + rg2 = from_json(o) + assert isinstance(rg2, ASTRemoteGraph) + assert rg2.dataset_id == 'my-dataset-id' + assert rg2.token is None + + +def test_serialization_remoteGraph_with_token(): + """Test RemoteGraph with auth token""" + rg = ASTRemoteGraph('my-dataset-id', token='secret-token') + o = rg.to_json() + assert o == { + 'type': 'RemoteGraph', + 'dataset_id': 'my-dataset-id', + 'token': 'secret-token' + } + rg2 = from_json(o) + assert isinstance(rg2, ASTRemoteGraph) + assert rg2.dataset_id == 'my-dataset-id' + assert rg2.token == 'secret-token' + + +def test_serialization_chainRef_empty(): + """Test ChainRef with empty chain""" + cr = ASTChainRef('mydata', []) + o = cr.to_json() + assert o == {'type': 'ChainRef', 'ref': 'mydata', 'chain': []} + cr2 = from_json(o) + assert isinstance(cr2, ASTChainRef) + assert cr2.ref == 'mydata' + assert cr2.chain == [] + + +def test_serialization_chainRef_with_ops(): + """Test ChainRef with operations""" + cr = ASTChainRef('data1', [n({'type': 'person'}), e_forward()]) + o = cr.to_json() + assert o['type'] == 'ChainRef' + assert o['ref'] == 'data1' + assert len(o['chain']) == 2 + cr2 = from_json(o) + assert isinstance(cr2, ASTChainRef) + assert cr2.ref == 'data1' + assert len(cr2.chain) == 2 + assert isinstance(cr2.chain[0], ASTNode) + assert isinstance(cr2.chain[1], ASTEdge) diff --git a/graphistry/tests/compute/test_ast_errors.py b/graphistry/tests/compute/test_ast_errors.py new file mode 100644 index 0000000000..fcbe12ee3a --- /dev/null +++ b/graphistry/tests/compute/test_ast_errors.py @@ -0,0 +1,79 @@ +"""Test error handling and messages in AST serialization""" + +import pytest +from graphistry.compute.ast import from_json +from graphistry.compute.exceptions import GFQLSyntaxError + + +class TestSerializationErrors: + """Test error handling in JSON serialization/deserialization""" + + def test_from_json_non_dict_input(self): + """Test clear error when input is not a dict""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json("not a dict") + + assert "AST JSON must be a dictionary" in str(exc_info.value) + + def test_from_json_none_input(self): + """Test clear error when input is None""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json(None) + + assert "AST JSON must be a dictionary" in str(exc_info.value) + + def test_from_json_missing_type(self): + """Test clear error when 'type' field is missing""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"no_type": "value"}) + + assert "AST JSON missing required 'type' field" in str(exc_info.value) + + def test_from_json_unknown_type(self): + """Test clear error for unknown type""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"type": "UnknownType"}) + + assert "Unknown AST type: UnknownType" in str(exc_info.value) + + def test_edge_missing_direction(self): + """Test clear error when Edge missing direction""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"type": "Edge"}) + + assert "Edge missing required 'direction' field" in str(exc_info.value) + + def test_edge_invalid_direction(self): + """Test clear error for invalid Edge direction""" + with pytest.raises(GFQLSyntaxError) as exc_info: + from_json({"type": "Edge", "direction": "invalid"}) + + assert "Edge has unknown direction: invalid" in str(exc_info.value) + + def test_querydag_missing_bindings(self): + """Test clear error when QueryDAG missing bindings""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "QueryDAG"}) + + assert "Let missing bindings" in str(exc_info.value) + + def test_remotegraph_missing_dataset_id(self): + """Test clear error when RemoteGraph missing dataset_id""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "RemoteGraph"}) + + assert "RemoteGraph missing dataset_id" in str(exc_info.value) + + def test_chainref_missing_ref(self): + """Test clear error when ChainRef missing ref""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "ChainRef"}) + + assert "ChainRef missing ref" in str(exc_info.value) + + def test_chainref_missing_chain(self): + """Test clear error when ChainRef missing chain""" + with pytest.raises(AssertionError) as exc_info: + from_json({"type": "ChainRef", "ref": "test"}) + + assert "ChainRef missing chain" in str(exc_info.value) diff --git a/graphistry/tests/compute/test_call_operations.py b/graphistry/tests/compute/test_call_operations.py new file mode 100644 index 0000000000..bcda6df310 --- /dev/null +++ b/graphistry/tests/compute/test_call_operations.py @@ -0,0 +1,394 @@ +"""Tests for GFQL Call operations.""" + +import pytest +import pandas as pd +from unittest.mock import Mock, patch, MagicMock + +from graphistry.tests.test_compute import CGFull +from graphistry.Engine import Engine, EngineAbstract +from graphistry.compute.ast import ASTCall, ASTLet, n +from graphistry.compute.chain_let import chain_let_impl +from graphistry.compute.call_safelist import validate_call_params +from graphistry.compute.call_executor import execute_call +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError, GFQLSyntaxError + + +class TestCallSafelist: + """Test method safelist validation.""" + + def test_allowed_method(self): + """Test validation of allowed methods.""" + params = validate_call_params('get_degrees', { + 'col': 'degree' + }) + assert params == {'col': 'degree'} + + def test_unknown_method(self): + """Test rejection of unknown methods.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('unknown_method', {}) + assert exc_info.value.code == ErrorCode.E303 + assert 'not in the safelist' in str(exc_info.value) + + def test_required_params(self): + """Test validation of required parameters.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('filter_nodes_by_dict', {}) + assert exc_info.value.code == ErrorCode.E105 + assert 'Missing required parameters' in str(exc_info.value) + + def test_unknown_params(self): + """Test rejection of unknown parameters.""" + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('get_degrees', { + 'col': 'degree', + 'unknown_param': 'value' + }) + assert exc_info.value.code == ErrorCode.E303 + assert 'Unknown parameters' in str(exc_info.value) + + def test_param_type_validation(self): + """Test parameter type validation.""" + # Valid types + params = validate_call_params('hop', { + 'hops': 2, + 'direction': 'forward', + 'to_fixed_point': True + }) + assert params['hops'] == 2 + + # Invalid type + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('hop', { + 'hops': 'two' # Should be int + }) + assert exc_info.value.code == ErrorCode.E201 + assert 'Invalid type' in str(exc_info.value) + + def test_enum_validation(self): + """Test enum parameter validation.""" + # Valid enum value + params = validate_call_params('hop', { + 'direction': 'forward' + }) + assert params['direction'] == 'forward' + + # Invalid enum value + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('hop', { + 'direction': 'sideways' + }) + assert exc_info.value.code == ErrorCode.E201 + + +class TestASTCall: + """Test ASTCall node validation and serialization.""" + + def test_basic_creation(self): + """Test creating a basic ASTCall.""" + call = ASTCall('get_degrees', {'col': 'degree'}) + assert call.function == 'get_degrees' + assert call.params == {'col': 'degree'} + + def test_empty_params(self): + """Test ASTCall with no parameters.""" + call = ASTCall('prune_self_edges') + assert call.function == 'prune_self_edges' + assert call.params == {} + + def test_validation(self): + """Test ASTCall field validation.""" + # Valid call + call = ASTCall('get_degrees', {'col': 'degree'}) + call.validate() # Should not raise + + # Invalid function type + with pytest.raises(GFQLTypeError) as exc_info: + call = ASTCall(123, {}) + call.validate() + assert exc_info.value.code == ErrorCode.E201 + assert 'function must be a string' in str(exc_info.value) + + # Invalid params type + with pytest.raises(GFQLTypeError) as exc_info: + call = ASTCall('get_degrees', 'not_a_dict') + call.validate() + assert exc_info.value.code == ErrorCode.E201 + assert 'params must be a dict' in str(exc_info.value) + + def test_to_json(self): + """Test ASTCall JSON serialization.""" + call = ASTCall('get_degrees', {'col': 'degree', 'engine': 'pandas'}) + json_data = call.to_json() + + assert json_data == { + 'type': 'Call', + 'function': 'get_degrees', + 'params': {'col': 'degree', 'engine': 'pandas'} + } + + def test_from_json(self): + """Test ASTCall JSON deserialization.""" + json_data = { + 'type': 'Call', + 'function': 'filter_nodes_by_dict', + 'params': {'filter_dict': {'type': 'user'}} + } + + call = ASTCall.from_json(json_data) + assert isinstance(call, ASTCall) + assert call.function == 'filter_nodes_by_dict' + assert call.params == {'filter_dict': {'type': 'user'}} + + def test_from_json_invalid(self): + """Test ASTCall from_json with invalid data.""" + # Missing function + with pytest.raises(AssertionError) as exc_info: + ASTCall.from_json({'type': 'Call'}) + assert 'Call missing function' in str(exc_info.value) + + # Wrong type - this would be caught earlier in the AST dispatch + # so we don't test it here + + +class TestCallExecution: + """Test call execution functionality.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_execute_get_degrees(self, sample_graph): + """Test executing get_degrees method.""" + result = execute_call( + sample_graph, + 'get_degrees', + {'col': 'degree'}, + Engine.PANDAS + ) + + # Result should be a Plottable + assert hasattr(result, '_nodes') + assert 'degree' in result._nodes.columns + assert len(result._nodes) == 4 + + def test_execute_filter_nodes(self, sample_graph): + """Test executing filter_nodes_by_dict method.""" + result = execute_call( + sample_graph, + 'filter_nodes_by_dict', + {'filter_dict': {'type': 'user'}}, + Engine.PANDAS + ) + + # Should filter to only user nodes + assert len(result._nodes) == 3 + assert all(result._nodes['type'] == 'user') + + def test_execute_materialize_nodes(self): + """Test executing materialize_nodes method.""" + # Start with edges only + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0] + }) + g = CGFull().edges(edges_df).bind(source='source', destination='target') + + # No nodes initially + assert g._nodes is None + + # Execute materialize_nodes + result = execute_call( + g, + 'materialize_nodes', + {}, + Engine.PANDAS + ) + + # Should have nodes now + assert result._nodes is not None + assert len(result._nodes) == 3 + + def test_execute_with_validation_error(self, sample_graph): + """Test that validation errors are properly raised.""" + with pytest.raises(GFQLTypeError) as exc_info: + execute_call( + sample_graph, + 'hop', + {'hops': 'invalid'}, # Should be int + Engine.PANDAS + ) + assert exc_info.value.code == ErrorCode.E201 + + def test_execute_unknown_method(self, sample_graph): + """Test execution of unknown method.""" + with pytest.raises(GFQLTypeError) as exc_info: + execute_call( + sample_graph, + 'unknown_method', + {}, + Engine.PANDAS + ) + assert exc_info.value.code == ErrorCode.E303 + + +class TestCallInDAG: + """Test ASTCall execution within DAGs.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0], + 'weight': [1.0, 2.0, 1.5, 3.0, 0.5] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_call_in_dag(self, sample_graph): + """Test executing ASTCall within a DAG.""" + dag = ASTLet({ + 'filtered': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_let_impl(sample_graph, dag, EngineAbstract.PANDAS) + + # Should have degree column + assert 'degree' in result._nodes.columns + # Should still have all nodes (get_degrees doesn't filter) + assert len(result._nodes) == 4 + + def test_call_referencing_binding(self, sample_graph): + """Test ASTCall that operates on whole graph (not in chain).""" + from graphistry.compute.ast import ASTChainRef + + # Call operations work on the whole graph, not as part of chains + dag = ASTLet({ + 'users': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_let_impl(sample_graph, dag, EngineAbstract.PANDAS) + + # Should have degree column on all nodes + assert len(result._nodes) == 4 # All nodes + assert 'degree' in result._nodes.columns + + def test_multiple_calls(self, sample_graph): + """Test multiple call operations in sequence.""" + # First add degrees + dag1 = ASTLet({ + 'with_degrees': ASTCall('get_degrees', {'col': 'deg'}) + }) + result1 = chain_let_impl(sample_graph, dag1, EngineAbstract.PANDAS) + assert 'deg' in result1._nodes.columns + + # Then filter - use the graph that has degrees + dag2 = ASTLet({ + 'filtered': ASTCall('filter_nodes_by_dict', {'filter_dict': {'deg': 2}}) + }) + result2 = chain_let_impl(result1, dag2, EngineAbstract.PANDAS) + + # Should have nodes with degree 2 + assert len(result2._nodes) > 0 + assert all(result2._nodes['deg'] == 2) + + @patch('graphistry.compute.call_executor.getattr') + def test_call_execution_error(self, mock_getattr, sample_graph): + """Test handling of execution errors in calls.""" + # Make the method raise an error + mock_method = Mock(side_effect=RuntimeError("Method failed")) + mock_getattr.return_value = mock_method + + dag = ASTLet({ + 'failing': ASTCall('get_degrees', {}) + }) + + with pytest.raises(RuntimeError) as exc_info: + chain_let_impl(sample_graph, dag, EngineAbstract.PANDAS) + assert "Failed to execute node 'failing'" in str(exc_info.value) + + +class TestGraphAlgorithmCalls: + """Test calls to graph algorithm methods.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3] + }) + + return CGFull()\ + .edges(edges_df)\ + .bind(source='source', destination='target') + + def test_compute_cugraph_params(self): + """Test compute_cugraph parameter validation.""" + # Valid params + params = validate_call_params('compute_cugraph', { + 'alg': 'pagerank', + 'out_col': 'pr_score', + 'directed': True + }) + assert params['alg'] == 'pagerank' + + # Missing required alg + with pytest.raises(GFQLTypeError) as exc_info: + validate_call_params('compute_cugraph', { + 'out_col': 'pr_score' + }) + assert exc_info.value.code == ErrorCode.E105 + + def test_compute_igraph_params(self): + """Test compute_igraph parameter validation.""" + params = validate_call_params('compute_igraph', { + 'alg': 'community_louvain', + 'directed': False + }) + assert params['alg'] == 'community_louvain' + + def test_layout_methods(self): + """Test layout method parameter validation.""" + # layout_cugraph + params = validate_call_params('layout_cugraph', { + 'layout': 'force_atlas2', + 'params': {'iterations': 100} + }) + assert params['layout'] == 'force_atlas2' + + # layout_igraph + params = validate_call_params('layout_igraph', { + 'layout': 'fruchterman_reingold', + 'directed': True + }) + assert params['layout'] == 'fruchterman_reingold' + + # fa2_layout + params = validate_call_params('fa2_layout', { + 'fa2_params': {'iterations': 500} + }) + assert params['fa2_params']['iterations'] == 500 diff --git a/graphistry/tests/compute/test_call_operations_gpu.py b/graphistry/tests/compute/test_call_operations_gpu.py new file mode 100644 index 0000000000..3f3b989353 --- /dev/null +++ b/graphistry/tests/compute/test_call_operations_gpu.py @@ -0,0 +1,243 @@ +"""GPU tests for GFQL Call operations.""" + +import os +import pytest +import pandas as pd + +from graphistry.tests.test_compute import CGFull +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTCall, ASTLet, n +from graphistry.compute.chain_let import chain_let_impl +from graphistry.compute.call_executor import execute_call +from graphistry.compute.validate_schema import validate_chain_schema +from graphistry.compute.exceptions import ErrorCode, GFQLTypeError + + +# Skip all tests if TEST_CUDF not set +skip_gpu = pytest.mark.skipif( + not ("TEST_CUDF" in os.environ and os.environ["TEST_CUDF"] == "1"), + reason="cudf tests need TEST_CUDF=1" +) + + +class TestCallOperationsGPU: + """Test Call operations with GPU/cudf.""" + + @skip_gpu + def test_call_with_cudf_dataframes(self): + """Test that Call operations work with cudf DataFrames.""" + import cudf + + # Create cudf dataframes + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2], + 'target': [1, 2, 0, 3], + 'weight': [1.0, 2.0, 3.0, 4.0] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + edges_gdf = cudf.from_pandas(edges_df) + nodes_gdf = cudf.from_pandas(nodes_df) + + # Create graph with cudf data + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Execute Call operation + result = execute_call(g, 'get_degrees', {'col': 'degree'}, Engine.CUDF) + + # Result should still have cudf nodes + assert hasattr(result._nodes, '__cuda_array_interface__') + assert 'degree' in result._nodes.columns + + @skip_gpu + def test_filter_with_cudf(self): + """Test filtering operations with cudf.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'], + 'score': [0.5, 0.8, 0.9, 0.3] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull().nodes(nodes_gdf).bind(node='node') + + # Filter nodes + result = execute_call( + g, + 'filter_nodes_by_dict', + {'filter_dict': {'type': 'user'}}, + Engine.CUDF + ) + + # Should still be cudf and filtered + assert hasattr(result._nodes, '__cuda_array_interface__') + assert len(result._nodes) == 3 + assert all(result._nodes['type'] == 'user') + + @skip_gpu + def test_compute_cugraph_call(self): + """Test compute_cugraph through Call operation.""" + import cudf + + # Create a simple graph + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull().edges(edges_gdf).bind(source='source', destination='target') + + # Skip if cugraph not available + try: + import cugraph + except ImportError: + pytest.skip("cugraph not installed") + + # Call compute_cugraph for pagerank + result = execute_call( + g, + 'compute_cugraph', + {'alg': 'pagerank', 'out_col': 'pr_score'}, + Engine.CUDF + ) + + # Should have pagerank scores + assert 'pr_score' in result._nodes.columns + assert hasattr(result._nodes, '__cuda_array_interface__') + + @skip_gpu + def test_layout_cugraph_call(self): + """Test layout_cugraph through Call operation.""" + import cudf + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0] + }) + edges_gdf = cudf.from_pandas(edges_df) + + g = CGFull().edges(edges_gdf).bind(source='source', destination='target') + + # Skip if cugraph not available + try: + import cugraph + except ImportError: + pytest.skip("cugraph not installed") + + # Call layout_cugraph + result = execute_call( + g, + 'layout_cugraph', + {'layout': 'force_atlas2'}, + Engine.CUDF + ) + + # Should have x,y coordinates + assert 'x' in result._nodes.columns + assert 'y' in result._nodes.columns + assert hasattr(result._nodes, '__cuda_array_interface__') + + @skip_gpu + def test_chain_let_with_gpu_calls(self): + """Test DAG execution with Call operations on GPU.""" + import cudf + + edges_df = pd.DataFrame({ + 'source': [0, 1, 2, 2, 3], + 'target': [1, 2, 0, 3, 0], + 'weight': [1.0, 2.0, 1.5, 3.0, 0.5] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'type': ['user', 'user', 'admin', 'user'] + }) + + edges_gdf = cudf.from_pandas(edges_df) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull()\ + .edges(edges_gdf)\ + .nodes(nodes_gdf)\ + .bind(source='source', destination='target', node='node') + + # Create DAG with Call operations + dag = ASTLet({ + 'filtered': n({'type': 'user'}), + 'with_degrees': ASTCall('get_degrees', {'col': 'degree'}) + }) + + result = chain_let_impl(g, dag, Engine.CUDF) + + # Should have GPU data with degrees + assert hasattr(result._nodes, '__cuda_array_interface__') + assert 'degree' in result._nodes.columns + + @skip_gpu + def test_schema_validation_with_cudf(self): + """Test schema validation works with cudf DataFrames.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2], + 'type': ['A', 'B', 'C'] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull().nodes(nodes_gdf).bind(node='node') + + # Valid call - column exists + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'A'}}) + errors = validate_chain_schema(g, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid call - column doesn't exist + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing': 'X'}}) + errors = validate_chain_schema(g, [call], collect_all=True) + assert len(errors) > 0 + assert any('missing' in str(e) for e in errors) + + @skip_gpu + def test_encode_with_gpu(self): + """Test visual encoding methods with GPU data.""" + import cudf + + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2, 3], + 'category': ['A', 'B', 'A', 'C'], + 'score': [0.1, 0.5, 0.8, 0.3] + }) + nodes_gdf = cudf.from_pandas(nodes_df) + + g = CGFull().nodes(nodes_gdf).bind(node='node') + + # Test encode_point_color + result = execute_call( + g, + 'encode_point_color', + {'column': 'category'}, + Engine.CUDF + ) + + # Should still have GPU data + assert hasattr(result._nodes, '__cuda_array_interface__') + + # Test encode_point_size + result2 = execute_call( + result, + 'encode_point_size', + {'column': 'score'}, + Engine.CUDF + ) + + assert hasattr(result2._nodes, '__cuda_array_interface__') + # Should have size encoding set + assert result2._point_size == 'score' diff --git a/graphistry/tests/compute/test_call_schema_validation.py b/graphistry/tests/compute/test_call_schema_validation.py new file mode 100644 index 0000000000..2cc2760802 --- /dev/null +++ b/graphistry/tests/compute/test_call_schema_validation.py @@ -0,0 +1,126 @@ +"""Test schema validation for Call operations.""" + +import pytest +import pandas as pd +from graphistry.tests.test_compute import CGFull +from graphistry.compute.ast import ASTCall, n +from graphistry.compute.chain import Chain +from graphistry.compute.validate_schema import validate_chain_schema +from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError + + +class TestCallSchemaValidation: + """Test schema validation for Call operations.""" + + @pytest.fixture + def sample_graph(self): + """Create a sample graph for testing.""" + edges_df = pd.DataFrame({ + 'source': [0, 1, 2], + 'target': [1, 2, 0], + 'weight': [1.0, 2.0, 3.0] + }) + nodes_df = pd.DataFrame({ + 'node': [0, 1, 2], + 'type': ['user', 'user', 'admin'], + 'score': [0.5, 0.8, 0.9] + }) + + return CGFull()\ + .edges(edges_df)\ + .nodes(nodes_df)\ + .bind(source='source', destination='target', node='node') + + def test_filter_nodes_requires_columns(self, sample_graph): + """Test that filter_nodes_by_dict validates required columns.""" + # Valid: filtering by existing column + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'user'}}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: filtering by non-existent column + call = ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing_col': 'value'}}) + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(sample_graph, [call], collect_all=False) + assert exc_info.value.code == ErrorCode.E301 + assert 'missing_col' in str(exc_info.value) + assert 'does not exist' in str(exc_info.value) + + def test_filter_edges_requires_columns(self, sample_graph): + """Test that filter_edges_by_dict validates required columns.""" + # Valid: filtering by existing edge column + call = ASTCall('filter_edges_by_dict', {'filter_dict': {'weight': 2.0}}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: filtering by non-existent edge column + call = ASTCall('filter_edges_by_dict', {'filter_dict': {'edge_type': 'friend'}}) + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(sample_graph, [call], collect_all=False) + assert exc_info.value.code == ErrorCode.E301 + assert 'edge_type' in str(exc_info.value) + + def test_encode_requires_columns(self, sample_graph): + """Test that encode methods validate required columns.""" + # Valid: encoding existing column + call = ASTCall('encode_point_color', {'column': 'type'}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + # Invalid: encoding non-existent column + call = ASTCall('encode_point_color', {'column': 'category'}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + # Note: encode methods don't require columns to exist (they create bindings) + # so this should not produce errors + assert len(errors) == 0 + + def test_chain_with_multiple_calls(self, sample_graph): + """Test validation of chains with multiple Call operations.""" + chain = Chain([ + ASTCall('filter_nodes_by_dict', {'filter_dict': {'type': 'user'}}), + ASTCall('get_degrees', {'col': 'degree'}), + ASTCall('filter_nodes_by_dict', {'filter_dict': {'degree': 2}}) + ]) + + # The second filter expects 'degree' column which doesn't exist yet + # But schema validation is static and doesn't track added columns + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + # Should have error for missing 'degree' column + assert any('degree' in str(e) for e in errors) + + def test_method_without_schema_effects(self, sample_graph): + """Test that methods without schema effects don't cause errors.""" + # materialize_nodes doesn't require any columns + call = ASTCall('materialize_nodes', {}) + errors = validate_chain_schema(sample_graph, [call], collect_all=True) + assert len(errors) == 0 + + def test_collect_all_mode(self, sample_graph): + """Test collect_all mode returns all errors.""" + chain = Chain([ + ASTCall('filter_nodes_by_dict', {'filter_dict': {'missing1': 'a', 'missing2': 'b'}}), + ASTCall('filter_edges_by_dict', {'filter_dict': {'missing3': 'c'}}) + ]) + + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + # Should collect all 3 missing column errors + assert len(errors) >= 3 + missing_cols = {'missing1', 'missing2', 'missing3'} + error_cols = set() + for e in errors: + for col in missing_cols: + if col in str(e): + error_cols.add(col) + assert error_cols == missing_cols + + def test_operation_index_in_errors(self, sample_graph): + """Test that errors include operation index.""" + chain = Chain([ + n({'type': 'user'}), # op 0 + ASTCall('filter_nodes_by_dict', {'filter_dict': {'bad_col': 1}}) # op 1 + ]) + + errors = validate_chain_schema(sample_graph, chain, collect_all=True) + call_errors = [e for e in errors if 'bad_col' in str(e)] + assert len(call_errors) > 0 + assert call_errors[0].context['operation_index'] == 1 diff --git a/graphistry/tests/compute/test_chain_let.py b/graphistry/tests/compute/test_chain_let.py new file mode 100644 index 0000000000..de284fadcc --- /dev/null +++ b/graphistry/tests/compute/test_chain_let.py @@ -0,0 +1,1272 @@ +"""Test chain let functionality + +For integration tests with real remote graphs, see test_chain_let_remote_integration.py +Enable remote tests with: TEST_REMOTE_INTEGRATION=1 +""" + +import pandas as pd +import pytest +from unittest.mock import patch, MagicMock +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, ASTNode, ASTObject, n, e +from graphistry.compute.chain_let import ( + extract_dependencies, build_dependency_graph, validate_dependencies, + detect_cycles, determine_execution_order +) +from graphistry.compute.execution_context import ExecutionContext +from graphistry.tests.test_compute import CGFull + + +class TestChainDagHelpers: + """Test the helper functions for DAG execution""" + + def test_extract_dependencies_no_deps(self): + """Test extracting dependencies from nodes with no dependencies""" + node = n({'type': 'person'}) + deps = extract_dependencies(node) + assert deps == set() + + remote = ASTRemoteGraph('dataset123') + deps = extract_dependencies(remote) + assert deps == set() + + def test_extract_dependencies_chain_ref(self): + """Test extracting dependencies from ASTChainRef""" + chain_ref = ASTChainRef('source', [n()]) + deps = extract_dependencies(chain_ref) + assert deps == {'source'} + + def test_extract_dependencies_nested(self): + """Test extracting dependencies from nested structures""" + # ChainRef with ChainRef in its chain + nested = ASTChainRef('a', [ASTChainRef('b', [n()])]) + deps = extract_dependencies(nested) + assert deps == {'a', 'b'} + + # Nested DAG + dag = ASTLet({ + 'inner': ASTChainRef('outer', [n()]) + }) + deps = extract_dependencies(dag) + assert deps == {'outer'} + + def test_build_dependency_graph(self): + """Test building dependency and dependent mappings""" + bindings = { + 'a': n(), + 'b': ASTChainRef('a', [n()]), + 'c': ASTChainRef('b', [n()]) + } + + dependencies, dependents = build_dependency_graph(bindings) + + assert dependencies == { + 'a': set(), + 'b': {'a'}, + 'c': {'b'} + } + assert dependents == { + 'a': {'b'}, + 'b': {'c'} + } + + def test_validate_dependencies_valid(self): + """Test validation passes for valid dependencies""" + bindings = { + 'a': n(), + 'b': ASTChainRef('a', [n()]) + } + dependencies = {'a': set(), 'b': {'a'}} + + # Should not raise + validate_dependencies(bindings, dependencies) + + def test_validate_dependencies_missing_ref(self): + """Test validation catches missing references""" + bindings = { + 'a': n() + } + dependencies = {'a': {'missing'}} + + with pytest.raises(ValueError) as exc_info: + validate_dependencies(bindings, dependencies) + + assert "references undefined nodes: ['missing']" in str(exc_info.value) + assert "Available nodes: ['a']" in str(exc_info.value) + + def test_validate_dependencies_self_ref(self): + """Test validation catches self-references""" + bindings = { + 'a': n() + } + dependencies = {'a': {'a'}} + + with pytest.raises(ValueError) as exc_info: + validate_dependencies(bindings, dependencies) + + assert "Self-reference cycle detected: 'a' depends on itself" in str(exc_info.value) + + def test_detect_cycles_no_cycle(self): + """Test cycle detection on acyclic graph""" + dependencies = { + 'a': set(), + 'b': {'a'}, + 'c': {'b'} + } + + cycle = detect_cycles(dependencies) + assert cycle is None + + def test_detect_cycles_simple_cycle(self): + """Test cycle detection on simple cycle""" + dependencies = { + 'a': {'b'}, + 'b': {'a'} + } + + cycle = detect_cycles(dependencies) + assert cycle == ['a', 'b', 'a'] or cycle == ['b', 'a', 'b'] + + def test_detect_cycles_longer_cycle(self): + """Test cycle detection on longer cycle""" + dependencies = { + 'a': {'b'}, + 'b': {'c'}, + 'c': {'a'}, + 'd': {'a'} + } + + cycle = detect_cycles(dependencies) + # Could start from any node in the cycle + assert len(cycle) == 4 # 3 nodes + repeat + assert cycle[0] == cycle[-1] # Cycle closes + + def test_determine_execution_order_empty(self): + """Test execution order for empty DAG""" + order = determine_execution_order({}) + assert order == [] + + def test_determine_execution_order_single(self): + """Test execution order for single node""" + bindings = {'only': n()} + order = determine_execution_order(bindings) + assert order == ['only'] + + def test_determine_execution_order_linear(self): + """Test execution order for linear dependencies""" + bindings = { + 'a': n(), + 'b': ASTChainRef('a', [n()]), + 'c': ASTChainRef('b', [n()]) + } + + order = determine_execution_order(bindings) + assert order == ['a', 'b', 'c'] + + def test_determine_execution_order_diamond(self): + """Test execution order for diamond pattern""" + bindings = { + 'top': n(), + 'left': ASTChainRef('top', [n()]), + 'right': ASTChainRef('top', [n()]), + 'bottom': ASTChainRef('left', [ASTChainRef('right', [n()])]) + } + + order = determine_execution_order(bindings) + # Top must come first, bottom must come last + assert order[0] == 'top' + assert order[-1] == 'bottom' + # Left and right can be in either order + assert set(order[1:3]) == {'left', 'right'} + + def test_determine_execution_order_disconnected(self): + """Test execution order for disconnected components""" + bindings = { + 'a1': n(), + 'a2': ASTChainRef('a1', [n()]), + 'b1': n(), + 'b2': ASTChainRef('b1', [n()]) + } + + order = determine_execution_order(bindings) + # Each component should be ordered correctly + assert order.index('a1') < order.index('a2') + assert order.index('b1') < order.index('b2') + + +class TestExecutionContext: + """Test ExecutionContext integration in chain_let""" + + def test_context_stores_results(self): + """Test that ExecutionContext stores node results""" + from graphistry.compute.chain_let import execute_node + + # Create a mock AST object that returns a known result + class MockNode: + def validate(self): + pass + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + mock_node = MockNode() + + # This should raise NotImplementedError but still store in context + try: + execute_node('test_node', mock_node, g, context, None) + except NotImplementedError: + pass + + # Even though execution failed, context.set_binding was called + # (we can't test this without implementing execution) + + def test_chain_ref_missing_reference(self): + """Test ASTChainRef with missing reference gives helpful error""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Create ASTChainRef that references non-existent binding + chain_ref = ASTChainRef('missing_ref', []) + + # Should raise ValueError with helpful message + with pytest.raises(ValueError) as exc_info: + execute_node('test', chain_ref, g, context, Engine.PANDAS) + + assert "references 'missing_ref' which has not been executed yet" in str(exc_info.value) + assert "Available bindings: []" in str(exc_info.value) + + def test_chain_ref_with_existing_reference(self): + """Test ASTChainRef successfully resolves existing reference""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Pre-populate context with a result + context.set_binding('previous_result', g) + + # Create ASTChainRef that references it (empty chain) + chain_ref = ASTChainRef('previous_result', []) + + # Should return the referenced result + result = execute_node('test', chain_ref, g, context, Engine.PANDAS) + assert result is g # Same object since empty chain + + # And store it under new name + assert context.get_binding('test') is g + + def test_context_passed_through_dag(self): + """Test that context is passed through DAG execution""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({}) + + # Empty DAG should work + result = g.gfql(dag) + assert result is not None + + def test_execution_order_verified(self): + """Test that execution order follows dependencies""" + # Create a DAG with known dependencies + dag = ASTLet({ + 'data': ASTRemoteGraph('dataset'), + 'filtered': ASTChainRef('data', []), + 'analyzed': ASTChainRef('filtered', []) + }) + + # Get execution order + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + + # Verify order respects dependencies + assert order == ['data', 'filtered', 'analyzed'] + + # Also test diamond pattern + dag_diamond = ASTLet({ + 'root': ASTRemoteGraph('data'), + 'left': ASTChainRef('root', []), + 'right': ASTChainRef('root', []), + 'merge': ASTChainRef('left', [ASTChainRef('right', [])]) + }) + + order_diamond = determine_execution_order(dag_diamond.bindings) + assert order_diamond[0] == 'root' + assert order_diamond[-1] == 'merge' + assert set(order_diamond[1:3]) == {'left', 'right'} + + def test_chain_ref_in_dag_execution(self): + """Test ASTChainRef works in let bindings execution (fails on chain ops)""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Create a simple mock that can be executed + class MockExecutable(ASTObject): + def validate(self): + pass + + def __call__(self, g, prev_node_wavefront, target_wave_front, engine): + raise NotImplementedError("Mock execution") + + def reverse(self): + return self + + # Create DAG with mock executable and chain ref + dag = ASTLet({ + 'first': MockExecutable(), + 'second': ASTChainRef('first', []) # Empty chain should work + }) + + # Try to execute - will fail on MockExecutable + try: + g.gfql(dag) + except RuntimeError as e: + # Should fail on first node (MockExecutable) + assert "Failed to execute node 'first'" in str(e) + assert "NotImplementedError" in str(e) + + +class TestEdgeExecution: + """Test ASTEdge execution in chain_let""" + + def test_edge_execution_basic(self): + """Test basic edge traversal in let bindings""" + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c', 'd'], + 'd': ['b', 'c', 'd', 'e'], + 'type': ['knows', 'works_with', 'knows', 'manages'] + }) + g = CGFull().edges(edges_df, 's', 'd') + g = g.materialize_nodes() + + dag = ASTLet({ + 'one_hop': e() # Default forward edge + }) + + result = g.gfql(dag) + assert result is not None + # Should have traversed edges + assert len(result._nodes) > 0 + + def test_edge_with_filter(self): + """Test edge traversal with filters""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd', 'e'], + 'type': ['person', 'person', 'company', 'person', 'company'] + }) + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c', 'd'], + 'd': ['b', 'c', 'd', 'e'], + 'rel': ['knows', 'works_at', 'invests', 'works_at'] + }) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'work_edges': e(edge_match={'rel': 'works_at'}) + }) + + result = g.gfql(dag) + # Should have filtered to work relationships + assert result is not None + + def test_edge_with_direction(self): + """Test edge traversal with different directions""" + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c'], + 'd': ['b', 'c', 'd'] + }) + g = CGFull().edges(edges_df, 's', 'd') + g = g.materialize_nodes() + + # Test reverse direction + from graphistry.compute.ast import ASTEdgeReverse + dag = ASTLet({ + 'reverse': ASTEdgeReverse() + }) + + result = g.gfql(dag) + assert result is not None + + def test_edge_with_name(self): + """Test edge operation adds name column""" + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c'], + 'd': ['b', 'c', 'd'] + }) + g = CGFull().edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'tagged_edges': e(name='important') + }) + + result = g.gfql(dag) + assert 'important' in result._edges.columns + + def test_node_edge_combination(self): + """Test DAG with both node and edge operations""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c'], + 'd': ['b', 'c', 'd'] + }) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'from_people': ASTChainRef('people', [e()]), + 'companies': n({'type': 'company'}) + }) + + # Should execute successfully + result = g.gfql(dag) + assert result is not None + + +class TestNodeExecution: + """Test ASTNode execution in chain_let""" + + def test_node_execution_empty_filter(self): + """Test ASTNode with empty filter returns original graph""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}), 's', 'd') + g = g.materialize_nodes() # Ensure nodes exist + context = ExecutionContext() + + # Empty node filter + node = n() + result = execute_node('test', node, g, context, Engine.PANDAS) + + # Should return graph with same data + assert len(result._nodes) == len(g._nodes) + assert len(result._edges) == len(g._edges) + + def test_node_execution_with_filter(self): + """Test ASTNode with filter_dict filters nodes""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + # Create graph with node attributes + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + context = ExecutionContext() + + # Filter for person nodes + node = n({'type': 'person'}) + result = execute_node('people', node, g, context, Engine.PANDAS) + + # Should only have person nodes + assert len(result._nodes) == 2 + assert set(result._nodes['id'].tolist()) == {'a', 'b'} + assert all(result._nodes['type'] == 'person') + + def test_node_execution_with_name(self): + """Test ASTNode adds name column when specified""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g = g.materialize_nodes() + context = ExecutionContext() + + # Node with name + node = n(name='tagged') + result = execute_node('test', node, g, context, Engine.PANDAS) + + # Should have 'tagged' column + assert 'tagged' in result._nodes.columns + assert all(result._nodes['tagged']) + + def test_node_in_dag_execution(self): + """Test ASTNode works in full DAG execution""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # DAG with node filter + dag = ASTLet({ + 'people': n({'type': 'person'}) + }) + + result = g.gfql(dag) + + # Should have filtered to people only + assert len(result._nodes) == 2 + assert set(result._nodes['type'].unique()) == {'person'} + + def test_dag_with_node_and_chainref(self): + """Test DAG execution with both node and chain reference""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'], + 'active': [True, False, True, True] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'b', 'c'], 'd': ['b', 'c', 'd', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # DAG: filter people, then filter active from those + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'active_people': ASTChainRef('people', [n({'active': True})]) + }) + + result = g.gfql(dag) + + # Should only have active people + assert len(result._nodes) == 1 + assert result._nodes['id'].iloc[0] == 'a' + assert result._nodes['type'].iloc[0] == 'person' + assert result._nodes['active'].iloc[0] # Just check truthiness + + +class TestErrorHandling: + """Test error handling and edge cases""" + + def test_invalid_dag_type(self): + """Test helpful error when dag parameter is wrong type""" + g = CGFull() + + with pytest.raises(TypeError) as exc_info: + g.gfql("not a dag") + assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) + + with pytest.raises(AssertionError) as exc_info: + g.gfql({'dict': 'not allowed'}) + assert "binding value must be ASTObject" in str(exc_info.value) + + def test_node_execution_error_wrapped(self): + """Test node execution errors are wrapped with context""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Create a node with invalid query syntax + dag = ASTLet({ + 'bad_query': n(query='invalid python syntax !@#') + }) + + with pytest.raises(RuntimeError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "Failed to execute node 'bad_query'" in error_msg + assert "Error:" in error_msg + + def test_cycle_detection_with_path(self): + """Test cycle detection provides the cycle path""" + dag = ASTLet({ + 'a': ASTChainRef('b', []), + 'b': ASTChainRef('c', []), + 'c': ASTChainRef('a', []) # Creates cycle a->b->c->a + }) + + g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + with pytest.raises(ValueError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "Circular dependency detected" in error_msg + assert "->" in error_msg # Shows the cycle path + + def test_complex_cycle_detection(self): + """Test detection of cycles in complex DAGs""" + # This DAG has no cycles, just complex dependencies + bindings = { + 'start': n(), + 'a': ASTChainRef('start', []), + 'b': ASTChainRef('a', []), + 'c': ASTChainRef('b', []), + 'd': ASTChainRef('c', []), + 'e': ASTChainRef('d', []), + 'f': ASTChainRef('b', []), # Second branch from b + 'g': ASTChainRef('f', []) # Note: removed nested ASTChainRef in chain + } + + # Test cycle detection directly + from graphistry.compute.chain_let import detect_cycles, build_dependency_graph + dependencies, _ = build_dependency_graph(bindings) + cycle = detect_cycles(dependencies) + + # Should find no cycle + assert cycle is None + + def test_missing_reference_with_suggestions(self): + """Test missing reference error includes available bindings""" + dag = ASTLet({ + 'data1': n(), + 'data2': n(), + 'result': ASTChainRef('data3', []) # data3 doesn't exist + }) + + g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + with pytest.raises(ValueError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "references undefined nodes: ['data3']" in error_msg + assert "Available nodes: ['data1', 'data2', 'result']" in error_msg + + +class TestExecutionMechanics: + """Test execution mechanics with granular tests""" + + def test_execute_node_stores_in_context(self): + """Test that execute_node stores results in context""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g = g.materialize_nodes() + context = ExecutionContext() + + # Execute a simple node + node = n() + result = execute_node('test_node', node, g, context, Engine.PANDAS) + + # Check result is stored in context + assert context.get_binding('test_node') is result + assert len(result._nodes) == 2 # nodes a and b + + def test_execute_node_with_different_ast_types(self): + """Test execute_node handles different AST object types""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Test ASTRemoteGraph is now implemented (will fail with missing auth) + # We'll test actual functionality with mocks in a separate test + + # Test nested ASTLet + nested_dag = ASTLet({'inner': n()}) + result = execute_node('nested', nested_dag, g, context, Engine.PANDAS) + assert result is not None + + @patch('graphistry.compute.chain_remote.chain_remote') + def test_remote_graph_execution(self, mock_chain_remote): + """Test ASTRemoteGraph executes correctly with mocked remote call""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + # Setup mock return value + mock_result = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + mock_chain_remote.return_value = mock_result + + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + context = ExecutionContext() + + # Execute remote graph + remote = ASTRemoteGraph('dataset123', token='secret-token') + result = execute_node('remote_data', remote, g, context, Engine.PANDAS) + assert result is mock_result # Verify correct result returned + + # Verify chain_remote was called with correct params + mock_chain_remote.assert_called_once() + call_args = mock_chain_remote.call_args + assert call_args[0][1] == [] # Empty chain + assert call_args[1]['dataset_id'] == 'dataset123' + assert call_args[1]['api_token'] == 'secret-token' + assert call_args[1]['output_type'] == 'all' + + # Verify result is stored in context + assert context.get_binding('remote_data') is mock_result + + def test_chain_ref_resolution_order(self): + """Test ASTChainRef resolves references in correct order""" + from graphistry.compute.chain_let import execute_node + from graphistry.Engine import Engine + + nodes_df = pd.DataFrame({'id': ['a', 'b', 'c'], 'value': [1, 2, 3]}) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + context = ExecutionContext() + + # Store initial result + filtered = g.filter_nodes_by_dict({'value': 2}) + context.set_binding('filtered_data', filtered) + + # Create chain ref that adds more filtering + chain_ref = ASTChainRef('filtered_data', [n({'id': 'b'})]) + result = execute_node('final', chain_ref, g, context, Engine.PANDAS) + + # Should have only node 'b' + assert len(result._nodes) == 1 + assert result._nodes['id'].iloc[0] == 'b' + + def test_execution_context_isolation(self): + """Test that each DAG execution has isolated context""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # First DAG execution + dag1 = ASTLet({'node1': n(name='first')}) + result1 = g.gfql(dag1) + assert result1 is not None # First execution succeeds + + # Second DAG execution should not see first's context + dag2 = ASTLet({ + 'node2': n(name='second'), + 'ref_fail': ASTChainRef('node1', []) # Should fail - node1 not in this context + }) + + with pytest.raises(ValueError) as exc_info: + g.gfql(dag2) + assert "references undefined nodes: ['node1']" in str(exc_info.value) + + def test_execution_order_logging(self): + """Test execution order is logged correctly""" + import logging + from graphistry.compute.chain_let import logger as dag_logger + + # Capture log output + logs = [] + handler = logging.Handler() + handler.emit = lambda record: logs.append(record) + dag_logger.addHandler(handler) + dag_logger.setLevel(logging.DEBUG) + + try: + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({ + 'first': n(), + 'second': ASTChainRef('first', []), + 'third': ASTChainRef('second', []) + }) + + g.gfql(dag) + + # Check execution order was logged + order_logs = [r for r in logs if 'DAG execution order' in str(r.getMessage())] + assert len(order_logs) > 0 + assert "['first', 'second', 'third']" in str(order_logs[0].getMessage()) + + # Check individual node execution was logged + node_logs = [r for r in logs if "Executing node" in str(r.getMessage())] + assert len(node_logs) >= 3 + finally: + dag_logger.removeHandler(handler) + + +class TestDiamondPatterns: + """Test diamond and complex dependency patterns""" + + def test_diamond_pattern_execution(self): + """Test diamond pattern executes correctly""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd', 'e'], + 'type': ['source', 'middle1', 'middle2', 'target', 'other'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c', 'd'], 'd': ['b', 'd', 'd', 'e']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Diamond: top -> (left, right) -> bottom + dag = ASTLet({ + 'top': n({'type': 'source'}), + 'left': ASTChainRef('top', [n(name='from_left')]), + 'right': ASTChainRef('top', [n(name='from_right')]), + 'bottom': ASTChainRef('left', []) + }) + + result = g.gfql(dag) + + # Result should have source node with from_left tag + assert len(result._nodes) == 1 + assert result._nodes['type'].iloc[0] == 'source' + assert 'from_left' in result._nodes.columns + assert result._nodes['from_left'].iloc[0] # Check truthiness + + def test_multi_branch_convergence(self): + """Test multiple branches converging""" + g = CGFull().edges(pd.DataFrame({ + 's': ['a', 'b', 'c', 'd', 'e'], + 'd': ['x', 'x', 'x', 'x', 'x'] + }), 's', 'd') + g = g.materialize_nodes() + + # Multiple branches converging - test execution order + from graphistry.compute.chain_let import determine_execution_order, ExecutionContext, execute_node + from graphistry.Engine import Engine + + dag = ASTLet({ + 'branch1': n(name='b1'), + 'branch2': n(name='b2'), + 'branch3': n(name='b3'), + 'converge': n() # Gets all nodes + }) + + # Test execution order - branches can execute in any order + order = determine_execution_order(dag.bindings) + assert len(order) == 4 + assert order[-1] == 'converge' # Converge must be last + + # Execute and check final result + result = g.gfql(dag) + assert len(result._nodes) == 6 # a,b,c,d,e,x + + def test_parallel_independent_branches(self): + """Test parallel branches execute independently""" + nodes_df = pd.DataFrame({ + 'id': list('abcdefgh'), + 'branch': ['A', 'A', 'A', 'A', 'B', 'B', 'B', 'B'] + }) + edges_df = pd.DataFrame({'s': list('abcdefg'), 'd': list('bcdefgh')}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Two independent branches + dag = ASTLet({ + 'branch_a': n({'branch': 'A'}), + 'branch_b': n({'branch': 'B'}), + 'a_subset': ASTChainRef('branch_a', [n(query="id in ['a', 'b']")]), + 'b_subset': ASTChainRef('branch_b', [n(query="id in ['e', 'f']")]) + }) + + # Check execution order allows parallel execution + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + + # branch_a and branch_b can execute in any order + assert order.index('branch_a') < order.index('a_subset') + assert order.index('branch_b') < order.index('b_subset') + + # Execute DAG + result = g.gfql(dag) + + # Result should be from last executed node (b_subset) + assert len(result._nodes) == 2 + assert set(result._nodes['id'].tolist()) == {'e', 'f'} + + def test_deep_dependency_chain(self): + """Test deep linear dependency chain""" + g = CGFull().edges(pd.DataFrame({'s': list('abcdef'), 'd': list('bcdefg')}), 's', 'd') + g = g.materialize_nodes() + + # Create deep chain: n1 -> n2 -> n3 -> ... -> n10 + # Using empty chains to avoid execution issues + dag_dict = {'n1': n(name='level1')} + for i in range(2, 11): + dag_dict[f'n{i}'] = ASTChainRef(f'n{i - 1}', []) + + dag = ASTLet(dag_dict) + + # Test execution order is correct + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + + # Should be in sequential order + expected_order = [f'n{i}' for i in range(1, 11)] + assert order == expected_order + + # Execute DAG + result = g.gfql(dag) + + # Result should have level1 tag from n1 + assert 'level1' in result._nodes.columns + + def test_fan_out_fan_in_pattern(self): + """Test fan-out then fan-in pattern""" + g = CGFull().edges(pd.DataFrame({ + 's': ['root', 'a1', 'a2', 'b1', 'b2', 'b3'], + 'd': ['hub', 'end', 'end', 'end', 'end', 'end'] + }), 's', 'd') + g = g.materialize_nodes() + + # Test execution order for fan-out/fan-in + from graphistry.compute.chain_let import determine_execution_order + + dag = ASTLet({ + 'start': n({'id': 'root'}), + 'expand1': ASTChainRef('start', []), + 'expand2': ASTChainRef('start', []), + 'expand3': ASTChainRef('start', []), + 'collect': n() # Gets all nodes from original graph + }) + + # Check execution order + order = determine_execution_order(dag.bindings) + # 'start' must come before expand nodes + assert order.index('start') < order.index('expand1') + assert order.index('start') < order.index('expand2') + assert order.index('start') < order.index('expand3') + # 'collect' has no dependencies so can be anywhere + + # Execute DAG + result = g.gfql(dag) + # Result is from last executed node (one of the expand nodes) + # which references 'start' (filtered to just 'root') + assert len(result._nodes) == 1 + assert result._nodes['id'].iloc[0] == 'root' + + +class TestIntegration: + """Integration tests for complex DAG scenarios""" + + def test_empty_dag(self): + """Test empty DAG returns original graph""" + g = CGFull().edges(pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}), 's', 'd') + dag = ASTLet({}) + + result = g.gfql(dag) + + # Should return original graph + assert len(result._edges) == len(g._edges) + pd.testing.assert_frame_equal(result._edges, g._edges) + + def test_large_dag_10_nodes(self): + """Test DAG with 10+ nodes executes successfully""" + # Create a complex graph with attributes + nodes_data = [] + edges_data = [] + for i in range(20): + nodes_data.append({ + 'id': f'n{i}', + 'value': i, + 'type': 'even' if i % 2 == 0 else 'odd' + }) + for j in range(i + 1, min(i + 3, 20)): + edges_data.append({'s': f'n{i}', 'd': f'n{j}'}) + + g = CGFull().nodes(pd.DataFrame(nodes_data), 'id').edges(pd.DataFrame(edges_data), 's', 'd') + + # Create a 10+ node DAG with various patterns + dag = ASTLet({ + # Layer 1: Initial filters using filter_dict + 'high_value': n(name='high'), + 'even': n({'type': 'even'}), + 'odd': n({'type': 'odd'}), + + # Layer 2: References + 'high_even': ASTChainRef('even', []), + 'high_odd': ASTChainRef('odd', []), + + # Layer 3: More nodes + 'n1': n(name='tag1'), + 'n2': n(name='tag2'), + 'n3': n(name='tag3'), + 'n4': n(name='tag4'), + + # Layer 4: Final node + 'final': n(name='final_tag') + }) + + # Should execute without error + result = g.gfql(dag) + assert result is not None + # The DAG has 10 nodes, so it meets our 10+ node requirement + assert len(dag.bindings) == 10 + + # Verify execution order is valid + from graphistry.compute.chain_let import determine_execution_order + order = determine_execution_order(dag.bindings) + assert len(order) == 10 + # References come after their dependencies + assert order.index('even') < order.index('high_even') + assert order.index('odd') < order.index('high_odd') + + def test_mock_remote_graph_placeholder(self): + """Test DAG with RemoteGraph requires authentication""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + dag = ASTLet({ + 'remote1': ASTRemoteGraph('dataset1'), + 'remote2': ASTRemoteGraph('dataset2', token='mock-token'), + 'combined': n() # Would combine results + }) + + # Should raise error due to missing authentication + with pytest.raises(RuntimeError) as exc_info: + g.gfql(dag) + + assert "Must call login() first" in str(exc_info.value) + + def test_memory_efficient_execution(self): + """Test that intermediate results are stored efficiently""" + + # Create a simple DAG + g = CGFull().edges(pd.DataFrame({'s': list('abc'), 'd': list('bcd')}), 's', 'd') + g = g.materialize_nodes() + + dag = ASTLet({ + 'step1': n(name='tag1'), + 'step2': n(name='tag2'), + 'step3': n(name='tag3') + }) + + # Execute and verify context usage + result = g.gfql(dag) + + # Each step should produce a result + assert result is not None + # Result has the last tag + assert 'tag3' in result._nodes.columns + + def test_error_propagation_with_context(self): + """Test errors include helpful context about which node failed""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + dag = ASTLet({ + 'good1': n(), + 'good2': n(), + 'bad': n(query='invalid syntax !@#'), + 'never_reached': n() + }) + + with pytest.raises(RuntimeError) as exc_info: + g.gfql(dag) + + error_msg = str(exc_info.value) + assert "Failed to execute node 'bad'" in error_msg + assert "Error:" in error_msg + + +class TestCrossValidation: + """Cross-validation tests to verify implementation correctness""" + + def test_dag_vs_chain_consistency(self): + """Test that simple DAG produces same result as chain for linear flow""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Using chain + chain_result = g.chain([n({'type': 'person'})]) + + # Using DAG + dag = ASTLet({ + 'people': n({'type': 'person'}) + }) + dag_result = g.gfql(dag) + + # Should produce same nodes + assert len(chain_result._nodes) == len(dag_result._nodes) + assert set(chain_result._nodes['id'].tolist()) == set(dag_result._nodes['id'].tolist()) + + def test_execution_order_deterministic(self): + """Test that execution order is deterministic for same DAG""" + from graphistry.compute.chain_let import determine_execution_order + + dag = ASTLet({ + 'a': n(), + 'b': n(), + 'c': ASTChainRef('a', []), + 'd': ASTChainRef('b', []), + 'e': ASTChainRef('c', []), + 'f': ASTChainRef('d', []) + }) + + # Get order multiple times + orders = [] + for i in range(5): + order = determine_execution_order(dag.bindings) + orders.append(order) + + # All should be the same + for order in orders[1:]: + assert order == orders[0] + + def test_context_bindings_accessible(self): + """Test that all intermediate results are accessible in context""" + from graphistry.compute.chain_let import chain_let_impl + + g = CGFull().edges(pd.DataFrame({'s': list('abc'), 'd': list('bcd')}), 's', 'd') + g = g.materialize_nodes() + + # Create a mock context to track all bindings + bindings_tracker = {} + + class TrackingContext(ExecutionContext): + def set_binding(self, name, value): + super().set_binding(name, value) + bindings_tracker[name] = value + + # Monkey patch the execution to use our tracking context + original_chain_let_impl = chain_let_impl + + def tracking_chain_let_impl(g, dag, engine): + # Call original but capture context usage + return original_chain_let_impl(g, dag, engine) + + dag = ASTLet({ + 'step1': n(name='tag1'), + 'step2': n(name='tag2'), + 'step3': ASTChainRef('step1', []) + }) + + result = g.gfql(dag) + + # We can't easily intercept the context, but we can verify the result + assert result is not None + + def test_error_doesnt_corrupt_state(self): + """Test that errors don't leave DAG execution in bad state""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # First execution with error + bad_dag = ASTLet({ + 'bad': n(query='invalid syntax !!!') + }) + + try: + g.gfql(bad_dag) + except RuntimeError: + pass # Expected + + # Second execution should work fine + good_dag = ASTLet({ + 'good': n() + }) + + result = g.gfql(good_dag) + assert result is not None + + def test_node_filter_consistency(self): + """Test node filtering is consistent between chain and chain_let""" + nodes_df = pd.DataFrame({ + 'id': list('abcdef'), + 'value': [10, 20, 30, 40, 50, 60], + 'active': [True, False, True, False, True, False] + }) + edges_df = pd.DataFrame({'s': list('abcde'), 'd': list('bcdef')}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Test filter_dict + dag1 = ASTLet({'result': n({'active': True})}) + result1 = g.gfql(dag1) + assert len(result1._nodes) == 3 + assert all(result1._nodes['active']) + + # Test with name + dag2 = ASTLet({'result': n({'active': True}, name='is_active')}) + result2 = g.gfql(dag2) + assert 'is_active' in result2._nodes.columns + assert all(result2._nodes['is_active']) + + +class TestChainDagInternal: + """Test internal chain_let functionality (via gfql)""" + + def test_chain_let_via_gfql(self): + """Test that DAG execution works via gfql""" + g = CGFull() + assert hasattr(g, 'gfql') + assert callable(g.gfql) + + # chain_dag should not be in public API + assert not hasattr(g, 'chain_dag') + + def test_chain_let_empty(self): + """Test chain_let with empty DAG""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({}) + + # Empty DAG should return original graph + result = g.gfql(dag) + assert result is not None + + def test_chain_let_single_node_works(self): + """Test chain_let with single node now works""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g = g.materialize_nodes() + + dag = ASTLet({ + 'all_nodes': n() + }) + + # Should work now that node execution is implemented + result = g.gfql(dag) + assert result is not None + assert len(result._nodes) == 2 # nodes a and b + + def test_chain_let_remote_not_implemented(self): + """Test chain_let with RemoteGraph requires authentication""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({ + 'remote': ASTRemoteGraph('dataset123') + }) + + # Should raise RuntimeError due to missing authentication + with pytest.raises(RuntimeError) as exc_info: + g.gfql(dag) + + assert "Failed to execute node 'remote' in let bindings" in str(exc_info.value) + assert "Must call login() first" in str(exc_info.value) + + def test_chain_let_multi_node_works(self): + """Test chain_let with multiple nodes now works""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({ + 'first': n(), + 'second': n() + }) + + # Should work now that node execution is implemented + result = g.gfql(dag) + assert result is not None + + # Result should be from last node ('second') + # Both nodes have empty filters so should have all data + assert len(result._nodes) == 2 # nodes a and b + + def test_chain_let_validates(self): + """Test chain_let validates the DAG""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Invalid DAG should raise during validation + with pytest.raises(TypeError) as exc_info: + g.gfql("not a dag") + + assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) + + def test_chain_let_output_selection(self): + """Test output parameter selects specific binding""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}), + 'all_nodes': n() + }) + + # Default: returns last executed + result_default = g.gfql(dag) + # Could be any of the three since they have no dependencies + assert result_default is not None + + # Select specific outputs + result_people = g.gfql(dag, output='people') + assert len(result_people._nodes) == 2 + assert all(result_people._nodes['type'] == 'person') + + result_companies = g.gfql(dag, output='companies') + assert len(result_companies._nodes) == 2 + assert all(result_companies._nodes['type'] == 'company') + + result_all = g.gfql(dag, output='all_nodes') + assert len(result_all._nodes) == 4 + + def test_chain_let_output_not_found(self): + """Test error when output binding not found""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + dag = ASTLet({'node1': n()}) + + with pytest.raises(ValueError) as exc_info: + g.gfql(dag, output='missing') + + error_msg = str(exc_info.value) + assert "Output binding 'missing' not found" in error_msg + assert "Available bindings: ['node1']" in error_msg diff --git a/graphistry/tests/compute/test_chain_let_gpu.py b/graphistry/tests/compute/test_chain_let_gpu.py new file mode 100644 index 0000000000..33faf19694 --- /dev/null +++ b/graphistry/tests/compute/test_chain_let_gpu.py @@ -0,0 +1,196 @@ +import os +import pytest +import pandas as pd + +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, n +from graphistry.compute.chain_let import chain_let_impl +from graphistry.compute.execution_context import ExecutionContext +from graphistry.tests.test_compute import CGFull + +# Skip all tests if TEST_CUDF not set +skip_gpu = pytest.mark.skipif( + not ("TEST_CUDF" in os.environ and os.environ["TEST_CUDF"] == "1"), + reason="cudf tests need TEST_CUDF=1" +) + + +class TestChainLetGPU: + """Test chain_let with GPU/cudf""" + + @skip_gpu + def test_execution_context_stores_cudf(self): + """Test that ExecutionContext can store cudf DataFrames""" + import cudf + context = ExecutionContext() + + # Create a cudf DataFrame + gdf = cudf.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']}) + + # Store it + context.set_binding('gpu_data', gdf) + + # Retrieve it + retrieved = context.get_binding('gpu_data') + + # Verify it's still a cudf DataFrame + assert isinstance(retrieved, cudf.DataFrame) + assert retrieved.equals(gdf) + + @skip_gpu + def test_chain_let_with_cudf_edges(self): + """Test chain_let with cudf edge DataFrame""" + import cudf + # Create cudf edges + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + edges_gdf = cudf.from_pandas(edges_df) + + # Create graph with cudf edges + g = CGFull().edges(edges_gdf, 's', 'd') + + # Verify edges are cudf + assert isinstance(g._edges, cudf.DataFrame) + + # Empty DAG should work + dag = ASTLet({}) + result = g.gfql(dag) + + # Result should preserve GPU mode + assert isinstance(result._edges, cudf.DataFrame) + + @skip_gpu + def test_chain_let_engine_cudf(self): + """Test chain_let with explicit engine='cudf'""" + import cudf + # Start with pandas + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Empty DAG with cudf engine + dag = ASTLet({}) + result = g.gfql(dag, engine='cudf') + + # Should have materialized nodes + assert result._nodes is not None + # Nodes should be cudf (due to materialize_nodes with cudf engine) + assert isinstance(result._nodes, cudf.DataFrame) + + @skip_gpu + def test_chain_let_auto_detects_gpu(self): + """Test chain_let auto-detects GPU mode from edges""" + import cudf + # Create graph with GPU edges + edges_gdf = cudf.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Create a simple DAG (will fail on execution but that's ok) + dag = ASTLet({ + 'step1': n() + }) + + # Try to execute + try: + g.gfql(dag) # engine='auto' by default + except RuntimeError as e: + # Should fail on execution, but engine should be detected + assert "Failed to execute node 'step1'" in str(e) + + # The important part is it didn't fail on engine detection + # or materialize_nodes with GPU data + + @skip_gpu + def test_resolve_engine_with_gpu(self): + """Test that resolve_engine correctly identifies GPU mode""" + import cudf + from graphistry.Engine import resolve_engine, EngineAbstract + + # Create graph with cudf edges + edges_gdf = cudf.DataFrame({'s': ['a'], 'd': ['b']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Resolve should detect cudf + engine = resolve_engine(EngineAbstract.AUTO, g) + assert engine.value == 'cudf' + + @skip_gpu + def test_materialize_nodes_preserves_gpu(self): + """Test materialize_nodes works with GPU""" + import cudf + # Create graph with cudf edges + edges_gdf = cudf.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Materialize nodes + g2 = g.materialize_nodes() + + # Both edges and nodes should be cudf + assert isinstance(g2._edges, cudf.DataFrame) + assert isinstance(g2._nodes, cudf.DataFrame) + + # Check node content + expected_nodes = ['a', 'b', 'c', 'd'] + assert sorted(g2._nodes['id'].to_pandas().tolist()) == expected_nodes + + @skip_gpu + def test_chain_ref_with_gpu_data(self): + """Test ASTChainRef resolution works with GPU data""" + import cudf + from graphistry.compute.chain_let import execute_node + from graphistry.compute.execution_context import ExecutionContext + from graphistry.Engine import Engine + + # Create graph with cudf data + edges_gdf = cudf.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Create context and store GPU result + context = ExecutionContext() + context.set_binding('gpu_result', g) + + # Create chain ref to GPU data + chain_ref = ASTChainRef('gpu_result', []) + + # Execute should preserve GPU + result = execute_node('test', chain_ref, g, context, Engine.CUDF) + + # Result should still have GPU data + assert isinstance(result._edges, cudf.DataFrame) + assert result._edges.equals(edges_gdf) + + @skip_gpu + def test_dag_execution_preserves_gpu(self): + """Test full DAG execution preserves GPU mode""" + import cudf + + # Create graph with GPU data + edges_gdf = cudf.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().edges(edges_gdf, 's', 'd') + + # Create a simple DAG + dag = ASTLet({}) # Empty DAG + + # Execute + result = g.gfql(dag) + + # Should preserve GPU mode + assert isinstance(result._edges, cudf.DataFrame) + assert isinstance(result._nodes, cudf.DataFrame) + + @skip_gpu + def test_context_binding_with_mixed_engines(self): + """Test ExecutionContext can handle mixed pandas/cudf results""" + import cudf + from graphistry.compute.execution_context import ExecutionContext + + context = ExecutionContext() + + # Create both pandas and cudf graphs + g_pandas = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + edges_gdf = cudf.DataFrame({'s': ['x'], 'd': ['y']}) + g_cudf = CGFull().edges(edges_gdf, 's', 'd') + + # Store both + context.set_binding('pandas_graph', g_pandas) + context.set_binding('cudf_graph', g_cudf) + + # Retrieve and verify types preserved + assert isinstance(context.get_binding('pandas_graph')._edges, pd.DataFrame) + assert isinstance(context.get_binding('cudf_graph')._edges, cudf.DataFrame) diff --git a/graphistry/tests/compute/test_chain_let_remote_integration.py b/graphistry/tests/compute/test_chain_let_remote_integration.py new file mode 100644 index 0000000000..6d3abdc1c8 --- /dev/null +++ b/graphistry/tests/compute/test_chain_let_remote_integration.py @@ -0,0 +1,212 @@ +"""Integration tests for remote graph functionality in chain_let. + +These tests require a real Graphistry server and authentication. +Enable with: TEST_REMOTE_INTEGRATION=1 + +Additional optional env vars: +- GRAPHISTRY_USERNAME: Username for authentication +- GRAPHISTRY_PASSWORD: Password for authentication +- GRAPHISTRY_API_KEY: API key (alternative to username/password) +- GRAPHISTRY_SERVER: Server URL (defaults to hub.graphistry.com) +- GRAPHISTRY_TEST_DATASET_ID: Known dataset ID to test with +""" + +import os +import pytest +import pandas as pd +from unittest.mock import patch + +from graphistry import PyGraphistry +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, n +from graphistry.tests.test_compute import CGFull + + +# Check if remote integration tests are enabled +REMOTE_INTEGRATION_ENABLED = os.environ.get("TEST_REMOTE_INTEGRATION") == "1" +skip_remote = pytest.mark.skipif( + not REMOTE_INTEGRATION_ENABLED, + reason="Remote integration tests need TEST_REMOTE_INTEGRATION=1" +) + + +@skip_remote +class TestRemoteGraphIntegration: + """Integration tests that connect to a real Graphistry server.""" + + @classmethod + def setup_class(cls): + """Set up authentication for remote tests.""" + # Configure PyGraphistry with env vars if available + server = os.environ.get("GRAPHISTRY_SERVER", "hub.graphistry.com") + protocol = "https" if "443" in server or "https" in server else "http" + + if os.environ.get("GRAPHISTRY_API_KEY"): + PyGraphistry.register( + api=3, + protocol=protocol, + server=server, + api_key=os.environ["GRAPHISTRY_API_KEY"] + ) + elif os.environ.get("GRAPHISTRY_USERNAME") and os.environ.get("GRAPHISTRY_PASSWORD"): + PyGraphistry.register( + api=3, + protocol=protocol, + server=server, + username=os.environ["GRAPHISTRY_USERNAME"], + password=os.environ["GRAPHISTRY_PASSWORD"] + ) + else: + pytest.skip("Need GRAPHISTRY_API_KEY or GRAPHISTRY_USERNAME/PASSWORD for remote tests") + + def test_remote_graph_fetch_real_dataset(self): + """Test fetching a real dataset from Graphistry server.""" + # First, upload a test dataset to get a real dataset_id + test_edges = pd.DataFrame({ + 'src': ['a', 'b', 'c'], + 'dst': ['b', 'c', 'a'], + 'weight': [1.0, 2.0, 3.0] + }) + test_nodes = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'label': ['Node A', 'Node B', 'Node C'] + }) + + g = CGFull().edges(test_edges, 'src', 'dst').nodes(test_nodes, 'id') + uploaded = g.upload() + dataset_id = uploaded._dataset_id + assert dataset_id is not None + + # Now test fetching it via ASTRemoteGraph + dag = ASTLet({ + 'remote_data': ASTRemoteGraph(dataset_id) + }) + + # CGFull() creates empty graph, need one with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + g2 = g_base.gfql(dag) + + # Verify we got the data back + assert len(g2._edges) == 3 + assert len(g2._nodes) == 3 + assert set(g2._edges['src'].values) == {'a', 'b', 'c'} + assert set(g2._nodes['label'].values) == {'Node A', 'Node B', 'Node C'} + + def test_remote_graph_with_token(self): + """Test using explicit token with RemoteGraph.""" + # Get current token + PyGraphistry.refresh() + token = PyGraphistry.api_token() + + if not token: + pytest.skip("No API token available") + + # Upload test data + g = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + uploaded = g.upload() + dataset_id = uploaded._dataset_id + + # Fetch with explicit token + dag = ASTLet({ + 'data': ASTRemoteGraph(dataset_id, token=token) + }) + + # Need graph with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag) + assert len(result._edges) == 1 + + def test_remote_graph_in_complex_dag(self): + """Test RemoteGraph as part of a complex DAG.""" + # Upload test dataset + edges_df = pd.DataFrame({ + 'src': ['a', 'b', 'c', 'd'], + 'dst': ['b', 'c', 'd', 'a'], + 'type': ['friend', 'friend', 'enemy', 'enemy'] + }) + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'category': ['person', 'person', 'bot', 'bot'] + }) + + g = CGFull().edges(edges_df, 'src', 'dst').nodes(nodes_df, 'id') + uploaded = g.upload() + dataset_id = uploaded._dataset_id + + # Create complex DAG with remote data + dag = ASTLet({ + 'remote': ASTRemoteGraph(dataset_id), + 'persons': ASTChainRef('remote', [n({'category': 'person'})]), + 'friends': ASTChainRef('persons', [n(edge_query="type == 'friend'")]) + }) + + # Execute and verify - need graph with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag, output='friends') + + # Should only have person nodes + assert all(result._nodes['category'] == 'person') + # Should only have friend edges between persons + assert len(result._edges) > 0 + + def test_remote_graph_error_handling(self): + """Test error handling for invalid dataset IDs.""" + dag = ASTLet({ + 'bad_remote': ASTRemoteGraph('invalid-dataset-id-12345') + }) + + # Need graph with edges for materialize_nodes + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + with pytest.raises(Exception) as exc_info: + g.gfql(dag) + + # Should get some kind of HTTP error or validation error + assert 'dataset' in str(exc_info.value).lower() or 'not found' in str(exc_info.value).lower() + + @pytest.mark.skipif( + not os.environ.get("GRAPHISTRY_TEST_DATASET_ID"), + reason="Need GRAPHISTRY_TEST_DATASET_ID env var for this test" + ) + def test_remote_graph_known_dataset(self): + """Test with a known dataset ID from env var.""" + dataset_id = os.environ["GRAPHISTRY_TEST_DATASET_ID"] + + dag = ASTLet({ + 'data': ASTRemoteGraph(dataset_id) + }) + + # Need graph with edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag) + + # Basic validation - should have some data + assert result._edges is not None or result._nodes is not None + if result._edges is not None: + print(f"Fetched {len(result._edges)} edges from {dataset_id}") + if result._nodes is not None: + print(f"Fetched {len(result._nodes)} nodes from {dataset_id}") + + +class TestRemoteGraphMocked: + """Tests with mocked remote calls (always run).""" + + @patch('graphistry.compute.chain_remote.chain_remote') + def test_remote_graph_execution_mocked(self, mock_chain_remote): + """Test that RemoteGraph calls chain_remote correctly.""" + # This test always runs, even without remote server + mock_result = CGFull().edges(pd.DataFrame({'s': ['x'], 'd': ['y']}), 's', 'd') + mock_chain_remote.return_value = mock_result + + dag = ASTLet({ + 'remote': ASTRemoteGraph('test-dataset-123', token='test-token') + }) + + # Mock result should be used, but we still need edges for materialize_nodes + g_base = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + result = g_base.gfql(dag) + assert result is not None # Verify result was returned + + # Verify chain_remote was called correctly + mock_chain_remote.assert_called_once() + call_args = mock_chain_remote.call_args + assert call_args[1]['dataset_id'] == 'test-dataset-123' + assert call_args[1]['api_token'] == 'test-token' diff --git a/graphistry/tests/compute/test_chain_schema_validation.py b/graphistry/tests/compute/test_chain_schema_validation.py index c3ab7c4e40..0494d1ba15 100644 --- a/graphistry/tests/compute/test_chain_schema_validation.py +++ b/graphistry/tests/compute/test_chain_schema_validation.py @@ -4,8 +4,9 @@ import pandas as pd from graphistry import edges, nodes from graphistry.compute.chain import Chain -from graphistry.compute.ast import n, e_forward +from graphistry.compute.ast import n, e_forward, ASTLet, ASTChainRef, ASTRemoteGraph from graphistry.compute.exceptions import ErrorCode, GFQLSchemaError +from graphistry.compute.validate.validate_schema import validate_chain_schema class TestChainSchemaValidation: @@ -129,3 +130,157 @@ def test_schema_validation_disabled(self): # result = self.g.chain([n({'missing': 'value'})], validate_schema=False) # Would return empty result instead of raising pass + + +class TestLetSchemaValidation: + """Test schema validation for new Let AST types.""" + + def setup_method(self): + """Set up test data.""" + # Simple test graph + self.edges_df = pd.DataFrame({ + 'src': ['a', 'b'], + 'dst': ['b', 'c'], + 'edge_type': ['friend', 'friend'] + }) + + self.nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'], + 'age': [25, 30, None] + }) + + self.g = edges(self.edges_df, 'src', 'dst').nodes(self.nodes_df, 'id') + + def test_let_valid_schema(self): + """Valid Let passes schema validation.""" + dag = ASTLet({ + 'persons': n({'type': 'person'}), + 'friends': e_forward({'edge_type': 'friend'}) + }) + + # Should not raise + errors = validate_chain_schema(self.g, [dag], collect_all=True) + assert errors == [] + + def test_let_invalid_node_column(self): + """Let with invalid node column fails.""" + dag = ASTLet({ + 'bad_nodes': n({'missing_column': 'value'}) + }) + + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(self.g, [dag], collect_all=False) + + assert exc_info.value.code == ErrorCode.E301 + assert 'missing_column' in str(exc_info.value) + assert exc_info.value.context.get('dag_binding') == 'bad_nodes' + + def test_let_collect_all_errors(self): + """Let can collect all validation errors.""" + dag = ASTLet({ + 'bad_nodes': n({'missing1': 'value'}), + 'bad_edges': e_forward({'missing2': 'value'}) + }) + + errors = validate_chain_schema(self.g, [dag], collect_all=True) + assert len(errors) == 2 + assert all(e.code == ErrorCode.E301 for e in errors) + + # Check binding context is added + binding_names = {e.context.get('dag_binding') for e in errors} + assert binding_names == {'bad_nodes', 'bad_edges'} + + def test_chainref_valid_schema(self): + """Valid ChainRef passes schema validation.""" + chain_ref = ASTChainRef('other_data', [ + n({'type': 'person'}), + e_forward({'edge_type': 'friend'}) + ]) + + # Should not raise + errors = validate_chain_schema(self.g, [chain_ref], collect_all=True) + assert errors == [] + + def test_chainref_invalid_chain_operation(self): + """ChainRef with invalid chain operation fails.""" + chain_ref = ASTChainRef('other_data', [ + n({'missing_column': 'value'}) + ]) + + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(self.g, [chain_ref], collect_all=False) + + assert exc_info.value.code == ErrorCode.E301 + assert 'missing_column' in str(exc_info.value) + assert exc_info.value.context.get('chain_ref') == 'other_data' + + def test_chainref_empty_chain(self): + """ChainRef with empty chain passes validation.""" + chain_ref = ASTChainRef('other_data', []) + + # Should not raise + errors = validate_chain_schema(self.g, [chain_ref], collect_all=True) + assert errors == [] + + def test_remotegraph_valid(self): + """Valid RemoteGraph passes schema validation.""" + remote = ASTRemoteGraph('dataset123') + + # Should not raise + errors = validate_chain_schema(self.g, [remote], collect_all=True) + assert errors == [] + + def test_remotegraph_valid_with_token(self): + """Valid RemoteGraph with token passes schema validation.""" + remote = ASTRemoteGraph('dataset123', token='secret-token') + + # Should not raise + errors = validate_chain_schema(self.g, [remote], collect_all=True) + assert errors == [] + + def test_remotegraph_empty_dataset_id(self): + """RemoteGraph with empty dataset_id fails.""" + remote = ASTRemoteGraph('') + + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(self.g, [remote], collect_all=False) + + assert exc_info.value.code == ErrorCode.E303 + assert 'dataset_id must be a non-empty string' in str(exc_info.value) + + def test_remotegraph_none_dataset_id(self): + """RemoteGraph with None dataset_id fails.""" + remote = ASTRemoteGraph(None) + + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(self.g, [remote], collect_all=False) + + assert exc_info.value.code == ErrorCode.E303 + assert 'dataset_id must be a non-empty string' in str(exc_info.value) + + def test_remotegraph_invalid_token_type(self): + """RemoteGraph with non-string token fails.""" + remote = ASTRemoteGraph('dataset123', token=123) # Wrong type + + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(self.g, [remote], collect_all=False) + + assert exc_info.value.code == ErrorCode.E303 + assert 'token must be a string' in str(exc_info.value) + + def test_nested_let_validation(self): + """Nested Let structures are validated recursively.""" + nested_dag = ASTLet({ + 'inner': ASTLet({ + 'bad_node': n({'missing_column': 'value'}) + }) + }) + + with pytest.raises(GFQLSchemaError) as exc_info: + validate_chain_schema(self.g, [nested_dag], collect_all=False) + + assert exc_info.value.code == ErrorCode.E301 + assert 'missing_column' in str(exc_info.value) + # Should have both outer and inner binding context + assert 'dag_binding' in exc_info.value.context diff --git a/graphistry/tests/compute/test_gfql.py b/graphistry/tests/compute/test_gfql.py new file mode 100644 index 0000000000..9c689ef521 --- /dev/null +++ b/graphistry/tests/compute/test_gfql.py @@ -0,0 +1,181 @@ +import pandas as pd +import pytest +from graphistry.compute.ast import ASTLet, ASTChainRef, n, e +from graphistry.compute.chain import Chain +from graphistry.tests.test_compute import CGFull + + +class TestGFQLAPI: + """Test unified GFQL API and migration""" + + def test_public_api_methods(self): + """Test what methods are available on the public API""" + g = CGFull() + + # Should have gfql + assert hasattr(g, 'gfql') + assert callable(g.gfql) + + # Should still have chain (with deprecation) + assert hasattr(g, 'chain') + assert callable(g.chain) + + # Should NOT have chain_dag in public API + assert not hasattr(g, 'chain_dag') + + +class TestGFQL: + """Test unified GFQL entrypoint""" + + def test_gfql_exists(self): + """Test that gfql method exists on CGFull""" + g = CGFull() + assert hasattr(g, 'gfql') + assert callable(g.gfql) + + def test_gfql_with_list(self): + """Test gfql with list executes as chain""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute as chain + result = g.gfql([n({'type': 'person'})]) + + assert len(result._nodes) == 2 + assert all(result._nodes['type'] == 'person') + + def test_gfql_with_chain_object(self): + """Test gfql with Chain object""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute with Chain + chain = Chain([n({'type': 'person'}), e(), n()]) + result = g.gfql(chain) + + assert result is not None + # Result depends on graph structure + + def test_gfql_with_dag(self): + """Test gfql with ASTLet""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute as DAG + dag = ASTLet({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}) + }) + + result = g.gfql(dag) + assert result is not None + + def test_gfql_with_dict_convenience(self): + """Test gfql with dict converts to DAG""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Dict should convert to DAG + result = g.gfql({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}) + }) + + assert result is not None + + def test_gfql_output_with_dag(self): + """Test gfql output parameter works with DAG""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd'], + 'type': ['person', 'person', 'company', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b', 'c'], 'd': ['b', 'c', 'd']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Execute with output selection + result = g.gfql({ + 'people': n({'type': 'person'}), + 'companies': n({'type': 'company'}) + }, output='people') + + assert len(result._nodes) == 2 + assert all(result._nodes['type'] == 'person') + + def test_gfql_output_ignored_for_chain(self): + """Test gfql output parameter ignored for chains""" + g = CGFull().edges(pd.DataFrame({'s': ['a'], 'd': ['b']}), 's', 'd') + + # Should work but output ignored + result = g.gfql([n()], output='ignored') + assert result is not None + + def test_gfql_with_single_ast_object(self): + """Test gfql with single ASTObject wraps in list""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # Single ASTObject should work + result = g.gfql(n({'type': 'person'})) + + assert len(result._nodes) == 2 + assert all(result._nodes['type'] == 'person') + + def test_gfql_invalid_query_type(self): + """Test gfql with invalid query type""" + g = CGFull() + + with pytest.raises(TypeError) as exc_info: + g.gfql("not a valid query") + + assert "Query must be ASTObject, List[ASTObject], Chain, ASTLet, or dict" in str(exc_info.value) + + def test_gfql_deprecation_and_migration(self): + """Test deprecation warnings and migration path""" + import warnings + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'type': ['person', 'person', 'company'] + }) + edges_df = pd.DataFrame({'s': ['a', 'b'], 'd': ['b', 'c']}) + g = CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + # chain() should show deprecation warning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + chain_result = g.chain([n({'type': 'person'})]) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert "chain() is deprecated" in str(w[0].message) + assert "Use gfql()" in str(w[0].message) + + assert len(chain_result._nodes) == 2 + + # chain_dag should no longer exist as public method + assert not hasattr(g, 'chain_dag'), "chain_dag should be removed from public API" + + # gfql should work for both patterns + gfql_chain = g.gfql([n({'type': 'person'})]) + assert len(gfql_chain._nodes) == 2 + + gfql_dag = g.gfql({'people': n({'type': 'person'})}) + assert len(gfql_dag._nodes) == 2 diff --git a/graphistry/tests/compute/test_gfql_validation.py b/graphistry/tests/compute/test_gfql_validation.py index 1a93888f91..982c546bf9 100644 --- a/graphistry/tests/compute/test_gfql_validation.py +++ b/graphistry/tests/compute/test_gfql_validation.py @@ -4,7 +4,7 @@ from graphistry import edges, nodes from graphistry.compute.ast import n, e_forward, e_reverse -from graphistry.compute.gfql.validate import ( +from graphistry.compute.gfql_validation.validate import ( validate_syntax, validate_schema, validate_query, extract_schema_from_dataframes, format_validation_errors, Schema, ValidationIssue diff --git a/graphistry/tests/compute/test_let.py b/graphistry/tests/compute/test_let.py new file mode 100644 index 0000000000..4acbb3ef20 --- /dev/null +++ b/graphistry/tests/compute/test_let.py @@ -0,0 +1,181 @@ +"""Tests for Let bindings and related AST nodes validation""" +import pytest +from graphistry.compute.ast import ASTLet, ASTRemoteGraph, ASTChainRef, n, e +from graphistry.compute.execution_context import ExecutionContext + + +class TestLetValidation: + """Test validation for Let bindings""" + + def test_let_valid(self): + """Valid Let should pass validation""" + dag = ASTLet({'a': n(), 'b': e()}) + dag.validate() # Should not raise + + def test_let_invalid_key_type(self): + """Let with non-string key should fail""" + with pytest.raises(AssertionError, match="binding key must be string"): + dag = ASTLet({123: n()}) + dag.validate() + + def test_let_invalid_value_type(self): + """Let with non-ASTObject value should fail""" + with pytest.raises(AssertionError, match="binding value must be ASTObject"): + dag = ASTLet({'a': 'not an AST object'}) + dag.validate() + + def test_let_nested_validation(self): + """Let should validate nested objects""" + # This should work - nested validation of valid objects + dag = ASTLet({ + 'a': n({'type': 'person'}), + 'b': ASTRemoteGraph('dataset123') + }) + dag.validate() + + +class TestRemoteGraphValidation: + """Test validation for RemoteGraph""" + + def test_remoteGraph_valid(self): + """Valid RemoteGraph should pass validation""" + rg = ASTRemoteGraph('my-dataset') + rg.validate() # Should not raise + + rg_with_token = ASTRemoteGraph('my-dataset', token='secret') + rg_with_token.validate() # Should not raise + + def test_remoteGraph_invalid_dataset_type(self): + """RemoteGraph with non-string dataset_id should fail""" + with pytest.raises(AssertionError, match="dataset_id must be a string"): + rg = ASTRemoteGraph(123) + rg.validate() + + def test_remoteGraph_empty_dataset(self): + """RemoteGraph with empty dataset_id should fail""" + with pytest.raises(AssertionError, match="dataset_id cannot be empty"): + rg = ASTRemoteGraph('') + rg.validate() + + def test_remoteGraph_invalid_token_type(self): + """RemoteGraph with non-string token should fail""" + with pytest.raises(AssertionError, match="token must be string or None"): + rg = ASTRemoteGraph('dataset', token=123) + rg.validate() + + +class TestChainRefValidation: + """Test validation for ChainRef""" + + def test_chainRef_valid(self): + """Valid ChainRef should pass validation""" + cr = ASTChainRef('myref', [n(), e()]) + cr.validate() # Should not raise + + cr_empty = ASTChainRef('myref', []) + cr_empty.validate() # Empty chain is valid + + def test_chainRef_invalid_ref_type(self): + """ChainRef with non-string ref should fail""" + with pytest.raises(AssertionError, match="ref must be a string"): + cr = ASTChainRef(123, []) + cr.validate() + + def test_chainRef_empty_ref(self): + """ChainRef with empty ref should fail""" + with pytest.raises(AssertionError, match="ref cannot be empty"): + cr = ASTChainRef('', []) + cr.validate() + + def test_chainRef_invalid_chain_type(self): + """ChainRef with non-list chain should fail""" + with pytest.raises(AssertionError, match="chain must be a list"): + cr = ASTChainRef('ref', 'not a list') + cr.validate() + + def test_chainRef_invalid_chain_element(self): + """ChainRef with non-ASTObject in chain should fail""" + with pytest.raises(AssertionError, match="must be ASTObject"): + cr = ASTChainRef('ref', [n(), 'not an AST object']) + cr.validate() + + def test_chainRef_nested_validation(self): + """ChainRef should validate nested operations""" + cr = ASTChainRef('ref', [n({'type': 'person'}), e()]) + cr.validate() # Should validate nested nodes + + +class TestExecutionContext: + """Test ExecutionContext functionality""" + + def test_context_basic_operations(self): + """Test basic get/set operations""" + ctx = ExecutionContext() + + # Set and get + ctx.set_binding('a', 'value_a') + assert ctx.get_binding('a') == 'value_a' + + # Has binding + assert ctx.has_binding('a') is True + assert ctx.has_binding('b') is False + + # Multiple bindings + ctx.set_binding('b', 123) + assert ctx.get_binding('b') == 123 + assert len(ctx.get_all_bindings()) == 2 + + def test_context_missing_binding(self): + """Test error on missing binding""" + ctx = ExecutionContext() + with pytest.raises(KeyError, match="No binding found for 'missing'"): + ctx.get_binding('missing') + + def test_context_invalid_name_type(self): + """Test error on non-string binding names""" + ctx = ExecutionContext() + + with pytest.raises(TypeError, match="Binding name must be string"): + ctx.set_binding(123, 'value') + + with pytest.raises(TypeError, match="Binding name must be string"): + ctx.get_binding(123) + + with pytest.raises(TypeError, match="Binding name must be string"): + ctx.has_binding(123) + + def test_context_clear(self): + """Test clearing all bindings""" + ctx = ExecutionContext() + ctx.set_binding('a', 1) + ctx.set_binding('b', 2) + assert len(ctx.get_all_bindings()) == 2 + + ctx.clear() + assert len(ctx.get_all_bindings()) == 0 + assert ctx.has_binding('a') is False + + def test_context_overwrite(self): + """Test overwriting existing binding""" + ctx = ExecutionContext() + ctx.set_binding('a', 'first') + assert ctx.get_binding('a') == 'first' + + ctx.set_binding('a', 'second') + assert ctx.get_binding('a') == 'second' + + +class TestChainRefReverse: + """Test reverse operation for ChainRef""" + + def test_chainRef_reverse(self): + """Test ChainRef reverse reverses operations""" + cr = ASTChainRef('data', [n(), e(), n()]) + reversed_cr = cr.reverse() + + assert isinstance(reversed_cr, ASTChainRef) + assert reversed_cr.ref == 'data' + assert len(reversed_cr.chain) == 3 + # Operations should be reversed + # Original: n, e, n + # Reversed: n, e.reverse(), n (each op is individually reversed) diff --git a/graphistry/tests/compute/test_validate.py b/graphistry/tests/compute/test_validate.py index 7b2c558c54..a54c211597 100644 --- a/graphistry/tests/compute/test_validate.py +++ b/graphistry/tests/compute/test_validate.py @@ -4,7 +4,7 @@ import pytest from typing import List -from graphistry.compute.gfql.validate import ( +from graphistry.compute.gfql_validation.validate import ( validate_syntax, validate_schema, validate_query, extract_schema, extract_schema_from_dataframes, format_validation_errors, suggest_fixes, diff --git a/graphistry/tests/test_compute_chain.py b/graphistry/tests/test_compute_chain.py index a5c7c33087..3c0ae9f526 100644 --- a/graphistry/tests/test_compute_chain.py +++ b/graphistry/tests/test_compute_chain.py @@ -4,7 +4,8 @@ import pandas as pd from common import NoAuthTestCase -from graphistry.compute.ast import n, e_forward, e_reverse, e_undirected, is_in +from graphistry.compute.ast import n, e_forward, e_reverse, e_undirected +from graphistry.compute import is_in from graphistry.tests.test_compute import CGFull from graphistry.tests.test_compute_hops import hops_graph from graphistry.util import setup_logger diff --git a/graphistry/tests/test_compute_filter_by_dict.py b/graphistry/tests/test_compute_filter_by_dict.py index 07224a23bb..ba021e2b76 100644 --- a/graphistry/tests/test_compute_filter_by_dict.py +++ b/graphistry/tests/test_compute_filter_by_dict.py @@ -1,7 +1,7 @@ import pandas as pd from functools import lru_cache -from graphistry.compute.ast import is_in, IsIn +from graphistry.compute import is_in, IsIn from graphistry.compute.filter_by_dict import filter_by_dict from graphistry.tests.test_compute import CGFull diff --git a/graphistry/tests/test_compute_hops.py b/graphistry/tests/test_compute_hops.py index cf2c8e6b02..348fffec3e 100644 --- a/graphistry/tests/test_compute_hops.py +++ b/graphistry/tests/test_compute_hops.py @@ -2,7 +2,7 @@ from common import NoAuthTestCase from functools import lru_cache -from graphistry.compute.ast import is_in +from graphistry.compute import is_in from graphistry.tests.test_compute import CGFull @lru_cache(maxsize=1)