diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ba16a4ea..e496b56ca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,20 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm * py.typed added, type checking active on PyGraphistry! * transform() and transform_umap() now require some parameters to be keyword-only +### Added + +* GFQL: Comparison predicates (`gt`, `lt`, `ge`, `le`, `eq`, `ne`, `between`) and `is_in` now support datetime, date, and time values with timezone awareness + +### Changed + +* GFQL: Temporal value classes (`DateTimeValue`, `DateValue`, `TimeValue`) are now exported from `graphistry.compute` + +### Internal + +* Remove unused imports across predicate modules +* Rename `numeric.py` to `comparison.py` for predicate module (more accurate name) +* Add proper type annotations for predicate normalization functions + ## [0.38.0 - 2025-06-17] ### Changed diff --git a/demos/gfql/temporal_predicates.ipynb b/demos/gfql/temporal_predicates.ipynb new file mode 100644 index 0000000000..22b229f660 --- /dev/null +++ b/demos/gfql/temporal_predicates.ipynb @@ -0,0 +1,461 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# GFQL DateTime Filtering Examples\n\nThis notebook shows how to filter graph data by dates and times using GFQL predicates." + }, + { + "cell_type": "markdown", + "source": "## Table of Contents\n\n**Key Temporal Filtering Concepts:**\n\n1. **Basic DateTime Filtering** - Filter by specific dates and times\n2. **Date-Only Filtering** - Ignore time components \n3. **Time-of-Day Filtering** - Filter by time patterns\n4. **Complex Temporal Queries** - Combine with other predicates\n5. **Temporal Value Classes** - Explicit temporal objects\n6. **Timezone-Aware Filtering** - Handle timezone conversions\n7. **Chain Operations** - Multi-hop temporal queries\n8. **Wire Protocol Dicts** - JSON-compatible configuration\n\n**Quick Reference:**\n- `gt()`, `lt()`, `ge()`, `le()` - Greater/less than comparisons\n- `between()` - Range queries\n- `is_in()` - Match specific values\n- `DateTimeValue`, `DateValue`, `TimeValue` - Explicit temporal types", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from datetime import datetime, date, time, timedelta\n", + "import pytz\n", + "import graphistry\n", + "from graphistry import n, e_forward, e_reverse, e_undirected\n", + "from graphistry.compute import (\n", + " gt, lt, ge, le, eq, ne, between, is_in,\n", + " DateTimeValue, DateValue, TimeValue\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup: Create Sample Data\n", + "\n", + "Let's create a sample dataset representing a transaction network with temporal data." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate sample transaction data\n", + "np.random.seed(42)\n", + "\n", + "# Create nodes (accounts)\n", + "n_accounts = 100\n", + "accounts_df = pd.DataFrame({\n", + " 'account_id': [f'ACC_{i:04d}' for i in range(n_accounts)],\n", + " 'account_type': np.random.choice(['checking', 'savings', 'business'], n_accounts),\n", + " 'created_date': pd.date_range('2020-01-01', periods=n_accounts, freq='W'),\n", + " 'last_active': pd.date_range('2023-01-01', periods=n_accounts, freq='D') + \n", + " pd.to_timedelta(np.random.randint(0, 365, n_accounts), unit='D')\n", + "})\n", + "\n", + "# Create edges (transactions)\n", + "n_transactions = 500\n", + "transactions_df = pd.DataFrame({\n", + " 'transaction_id': [f'TXN_{i:06d}' for i in range(n_transactions)],\n", + " 'source': np.random.choice(accounts_df['account_id'], n_transactions),\n", + " 'target': np.random.choice(accounts_df['account_id'], n_transactions),\n", + " 'amount': np.random.exponential(100, n_transactions).round(2),\n", + " 'timestamp': pd.date_range('2023-01-01', periods=n_transactions, freq='H') + \n", + " pd.to_timedelta(np.random.randint(0, 8760, n_transactions), unit='H'),\n", + " 'transaction_time': [time(np.random.randint(0, 24), np.random.randint(0, 60)) \n", + " for _ in range(n_transactions)],\n", + " 'transaction_type': np.random.choice(['transfer', 'payment', 'deposit'], n_transactions)\n", + "})\n", + "\n", + "print(f\"Created {len(accounts_df)} accounts and {len(transactions_df)} transactions\")\n", + "print(f\"\\nTransaction date range: {transactions_df['timestamp'].min()} to {transactions_df['timestamp'].max()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Create graphistry instance\n", + "g = graphistry.edges(transactions_df, 'source', 'target').nodes(accounts_df, 'account_id')\n", + "print(f\"Graph: {len(g._nodes)} nodes, {len(g._edges)} edges\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Basic DateTime Filtering\n", + "\n", + "Filter transactions based on datetime values using edge predicates." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter transactions after a specific date\n", + "# First, filter the edges directly\n", + "cutoff_date = datetime(2023, 7, 1)\n", + "recent_edges = g._edges[gt(pd.Timestamp(cutoff_date))(g._edges['timestamp'])]\n", + "recent_g = g.edges(recent_edges)\n", + "\n", + "print(f\"Transactions after {cutoff_date}: {len(recent_g._edges)}\")\n", + "recent_g._edges[['transaction_id', 'timestamp', 'amount']].head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Alternative: Use chain with edge operations\n", + "# Start from all nodes, then follow edges with temporal filter\n", + "recent_chain = g.chain([\n", + " n(), # Start with all nodes\n", + " e_forward({\n", + " \"timestamp\": gt(pd.Timestamp(cutoff_date))\n", + " })\n", + "])\n", + "\n", + "print(f\"Transactions after {cutoff_date} (chain): {len(recent_chain._edges)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter transactions in a specific month\n", + "march_edges = g._edges[\n", + " between(\n", + " datetime(2023, 3, 1),\n", + " datetime(2023, 3, 31, 23, 59, 59)\n", + " )(g._edges['timestamp'])\n", + "]\n", + "march_g = g.edges(march_edges)\n", + "\n", + "print(f\"Transactions in March 2023: {len(march_g._edges)}\")\n", + "march_g._edges[['transaction_id', 'timestamp', 'amount']].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Date-Only Filtering\n", + "\n", + "Filter nodes based on dates, ignoring time components." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter accounts created after a specific date\n", + "new_accounts = g.chain([\n", + " n(filter_dict={\n", + " \"created_date\": ge(date(2021, 1, 1))\n", + " })\n", + "])\n", + "\n", + "print(f\"Accounts created after 2021: {len(new_accounts._nodes)}\")\n", + "new_accounts._nodes[['account_id', 'created_date', 'account_type']].head()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Find accounts active in the last 90 days\n", + "ninety_days_ago = datetime.now().date() - timedelta(days=90)\n", + "active_accounts = g.chain([\n", + " n(filter_dict={\n", + " \"last_active\": gt(pd.Timestamp(ninety_days_ago))\n", + " })\n", + "])\n", + "\n", + "print(f\"Recently active accounts: {len(active_accounts._nodes)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Time-of-Day Filtering\n", + "\n", + "Filter transactions based on time of day." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# Find transactions during business hours (9 AM - 5 PM)\n", + "business_hours_edges = g._edges[\n", + " between(\n", + " time(9, 0, 0),\n", + " time(17, 0, 0)\n", + " )(g._edges['transaction_time'])\n", + "]\n", + "business_hours_g = g.edges(business_hours_edges)\n", + "\n", + "print(f\"Business hour transactions: {len(business_hours_g._edges)}\")\n", + "print(f\"Percentage of total: {len(business_hours_g._edges) / len(g._edges) * 100:.1f}%\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Find transactions at specific times (e.g., on the hour)\n", + "on_the_hour_times = [time(h, 0, 0) for h in range(24)]\n", + "on_hour_edges = g._edges[\n", + " is_in(on_the_hour_times)(g._edges['transaction_time'])\n", + "]\n", + "on_hour_g = g.edges(on_hour_edges)\n", + "\n", + "print(f\"Transactions on the hour: {len(on_hour_g._edges)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Complex Temporal Queries\n", + "\n", + "Combine temporal predicates with other filters for complex queries." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# Find large transactions (>$500) in Q4 2023\n", + "q4_mask = between(\n", + " datetime(2023, 10, 1),\n", + " datetime(2023, 12, 31, 23, 59, 59)\n", + ")(g._edges['timestamp'])\n", + "large_mask = gt(500)(g._edges['amount'])\n", + "\n", + "q4_large_edges = g._edges[q4_mask & large_mask]\n", + "q4_large_g = g.edges(q4_large_edges)\n", + "\n", + "print(f\"Large Q4 2023 transactions: {len(q4_large_g._edges)}\")\n", + "if len(q4_large_g._edges) > 0:\n", + " print(f\"Total value: ${q4_large_g._edges['amount'].sum():,.2f}\")\n", + " print(f\"Average: ${q4_large_g._edges['amount'].mean():,.2f}\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": "# Multi-hop query: Find accounts that received money recently\n# and then sent money to business accounts\nthirty_days_ago = datetime.now() - timedelta(days=30)\n\n# First, find recent transactions\nrecent_edges = g._edges[gt(pd.Timestamp(thirty_days_ago))(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\n# Use chain to find money flow pattern\nmoney_flow = recent_g.chain([\n # Start with any node\n n(),\n # Follow incoming edges (as destination)\n e_reverse(),\n # Go to source nodes\n n(),\n # Follow outgoing edges\n e_forward(),\n # To business accounts\n n(filter_dict={\"account_type\": \"business\"})\n])\n\nprint(f\"Money flow pattern found: {len(money_flow._nodes)} business accounts\")", + "execution_count": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Using Temporal Value Classes\n", + "\n", + "Use explicit temporal value classes for more control." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Create temporal values with specific properties\n", + "dt_value = DateTimeValue(\"2023-06-15T14:30:00\", \"UTC\")\n", + "date_value = DateValue(\"2023-06-15\")\n", + "time_value = TimeValue(\"14:30:00\")\n", + "\n", + "# Use in predicates\n", + "specific_edges = g._edges[gt(dt_value)(g._edges['timestamp'])]\n", + "specific_g = g.edges(specific_edges)\n", + "\n", + "print(f\"Transactions after {dt_value.value}: {len(specific_g._edges)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Timezone-Aware Filtering\n", + "\n", + "Handle timezone-aware datetime comparisons." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# Add timezone info to our data for this example\n", + "transactions_df_tz = transactions_df.copy()\n", + "transactions_df_tz['timestamp_utc'] = pd.to_datetime(transactions_df_tz['timestamp']).dt.tz_localize('UTC')\n", + "transactions_df_tz['timestamp_eastern'] = transactions_df_tz['timestamp_utc'].dt.tz_convert('US/Eastern')\n", + "\n", + "g_tz = graphistry.edges(transactions_df_tz, 'source', 'target')\n", + "\n", + "# Filter using Eastern time\n", + "eastern = pytz.timezone('US/Eastern')\n", + "eastern_cutoff = eastern.localize(datetime(2023, 7, 1, 9, 0, 0)) # 9 AM Eastern\n", + "\n", + "eastern_morning_edges = g_tz._edges[\n", + " gt(pd.Timestamp(eastern_cutoff))(g_tz._edges['timestamp_eastern'])\n", + "]\n", + "eastern_morning_g = g_tz.edges(eastern_morning_edges)\n", + "\n", + "print(f\"Transactions after 9 AM Eastern on July 1, 2023: {len(eastern_morning_g._edges)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Chain Operations with Temporal Edge Filters\n", + "\n", + "Demonstrate using temporal predicates in chain operations with proper edge filtering." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Find paths through recent high-value transactions\n", + "recent_high_value = g.chain([\n", + " # Start from all nodes\n", + " n(),\n", + " # Follow edges with temporal and amount filters\n", + " e_forward({\n", + " \"timestamp\": gt(datetime.now() - timedelta(days=7)),\n", + " \"amount\": gt(200)\n", + " }),\n", + " # Reach destination nodes\n", + " n()\n", + "])\n", + "\n", + "print(f\"Recent high-value transaction paths:\")\n", + "print(f\" Nodes: {len(recent_high_value._nodes)}\")\n", + "print(f\" Edges: {len(recent_high_value._edges)}\")" + ] + }, + { + "cell_type": "code", + "source": "# Wire protocol dicts in is_in predicates\n# Useful for checking against multiple specific timestamps\n\nimportant_dates = [\n {\"type\": \"datetime\", \"value\": \"2023-01-01T00:00:00\", \"timezone\": \"UTC\"}, # New Year\n {\"type\": \"datetime\", \"value\": \"2023-07-04T00:00:00\", \"timezone\": \"UTC\"}, # July 4th\n {\"type\": \"datetime\", \"value\": \"2023-12-25T00:00:00\", \"timezone\": \"UTC\"}, # Christmas\n]\n\n# Note: This checks for exact timestamp matches\n# For date matching, you'd need to extract the date portion\nholiday_pred = is_in(important_dates)\n\n# For demonstration, let's check if any transactions happened exactly at midnight on these days\n# (In real data, you'd probably want to check date ranges instead)\nprint(f\"Checking for transactions at midnight on holidays...\")\nprint(f\"(This is likely 0 unless transactions were specifically created at midnight)\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": "## Summary\n\nThis notebook demonstrated:\n\n1. **DateTime filtering** with `gt`, `lt`, `between` predicates on edges\n2. **Date-only filtering** for day-level granularity on nodes\n3. **Time-of-day filtering** for patterns like business hours\n4. **Complex queries** combining temporal and non-temporal predicates\n5. **Multi-hop queries** with temporal constraints using chain operations\n6. **Temporal value classes** for explicit control\n7. **Timezone-aware** filtering\n8. **Wire protocol dictionaries** for JSON-compatible predicate configuration\n9. **Proper chain syntax** with edge filters in `e_forward()` and node filters in `n()`\n\nKey takeaways:\n- Temporal predicates work seamlessly with pandas datetime types\n- Wire protocol dicts enable configuration-driven filtering: `gt({\"type\": \"datetime\", \"value\": \"2023-01-01T00:00:00\", \"timezone\": \"UTC\"})`\n- Timezone awareness is built-in for accurate cross-timezone comparisons\n- Complex temporal patterns can be expressed through chain operations\n\nTemporal predicates in GFQL provide a powerful way to analyze time-series aspects of graph data, enabling complex temporal queries while maintaining the expressiveness of graph traversals.", + "metadata": {} + }, + { + "cell_type": "code", + "source": "# Build predicates programmatically with wire protocol dicts\ndef create_date_filter(year, month, day, comparison=\"gt\"):\n \"\"\"Create a date filter using wire protocol format\"\"\"\n date_dict = {\n \"type\": \"date\",\n \"value\": f\"{year:04d}-{month:02d}-{day:02d}\"\n }\n \n if comparison == \"gt\":\n return gt(date_dict)\n elif comparison == \"lt\":\n return lt(date_dict)\n elif comparison == \"ge\":\n return ge(date_dict)\n elif comparison == \"le\":\n return le(date_dict)\n else:\n raise ValueError(f\"Unknown comparison: {comparison}\")\n\n# Use the programmatic filter\nfilter_2023 = create_date_filter(2023, 1, 1, \"ge\")\naccounts_2023 = g.chain([\n n(filter_dict={\n \"created_date\": filter_2023\n })\n])\n\nprint(f\"Accounts created in 2023 or later: {len(accounts_2023._nodes)}\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": "# Example: Load predicate configuration from JSON\nimport json\n\n# Simulate loading from a JSON config file\nconfig_json = '''\n{\n \"filters\": {\n \"recent_transactions\": {\n \"timestamp\": {\n \"type\": \"gt\",\n \"value\": {\n \"type\": \"datetime\",\n \"value\": \"2023-10-01T00:00:00\",\n \"timezone\": \"UTC\"\n }\n }\n },\n \"business_hours\": {\n \"transaction_time\": {\n \"type\": \"between\",\n \"start\": {\"type\": \"time\", \"value\": \"09:00:00\"},\n \"end\": {\"type\": \"time\", \"value\": \"17:00:00\"}\n }\n }\n }\n}\n'''\n\nconfig = json.loads(config_json)\n\n# Use the wire protocol dict directly\nrecent_filter = config[\"filters\"][\"recent_transactions\"][\"timestamp\"][\"value\"]\nrecent_edges = g._edges[gt(recent_filter)(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\nprint(f\"Recent transactions (from JSON config): {len(recent_g._edges)}\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": "# Wire protocol dictionaries work directly in Python\n# These are equivalent:\npred1 = gt(pd.Timestamp(\"2023-07-01\"))\npred2 = gt({\"type\": \"datetime\", \"value\": \"2023-07-01T00:00:00\", \"timezone\": \"UTC\"})\n\n# Test they produce the same results\nresult1 = pred1(g._edges['timestamp'])\nresult2 = pred2(g._edges['timestamp'])\nprint(f\"Results are identical: {result1.equals(result2)}\")\nprint(f\"Transactions after July 1, 2023: {result1.sum()}\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": "## 8. Using Wire Protocol Dictionaries\n\nYou can pass wire protocol dictionaries directly to temporal predicates. This is useful for:\n- Loading predicate configurations from JSON files\n- Building predicates programmatically\n- Sharing predicate definitions between systems", + "metadata": {} + }, + { + "cell_type": "markdown", + "source": "## What's Next?\n\n### Learn More\n- **[GFQL Documentation](https://hub.graphistry.com/docs/api/3/gfql/)** - Complete GFQL reference\n- **[Datetime Filtering Guide](../../docs/source/gfql/datetime_filtering.md)** - Detailed temporal predicate documentation\n- **[Wire Protocol Reference](../../docs/source/gfql/wire_protocol_examples.md)** - JSON configuration examples\n\n### Try These Examples\n1. **Anomaly Detection**: Find transactions outside business hours or on weekends\n2. **Pattern Analysis**: Identify periodic transaction patterns\n3. **Time-based Cohorts**: Group accounts by creation date\n4. **Temporal Paths**: Find time-ordered sequences of events\n\n### Advanced Topics\n- Combine temporal predicates with graph algorithms\n- Build time-based graph visualizations\n- Create reusable temporal filter configurations\n- Integrate with streaming data pipelines", + "metadata": {} + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": "# Complex multi-hop with temporal constraints\n# Find 2-hop paths through recent transactions\ntwo_hop_recent = g.chain([\n # Start from business accounts\n n(filter_dict={\"account_type\": \"business\"}),\n # First hop: recent outgoing transactions\n e_forward({\n \"timestamp\": gt(datetime.now() - timedelta(days=30))\n }, name=\"hop1\"),\n # Intermediate nodes\n n(),\n # Second hop: any transaction\n e_forward(name=\"hop2\"),\n # Final nodes\n n()\n])\n\nprint(f\"2-hop paths from business accounts through recent transactions:\")\nprint(f\" Total edges: {len(two_hop_recent._edges)}\")\nprint(f\" Hop 1 edges: {two_hop_recent._edges['hop1'].sum()}\")\nprint(f\" Hop 2 edges: {two_hop_recent._edges['hop2'].sum()}\")", + "execution_count": 16 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook demonstrated:\n", + "\n", + "1. **DateTime filtering** with `gt`, `lt`, `between` predicates on edges\n", + "2. **Date-only filtering** for day-level granularity on nodes\n", + "3. **Time-of-day filtering** for patterns like business hours\n", + "4. **Complex queries** combining temporal and non-temporal predicates\n", + "5. **Multi-hop queries** with temporal constraints using chain operations\n", + "6. **Temporal value classes** for explicit control\n", + "7. **Timezone-aware** filtering\n", + "8. **Proper chain syntax** with edge filters in `e_forward()` and node filters in `n()`\n", + "\n", + "Temporal predicates in GFQL provide a powerful way to analyze time-series aspects of graph data, enabling complex temporal queries while maintaining the expressiveness of graph traversals." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 178284f0b6..fcfbb47d53 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -68,3 +68,21 @@ services: - "8888:8888" volumes: - ../jupyter_dev:/home/jovyan/jupyter_dev + + test-notebooks: + << : *production_opts + image: graphistry/test-notebooks:${DOCKER_TAG:-latest} + build: + << : *build_kwargs + context: .. + dockerfile: ./docker/test-notebooks.Dockerfile + cache_from: + - graphistry/test-notebooks:${DOCKER_TAG:-latest} + container_name: "test-notebooks" + environment: + - TEST_TYPE=${TEST_TYPE:-nbval} + - NOTEBOOK_PATH=${NOTEBOOK_PATH:-demos} + - TIMEOUT=${TIMEOUT:-300} + - PARALLEL=${PARALLEL:-auto} + volumes: + - ..:/opt/pygraphistry diff --git a/docs/source/api/ai.rst b/docs/source/api/ai.rst index 7460dcd0b5..a8ad9ea41b 100644 --- a/docs/source/api/ai.rst +++ b/docs/source/api/ai.rst @@ -58,4 +58,5 @@ HeterographEmbedModuleMixin .. automodule:: graphistry.embed_utils :members: :undoc-members: + :inherited-members: :show-inheritance: diff --git a/docs/source/api/compute.rst b/docs/source/api/compute.rst index fb361e6ca2..5fbf09cc54 100644 --- a/docs/source/api/compute.rst +++ b/docs/source/api/compute.rst @@ -6,6 +6,7 @@ ComputeMixin module .. automodule:: graphistry.compute.ComputeMixin :members: :undoc-members: + :inherited-members: :show-inheritance: Collapse diff --git a/docs/source/api/gfql/predicates.rst b/docs/source/api/gfql/predicates.rst index fedacd84df..320b63ef86 100644 --- a/docs/source/api/gfql/predicates.rst +++ b/docs/source/api/gfql/predicates.rst @@ -53,3 +53,12 @@ Temporal :undoc-members: :show-inheritance: :noindex: + +Temporal Values +--------------- + +.. automodule:: graphistry.compute.predicates.temporal_values + :members: + :undoc-members: + :show-inheritance: + :noindex: diff --git a/docs/source/api/layout/index.rst b/docs/source/api/layout/index.rst index e9ad602be9..defa7a14eb 100644 --- a/docs/source/api/layout/index.rst +++ b/docs/source/api/layout/index.rst @@ -38,4 +38,5 @@ LayoutsMixin .. automodule:: graphistry.layouts.LayoutsMixin :members: :undoc-members: + :inherited-members: :show-inheritance: \ No newline at end of file diff --git a/docs/source/api/plotter.rst b/docs/source/api/plotter.rst index 140d888cb1..361135c339 100644 --- a/docs/source/api/plotter.rst +++ b/docs/source/api/plotter.rst @@ -18,20 +18,18 @@ Plotter Class :inherited-members: :show-inheritance: - -PlotterBase Class +Plottable Interface ---------------------- -.. automodule:: graphistry.PlotterBase +.. automodule:: graphistry.Plottable :members: :undoc-members: :inherited-members: :show-inheritance: - -Plottable Interface +PlotterBase Class ---------------------- -.. automodule:: graphistry.Plottable +.. automodule:: graphistry.PlotterBase :members: :undoc-members: :inherited-members: - :show-inheritance: \ No newline at end of file + :show-inheritance: diff --git a/docs/source/api/plugins/compute/cugraph.rst b/docs/source/api/plugins/compute/cugraph.rst index b49bad469f..775ced993e 100644 --- a/docs/source/api/plugins/compute/cugraph.rst +++ b/docs/source/api/plugins/compute/cugraph.rst @@ -9,6 +9,7 @@ cuGraph is a GPU-accelerated graph library that leverages the Nvidia RAPIDS ecos .. automodule:: graphistry.plugins.cugraph :members: :undoc-members: + :inherited-members: :show-inheritance: .. rubric:: Constants diff --git a/docs/source/api/plugins/compute/graphviz.rst b/docs/source/api/plugins/compute/graphviz.rst index 099abdab06..81c760aa4f 100644 --- a/docs/source/api/plugins/compute/graphviz.rst +++ b/docs/source/api/plugins/compute/graphviz.rst @@ -9,6 +9,7 @@ graphviz is a popular graph visualization library that PyGraphistry can interfac .. automodule:: graphistry.plugins.graphviz :members: :undoc-members: + :inherited-members: :show-inheritance: .. rubric:: Constants diff --git a/docs/source/api/plugins/compute/igraph.rst b/docs/source/api/plugins/compute/igraph.rst index 0290b43f61..d887e0e096 100644 --- a/docs/source/api/plugins/compute/igraph.rst +++ b/docs/source/api/plugins/compute/igraph.rst @@ -9,6 +9,7 @@ igraph is a popular graph library that PyGraphistry can interface with. This all .. automodule:: graphistry.plugins.igraph :members: :undoc-members: + :inherited-members: :show-inheritance: .. rubric:: Constants diff --git a/docs/source/api/plugins/db/cosmos.rst b/docs/source/api/plugins/db/cosmos.rst index 31075541a3..cacb7930aa 100644 --- a/docs/source/api/plugins/db/cosmos.rst +++ b/docs/source/api/plugins/db/cosmos.rst @@ -10,4 +10,5 @@ Azure Cosmos DB supports Gremlin graph queries .. autoclass:: graphistry.gremlin.CosmosMixin :members: :undoc-members: + :inherited-members: :show-inheritance: \ No newline at end of file diff --git a/docs/source/api/plugins/db/gremlin.rst b/docs/source/api/plugins/db/gremlin.rst index ac54c20b27..25f8478bd9 100644 --- a/docs/source/api/plugins/db/gremlin.rst +++ b/docs/source/api/plugins/db/gremlin.rst @@ -12,4 +12,5 @@ As an open source technology, multiple databases support it. .. autoclass:: graphistry.gremlin.GremlinMixin :members: :undoc-members: + :inherited-members: :show-inheritance: \ No newline at end of file diff --git a/docs/source/api/plugins/db/neptune.rst b/docs/source/api/plugins/db/neptune.rst index c50a9b3b1f..c0687e3592 100644 --- a/docs/source/api/plugins/db/neptune.rst +++ b/docs/source/api/plugins/db/neptune.rst @@ -10,4 +10,5 @@ Amazon Neptune is a managed graph database by Amazon. It supports OpenCypher, RD .. autoclass:: graphistry.gremlin.NeptuneMixin :members: :undoc-members: + :inherited-members: :show-inheritance: diff --git a/docs/source/gfql/datetime_filtering.md b/docs/source/gfql/datetime_filtering.md new file mode 100644 index 0000000000..fe6e0ea93c --- /dev/null +++ b/docs/source/gfql/datetime_filtering.md @@ -0,0 +1,353 @@ +# Working with Dates and Times + +GFQL predicates support filtering by datetime, date, and time values. This guide covers common patterns and gotchas when working with temporal data. + +## Required Imports + +```python +# Core imports +import graphistry +from graphistry import n, e_forward, e_reverse, e_undirected + +# Temporal predicates +from graphistry.compute import ( + gt, lt, ge, le, eq, ne, between, is_in, + DateTimeValue, DateValue, TimeValue +) + +# Standard datetime types +import pandas as pd +from datetime import datetime, date, time, timedelta +import pytz # For timezone support +``` + +## Supported Types and Standards + +### Native Python/Pandas Types +Comparison predicates (`gt`, `lt`, `ge`, `le`, `eq`, `ne`, `between`) and `is_in` work with: +- **`datetime` / `pd.Timestamp`** - Full datetime with optional timezone (ISO 8601 format) +- **`date`** - Date only (year, month, day) +- **`time`** - Time only (hour, minute, second, microsecond) + +### Standards Compliance +- **ISO 8601**: All datetime strings follow ISO 8601 format (e.g., "2023-01-01T12:00:00Z") +- **IANA Timezones**: Timezone names follow IANA Time Zone Database (e.g., "US/Eastern", "UTC") +- **RFC 3339**: Compatible with RFC 3339 for internet timestamps + +### Wire Protocol Types +For JSON serialization and cross-system compatibility: +- **DateTimeWire**: `{"type": "datetime", "value": "ISO-8601-string", "timezone": "IANA-timezone"}` +- **DateWire**: `{"type": "date", "value": "YYYY-MM-DD"}` +- **TimeWire**: `{"type": "time", "value": "HH:MM:SS[.ffffff]"}` + +Note: The `timezone` field is optional for DateTimeWire and defaults to "UTC" if omitted. + +## Basic Usage + +### Datetime Filtering + +Filter nodes or edges based on datetime values: + +```python +import pandas as pd +from datetime import datetime +from graphistry import n +from graphistry.compute import gt, lt, between + +# Filter nodes created after a specific datetime +recent_nodes = g.chain([ + n(filter_dict={"created_at": gt(pd.Timestamp("2023-01-01 12:00:00"))}) +]) + +# Filter edges within a date range +date_range_edges = g.chain([ + n(edge_match={"timestamp": between( + datetime(2023, 1, 1), + datetime(2023, 12, 31) + )}) +]) +``` + +### Date-Only Filtering + +For date comparisons (ignoring time): + +```python +from datetime import date +from graphistry.compute import eq, ge + +# Filter nodes by exact date +specific_date = g.chain([ + n(filter_dict={"event_date": eq(date(2023, 6, 15))}) +]) + +# Filter nodes on or after a date +after_date = g.chain([ + n(filter_dict={"start_date": ge(date(2023, 1, 1))}) +]) +``` + +### Time-Only Filtering + +Filter based on time of day: + +```python +from datetime import time +from graphistry.compute import is_in, between + +# Filter events at specific times +morning_events = g.chain([ + n(filter_dict={"event_time": is_in([ + time(9, 0, 0), + time(9, 30, 0), + time(10, 0, 0) + ])}) +]) + +# Filter events in time range +business_hours = g.chain([ + n(filter_dict={"timestamp": between( + time(9, 0, 0), + time(17, 0, 0) + )}) +]) +``` + +## Timezone Support + +Temporal predicates fully support timezone-aware datetime comparisons: + +```python +import pytz + +# Create timezone-aware timestamp +eastern = pytz.timezone('US/Eastern') +utc = pytz.UTC + +# Filter with timezone-aware datetime +tz_aware_filter = g.chain([ + n(filter_dict={ + "timestamp": gt(pd.Timestamp("2023-01-01 12:00:00", tz=eastern)) + }) +]) + +# Comparisons automatically handle timezone conversions +``` + +## Advanced Usage + +### Mixed Temporal and Non-Temporal Predicates + +Combine temporal predicates with other filters: + +```python +from graphistry.compute import gt, lt, eq + +# Complex filter with multiple conditions +complex_filter = g.chain([ + n(filter_dict={ + "created_at": gt(datetime(2023, 1, 1)), + "status": eq("active"), + "priority": gt(5) + }) +]) +``` + +### Using Wire Protocol Dictionaries Directly + +You can pass wire protocol dictionaries directly to predicates, which is useful for programmatic predicate creation or when working with JSON configurations: + +```python +# Pass wire protocol dictionaries directly +filter_with_dict = g.chain([ + n(filter_dict={"timestamp": gt({ + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + })}) +]) + +# Works with all temporal predicates +date_range_filter = g.chain([ + n(filter_dict={"event_date": between( + {"type": "date", "value": "2023-01-01"}, + {"type": "date", "value": "2023-12-31"} + )}) +]) + +# And with is_in for multiple values +time_filter = g.chain([ + n(filter_dict={"event_time": is_in([ + {"type": "time", "value": "09:00:00"}, + {"type": "time", "value": "12:00:00"}, + {"type": "time", "value": "17:00:00"} + ])}) +]) +``` + +This is the same format used by the wire protocol, making it easy to: +- Store predicate configurations in JSON files +- Build predicates programmatically from external data sources +- Share predicate definitions between Python and other systems + +### Temporal Predicates in Multi-Hop Queries + +Use temporal filters in complex graph traversals: + +```python +# Find all transactions after a date, then their related accounts +recent_transactions = g.chain([ + n(filter_dict={"type": eq("transaction"), + "date": gt(date(2023, 6, 1))}), + n(edge_match={"relationship": eq("involves")}), + n(filter_dict={"type": eq("account")}) +]) +``` + +## Temporal Value Classes + +PyGraphistry provides three temporal value classes for precise control: + +### DateTimeValue + +For full datetime with optional timezone: + +```python +from graphistry.compute import DateTimeValue, gt + +# Create datetime value with timezone +dt_value = DateTimeValue("2023-01-01T12:00:00", "US/Eastern") + +# Use in predicate +filter_dt = g.chain([ + n(filter_dict={"timestamp": gt(dt_value)}) +]) +``` + +### DateValue + +For date-only comparisons: + +```python +from graphistry.compute import DateValue, between + +# Create date values +start = DateValue("2023-01-01") +end = DateValue("2023-12-31") + +# Use in between predicate +year_filter = g.chain([ + n(filter_dict={"event_date": between(start, end)}) +]) +``` + +### TimeValue + +For time-of-day comparisons: + +```python +from graphistry.compute import TimeValue, is_in + +# Create time values +morning = TimeValue("09:00:00") +noon = TimeValue("12:00:00") + +# Filter by specific times +time_filter = g.chain([ + n(filter_dict={"daily_event": is_in([morning, noon])}) +]) +``` + +## Best Practices + +1. **Use Explicit Types**: Always use `pd.Timestamp`, `datetime`, `date`, or `time` objects instead of strings to avoid ambiguity. + +2. **Timezone Awareness**: When working with timestamps across timezones, always specify timezones explicitly. + +3. **Performance**: Temporal comparisons are optimized for pandas DataFrames. For large datasets, ensure your datetime columns are properly typed. + +4. **JSON Serialization**: When serializing queries, temporal values are automatically converted to tagged dictionaries that preserve type and timezone information. + +## Unsupported Features + +### Duration/Interval Support +Currently, PyGraphistry does not support duration or interval types (e.g., ISO 8601 durations like "P1D" or "PT2H"). For duration-based queries: + +```python +# Instead of duration literals, calculate explicit timestamps +from datetime import datetime, timedelta + +# Find events within last 7 days +now = datetime.now() +week_ago = now - timedelta(days=7) +recent_events = g.chain([ + n(filter_dict={"timestamp": gt(pd.Timestamp(week_ago))}) +]) + +# For recurring intervals, use multiple conditions +business_days = g.chain([ + n(filter_dict={ + "timestamp": between( + pd.Timestamp("2023-01-01"), + pd.Timestamp("2023-12-31") + ) + }) +]) +``` + +## Common Patterns + +### Filter Recent Data + +```python +from datetime import datetime, timedelta + +# Get data from last 30 days +thirty_days_ago = datetime.now() - timedelta(days=30) +recent_data = g.chain([ + n(filter_dict={"timestamp": gt(pd.Timestamp(thirty_days_ago))}) +]) +``` + +### Business Hours Filtering + +```python +# Filter events during business hours +business_hours = g.chain([ + n(filter_dict={ + "timestamp": between(time(9, 0, 0), time(17, 0, 0)) + }) +]) +``` + +### Quarterly Data Analysis + +```python +# Q1 2023 data +q1_2023 = g.chain([ + n(filter_dict={ + "date": between( + date(2023, 1, 1), + date(2023, 3, 31) + ) + }) +]) +``` + +## Error Handling + +Temporal predicates include validation to prevent common errors: + +```python +# This will raise an error - strings are ambiguous +# bad_filter = gt("2023-01-01") # Don't do this + +# Instead, be explicit +good_filter = gt(pd.Timestamp("2023-01-01")) # Do this instead +``` + +## See Also + +- [GFQL Predicates API Reference](../api/gfql/predicates.rst) +- [GFQL Chain Operations](chain.rst) +- [Wire Protocol Reference](wire_protocol_examples.md) \ No newline at end of file diff --git a/docs/source/gfql/overview.rst b/docs/source/gfql/overview.rst index a08a2ae0b4..bbedaf9414 100644 --- a/docs/source/gfql/overview.rst +++ b/docs/source/gfql/overview.rst @@ -94,6 +94,30 @@ Example: Find nodes up to 2 hops away from node `"a"` and label each hop. first_hop_edges = g_2_hops._edges[ g_2_hops._edges["hop1"] == True ] print('Number of first-hop edges:', len(first_hop_edges)) +**Filter by Date/Time** + +Example: Find recent transactions using temporal predicates. + +.. code-block:: python + + from graphistry import n, e_forward + from graphistry.compute import gt, between + from datetime import datetime, date, time + import pandas as pd + + # Find transactions after a specific date + recent = g.chain([ + n(edge_match={"timestamp": gt(pd.Timestamp("2023-01-01"))}) + ]) + + # Find transactions in a date range during business hours + business_hours_txns = g.chain([ + n(edge_match={ + "date": between(date(2023, 6, 1), date(2023, 6, 30)), + "time": between(time(9, 0), time(17, 0)) + }) + ]) + **Query for Transaction Nodes Between Risky Nodes** Example: Find transaction nodes between two kinds of risky nodes. diff --git a/docs/source/gfql/predicates/quick.rst b/docs/source/gfql/predicates/quick.rst index 0fe7f7f09e..7eeda5a7b2 100644 --- a/docs/source/gfql/predicates/quick.rst +++ b/docs/source/gfql/predicates/quick.rst @@ -40,6 +40,15 @@ The following table lists the available operators, their descriptions, and examp - Between ``lower`` and ``upper`` (inclusive). - ``n({ "age": between(18, 65) })`` +.. note:: + All numeric comparison operators (``gt``, ``lt``, ``ge``, ``le``, ``eq``, ``ne``, ``between``) also support temporal values: + + - **DateTime**: ``n({ "created_at": gt(pd.Timestamp("2023-01-01 12:00:00")) })`` + - **Date**: ``n({ "event_date": eq(date(2023, 6, 15)) })`` + - **Time**: ``n({ "daily_time": between(time(9, 0), time(17, 0)) })`` + + See :doc:`/gfql/datetime_filtering` for datetime filtering examples. + **Categorical Operators** .. list-table:: diff --git a/docs/source/gfql/wire_protocol_examples.md b/docs/source/gfql/wire_protocol_examples.md new file mode 100644 index 0000000000..f2ab61df22 --- /dev/null +++ b/docs/source/gfql/wire_protocol_examples.md @@ -0,0 +1,516 @@ +# Temporal Predicates Wire Protocol Reference + +This document provides a comprehensive reference for how temporal predicates serialize to JSON in the GFQL wire protocol. The wire protocol enables interoperability between Python and other systems. + +## Overview + +The wire protocol uses tagged dictionaries to preserve type information during JSON serialization. This enables: +- Cross-language compatibility +- Configuration-driven predicate creation +- Network transport of queries +- Storage of predicate definitions + +**Key Concept**: Wire protocol dictionaries can be used directly in the Python API: + +```python +# These are equivalent: +pred1 = gt(100) +pred2 = gt(pd.Timestamp("2023-01-01")) +pred3 = gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"}) +``` + +## 1. DateTime Comparisons + +### Python API +```python +import pandas as pd +from datetime import datetime +from graphistry import n +from graphistry.compute import gt, between + +# Using pandas Timestamp +filter1 = n(filter_dict={ + "created_at": gt(pd.Timestamp("2023-01-01 12:00:00")) +}) + +# Using Python datetime +filter2 = n(edge_match={ + "timestamp": between( + datetime(2023, 1, 1), + datetime(2023, 12, 31, 23, 59, 59) + ) +}) +``` + +### Wire Protocol (JSON) +```json +// GT with datetime +{ + "type": "ASTNode", + "filter_dict": { + "created_at": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + } + } + } +} + +// Between with datetime range +{ + "type": "ASTNode", + "edge_match": { + "timestamp": { + "type": "Between", + "lower": { + "type": "datetime", + "value": "2023-01-01T00:00:00", + "timezone": "UTC" + }, + "upper": { + "type": "datetime", + "value": "2023-12-31T23:59:59", + "timezone": "UTC" + }, + "inclusive": true + } + } +} +``` + +### Round-trip Example +```python +# Create predicate +from graphistry.compute import gt +pred = gt(pd.Timestamp("2023-01-01 12:00:00")) + +# Serialize to JSON +json_data = pred.to_json() +print(json_data) +# Output: { +# 'type': 'GT', +# 'val': { +# 'type': 'datetime', +# 'value': '2023-01-01T12:00:00', +# 'timezone': 'UTC' +# } +# } + +# Deserialize from JSON +from graphistry.compute.predicates.numeric import GT +pred2 = GT.from_json(json_data) +# pred2 is functionally equivalent to pred +``` + +## 2. Date-Only Comparisons + +### Python API +```python +from datetime import date +from graphistry.compute import eq, ge + +# Date equality +filter1 = n(filter_dict={ + "event_date": eq(date(2023, 6, 15)) +}) + +# Date range check +filter2 = n(filter_dict={ + "start_date": ge(date(2023, 1, 1)) +}) +``` + +### Wire Protocol (JSON) +```json +// Date equality +{ + "type": "ASTNode", + "filter_dict": { + "event_date": { + "type": "EQ", + "val": { + "type": "date", + "value": "2023-06-15" + } + } + } +} + +// Date greater than or equal +{ + "type": "ASTNode", + "filter_dict": { + "start_date": { + "type": "GE", + "val": { + "type": "date", + "value": "2023-01-01" + } + } + } +} +``` + +## 3. Time-Only Comparisons + +### Python API +```python +from datetime import time +from graphistry.compute import is_in, between + +# Specific times +filter1 = n(filter_dict={ + "event_time": is_in([ + time(9, 0, 0), + time(12, 0, 0), + time(17, 0, 0) + ]) +}) + +# Time range +filter2 = n(edge_match={ + "daily_schedule": between( + time(9, 0, 0), + time(17, 30, 0) + ) +}) +``` + +### Wire Protocol (JSON) +```json +// IsIn with times +{ + "type": "ASTNode", + "filter_dict": { + "event_time": { + "type": "IsIn", + "options": [ + {"type": "time", "value": "09:00:00"}, + {"type": "time", "value": "12:00:00"}, + {"type": "time", "value": "17:00:00"} + ] + } + } +} + +// Time range +{ + "type": "ASTNode", + "edge_match": { + "daily_schedule": { + "type": "Between", + "lower": {"type": "time", "value": "09:00:00"}, + "upper": {"type": "time", "value": "17:30:00"}, + "inclusive": true + } + } +} +``` + +## 4. Timezone-Aware DateTime + +### Python API +```python +import pytz +from graphistry.compute import DateTimeValue, gt + +# Using timezone-aware timestamp +eastern = pytz.timezone('US/Eastern') +filter1 = n(filter_dict={ + "timestamp": gt( + pd.Timestamp("2023-01-01 09:00:00", tz=eastern) + ) +}) + +# Using DateTimeValue with explicit timezone +dt_val = DateTimeValue("2023-01-01T09:00:00", "US/Eastern") +filter2 = n(filter_dict={ + "timestamp": gt(dt_val) +}) +``` + +### Wire Protocol (JSON) +```json +// Timezone-aware datetime +{ + "type": "ASTNode", + "filter_dict": { + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-01-01T09:00:00", + "timezone": "US/Eastern" + } + } + } +} +``` + +## 5. Complex Chain with Temporal Predicates + +### Python API +```python +from graphistry import n, e_forward +from graphistry.compute import gt, eq, between +from datetime import datetime, timedelta + +# Multi-hop query with temporal filters +chain = g.chain([ + # Recent transactions + n(edge_match={ + "timestamp": gt(datetime.now() - timedelta(days=7)), + "amount": gt(1000) + }), + # To active accounts + n(filter_dict={ + "status": eq("active"), + "last_login": between( + datetime.now() - timedelta(days=30), + datetime.now() + ) + }), + # Outgoing transfers + e_forward(edge_match={ + "type": eq("transfer"), + "timestamp": gt(datetime.now() - timedelta(days=1)) + }) +]) +``` + +### Wire Protocol (JSON) +```json +{ + "type": "Chain", + "queries": [ + { + "type": "ASTNode", + "edge_match": { + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-12-18T10:30:00", + "timezone": "UTC" + } + }, + "amount": { + "type": "GT", + "val": 1000 + } + } + }, + { + "type": "ASTNode", + "filter_dict": { + "status": { + "type": "EQ", + "val": "active" + }, + "last_login": { + "type": "Between", + "lower": { + "type": "datetime", + "value": "2023-11-25T10:30:00", + "timezone": "UTC" + }, + "upper": { + "type": "datetime", + "value": "2023-12-25T10:30:00", + "timezone": "UTC" + }, + "inclusive": true + } + } + }, + { + "type": "ASTEdge", + "direction": "forward", + "edge_match": { + "type": { + "type": "EQ", + "val": "transfer" + }, + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-12-24T10:30:00", + "timezone": "UTC" + } + } + } + } + ] +} +``` + +## 6. Temporal Value Classes Direct Usage + +### Python API +```python +from graphistry.compute import ( + DateTimeValue, DateValue, TimeValue, + temporal_value_from_json, gt +) + +# Create temporal values +dt_val = DateTimeValue("2023-06-15T14:30:00", "Europe/London") +date_val = DateValue("2023-06-15") +time_val = TimeValue("14:30:00") + +# Use in predicates +filter1 = n(filter_dict={"timestamp": gt(dt_val)}) +filter2 = n(filter_dict={"event_date": eq(date_val)}) +filter3 = n(filter_dict={"daily_time": eq(time_val)}) + +# Create from JSON +json_dt = { + "type": "datetime", + "value": "2023-06-15T14:30:00", + "timezone": "Europe/London" +} +dt_from_json = temporal_value_from_json(json_dt) +``` + +### Wire Protocol (JSON) +```json +// DateTimeValue serialization +{ + "type": "datetime", + "value": "2023-06-15T14:30:00", + "timezone": "Europe/London" +} + +// DateValue serialization +{ + "type": "date", + "value": "2023-06-15" +} + +// TimeValue serialization +{ + "type": "time", + "value": "14:30:00" +} +``` + +## 7. Full Round-Trip Example + +```python +# 1. Create a complex query with temporal predicates +from graphistry import n, Chain +from graphistry.compute import gt, between, is_in +from datetime import datetime, date, time +import pandas as pd + +query = Chain([ + n(filter_dict={ + "created": gt(pd.Timestamp("2023-01-01")), + "event_date": between(date(2023, 6, 1), date(2023, 6, 30)), + "event_time": is_in([time(9, 0), time(12, 0), time(17, 0)]) + }) +]) + +# 2. Serialize to JSON +json_query = query.to_json() +print(json_query) + +# 3. Send over wire (simulated) +import json +wire_data = json.dumps(json_query) +received_data = json.loads(wire_data) + +# 4. Deserialize on receiving end +from graphistry.compute.ast import Chain +reconstructed_query = Chain.from_json(received_data) + +# 5. Apply to graph data +result = g.chain(reconstructed_query.queries) +``` + +## Wire Protocol Structure + +### Temporal Value Types + +All temporal values in the wire protocol follow this pattern: + +```typescript +// DateTime with timezone +interface DateTimeWire { + type: "datetime"; + value: string; // ISO 8601 format + timezone?: string; // IANA timezone (default: "UTC") +} + +// Date only +interface DateWire { + type: "date"; + value: string; // YYYY-MM-DD format +} + +// Time only +interface TimeWire { + type: "time"; + value: string; // HH:MM:SS[.ffffff] format +} +``` + +### Predicate Structure + +Predicates containing temporal values serialize as: + +```typescript +interface TemporalPredicate { + type: "GT" | "LT" | "GE" | "LE" | "EQ" | "NE"; + val: DateTimeWire | DateWire | TimeWire; +} + +interface BetweenPredicate { + type: "Between"; + lower: DateTimeWire | DateWire | TimeWire; + upper: DateTimeWire | DateWire | TimeWire; + inclusive: boolean; +} + +interface IsInPredicate { + type: "IsIn"; + options: Array; +} +``` + +## Key Points + +1. **Type Safety**: Raw strings are rejected in the Python API to avoid ambiguity +2. **Automatic Conversion**: Python datetime objects are automatically converted to appropriate temporal values +3. **Timezone Preservation**: Timezone information is preserved through serialization +4. **Tagged Format**: JSON uses tagged dictionaries to preserve type information +5. **Direct Usage**: Wire protocol dicts can be passed directly to Python predicates + +## Error Handling + +```python +# This will raise an error - ambiguous string +try: + bad_filter = n(filter_dict={"date": gt("2023-01-01")}) +except ValueError as e: + print(e) + # Output: Raw string '2023-01-01' is ambiguous. Use: + # - gt(pd.Timestamp('2023-01-01')) for datetime + # - gt({'type': 'datetime', 'value': '2023-01-01'}) for explicit type + +# Correct approaches +good_filter1 = n(filter_dict={"date": gt(pd.Timestamp("2023-01-01"))}) +good_filter2 = n(filter_dict={"date": gt(datetime(2023, 1, 1))}) +good_filter3 = n(filter_dict={"date": gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"})}) +``` + +## Performance Considerations + +- Temporal predicates leverage pandas' optimized datetime operations +- Timezone conversions are handled efficiently +- For large datasets, ensure datetime columns are properly typed (not object dtype) +- Use `pd.Timestamp` for best performance when creating many predicates programmatically \ No newline at end of file diff --git a/docs/source/notebooks/gfql.rst b/docs/source/notebooks/gfql.rst index dab38cf40e..d03accb479 100644 --- a/docs/source/notebooks/gfql.rst +++ b/docs/source/notebooks/gfql.rst @@ -7,6 +7,7 @@ GFQL Graph queries :titlesonly: Intro to graph queries with hop and chain <../demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb> + DateTime Filtering Examples <../demos/gfql/temporal_predicates.ipynb> GPU Benchmarking <../demos/gfql/benchmark_hops_cpu_gpu.ipynb> GFQL Remote mode <../demos/gfql/gfql_remote.ipynb> Python Remote mode <../demos/gfql/python_remote.ipynb> diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 1f65fd8359..7c9f36b1d0 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -18,7 +18,14 @@ is_year_end, IsYearEnd, is_leap_year, IsLeapYear ) -from .predicates.numeric import ( +from .ast_temporal import ( + TemporalValue, + DateTimeValue, + DateValue, + TimeValue, + temporal_value_from_json +) +from .predicates.comparison import ( gt, GT, lt, LT, ge, GE, diff --git a/graphistry/compute/ast.py b/graphistry/compute/ast.py index 59601cf474..6d99e48e76 100644 --- a/graphistry/compute/ast.py +++ b/graphistry/compute/ast.py @@ -27,7 +27,7 @@ is_year_end, IsYearEnd, is_leap_year, IsLeapYear ) -from .predicates.numeric import ( +from .predicates.comparison import ( gt, GT, lt, LT, ge, GE, diff --git a/graphistry/compute/ast_temporal.py b/graphistry/compute/ast_temporal.py new file mode 100644 index 0000000000..cb69dfdf0b --- /dev/null +++ b/graphistry/compute/ast_temporal.py @@ -0,0 +1,155 @@ +from abc import ABC, abstractmethod +from datetime import datetime, date, time +from typing import Dict, Any, Union, TYPE_CHECKING +import pandas as pd +from dateutil import parser as date_parser # type: ignore[import] +import pytz # type: ignore[import] + +from graphistry.utils.json import JSONVal + +if TYPE_CHECKING: + from ..models.gfql.types.temporal import DateTimeWire, DateWire, TimeWire, TemporalWire + + +class TemporalValue(ABC): + """Base class for temporal values with tagging support""" + + @abstractmethod + def to_json(self) -> 'TemporalWire': + """Serialize to JSON-compatible dictionary""" + pass + + @abstractmethod + def as_pandas_value(self) -> Any: + """Convert to pandas-compatible value for comparison""" + pass + + +class DateTimeValue(TemporalValue): + """Tagged datetime value with timezone support""" + + def __init__(self, value: str, timezone: str = "UTC"): + self.value = value + self.timezone = timezone + self._parsed = self._parse_iso8601(value, timezone) + + @classmethod + def from_pandas_timestamp(cls, ts: pd.Timestamp) -> 'DateTimeValue': + """Create from pandas Timestamp""" + tz = str(ts.tz) if ts.tz else "UTC" + value = ts.isoformat() + return cls(value, tz) + + @classmethod + def from_datetime(cls, dt: datetime) -> 'DateTimeValue': + """Create from Python datetime""" + tz = str(dt.tzinfo) if dt.tzinfo else "UTC" + value = dt.isoformat() + return cls(value, tz) + + def _parse_iso8601(self, value: str, timezone: str) -> pd.Timestamp: + """Parse ISO8601 datetime string with timezone""" + # Parse the datetime + dt = date_parser.isoparse(value) + + # Handle timezone + if dt.tzinfo is None: + # Naive datetime - localize to specified timezone + tz = pytz.timezone(timezone) + dt = tz.localize(dt) + else: + # Already has timezone - convert to specified timezone + tz = pytz.timezone(timezone) + dt = dt.astimezone(tz) + + return pd.Timestamp(dt) + + def to_json(self) -> 'DateTimeWire': + """Return dict for tagged temporal value""" + from ..models.gfql.types.temporal import DateTimeWire + result: DateTimeWire = { + "type": "datetime", + "value": self.value, + "timezone": self.timezone + } + return result + + def as_pandas_value(self) -> pd.Timestamp: + return self._parsed + + +class DateValue(TemporalValue): + """Tagged date value""" + + def __init__(self, value: str): + self.value = value + self._parsed = self._parse_date(value) + + @classmethod + def from_date(cls, d: date) -> 'DateValue': + """Create from Python date""" + return cls(d.isoformat()) + + def _parse_date(self, value: str) -> date: + """Parse date string in ISO format (YYYY-MM-DD)""" + return date_parser.isoparse(value).date() + + def to_json(self) -> 'DateWire': + """Return dict for tagged temporal value""" + from ..models.gfql.types.temporal import DateWire + result: DateWire = { + "type": "date", + "value": self.value + } + return result + + def as_pandas_value(self) -> pd.Timestamp: + # Convert date to pandas Timestamp at midnight + return pd.Timestamp(self._parsed) + + +class TimeValue(TemporalValue): + """Tagged time value""" + + def __init__(self, value: str): + self.value = value + self._parsed = self._parse_time(value) + + @classmethod + def from_time(cls, t: time) -> 'TimeValue': + """Create from Python time""" + return cls(t.isoformat()) + + def _parse_time(self, value: str) -> time: + """Parse time string in ISO format (HH:MM:SS)""" + # Handle time-only strings + if 'T' not in value and ' ' not in value: + # Pure time string like "14:30:00" + return datetime.strptime(value, "%H:%M:%S").time() + else: + # Extract time from full datetime + return date_parser.isoparse(value).time() + + def to_json(self) -> 'TimeWire': + """Return dict for tagged temporal value""" + from ..models.gfql.types.temporal import TimeWire + result: TimeWire = { + "type": "time", + "value": self.value + } + return result + + def as_pandas_value(self) -> time: + return self._parsed + + +def temporal_value_from_json(d: Dict[str, Any]) -> TemporalValue: + """Factory function to create temporal value from JSON dict""" + if d["type"] == "datetime": + return DateTimeValue(d["value"], d.get("timezone", "UTC")) + elif d["type"] == "date": + return DateValue(d["value"]) + elif d["type"] == "time": + return TimeValue(d["value"]) + else: + raise ValueError(f"Unknown temporal value type: {d['type']}") diff --git a/graphistry/compute/predicates/ASTPredicate.py b/graphistry/compute/predicates/ASTPredicate.py index 521ae1a947..53f780169e 100644 --- a/graphistry/compute/predicates/ASTPredicate.py +++ b/graphistry/compute/predicates/ASTPredicate.py @@ -1,6 +1,4 @@ from abc import abstractmethod -import pandas as pd -from typing import Any, TYPE_CHECKING from graphistry.compute.ASTSerializable import ASTSerializable from graphistry.compute.typing import SeriesT diff --git a/graphistry/compute/predicates/__init__.py b/graphistry/compute/predicates/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/graphistry/compute/predicates/categorical.py b/graphistry/compute/predicates/categorical.py index c9d01f91c0..a6baafa871 100644 --- a/graphistry/compute/predicates/categorical.py +++ b/graphistry/compute/predicates/categorical.py @@ -1,6 +1,4 @@ -from typing import Any from typing_extensions import Literal -import pandas as pd from .ASTPredicate import ASTPredicate from graphistry.compute.typing import SeriesT diff --git a/graphistry/compute/predicates/comparison.py b/graphistry/compute/predicates/comparison.py new file mode 100644 index 0000000000..68e97fbec3 --- /dev/null +++ b/graphistry/compute/predicates/comparison.py @@ -0,0 +1,354 @@ +from typing import Dict, Union, TYPE_CHECKING +import pandas as pd +import numpy as np +from datetime import datetime, date, time + +from .ASTPredicate import ASTPredicate +from ..ast_temporal import TemporalValue, DateTimeValue, DateValue, TimeValue +from ...models.gfql.coercions.temporal import to_ast +from ...models.gfql.types.guards import is_native_numeric, is_any_temporal, is_string +from ...models.gfql.types.predicates import ComparisonInput, BetweenBoundInput +from ...utils.json import JSONVal +from graphistry.compute.typing import SeriesT + +if TYPE_CHECKING: + pass + + +class ComparisonPredicate(ASTPredicate): + """Base class for comparison predicates that support both numeric and temporal values""" + + def __init__(self, val: ComparisonInput) -> None: + self.val = self._normalize_value(val) + + def _normalize_value(self, val: ComparisonInput) -> Union[int, float, np.number, TemporalValue]: + """Convert various input types to internal representation""" + # Comparison predicates need: + # - Numerics as-is + # - Temporals as AST objects (for timezone handling) + # - Strings rejected (ambiguous) + if is_native_numeric(val): + return val + elif is_any_temporal(val): + return to_ast(val) + elif is_string(val): + raise ValueError( + f"Raw string '{val}' is ambiguous. Use:\n" + f" - pd.Timestamp('{val}') for datetime\n" + f" - {{'type': 'datetime', 'value': '{val}'}} for explicit type" + ) + else: + raise TypeError(f"Unsupported type for {self.__class__.__name__}: {type(val)}") + + def _temporal_comparison(self, s: SeriesT, temporal_val: TemporalValue, op: str) -> SeriesT: + """Handle temporal comparisons with proper type handling""" + if isinstance(temporal_val, DateTimeValue): + # Normalize series to target timezone for comparison + if hasattr(s, 'dt'): + if s.dt.tz is None: + s_tz = s.dt.tz_localize('UTC').dt.tz_convert(temporal_val.timezone) + else: + s_tz = s.dt.tz_convert(temporal_val.timezone) + else: + s_tz = s # type: ignore + + if op == '>': + return s_tz > temporal_val.as_pandas_value() + elif op == '<': + return s_tz < temporal_val.as_pandas_value() + elif op == '>=': + return s_tz >= temporal_val.as_pandas_value() + elif op == '<=': + return s_tz <= temporal_val.as_pandas_value() + elif op == '==': + return s_tz == temporal_val.as_pandas_value() + elif op == '!=': + return s_tz != temporal_val.as_pandas_value() + + elif isinstance(temporal_val, DateValue): + # Extract date from datetime series if needed + if hasattr(s, 'dt'): + s_date = s.dt.date + else: + s_date = s + + parsed_date = temporal_val._parsed + if op == '>': + return s_date > parsed_date + elif op == '<': + return s_date < parsed_date + elif op == '>=': + return s_date >= parsed_date + elif op == '<=': + return s_date <= parsed_date + elif op == '==': + return s_date == parsed_date + elif op == '!=': + return s_date != parsed_date + + elif isinstance(temporal_val, TimeValue): + # Extract time from datetime series if needed + if hasattr(s, 'dt'): + s_time = s.dt.time + else: + s_time = s + + parsed_time = temporal_val._parsed + if op == '>': + return s_time > parsed_time + elif op == '<': + return s_time < parsed_time + elif op == '>=': + return s_time >= parsed_time + elif op == '<=': + return s_time <= parsed_time + elif op == '==': + return s_time == parsed_time + elif op == '!=': + return s_time != parsed_time + + raise TypeError(f"Unknown temporal value type: {type(temporal_val)}") + + def validate(self) -> None: + """Validate both numeric and temporal values""" + if isinstance(self.val, (int, float)): + pass # Numeric values are always valid + elif isinstance(self.val, TemporalValue): + pass # Temporal values validated on construction + else: + raise TypeError(f"Invalid value type: {type(self.val)}") + + def to_json(self, validate=True) -> dict: + """Serialize maintaining backward compatibility""" + if validate: + self.validate() + + result: Dict[str, JSONVal] = {"type": self.__class__.__name__} + + if isinstance(self.val, TemporalValue): + # to_json() returns a dict, not a string + val_dict = self.val.to_json() + result["val"] = val_dict + else: + result["val"] = self.val + + return result + + +# For backward compatibility - keep but deprecated +class NumericASTPredicate(ComparisonPredicate): + """Deprecated: Use ComparisonPredicate instead""" + pass + +### + +class GT(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Greater than comparison""" + if isinstance(self.val, (int, float)): + return s > self.val + elif isinstance(self.val, TemporalValue): + return self._temporal_comparison(s, self.val, '>') + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def gt(val: ComparisonInput) -> GT: + """ + Return whether a given value is greater than a threshold + """ + return GT(val) + +class LT(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Less than comparison""" + if isinstance(self.val, (int, float)): + return s < self.val + elif isinstance(self.val, TemporalValue): + return self._temporal_comparison(s, self.val, '<') + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def lt(val: ComparisonInput) -> LT: + """ + Return whether a given value is less than a threshold + """ + return LT(val) + +class GE(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Greater than or equal comparison""" + if isinstance(self.val, (int, float)): + return s >= self.val + elif isinstance(self.val, TemporalValue): + return self._temporal_comparison(s, self.val, '>=') + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def ge(val: ComparisonInput) -> GE: + """ + Return whether a given value is greater than or equal to a threshold + """ + return GE(val) + +class LE(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Less than or equal comparison""" + if isinstance(self.val, (int, float)): + return s <= self.val + elif isinstance(self.val, TemporalValue): + return self._temporal_comparison(s, self.val, '<=') + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def le(val: ComparisonInput) -> LE: + """ + Return whether a given value is less than or equal to a threshold + """ + return LE(val) + +class EQ(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Equal comparison""" + if isinstance(self.val, (int, float)): + return s == self.val + elif isinstance(self.val, TemporalValue): + return self._temporal_comparison(s, self.val, '==') + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def eq(val: ComparisonInput) -> EQ: + """ + Return whether a given value is equal to a threshold + """ + return EQ(val) + +class NE(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Not equal comparison""" + if isinstance(self.val, (int, float)): + return s != self.val + elif isinstance(self.val, TemporalValue): + return self._temporal_comparison(s, self.val, '!=') + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def ne(val: ComparisonInput) -> NE: + """ + Return whether a given value is not equal to a threshold + """ + return NE(val) + +class Between(ASTPredicate): + def __init__(self, lower: BetweenBoundInput, + upper: BetweenBoundInput, + inclusive: bool = True) -> None: + self.lower = self._normalize_value(lower) + self.upper = self._normalize_value(upper) + self.inclusive = inclusive + + def _normalize_value(self, val: BetweenBoundInput) -> Union[int, float, np.number, TemporalValue]: + """Convert various input types to internal representation""" + # Same normalization as ComparisonPredicate + if is_native_numeric(val): + return val + elif is_any_temporal(val): + return to_ast(val) + elif is_string(val): + raise ValueError( + f"Raw string '{val}' is ambiguous. Use:\n" + f" - pd.Timestamp('{val}') for datetime\n" + f" - {{'type': 'datetime', 'value': '{val}'}} for explicit type" + ) + else: + raise TypeError(f"Unsupported type for {self.__class__.__name__}: {type(val)}") + + def __call__(self, s: SeriesT) -> SeriesT: + # Check if both bounds are same type + lower_is_numeric = isinstance(self.lower, (int, float)) + upper_is_numeric = isinstance(self.upper, (int, float)) + lower_is_temporal = isinstance(self.lower, TemporalValue) + upper_is_temporal = isinstance(self.upper, TemporalValue) + + if lower_is_numeric and upper_is_numeric: + # Numeric comparison + if self.inclusive: + return (s >= self.lower) & (s <= self.upper) + else: + return (s > self.lower) & (s < self.upper) + + elif lower_is_temporal and upper_is_temporal: + # Temporal comparison + # Create comparison predicates and use them + ge_pred = GE(self.lower) + le_pred = LE(self.upper) + gt_pred = GT(self.lower) + lt_pred = LT(self.upper) + + if self.inclusive: + return ge_pred(s) & le_pred(s) + else: + return gt_pred(s) & lt_pred(s) + + else: + raise TypeError("Between requires both bounds to be same type (numeric or temporal)") + + def validate(self) -> None: + # Check types match + lower_is_numeric = isinstance(self.lower, (int, float)) + upper_is_numeric = isinstance(self.upper, (int, float)) + lower_is_temporal = isinstance(self.lower, TemporalValue) + upper_is_temporal = isinstance(self.upper, TemporalValue) + + if not ((lower_is_numeric and upper_is_numeric) or (lower_is_temporal and upper_is_temporal)): + raise TypeError("Between requires both bounds to be same type") + + assert isinstance(self.inclusive, bool) + + def to_json(self, validate=True) -> dict: + """Serialize maintaining backward compatibility""" + if validate: + self.validate() + + result = {"type": self.__class__.__name__, "inclusive": self.inclusive} + + # Serialize lower/upper based on type + if isinstance(self.lower, TemporalValue): + result["lower"] = self.lower.to_json() + else: + result["lower"] = self.lower + + if isinstance(self.upper, TemporalValue): + result["upper"] = self.upper.to_json() + else: + result["upper"] = self.upper + + return result + +def between(lower: BetweenBoundInput, + upper: BetweenBoundInput, + inclusive: bool = True) -> Between: + """ + Return whether a given value is between a lower and upper threshold + """ + return Between(lower, upper, inclusive) + +class IsNA(ASTPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + return s.isna() + +def isna() -> IsNA: + """ + Return whether a given value is NA + """ + return IsNA() + + +class NotNA(ASTPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + return s.notna() + +def notna() -> NotNA: + """ + Return whether a given value is not NA + """ + return NotNA() diff --git a/graphistry/compute/predicates/from_json.py b/graphistry/compute/predicates/from_json.py index fd248ec73e..933de7a32b 100644 --- a/graphistry/compute/predicates/from_json.py +++ b/graphistry/compute/predicates/from_json.py @@ -3,7 +3,7 @@ from graphistry.compute.predicates.ASTPredicate import ASTPredicate from graphistry.compute.predicates.categorical import Duplicated from graphistry.compute.predicates.is_in import IsIn -from graphistry.compute.predicates.numeric import GT, LT, GE, LE, EQ, NE, Between, IsNA, NotNA +from graphistry.compute.predicates.comparison import GT, LT, GE, LE, EQ, NE, Between, IsNA, NotNA from graphistry.compute.predicates.str import ( Contains, Startswith, Endswith, Match, IsNumeric, IsAlpha, IsDecimal, IsDigit, IsLower, IsUpper, IsSpace, IsAlnum, IsTitle, IsNull, NotNull @@ -36,7 +36,10 @@ def from_json(d: Dict[str, JSONVal]) -> ASTPredicate: assert d['type'] in type_to_predicate assert isinstance(d['type'], str) pred = type_to_predicate[d['type']] + + # Use default from_json method out = pred.from_json(d) + assert isinstance(out, ASTPredicate) out.validate() return out diff --git a/graphistry/compute/predicates/is_in.py b/graphistry/compute/predicates/is_in.py index 6c2959a7c4..27b566d7c1 100644 --- a/graphistry/compute/predicates/is_in.py +++ b/graphistry/compute/predicates/is_in.py @@ -1,21 +1,126 @@ -from typing import Any, List +from typing import Any, List, Dict import pandas as pd +import numpy as np +from datetime import datetime, date, time from graphistry.utils.json import assert_json_serializable from .ASTPredicate import ASTPredicate +from ..ast_temporal import TemporalValue, DateTimeValue, DateValue, TimeValue +from ...models.gfql.coercions.temporal import to_native +from ...models.gfql.types.guards import is_basic_scalar, is_any_temporal +from ...models.gfql.types.predicates import IsInElementInput from graphistry.compute.typing import SeriesT class IsIn(ASTPredicate): - def __init__(self, options: List[Any]) -> None: - self.options = options + def __init__(self, options: List[IsInElementInput]) -> None: + self.options = self._normalize_options(options) + + def _normalize_options(self, options: List[IsInElementInput]) -> List[Any]: + """Normalize options list to handle temporal values + + Returns List[Any] because normalized values include pandas types + and other hashable types for membership testing. + """ + normalized = [] + for val in options: + normalized.append(self._normalize_value(val)) + return normalized + + def _normalize_value(self, val: IsInElementInput) -> Any: + """Convert various input types to internal representation + + Returns Any because IsIn accepts any hashable type for membership testing, + including normalized pandas types. + """ + # IsIn predicate needs: + # - Basic scalars (including strings) as-is + # - Temporals as native/pandas types (for .isin() method) + # - Everything else passes through + if is_basic_scalar(val): + return val + elif is_any_temporal(val): + return to_native(val) + else: + # Everything else passes through (including dicts) + return val def __call__(self, s: SeriesT) -> SeriesT: + # Check if we have any temporal values in options + has_temporal = any( + isinstance(opt, (pd.Timestamp, date, time)) + for opt in self.options + ) + + if has_temporal and hasattr(s, 'dt'): + # For datetime series with time-only values in options, + # we need special handling + time_opts = [opt for opt in self.options if isinstance(opt, time)] + other_opts = [opt for opt in self.options if not isinstance(opt, time)] + + if time_opts: + # Check time component + time_matches = s.dt.time.isin(time_opts) + if other_opts: + # Also check other values + other_matches = s.isin(other_opts) + return time_matches | other_matches + else: + return time_matches + return s.isin(self.options) def validate(self) -> None: assert isinstance(self.options, list) - assert_json_serializable(self.options) + # Check that normalized options are still JSON serializable + # (temporal values are converted to pandas types which are serializable) + try: + # Create a test list with JSON-compatible versions + json_test = [] + for opt in self.options: + if isinstance(opt, pd.Timestamp): + json_test.append(opt.isoformat()) + elif isinstance(opt, (date, time)): + json_test.append(str(opt)) + else: + json_test.append(opt) + assert_json_serializable(json_test) + except Exception as e: + raise ValueError(f"Options not JSON serializable: {e}") + + def to_json(self, validate=True) -> dict: + """Override to handle temporal values in options""" + if validate: + self.validate() + + # Convert temporal values back to tagged dicts for serialization + json_options = [] + for opt in self.options: + if isinstance(opt, pd.Timestamp): + # Convert back to tagged dict + json_options.append({ + "type": "datetime", + "value": opt.isoformat(), + "timezone": str(opt.tz) if opt.tz else "UTC" + }) + elif isinstance(opt, date) and not isinstance(opt, datetime): + json_options.append({ + "type": "date", + "value": opt.isoformat() + }) + elif isinstance(opt, time): + json_options.append({ + "type": "time", + "value": opt.isoformat() + }) + else: + json_options.append(opt) + + return { + 'type': self.__class__.__name__, + 'options': json_options + } + -def is_in(options: List[Any]) -> IsIn: +def is_in(options: List[IsInElementInput]) -> IsIn: return IsIn(options) diff --git a/graphistry/compute/predicates/numeric.py b/graphistry/compute/predicates/numeric.py deleted file mode 100644 index d2289d1b6c..0000000000 --- a/graphistry/compute/predicates/numeric.py +++ /dev/null @@ -1,137 +0,0 @@ -from typing import Any, Union -import pandas as pd - -from .ASTPredicate import ASTPredicate -from graphistry.compute.typing import SeriesT - - -class NumericASTPredicate(ASTPredicate): - def __init__(self, val: Union[int, float]) -> None: - self.val = val - - def validate(self) -> None: - assert isinstance(self.val, (int, float)) - -### - -class GT(NumericASTPredicate): - def __init__(self, val: float) -> None: - self.val = val - - def __call__(self, s: SeriesT) -> SeriesT: - return s > self.val - -def gt(val: float) -> GT: - """ - Return whether a given value is greater than a threshold - """ - return GT(val) - -class LT(NumericASTPredicate): - def __init__(self, val: float) -> None: - self.val = val - - def __call__(self, s: SeriesT) -> SeriesT: - return s < self.val - -def lt(val: float) -> LT: - """ - Return whether a given value is less than a threshold - """ - return LT(val) - -class GE(NumericASTPredicate): - def __init__(self, val: float) -> None: - self.val = val - - def __call__(self, s: SeriesT) -> SeriesT: - return s >= self.val - -def ge(val: float) -> GE: - """ - Return whether a given value is greater than or equal to a threshold - """ - return GE(val) - -class LE(NumericASTPredicate): - def __init__(self, val: float) -> None: - self.val = val - - def __call__(self, s: SeriesT) -> SeriesT: - return s <= self.val - -def le(val: float) -> LE: - """ - Return whether a given value is less than or equal to a threshold - """ - return LE(val) - -class EQ(NumericASTPredicate): - def __init__(self, val: float) -> None: - self.val = val - - def __call__(self, s: SeriesT) -> SeriesT: - return s == self.val - -def eq(val: float) -> EQ: - """ - Return whether a given value is equal to a threshold - """ - return EQ(val) - -class NE(NumericASTPredicate): - def __init__(self, val: float) -> None: - self.val = val - - def __call__(self, s: SeriesT) -> SeriesT: - return s != self.val - -def ne(val: float) -> NE: - """ - Return whether a given value is not equal to a threshold - """ - return NE(val) - -class Between(ASTPredicate): - def __init__(self, lower: float, upper: float, inclusive: bool = True) -> None: - self.lower = lower - self.upper = upper - self.inclusive = inclusive - - def __call__(self, s: SeriesT) -> SeriesT: - if self.inclusive: - return (s >= self.lower) & (s <= self.upper) - else: - return (s > self.lower) & (s < self.upper) - - def validate(self) -> None: - assert isinstance(self.lower, (int, float)) - assert isinstance(self.upper, (int, float)) - assert isinstance(self.inclusive, bool) - -def between(lower: float, upper: float, inclusive: bool = True) -> Between: - """ - Return whether a given value is between a lower and upper threshold - """ - return Between(lower, upper, inclusive) - -class IsNA(ASTPredicate): - def __call__(self, s: SeriesT) -> SeriesT: - return s.isna() - -def isna() -> IsNA: - """ - Return whether a given value is NA - """ - return IsNA() - - -class NotNA(ASTPredicate): - def __call__(self, s: SeriesT) -> SeriesT: - return s.notna() - -def notna() -> NotNA: - """ - Return whether a given value is not NA - """ - return NotNA() diff --git a/graphistry/compute/predicates/str.py b/graphistry/compute/predicates/str.py index 25805b9b3f..3819deeee8 100644 --- a/graphistry/compute/predicates/str.py +++ b/graphistry/compute/predicates/str.py @@ -1,5 +1,4 @@ -from typing import Any, Optional -import pandas as pd +from typing import Optional from .ASTPredicate import ASTPredicate from graphistry.compute.typing import SeriesT diff --git a/graphistry/compute/predicates/temporal.py b/graphistry/compute/predicates/temporal.py index 9421190e2b..d48d437d56 100644 --- a/graphistry/compute/predicates/temporal.py +++ b/graphistry/compute/predicates/temporal.py @@ -1,5 +1,3 @@ -from typing import Any, Optional -import pandas as pd from .ASTPredicate import ASTPredicate from graphistry.compute.typing import SeriesT diff --git a/graphistry/models/gfql/coercions/numeric.py b/graphistry/models/gfql/coercions/numeric.py new file mode 100644 index 0000000000..304c0a394a --- /dev/null +++ b/graphistry/models/gfql/coercions/numeric.py @@ -0,0 +1,32 @@ +""" +Pure numeric type transformations + +Currently numeric types don't need much transformation, +but this provides a consistent interface. +""" + +from typing import Union +import numpy as np + +from ..types.numeric import NativeNumeric + + +def to_native(val: NativeNumeric) -> NativeNumeric: + """Numeric values are already native, just pass through""" + return val + + +def to_ast(val: NativeNumeric) -> NativeNumeric: + """Numeric values don't have special AST representation, pass through""" + return val + + +def to_wire(val: NativeNumeric) -> Union[int, float]: + """Convert numeric to JSON-compatible type""" + if isinstance(val, np.number): + # Convert numpy types to Python native + if isinstance(val, (np.integer, np.int_)): + return int(val) + else: + return float(val) + return val diff --git a/graphistry/models/gfql/coercions/temporal.py b/graphistry/models/gfql/coercions/temporal.py new file mode 100644 index 0000000000..4be9f714cb --- /dev/null +++ b/graphistry/models/gfql/coercions/temporal.py @@ -0,0 +1,94 @@ +""" +Pure temporal type transformations + +Functions to convert between Native, Wire, and AST representations. +""" + +from typing import Union +from datetime import datetime, date, time +import pandas as pd + +from ....compute.ast_temporal import TemporalValue, DateTimeValue, DateValue, TimeValue +from ..types.temporal import ( + NativeTemporal, NativeDateTime, NativeDate, NativeTime, + TemporalWire, DateTimeWire, DateWire, TimeWire +) + + +# ============= To AST Transforms ============= + + +def to_ast(val: Union[NativeTemporal, TemporalWire, TemporalValue]) -> TemporalValue: + """Convert any temporal representation to AST (TemporalValue)""" + # Already AST + if isinstance(val, TemporalValue): + return val + + # From native + elif isinstance(val, pd.Timestamp): + return DateTimeValue.from_pandas_timestamp(val) + elif isinstance(val, datetime): + return DateTimeValue.from_datetime(val) + elif isinstance(val, date): + return DateValue.from_date(val) + elif isinstance(val, time): + return TimeValue.from_time(val) + + # From wire + elif isinstance(val, dict) and "type" in val: + if val["type"] == "datetime": + timezone = val.get("timezone", "UTC") + assert isinstance(timezone, str) + return DateTimeValue(val["value"], timezone) + elif val["type"] == "date": + return DateValue(val["value"]) + elif val["type"] == "time": + return TimeValue(val["value"]) + else: + raise ValueError(f"Unknown temporal wire type: {val['type']}") + + else: + raise TypeError(f"Cannot convert {type(val)} to AST temporal") + + +# ============= To Native Transforms ============= + + +def to_native(val: Union[NativeTemporal, TemporalWire, TemporalValue]) -> Union[pd.Timestamp, time]: + """Convert any temporal representation to native Python/Pandas type""" + # Already native pandas + if isinstance(val, pd.Timestamp): + return val + + # Native Python to pandas + elif isinstance(val, (datetime, date)): + return pd.Timestamp(val) + elif isinstance(val, time): + return val # time stays as time + + # From AST + elif isinstance(val, TemporalValue): + return val.as_pandas_value() + + # From wire (via AST) + elif isinstance(val, dict) and "type" in val: + ast_val = to_ast(val) + return ast_val.as_pandas_value() + + else: + raise TypeError(f"Cannot convert {type(val)} to native temporal") + + +# ============= To Wire Transforms ============= + + +def to_wire(val: Union[NativeTemporal, TemporalValue]) -> TemporalWire: + """Convert temporal to wire format (for JSON serialization)""" + # From AST + if isinstance(val, TemporalValue): + return val.to_json() + + # From native (via AST) + else: + ast_val = to_ast(val) + return ast_val.to_json() diff --git a/graphistry/models/gfql/types/guards.py b/graphistry/models/gfql/types/guards.py new file mode 100644 index 0000000000..df74db52d2 --- /dev/null +++ b/graphistry/models/gfql/types/guards.py @@ -0,0 +1,85 @@ +""" +Type detection utilities + +Clear functions to check what kind of type a value is. +""" + +from typing import Any, TYPE_CHECKING, Union, Dict + +if TYPE_CHECKING: + from typing_extensions import TypeGuard +else: + try: + from typing import TypeGuard + except ImportError: + TypeGuard = bool +from datetime import datetime, date, time +import pandas as pd +import numpy as np + +from ....compute.ast_temporal import TemporalValue + + +# ============= Temporal Detection ============= + +def is_native_temporal(val: Any) -> bool: + """Check if value is a native Python/Pandas temporal type""" + return isinstance(val, (pd.Timestamp, datetime, date, time)) + + +def is_ast_temporal(val: Any) -> bool: + """Check if value is an AST TemporalValue""" + return isinstance(val, TemporalValue) + + +def is_wire_temporal(val: Any) -> bool: + """Check if value is a wire format temporal (tagged dict)""" + return ( + isinstance(val, dict) + and "type" in val + and val["type"] in ["datetime", "date", "time"] + ) + + +def is_any_temporal(val: Any) -> bool: + """Check if value is any kind of temporal (native, AST, or wire)""" + return is_native_temporal(val) or is_ast_temporal(val) or is_wire_temporal(val) + + +# ============= Numeric Detection ============= + +def is_native_numeric(val: Any) -> bool: + """Check if value is a native numeric type""" + return isinstance(val, (int, float, np.number)) + + +def is_any_numeric(val: Any) -> bool: + """Check if value is any kind of numeric (currently same as native)""" + return is_native_numeric(val) + + +# ============= Other Detection ============= + +def is_string(val: Any) -> bool: + """Check if value is a string""" + return isinstance(val, str) + + +def is_none(val: Any) -> bool: + """Check if value is None""" + return val is None + + +def is_basic_scalar(val: Any) -> bool: + """Check if value is a basic scalar (int, float, str, None)""" + return isinstance(val, (int, float, str, np.number, type(None))) + + +def is_dict(val: Any) -> bool: + """Check if value is a dictionary""" + return isinstance(val, dict) + + +def is_tagged_dict(val: Any) -> bool: + """Check if value is a tagged dictionary (has 'type' field)""" + return isinstance(val, dict) and "type" in val diff --git a/graphistry/models/gfql/types/numeric.py b/graphistry/models/gfql/types/numeric.py new file mode 100644 index 0000000000..b56d5fa076 --- /dev/null +++ b/graphistry/models/gfql/types/numeric.py @@ -0,0 +1,33 @@ +""" +Type definitions for numeric values in GFQL + +Numeric data representations: +1. Native types - Host language numeric types (int, float, numpy types) +2. AST types - For numeric predicates, native types are used directly +3. Wire types - JSON number format (no special encoding needed) +""" + +from typing import Union +import numpy as np + + +# ============= Native Numeric Types ============= +# Host language numeric types + +NativeInt = int +NativeFloat = float +NativeNumeric = Union[int, float, np.number] + + +# ============= Wire Types (JSON) ============= +# For numeric values, JSON numbers map directly to native types +# No special wire format needed - JSON handles int/float natively + +WireNumeric = Union[int, float] + + +# ============= AST Types ============= +# Numeric predicates use native types directly in the AST +# No special wrapper needed unlike temporal values + +ASTNumeric = NativeNumeric diff --git a/graphistry/models/gfql/types/predicates.py b/graphistry/models/gfql/types/predicates.py new file mode 100644 index 0000000000..181909cba2 --- /dev/null +++ b/graphistry/models/gfql/types/predicates.py @@ -0,0 +1,39 @@ +""" +Type definitions for GFQL predicate inputs + +Defines what types users can pass to predicates. +Implementation details (like what gets stored internally) belong in compute/predicates. +""" + +from typing import Union, Any, Dict +import numpy as np + +from .temporal import NativeTemporal, TemporalWire +from .numeric import NativeNumeric + + +# ============= Basic Types ============= +# Simple scalar types + +BasicScalar = Union[int, float, str, np.number, None] + + +# ============= Predicate Input Types ============= +# What users can provide to each predicate type + +# Comparison predicates (GT, LT, GE, LE, EQ, NE) - strict, no strings +ComparisonInput = Union[ + NativeNumeric, # Python int, float, np.number + NativeTemporal, # Python datetime, date, time, pd.Timestamp + TemporalWire, # Wire format: {"type": "datetime", ...} +] + +# IsIn predicate - permissive, allows strings and arbitrary values +IsInElementInput = Union[ + BasicScalar, # Includes strings and None + NativeTemporal, # Python datetime types + TemporalWire, # Wire format temporal +] + +# Between predicate - each bound follows comparison rules +BetweenBoundInput = ComparisonInput diff --git a/graphistry/models/gfql/types/temporal.py b/graphistry/models/gfql/types/temporal.py new file mode 100644 index 0000000000..cc21ac514a --- /dev/null +++ b/graphistry/models/gfql/types/temporal.py @@ -0,0 +1,53 @@ +""" +Type definitions for temporal values in GFQL + +Temporal data has three representations: +1. Native types - Host language types (Python datetime, pandas.Timestamp, etc.) +2. AST types - Domain model objects used in the query AST (TemporalValue classes) +3. Wire types - JSON serialization format for client/server communication + +Conversion functions follow the pattern: to/from_native, to/from_wire, to/from_ast +""" + +from typing import Union, TypedDict, Literal +from datetime import datetime, date, time +import pandas as pd + + +# ============= Native Temporal Types ============= +# Host language types that users work with directly + +NativeDateTime = Union[pd.Timestamp, datetime] +NativeDate = date # Note: NOT a Union - important for overload ordering +NativeTime = time +NativeTemporal = Union[NativeDateTime, NativeDate, NativeTime] + + +# ============= Wire Types (JSON) ============= +# Tagged dictionaries for JSON serialization/deserialization + +class DateTimeWire(TypedDict, total=False): + """Wire format for datetime with timezone""" + type: Literal["datetime"] + value: str # ISO 8601 datetime string + timezone: str # Optional IANA timezone (default: UTC) + + +class DateWire(TypedDict): + """Wire format for date only""" + type: Literal["date"] + value: str # ISO 8601 date (YYYY-MM-DD) + + +class TimeWire(TypedDict): + """Wire format for time only""" + type: Literal["time"] + value: str # ISO 8601 time (HH:MM:SS[.ffffff]) + + +# Union of all temporal wire types +TemporalWire = Union[DateTimeWire, DateWire, TimeWire] + + +# Note: AST types (TemporalValue, DateTimeValue, etc.) are defined +# in graphistry.compute.ast_temporal to avoid circular imports diff --git a/graphistry/tests/compute/predicates/test_is_in.py b/graphistry/tests/compute/predicates/test_is_in.py index 8648f3487c..53f47c9edf 100644 --- a/graphistry/tests/compute/predicates/test_is_in.py +++ b/graphistry/tests/compute/predicates/test_is_in.py @@ -1,3 +1,8 @@ +import pandas as pd +import pytest +from datetime import datetime, date, time +import pytz + from graphistry.compute.predicates.is_in import IsIn, is_in @@ -14,3 +19,148 @@ def test_is_in(): d2 = IsIn.from_json(o) assert isinstance(d2, IsIn) assert d2.options == [1, 2, 3] + + +def test_is_in_with_datetime(): + """Test IsIn with datetime values""" + dt1 = datetime(2023, 1, 1, 12, 0, 0) + dt2 = datetime(2023, 1, 2, 12, 0, 0) + dt3 = datetime(2023, 1, 3, 12, 0, 0) + + # Test with datetime objects + pred = is_in([dt1, dt2]) + assert isinstance(pred, IsIn) + # Should be converted to pd.Timestamp + assert all(isinstance(opt, pd.Timestamp) for opt in pred.options) + + # Test with pandas series + s = pd.Series([dt1, dt2, dt3, datetime(2023, 1, 4)]) + result = pred(s) + assert result.tolist() == [True, True, False, False] + + +def test_is_in_with_pandas_timestamp(): + """Test IsIn with pandas Timestamp values""" + ts1 = pd.Timestamp('2023-01-01 12:00:00') + ts2 = pd.Timestamp('2023-01-02 12:00:00') + ts3 = pd.Timestamp('2023-01-03 12:00:00') + + pred = is_in([ts1, ts2]) + assert isinstance(pred, IsIn) + assert pred.options == [ts1, ts2] + + # Test with series + s = pd.Series([ts1, ts2, ts3]) + result = pred(s) + assert result.tolist() == [True, True, False] + + +def test_is_in_with_date(): + """Test IsIn with date values""" + d1 = date(2023, 1, 1) + d2 = date(2023, 1, 2) + + pred = is_in([d1, d2]) + assert isinstance(pred, IsIn) + # Should be converted to pd.Timestamp + assert all(isinstance(opt, pd.Timestamp) for opt in pred.options) + + # Test with datetime series (will compare dates) + s = pd.Series(pd.to_datetime(['2023-01-01', '2023-01-02', '2023-01-03'])) + result = pred(s) + assert result.tolist() == [True, True, False] + + +def test_is_in_with_time(): + """Test IsIn with time values""" + t1 = time(12, 0, 0) + t2 = time(13, 0, 0) + + pred = is_in([t1, t2]) + assert isinstance(pred, IsIn) + # Time values should remain as time objects + assert all(isinstance(opt, time) for opt in pred.options) + + # Test with datetime series (will extract time component) + dt_series = pd.Series(pd.to_datetime([ + '2023-01-01 12:00:00', + '2023-01-01 13:00:00', + '2023-01-01 14:00:00' + ])) + result = pred(dt_series) + assert result.tolist() == [True, True, False] + + +def test_is_in_with_timezone(): + """Test IsIn with timezone-aware datetime values""" + utc = pytz.UTC + + # Create timezone-aware timestamps + ts1_utc = pd.Timestamp('2023-01-01 12:00:00', tz=utc) + ts2_utc = pd.Timestamp('2023-01-01 17:00:00', tz=utc) # Same as 12:00 EST + + pred = is_in([ts1_utc]) + + # Test with timezone-aware series + s = pd.Series([ts1_utc, ts2_utc, pd.Timestamp('2023-01-01 13:00:00', tz=utc)]) + result = pred(s) + assert result.tolist() == [True, False, False] + + +def test_is_in_with_tagged_dict(): + """Test IsIn with tagged dictionary values (JSON deserialization)""" + # Test datetime tagged dict + dt_dict = { + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + } + + pred = is_in([dt_dict, {"type": "datetime", "value": "2023-01-02T12:00:00"}]) + assert isinstance(pred, IsIn) + assert all(isinstance(opt, pd.Timestamp) for opt in pred.options) + + # Test with series + s = pd.Series(pd.to_datetime(['2023-01-01 12:00:00', '2023-01-02 12:00:00', '2023-01-03 12:00:00'])) + s = s.dt.tz_localize('UTC') + result = pred(s) + assert result.tolist() == [True, True, False] + + +def test_is_in_mixed_types(): + """Test IsIn with mixed temporal and non-temporal values""" + dt = datetime(2023, 1, 1) + pred = is_in(["hello", 42, dt, None]) + + # Test with mixed series + s = pd.Series(["hello", 42, pd.Timestamp(dt), None, "world"]) + result = pred(s) + assert result.tolist() == [True, True, True, True, False] + + +def test_is_in_validation(): + """Test IsIn validation with temporal values""" + dt = datetime(2023, 1, 1) + pred = is_in([dt, "test", 123]) + + # Should not raise during validation + pred.validate() + + +def test_is_in_json_serialization_with_temporal(): + """Test JSON serialization/deserialization with temporal values""" + dt = datetime(2023, 1, 1, 12, 30, 45) + pred = is_in([dt, "test"]) + + # Serialize to JSON + json_dict = pred.to_json() + assert json_dict['type'] == 'IsIn' + assert 'options' in json_dict + + # Deserialize from JSON + pred2 = IsIn.from_json(json_dict) + assert isinstance(pred2, IsIn) + # The datetime should be preserved (as pd.Timestamp) + assert len(pred2.options) == 2 + assert isinstance(pred2.options[0], pd.Timestamp) + assert pred2.options[1] == "test" diff --git a/graphistry/tests/compute/predicates/test_numeric.py b/graphistry/tests/compute/predicates/test_numeric.py index b6ce762c60..ac7d0ea4c7 100644 --- a/graphistry/tests/compute/predicates/test_numeric.py +++ b/graphistry/tests/compute/predicates/test_numeric.py @@ -1,7 +1,11 @@ -from graphistry.compute.predicates.numeric import GT, gt +import pandas as pd +import pytest +from datetime import datetime, date, time +from graphistry.compute.predicates.comparison import GT, gt, LT, lt, GE, ge, LE, le, EQ, eq, NE, ne, Between, between +from graphistry.compute.ast_temporal import DateTimeValue, DateValue, TimeValue def test_gt(): - + # Test numeric GT (existing test) d = gt(1) assert isinstance(d, GT) assert d.val == 1 @@ -14,3 +18,181 @@ def test_gt(): d2 = GT.from_json(o) assert isinstance(d2, GT) assert d2.val == 1 + +def test_gt_temporal(): + """Test GT with temporal values""" + # Test with pd.Timestamp + ts = pd.Timestamp('2024-01-01', tz='UTC') + d = gt(ts) + assert isinstance(d, GT) + assert isinstance(d.val, DateTimeValue) + + # Test serialization + o = d.to_json() + assert o['type'] == 'GT' + assert isinstance(o['val'], dict) + assert o['val']['type'] == 'datetime' + assert o['val']['value'] == '2024-01-01T00:00:00+00:00' + assert o['val']['timezone'] == 'UTC' + + # Test deserialization + d2 = GT.from_json(o) + assert isinstance(d2, GT) + assert isinstance(d2.val, DateTimeValue) + +def test_lt_temporal(): + """Test LT with temporal values""" + # Test with datetime + dt = datetime(2024, 1, 1, 12, 0, 0) + d = lt(dt) + assert isinstance(d, LT) + assert isinstance(d.val, DateTimeValue) + + # Test with date + date_val = date(2024, 1, 1) + d = lt(date_val) + assert isinstance(d, LT) + assert isinstance(d.val, DateValue) + + # Test with time + time_val = time(12, 0, 0) + d = lt(time_val) + assert isinstance(d, LT) + assert isinstance(d.val, TimeValue) + +def test_temporal_with_series(): + """Test temporal predicates with pandas Series""" + # Create datetime series + dates = pd.date_range('2024-01-01', periods=5, freq='D', tz='UTC') + s = pd.Series(dates) + + # Test GT + cutoff = pd.Timestamp('2024-01-03', tz='UTC') + gt_pred = gt(cutoff) + result = gt_pred(s) + expected = pd.Series([False, False, False, True, True]) + assert result.equals(expected) + + # Test LT + lt_pred = lt(cutoff) + result = lt_pred(s) + expected = pd.Series([True, True, False, False, False]) + assert result.equals(expected) + + # Test GE + ge_pred = ge(cutoff) + result = ge_pred(s) + expected = pd.Series([False, False, True, True, True]) + assert result.equals(expected) + + # Test LE + le_pred = le(cutoff) + result = le_pred(s) + expected = pd.Series([True, True, True, False, False]) + assert result.equals(expected) + + # Test EQ + eq_pred = eq(cutoff) + result = eq_pred(s) + expected = pd.Series([False, False, True, False, False]) + assert result.equals(expected) + + # Test NE + ne_pred = ne(cutoff) + result = ne_pred(s) + expected = pd.Series([True, True, False, True, True]) + assert result.equals(expected) + +def test_between_temporal(): + """Test Between with temporal values""" + # Test with timestamps + start = pd.Timestamp('2024-01-02', tz='UTC') + end = pd.Timestamp('2024-01-04', tz='UTC') + + # Create predicate + between_pred = between(start, end, inclusive=True) + assert isinstance(between_pred, Between) + assert isinstance(between_pred.lower, DateTimeValue) + assert isinstance(between_pred.upper, DateTimeValue) + + # Test serialization + o = between_pred.to_json() + assert o['type'] == 'Between' + assert o['inclusive'] is True + assert isinstance(o['lower'], dict) + assert o['lower']['type'] == 'datetime' + assert isinstance(o['upper'], dict) + assert o['upper']['type'] == 'datetime' + + # Test with series + dates = pd.date_range('2024-01-01', periods=5, freq='D', tz='UTC') + s = pd.Series(dates) + result = between_pred(s) + expected = pd.Series([False, True, True, True, False]) + assert result.equals(expected) + + # Test exclusive + between_pred_excl = between(start, end, inclusive=False) + result = between_pred_excl(s) + expected = pd.Series([False, False, True, False, False]) + assert result.equals(expected) + +def test_temporal_with_tagged_dict(): + """Test predicates with tagged dict input""" + # Test GT with tagged dict + d = gt({ + "type": "datetime", + "value": "2024-01-01T00:00:00Z", + "timezone": "UTC" + }) + assert isinstance(d, GT) + assert isinstance(d.val, DateTimeValue) + assert d.val.value == "2024-01-01T00:00:00Z" + assert d.val.timezone == "UTC" + + # Test date dict + d = lt({ + "type": "date", + "value": "2024-01-01" + }) + assert isinstance(d, LT) + assert isinstance(d.val, DateValue) + + # Test time dict + d = eq({ + "type": "time", + "value": "12:00:00" + }) + assert isinstance(d, EQ) + assert isinstance(d.val, TimeValue) + +def test_temporal_error_cases(): + """Test error handling for temporal predicates""" + # Test raw string rejection + with pytest.raises(ValueError, match="ambiguous"): + gt("2024-01-01") + + with pytest.raises(ValueError, match="ambiguous"): + between("2024-01-01", "2024-12-31") + + # Test type mismatch in Between - error happens during validation or call + between_pred = between(100, pd.Timestamp('2024-01-01')) + with pytest.raises(TypeError, match="same type"): + between_pred.validate() + + # Test unknown temporal type - should fail as unsupported type + with pytest.raises(TypeError, match="Unsupported type"): + gt({"type": "duration", "value": "P1D"}) + +def test_timezone_handling(): + """Test timezone normalization in comparisons""" + # Create series with different timezone + dates = pd.date_range('2024-01-01', periods=3, freq='D', tz='US/Eastern') + s = pd.Series(dates) + + # Compare with UTC timestamp + cutoff = pd.Timestamp('2024-01-02T05:00:00', tz='UTC') # Equivalent to 2024-01-02 00:00:00 EST + gt_pred = gt(cutoff) + result = gt_pred(s) + expected = pd.Series([False, False, True]) + assert result.equals(expected) diff --git a/graphistry/tests/compute/predicates/test_temporal_values.py b/graphistry/tests/compute/predicates/test_temporal_values.py new file mode 100644 index 0000000000..62dfd2de3b --- /dev/null +++ b/graphistry/tests/compute/predicates/test_temporal_values.py @@ -0,0 +1,103 @@ +import pytest +import pandas as pd +from datetime import datetime, date, time +import pytz + +from graphistry.compute.ast_temporal import ( + DateTimeValue, DateValue, TimeValue, temporal_value_from_json +) + + +class TestDateTimeValue: + def test_parse_iso8601_with_timezone(self): + dt = DateTimeValue("2024-01-01T12:00:00+00:00", "UTC") + assert dt.value == "2024-01-01T12:00:00+00:00" + assert dt.timezone == "UTC" + assert isinstance(dt.as_pandas_value(), pd.Timestamp) + assert dt.as_pandas_value().hour == 12 + + def test_parse_iso8601_naive(self): + dt = DateTimeValue("2024-01-01T12:00:00", "UTC") + assert dt.timezone == "UTC" + assert dt.as_pandas_value().hour == 12 + assert dt.as_pandas_value().tz.zone == "UTC" + + def test_timezone_conversion(self): + # Create datetime in UTC + dt_utc = DateTimeValue("2024-01-01T12:00:00+00:00", "UTC") + # Create same instant in EST (UTC-5) + dt_est = DateTimeValue("2024-01-01T12:00:00+00:00", "US/Eastern") + + # Should be same instant but displayed in EST + assert dt_est.as_pandas_value().hour == 7 # 12 UTC = 7 EST + assert dt_utc.as_pandas_value().timestamp() == dt_est.as_pandas_value().timestamp() + + def test_to_json(self): + dt = DateTimeValue("2024-01-01T12:00:00Z", "UTC") + json_data = dt.to_json() + assert json_data == { + "type": "datetime", + "value": "2024-01-01T12:00:00Z", + "timezone": "UTC" + } + + +class TestDateValue: + def test_parse_date(self): + d = DateValue("2024-01-01") + assert d.value == "2024-01-01" + assert d._parsed == date(2024, 1, 1) + assert isinstance(d.as_pandas_value(), pd.Timestamp) + assert d.as_pandas_value().date() == date(2024, 1, 1) + + def test_to_json(self): + d = DateValue("2024-01-01") + json_data = d.to_json() + assert json_data == { + "type": "date", + "value": "2024-01-01" + } + + +class TestTimeValue: + def test_parse_time(self): + t = TimeValue("14:30:00") + assert t.value == "14:30:00" + assert t._parsed == time(14, 30, 0) + assert isinstance(t.as_pandas_value(), time) + assert t.as_pandas_value().hour == 14 + assert t.as_pandas_value().minute == 30 + + def test_to_json(self): + t = TimeValue("14:30:00") + json_data = t.to_json() + assert json_data == { + "type": "time", + "value": "14:30:00" + } + + +class TestTemporalValueFromJson: + def test_datetime_from_json(self): + json_data = {"type": "datetime", "value": "2024-01-01T12:00:00Z", "timezone": "UTC"} + dt = temporal_value_from_json(json_data) + assert isinstance(dt, DateTimeValue) + assert dt.value == "2024-01-01T12:00:00Z" + assert dt.timezone == "UTC" + + def test_date_from_json(self): + json_data = {"type": "date", "value": "2024-01-01"} + d = temporal_value_from_json(json_data) + assert isinstance(d, DateValue) + assert d.value == "2024-01-01" + + def test_time_from_json(self): + json_data = {"type": "time", "value": "14:30:00"} + t = temporal_value_from_json(json_data) + assert isinstance(t, TimeValue) + assert t.value == "14:30:00" + + def test_invalid_type(self): + json_data = {"type": "invalid", "value": "something"} + with pytest.raises(ValueError, match="Unknown temporal value type"): + temporal_value_from_json(json_data) diff --git a/graphistry/tests/compute/predicates/test_wire_dict_temporal.py b/graphistry/tests/compute/predicates/test_wire_dict_temporal.py new file mode 100644 index 0000000000..24e1f2867d --- /dev/null +++ b/graphistry/tests/compute/predicates/test_wire_dict_temporal.py @@ -0,0 +1,103 @@ +"""Test that wire protocol dictionaries work directly in temporal predicates""" + +import pandas as pd +from datetime import time + +from graphistry.compute import gt, ge, between, is_in + + +class TestWireProtocolDicts: + """Test using wire protocol dictionaries directly in predicates""" + + def test_datetime_wire_dict(self): + """Test datetime wire protocol dicts in comparison predicates""" + df = pd.DataFrame({ + 'timestamp': pd.date_range('2023-01-01', periods=5, + freq='D', tz='UTC') + }) + + # Test with gt + wire_dict = {"type": "datetime", "value": "2023-01-03T00:00:00", + "timezone": "UTC"} + pred = gt(wire_dict) + result = pred(df['timestamp']) + assert result.tolist() == [False, False, False, True, True] + + # Test with between + start_dict = {"type": "datetime", "value": "2023-01-02T00:00:00", + "timezone": "UTC"} + end_dict = {"type": "datetime", "value": "2023-01-04T00:00:00", + "timezone": "UTC"} + pred2 = between(start_dict, end_dict) + result2 = pred2(df['timestamp']) + assert result2.tolist() == [False, True, True, True, False] + + def test_date_wire_dict(self): + """Test date wire protocol dicts""" + df = pd.DataFrame({ + 'date': pd.date_range('2023-01-01', periods=5, freq='D').date + }) + + wire_dict = {"type": "date", "value": "2023-01-03"} + pred = ge(wire_dict) + result = pred(df['date']) + assert result.tolist() == [False, False, True, True, True] + + def test_time_wire_dict(self): + """Test time wire protocol dicts""" + times = [time(9, 0), time(12, 0), time(15, 0), time(18, 0)] + df = pd.DataFrame({'time': times}) + + wire_dict = {"type": "time", "value": "12:00:00"} + pred = gt(wire_dict) + result = pred(df['time']) + assert result.tolist() == [False, False, True, True] + + def test_is_in_wire_dicts(self): + """Test is_in with wire protocol dicts""" + df = pd.DataFrame({ + 'timestamp': pd.date_range('2023-01-01', periods=5, + freq='D', tz='UTC') + }) + + wire_dicts = [ + {"type": "datetime", "value": "2023-01-02T00:00:00", + "timezone": "UTC"}, + {"type": "datetime", "value": "2023-01-04T00:00:00", + "timezone": "UTC"} + ] + pred = is_in(wire_dicts) + result = pred(df['timestamp']) + assert result.tolist() == [False, True, False, True, False] + + def test_wire_dict_serialization(self): + """Test that wire dicts serialize correctly in to_json()""" + wire_dict = {"type": "datetime", "value": "2023-01-01T00:00:00", + "timezone": "UTC"} + pred = gt(wire_dict) + + json_obj = pred.to_json() + assert json_obj["type"] == "GT" + assert json_obj["val"]["type"] == "datetime" + assert json_obj["val"]["value"] == "2023-01-01T00:00:00" + assert json_obj["val"]["timezone"] == "UTC" + + def test_mixed_types_with_wire_dicts(self): + """Test mixing wire dicts with regular values""" + df = pd.DataFrame({ + 'timestamp': pd.date_range('2023-01-01', periods=5, + freq='D', tz='UTC'), + 'value': [1, 2, 3, 4, 5] + }) + + # Numeric comparison still works normally + pred_num = gt(3) + result_num = pred_num(df['value']) + assert result_num.tolist() == [False, False, False, True, True] + + # Temporal comparison with wire dict + wire_dict = {"type": "datetime", "value": "2023-01-03T00:00:00", + "timezone": "UTC"} + pred_temporal = gt(wire_dict) + result_temporal = pred_temporal(df['timestamp']) + assert result_temporal.tolist() == [False, False, False, True, True] diff --git a/graphistry/tests/compute/test_chain.py b/graphistry/tests/compute/test_chain.py index 14eb3966ac..25e8f11eb0 100644 --- a/graphistry/tests/compute/test_chain.py +++ b/graphistry/tests/compute/test_chain.py @@ -5,7 +5,7 @@ from graphistry.compute.ast import ASTEdgeUndirected, ASTNode, ASTEdge, n, e, e_undirected, e_forward from graphistry.compute.chain import Chain from graphistry.compute.predicates.is_in import IsIn, is_in -from graphistry.compute.predicates.numeric import gt +from graphistry.compute.predicates.comparison import gt from graphistry.tests.test_compute import CGFull diff --git a/graphistry/tests/compute/test_datetime_integration.py b/graphistry/tests/compute/test_datetime_integration.py new file mode 100644 index 0000000000..ddac7595e9 --- /dev/null +++ b/graphistry/tests/compute/test_datetime_integration.py @@ -0,0 +1,156 @@ +"""Integration tests for GFQL datetime predicates end-to-end usage""" + +import pandas as pd +import pytest +from datetime import datetime, date, time +import pytz + +from graphistry import n +from graphistry.compute import ( + DateTimeValue, DateValue, TimeValue, + gt, lt, between, eq, is_in +) + + +def test_datetime_predicates_in_chain(): + """Test datetime predicates in a chain operation""" + # Create sample data with datetime columns + df = pd.DataFrame({ + 'timestamp': pd.to_datetime([ + '2023-01-01 10:00:00', + '2023-01-02 14:30:00', + '2023-01-03 09:00:00', + '2023-01-04 16:45:00' + ]), + 'value': [10, 20, 30, 40] + }) + + # Test GT with datetime + dt_threshold = datetime(2023, 1, 2, 12, 0, 0) + result = df[gt(pd.Timestamp(dt_threshold))(df['timestamp'])] + assert len(result) == 3 # All timestamps after 2023-01-02 12:00:00 + assert result['value'].tolist() == [20, 30, 40] + + # Test Between with dates (converts to midnight timestamps) + start_date = date(2023, 1, 2) # 2023-01-02 00:00:00 + end_date = date(2023, 1, 3) # 2023-01-03 00:00:00 + result = df[between(pd.Timestamp(start_date), pd.Timestamp(end_date))(df['timestamp'])] + # Only 2023-01-02 14:30:00 falls between these dates (inclusive) + assert len(result) == 1 + assert result['value'].tolist() == [20] + + +def test_datetime_with_timezone(): + """Test timezone-aware datetime comparisons""" + utc = pytz.UTC + + # Create timezone-aware data + df = pd.DataFrame({ + 'timestamp': [ + pd.Timestamp('2023-01-01 10:00:00', tz=utc), + pd.Timestamp('2023-01-01 15:00:00', tz=utc), # 10:00 EST + pd.Timestamp('2023-01-01 18:00:00', tz=utc), # 13:00 EST + ], + 'event': ['A', 'B', 'C'] + }) + + # Test with UTC timestamp + threshold = pd.Timestamp('2023-01-01 14:00:00', tz=utc) + result = df[gt(threshold)(df['timestamp'])] + assert result['event'].tolist() == ['B', 'C'] + + +def test_time_predicates(): + """Test time-only predicates""" + df = pd.DataFrame({ + 'timestamp': pd.to_datetime([ + '2023-01-01 09:00:00', + '2023-01-01 12:30:00', + '2023-01-01 15:45:00', + '2023-01-02 09:00:00', + '2023-01-02 18:00:00' + ]), + 'event': ['morning1', 'lunch', 'afternoon', 'morning2', 'evening'] + }) + + # Test IsIn with time values + morning_times = [time(9, 0, 0), time(9, 30, 0)] + pred = is_in(morning_times) + result = df[pred(df['timestamp'])] + assert result['event'].tolist() == ['morning1', 'morning2'] + + +def test_mixed_temporal_types(): + """Test predicates with mixed temporal types""" + df = pd.DataFrame({ + 'date': pd.to_datetime(['2023-01-01', '2023-01-02', '2023-01-03']).date, + 'datetime': pd.to_datetime([ + '2023-01-01 10:00:00', + '2023-01-02 14:00:00', + '2023-01-03 16:00:00' + ]), + 'id': [1, 2, 3] + }) + + # Test date comparison + target_date = date(2023, 1, 2) + result = df[eq(target_date)(pd.to_datetime(df['date']))] + assert result['id'].tolist() == [2] + + +def test_json_serialization_roundtrip(): + """Test JSON serialization/deserialization with temporal predicates""" + # Create predicate with temporal value + dt = datetime(2023, 1, 1, 12, 0, 0) + pred = gt(pd.Timestamp(dt)) + + # Serialize to JSON + json_data = pred.to_json() + assert json_data['type'] == 'GT' + + # Deserialize and verify it still works + from graphistry.compute.predicates.comparison import GT + pred2 = GT.from_json(json_data) + + # Test on data + df = pd.DataFrame({ + 'timestamp': pd.to_datetime(['2023-01-01 10:00:00', '2023-01-01 14:00:00']), + 'value': [1, 2] + }) + result = pred2(df['timestamp']) + assert result.tolist() == [False, True] + + +def test_temporal_values_api(): + """Test the temporal value classes directly""" + # DateTimeValue + dt_val = DateTimeValue("2023-01-01T12:00:00", "UTC") + assert dt_val.to_json() == { + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + } + + # DateValue + date_val = DateValue("2023-01-01") + assert date_val.to_json() == { + "type": "date", + "value": "2023-01-01" + } + + # TimeValue + time_val = TimeValue("14:30:00") + assert time_val.to_json() == { + "type": "time", + "value": "14:30:00" + } + + # Factory method + from graphistry.compute.ast_temporal import temporal_value_from_json + dt_val2 = temporal_value_from_json({ + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "US/Eastern" + }) + assert isinstance(dt_val2, DateTimeValue) + assert dt_val2.timezone == "US/Eastern" diff --git a/graphistry/tests/compute/test_temporal_ast_integration.py b/graphistry/tests/compute/test_temporal_ast_integration.py new file mode 100644 index 0000000000..b1283026e5 --- /dev/null +++ b/graphistry/tests/compute/test_temporal_ast_integration.py @@ -0,0 +1,179 @@ +"""Integration tests for temporal predicates in AST and wire protocol""" + +import pandas as pd +import pytest +from datetime import datetime, date, time + +from graphistry import n, e_forward +from graphistry.compute import gt, lt, between +from graphistry.compute.ast import ASTNode, ASTEdge +from graphistry.compute.chain import Chain +from graphistry.compute.ast_temporal import DateTimeValue, DateValue, TimeValue + + +class TestTemporalASTIntegration: + """Test temporal predicates work correctly in AST nodes and edges""" + + def test_temporal_in_node_filter_dict(self): + """Test temporal predicates can be used in node filter_dict""" + dt = datetime(2023, 1, 1, 12, 0, 0) + pred = gt(pd.Timestamp(dt)) + + # Create AST node with temporal predicate + node = ASTNode(filter_dict={'timestamp': pred}) + + # Test JSON serialization + json_data = node.to_json() + + # Verify structure + assert 'filter_dict' in json_data + assert 'timestamp' in json_data['filter_dict'] + assert json_data['filter_dict']['timestamp']['type'] == 'GT' + assert json_data['filter_dict']['timestamp']['val']['type'] == 'datetime' + assert json_data['filter_dict']['timestamp']['val']['value'] == '2023-01-01T12:00:00' + + # Test deserialization + node2 = ASTNode.from_json(json_data) + assert node2.filter_dict is not None + assert 'timestamp' in node2.filter_dict + + def test_temporal_in_edge_matches(self): + """Test temporal predicates in all edge match types""" + start_date = date(2023, 1, 1) + end_date = date(2023, 12, 31) + date_range = between(pd.Timestamp(start_date), pd.Timestamp(end_date)) + + # Create AST edge with temporal predicates + edge = ASTEdge( + direction='forward', + edge_match={'created_date': date_range}, + source_node_match={'last_login': gt(pd.Timestamp(datetime(2023, 6, 1)))}, + destination_node_match={'expires': lt(pd.Timestamp(datetime(2024, 1, 1)))} + ) + + # Test JSON serialization + json_data = edge.to_json() + + # Verify all match types have temporal predicates + assert json_data['edge_match']['created_date']['type'] == 'Between' + assert json_data['edge_match']['created_date']['lower']['type'] == 'datetime' + assert json_data['edge_match']['created_date']['upper']['type'] == 'datetime' + + assert json_data['source_node_match']['last_login']['type'] == 'GT' + assert json_data['source_node_match']['last_login']['val']['type'] == 'datetime' + + assert json_data['destination_node_match']['expires']['type'] == 'LT' + assert json_data['destination_node_match']['expires']['val']['type'] == 'datetime' + + # Test deserialization + edge2 = ASTEdge.from_json(json_data) + assert edge2.edge_match is not None + assert edge2.source_node_match is not None + assert edge2.destination_node_match is not None + + def test_temporal_in_chain(self): + """Test temporal predicates in a complete chain""" + chain = Chain([ + n({'created': gt(pd.Timestamp('2023-01-01'))}), + e_forward( + edge_match={'timestamp': between( + pd.Timestamp('2023-01-01'), + pd.Timestamp('2023-12-31') + )}, + destination_node_match={'active_until': gt(pd.Timestamp('2023-06-01'))} + ), + n({'status': 'active'}) + ]) + + # Test JSON serialization + json_data = chain.to_json() + + # Verify chain structure + assert len(json_data['chain']) == 3 + + # Check first node has temporal predicate + node1 = json_data['chain'][0] + assert node1['filter_dict']['created']['type'] == 'GT' + assert node1['filter_dict']['created']['val']['type'] == 'datetime' + + # Check edge has temporal predicates + edge = json_data['chain'][1] + assert edge['edge_match']['timestamp']['type'] == 'Between' + assert edge['destination_node_match']['active_until']['type'] == 'GT' + + # Test deserialization + chain2 = Chain.from_json(json_data) + assert len(chain2.chain) == 3 + + def test_temporal_value_objects_in_ast(self): + """Test using temporal value objects directly""" + dt_val = DateTimeValue("2023-01-01T12:00:00", "US/Eastern") + date_val = DateValue("2023-01-01") + time_val = TimeValue("12:00:00") + + # Create predicates with temporal value objects + node = ASTNode(filter_dict={ + 'datetime_col': gt(dt_val), + 'date_col': lt(date_val), + 'time_col': between(time_val, TimeValue("18:00:00")) + }) + + # Test serialization + json_data = node.to_json() + + # Verify timezone is preserved + assert json_data['filter_dict']['datetime_col']['val']['timezone'] == 'US/Eastern' + + # Verify date and time types + assert json_data['filter_dict']['date_col']['val']['type'] == 'date' + assert json_data['filter_dict']['time_col']['lower']['type'] == 'time' + assert json_data['filter_dict']['time_col']['upper']['type'] == 'time' + + def test_mixed_predicate_types(self): + """Test mixing temporal, numeric, and other predicate types""" + from graphistry.compute.predicates.is_in import is_in + + node = ASTNode(filter_dict={ + 'score': gt(0.5), # numeric predicate + 'created': gt(pd.Timestamp('2023-01-01')), # temporal predicate + 'category': 'A', # regular value + 'tags': is_in(['tag1', 'tag2']) # categorical predicate + }) + + json_data = node.to_json() + + # Verify each type is handled correctly + assert json_data['filter_dict']['score']['type'] == 'GT' + assert isinstance(json_data['filter_dict']['score']['val'], (int, float)) + + assert json_data['filter_dict']['created']['type'] == 'GT' + assert json_data['filter_dict']['created']['val']['type'] == 'datetime' + + assert json_data['filter_dict']['category'] == 'A' + + assert json_data['filter_dict']['tags']['type'] == 'IsIn' + assert json_data['filter_dict']['tags']['options'] == ['tag1', 'tag2'] + + def test_timezone_aware_datetime_in_ast(self): + """Test timezone-aware datetime handling""" + import pytz + + # Create timezone-aware timestamp + eastern = pytz.timezone('US/Eastern') + dt = eastern.localize(datetime(2023, 1, 1, 12, 0, 0)) + + node = ASTNode(filter_dict={ + 'timestamp': gt(pd.Timestamp(dt)) + }) + + json_data = node.to_json() + + # Timezone should be preserved + assert json_data['filter_dict']['timestamp']['val']['timezone'] == 'US/Eastern' + + # Deserialize and verify + node2 = ASTNode.from_json(json_data) + pred = node2.filter_dict['timestamp'] + assert hasattr(pred, 'val') + assert isinstance(pred.val, DateTimeValue) + assert pred.val.timezone == 'US/Eastern' diff --git a/graphistry/tests/test_compute_filter_by_dict_temporal.py b/graphistry/tests/test_compute_filter_by_dict_temporal.py new file mode 100644 index 0000000000..960dc2ff0e --- /dev/null +++ b/graphistry/tests/test_compute_filter_by_dict_temporal.py @@ -0,0 +1,178 @@ +"""Test temporal predicates work with filter_by_dict""" + +import pandas as pd +import pytest +from datetime import datetime, date, time + +from graphistry.compute import gt, lt, between, eq +from graphistry.compute.filter_by_dict import filter_by_dict +from graphistry.tests.test_compute import CGFull + + +class TestFilterByDictTemporal: + """Test temporal predicates in filter_by_dict operations""" + + @pytest.fixture + def temporal_graph(self): + """Create a graph with temporal data""" + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c', 'd', 'e'], + 'created': pd.to_datetime([ + '2023-01-01 10:00:00', + '2023-03-15 14:30:00', + '2023-06-01 09:00:00', + '2023-09-20 16:45:00', + '2023-12-25 12:00:00' + ]), + 'expires': pd.to_datetime([ + '2024-01-01', + '2024-03-15', + '2024-06-01', + '2024-09-20', + '2024-12-25' + ]), + 'status': ['active', 'active', 'expired', 'active', 'pending'] + }) + + edges_df = pd.DataFrame({ + 's': ['a', 'b', 'c', 'd'], + 'd': ['b', 'c', 'd', 'e'], + 'timestamp': pd.to_datetime([ + '2023-01-15 11:00:00', + '2023-04-01 15:00:00', + '2023-07-10 10:30:00', + '2023-10-05 17:00:00' + ]), + 'weight': [1.0, 2.5, 3.0, 4.5] + }) + + return CGFull().nodes(nodes_df, 'id').edges(edges_df, 's', 'd') + + def test_temporal_gt_predicate(self, temporal_graph): + """Test greater than with datetime""" + g = temporal_graph + threshold = pd.Timestamp('2023-06-01') + + # Filter nodes created after June 1st + filtered = filter_by_dict(g._nodes, {'created': gt(threshold)}) + assert len(filtered) == 3 # nodes c, d, e + assert set(filtered['id'].tolist()) == {'c', 'd', 'e'} + + def test_temporal_lt_predicate(self, temporal_graph): + """Test less than with datetime""" + g = temporal_graph + threshold = pd.Timestamp('2023-06-01') + + # Filter nodes created before June 1st + filtered = filter_by_dict(g._nodes, {'created': lt(threshold)}) + assert len(filtered) == 2 # nodes a, b + assert set(filtered['id'].tolist()) == {'a', 'b'} + + def test_temporal_between_predicate(self, temporal_graph): + """Test between with datetime range""" + g = temporal_graph + start = pd.Timestamp('2023-03-01') + end = pd.Timestamp('2023-10-01') + + # Filter nodes created between March and October + filtered = filter_by_dict(g._nodes, {'created': between(start, end)}) + assert len(filtered) == 3 # nodes b, c, d + assert set(filtered['id'].tolist()) == {'b', 'c', 'd'} + + def test_temporal_eq_predicate(self, temporal_graph): + """Test equality with date (comparing just the date part)""" + g = temporal_graph + target_date = date(2023, 1, 1) + + # Filter nodes created on Jan 1st (regardless of time) + # Convert date to datetime for comparison + filtered = filter_by_dict(g._nodes, { + 'created': between( + pd.Timestamp(target_date), + pd.Timestamp(target_date) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1) + ) + }) + assert len(filtered) == 1 + assert filtered['id'].tolist() == ['a'] + + def test_mixed_temporal_and_regular_filters(self, temporal_graph): + """Test combining temporal predicates with regular filters""" + g = temporal_graph + + # Active nodes created after March 1st + filtered = filter_by_dict(g._nodes, { + 'created': gt(pd.Timestamp('2023-03-01')), + 'status': 'active' + }) + assert len(filtered) == 2 # nodes b, d + assert set(filtered['id'].tolist()) == {'b', 'd'} + + def test_temporal_edge_filtering(self, temporal_graph): + """Test temporal predicates on edges""" + g = temporal_graph + + # Edges with timestamps in Q2/Q3 2023 + filtered = filter_by_dict(g._edges, { + 'timestamp': between( + pd.Timestamp('2023-04-01'), + pd.Timestamp('2023-09-30') + ) + }) + assert len(filtered) == 2 + assert set(filtered['s'].tolist()) == {'b', 'c'} + + def test_node_filter_method(self, temporal_graph): + """Test filter_nodes_by_dict with temporal predicates""" + g = temporal_graph + + # Use the graph method + g2 = g.filter_nodes_by_dict({ + 'created': gt(pd.Timestamp('2023-06-01')), + 'expires': lt(pd.Timestamp('2024-12-01')) + }) + + assert len(g2._nodes) == 2 # nodes c and d match both conditions + assert set(g2._nodes['id'].tolist()) == {'c', 'd'} + + def test_edge_filter_method(self, temporal_graph): + """Test filter_edges_by_dict with temporal predicates""" + g = temporal_graph + + # Use the graph method + g2 = g.filter_edges_by_dict({ + 'timestamp': between( + pd.Timestamp('2023-01-01'), + pd.Timestamp('2023-06-30') + ), + 'weight': gt(1.5) + }) + + assert len(g2._edges) == 1 + assert g2._edges['s'].tolist() == ['b'] + assert g2._edges['weight'].tolist() == [2.5] + + def test_timezone_aware_filtering(self): + """Test filtering with timezone-aware timestamps""" + import pytz + + utc = pytz.UTC + # eastern = pytz.timezone('US/Eastern') # Not used + + # Create timezone-aware data + nodes_df = pd.DataFrame({ + 'id': ['a', 'b', 'c'], + 'timestamp': [ + pd.Timestamp('2023-01-01 10:00:00', tz=utc), + pd.Timestamp('2023-01-01 15:00:00', tz=utc), # 10:00 EST + pd.Timestamp('2023-01-01 18:00:00', tz=utc), # 13:00 EST + ] + }) + + g = CGFull().nodes(nodes_df, 'id') + + # Filter with UTC threshold + threshold = pd.Timestamp('2023-01-01 14:00:00', tz=utc) + g2 = g.filter_nodes_by_dict({'timestamp': gt(threshold)}) + + assert len(g2._nodes) == 2 + assert set(g2._nodes['id'].tolist()) == {'b', 'c'} diff --git a/graphistry/utils/lazy_import.py b/graphistry/utils/lazy_import.py index 6f56a92127..234c2b3179 100644 --- a/graphistry/utils/lazy_import.py +++ b/graphistry/utils/lazy_import.py @@ -97,7 +97,7 @@ def lazy_embed_import(): from dgl.dataloading import GraphDataLoader import torch.nn.functional as F from graphistry.networks import HeteroEmbed - from tqdm import trange + from tqdm import trange # type: ignore[import] return True, torch, nn, dgl, GraphDataLoader, HeteroEmbed, F, trange except ModuleNotFoundError: return False, None, None, None, None, None, None, None diff --git a/setup.py b/setup.py index 75b060efcf..2c31d3c14a 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def unique_flatten_dict(d): ] stubs = [ - 'pandas-stubs', 'types-requests', 'ipython', 'types-tqdm' + 'pandas-stubs', 'types-requests', 'ipython', 'types-tqdm', 'types-python-dateutil' ] test_workarounds = []