diff --git a/demos/gfql/GPU_memory_consumption_tutorial.ipynb b/demos/gfql/GPU_memory_consumption_tutorial.ipynb index c79fe460f6..b3312345ef 100644 --- a/demos/gfql/GPU_memory_consumption_tutorial.ipynb +++ b/demos/gfql/GPU_memory_consumption_tutorial.ipynb @@ -723,7 +723,7 @@ "# Step 4: Profile the GFQL 2-hop Traversal\n", "g1 = graphistry.edges(gdf, 'src_ip', 'dst_ip') # Example edge specification for Graphistry\n", "with rmm.statistics.profiler(name=\"GFQL 2-hop Traversal\"):\n", - " g2 = g1.chain([\n", + " g2 = g1.gfql([\n", " n(),\n", " e(edge_match={'event_type': 'data_transfer'},\n", " edge_query=\"bytes_transferred > 500\"),\n", @@ -905,4 +905,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file diff --git a/demos/gfql/benchmark_hops_cpu_gpu.ipynb b/demos/gfql/benchmark_hops_cpu_gpu.ipynb index cafd90815a..a4b2d31f2d 100644 --- a/demos/gfql/benchmark_hops_cpu_gpu.ipynb +++ b/demos/gfql/benchmark_hops_cpu_gpu.ipynb @@ -1,4820 +1,4820 @@ { - "cells": [ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "GZxoiU8sQDk_" + }, + "source": [ + "# GFQL CPU, GPU Benchmark\n", + "\n", + "This notebook examines GFQL progerty graph query performance on 1-8 hop queries using CPU + GPU modes on various real-world 100K - 100M edge graphs. The data comes from a variety of popular social networks. The single-threaded CPU mode benefits from GFQL's novel dataframe engine, and the GPU mode further adds single-GPU acceleration. Both the `chain()` and `hop()` methods are examined.\n", + "\n", + "The benchmark does not examine bigger-than-memory and distributed scenarios. The provided results here are from running on a free Google Colab T4 runtime, with a 2.2GHz Intel CPU (12 GB CPU RAM) and T4 Nvidia GPU (16 GB GPU RAM).\n", + "\n", + "## Data\n", + "From [SNAP](https://snap.stanford.edu/data/)\n", + "\n", + "| Network | Nodes | Edges |\n", + "|-------------|-----------|--------------|\n", + "| **Facebook**| 4,039 | 88,234 |\n", + "| **Twitter** | 81,306 | 2,420,766 |\n", + "| **GPlus** | 107,614 | 30,494,866 |\n", + "| **Orkut** | 3,072,441 | 117,185,082 |\n", + "\n", + "## Results\n", + "\n", + "Definitions:\n", + "\n", + "* GTEPS: Giga (billion) edges traversed per second\n", + "\n", + "* T edges / \\$: Estimated trillion edges traversed for 1\\$ USD based on observed GTEPS and a 3yr AWS reservation (as of 12/2023)\n", + "\n", + "Tasks:\n", + "\n", + "1. `chain()` - includes complex pre/post processing\n", + "\n", + " **Task**: `g.gfql([n({'id': some_id}), e_forward(hops=some_n)])`\n", + "\n", + "\n", + "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n", + "|-------------|--------------|-------------|-------------|----------------------------|--------------------------------|\n", + "| **Facebook**| 1.1X | 0.66 | 0.61 | 65.7 | 10.4 |\n", + "| **Twitter** | 17.4X | 0.17 | 2.81 | 16.7 | 48.1 |\n", + "| **GPlus** | 43.8X | 0.09 | 2.87 | 8.5 | 49.2 |\n", + "| **Orkut** | N/A | N/A | 12.15 | N/A | 208.3 |\n", + "| **AVG** | 20.7X | 0.30 | 4.61 | 30.3 | 79.0\n", + "| **MAX** | 43.8X | 0.66 | 12.15 | 65.7 | 208.3\n", + "\n", + "\n", + "2. `hop()` - core property search primitive similar to BFS\n", + "\n", + " **Task**: `g.hop(nodes=[some_id], direction='forward', hops=some_n)`\n", + "\n", + "\n", + "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n", + "|-------------|-------------|-----------|-----------|--------------------|--------------------------------|\n", + "| **Facebook**| 3X | 0.47 | 1.47 | 47.0 | 25.2 |\n", + "| **Twitter** | 42X | 0.50 | 10.51 | 50.2 | 180.2 |\n", + "| **GPlus** | 21X | 0.26 | 4.11 | 26.2 | 70.4 |\n", + "| **Orkut** | N/A | N/A | 41.50 | N/A | 711.4 |\n", + "| **AVG** | 22X | 0.41 | 14.4 | 41.1 | 246.8\n", + "| **MAX** | 42X | 0.50 | 41.50 | 50.2 | 711.4\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SAj8lhREEOwS" + }, + "source": [ + "## Optional: GPU setup - Google Colab" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4hrEEAAm7DTO" + }, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "W2MF6ZsjDv3B", + "outputId": "46088cbc-2db9-4529-f724-dc57ed85dfb7" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "GZxoiU8sQDk_" - }, - "source": [ - "# GFQL CPU, GPU Benchmark\n", - "\n", - "This notebook examines GFQL progerty graph query performance on 1-8 hop queries using CPU + GPU modes on various real-world 100K - 100M edge graphs. The data comes from a variety of popular social networks. The single-threaded CPU mode benefits from GFQL's novel dataframe engine, and the GPU mode further adds single-GPU acceleration. Both the `chain()` and `hop()` methods are examined.\n", - "\n", - "The benchmark does not examine bigger-than-memory and distributed scenarios. The provided results here are from running on a free Google Colab T4 runtime, with a 2.2GHz Intel CPU (12 GB CPU RAM) and T4 Nvidia GPU (16 GB GPU RAM).\n", - "\n", - "## Data\n", - "From [SNAP](https://snap.stanford.edu/data/)\n", - "\n", - "| Network | Nodes | Edges |\n", - "|-------------|-----------|--------------|\n", - "| **Facebook**| 4,039 | 88,234 |\n", - "| **Twitter** | 81,306 | 2,420,766 |\n", - "| **GPlus** | 107,614 | 30,494,866 |\n", - "| **Orkut** | 3,072,441 | 117,185,082 |\n", - "\n", - "## Results\n", - "\n", - "Definitions:\n", - "\n", - "* GTEPS: Giga (billion) edges traversed per second\n", - "\n", - "* T edges / \\$: Estimated trillion edges traversed for 1\\$ USD based on observed GTEPS and a 3yr AWS reservation (as of 12/2023)\n", - "\n", - "Tasks:\n", - "\n", - "1. `chain()` - includes complex pre/post processing\n", - "\n", - " **Task**: `g.chain([n({'id': some_id}), e_forward(hops=some_n)])`\n", - "\n", - "\n", - "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n", - "|-------------|--------------|-------------|-------------|----------------------------|--------------------------------|\n", - "| **Facebook**| 1.1X | 0.66 | 0.61 | 65.7 | 10.4 |\n", - "| **Twitter** | 17.4X | 0.17 | 2.81 | 16.7 | 48.1 |\n", - "| **GPlus** | 43.8X | 0.09 | 2.87 | 8.5 | 49.2 |\n", - "| **Orkut** | N/A | N/A | 12.15 | N/A | 208.3 |\n", - "| **AVG** | 20.7X | 0.30 | 4.61 | 30.3 | 79.0\n", - "| **MAX** | 43.8X | 0.66 | 12.15 | 65.7 | 208.3\n", - "\n", - "\n", - "2. `hop()` - core property search primitive similar to BFS\n", - "\n", - " **Task**: `g.hop(nodes=[some_id], direction='forward', hops=some_n)`\n", - "\n", - "\n", - "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n", - "|-------------|-------------|-----------|-----------|--------------------|--------------------------------|\n", - "| **Facebook**| 3X | 0.47 | 1.47 | 47.0 | 25.2 |\n", - "| **Twitter** | 42X | 0.50 | 10.51 | 50.2 | 180.2 |\n", - "| **GPlus** | 21X | 0.26 | 4.11 | 26.2 | 70.4 |\n", - "| **Orkut** | N/A | N/A | 41.50 | N/A | 711.4 |\n", - "| **AVG** | 22X | 0.41 | 14.4 | 41.1 | 246.8\n", - "| **MAX** | 42X | 0.50 | 41.50 | 50.2 | 711.4\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:50:30 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 54C P8 10W / 70W | 0MiB / 15360MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "# Report GPU used when GPU benchmarking\n", + "# ! nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "Aikh0x4ID_wK" + }, + "outputs": [], + "source": [ + "# if in google colab\n", + "#!git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n", + "#!python rapidsai-csp-utils/colab/pip-install.py" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 }, + "id": "Lwekdei1dH3N", + "outputId": "71f5b01d-7917-4283-8338-969167d6e1e8" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "SAj8lhREEOwS" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" }, - "source": [ - "## Optional: GPU setup - Google Colab" + "text/plain": [ + "'23.12.01'" ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import cudf\n", + "cudf.__version__" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QQpsrtwBT7sa" + }, + "source": [ + "## 1. Install & configure" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "cYjRbgkU9Sx8", + "outputId": "2cf25531-9b8b-4715-ccc7-e79094d84ebd" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "4hrEEAAm7DTO" - }, - "source": [] + "name": "stdout", + "output_type": "stream", + "text": [ + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n" + ] + } + ], + "source": [ + "#! pip install graphistry[igraph]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ff6Tt9DhkePl" + }, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 }, + "id": "S5_y0CbLkjft", + "outputId": "a68a9c4b-c9c5-4b8b-ea4f-7bf1e4ddf315" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "W2MF6ZsjDv3B", - "outputId": "46088cbc-2db9-4529-f724-dc57ed85dfb7" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:50:30 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 54C P8 10W / 70W | 0MiB / 15360MiB | 0% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "| No running processes found |\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "# Report GPU used when GPU benchmarking\n", - "# ! nvidia-smi" + "text/plain": [ + "'0.32.0+12.g72e778c'" ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "import graphistry\n", + "\n", + "from graphistry import (\n", + "\n", + " # graph operators\n", + " n, e_undirected, e_forward, e_reverse,\n", + "\n", + " # attribute predicates\n", + " is_in, ge, startswith, contains, match as match_re\n", + ")\n", + "graphistry.__version__" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "I7Fg75jsG4co" + }, + "outputs": [], + "source": [ + "import cudf" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "uLZKph2-a5M4" + }, + "outputs": [], + "source": [ + "#work around google colab shell encoding bugs\n", + "\n", + "import locale\n", + "locale.getpreferredencoding = lambda: \"UTF-8\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eU9SyauNUHtR" + }, + "source": [ + "## 2. Perf benchmarks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NA0Ym11fkB8j" + }, + "source": [ + "### Facebook: 88K edges" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 224 }, + "id": "vXuQogHekClJ", + "outputId": "64db92c0-2704-438b-d0e4-25865acbb5e9" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "Aikh0x4ID_wK" - }, - "outputs": [], - "source": [ - "# if in google colab\n", - "#!git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n", - "#!python rapidsai-csp-utils/colab/pip-install.py" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(88234, 2)\n" + ] }, { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 35 - }, - "id": "Lwekdei1dH3N", - "outputId": "71f5b01d-7917-4283-8338-969167d6e1e8" - }, - "outputs": [ - { - "data": { - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "string" - }, - "text/plain": [ - "'23.12.01'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sd
001
102
203
304
405
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], - "source": [ - "import cudf\n", - "cudf.__version__" + "text/plain": [ + " s d\n", + "0 0 1\n", + "1 0 2\n", + "2 0 3\n", + "3 0 4\n", + "4 0 5" ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv('https://raw.githubusercontent.com/graphistry/pygraphistry/master/demos/data/facebook_combined.txt', sep=' ', names=['s', 'd'])\n", + "print(df.shape)\n", + "df.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 224 }, + "id": "jEma7hvvkzkN", + "outputId": "dbf21342-6b80-429c-bd3f-b1494c6854c7" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "QQpsrtwBT7sa" - }, - "source": [ - "## 1. Install & configure" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(4039, 1) (88234, 2)\n" + ] }, { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cYjRbgkU9Sx8", - "outputId": "2cf25531-9b8b-4715-ccc7-e79094d84ebd" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n" - ] - } + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
id
00
11
22
33
44
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], - "source": [ - "#! pip install graphistry[igraph]" + "text/plain": [ + " id\n", + "0 0\n", + "1 1\n", + "2 2\n", + "3 3\n", + "4 4" ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fg = graphistry.edges(df, 's', 'd').materialize_nodes()\n", + "print(fg._nodes.shape, fg._edges.shape)\n", + "fg._nodes.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "5lEdCBw9lzd7", + "outputId": "ed7451e0-401e-4edc-c8de-79c5afd0c95b" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "Ff6Tt9DhkePl" - }, - "source": [ - "### Imports" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 13.6 s, sys: 2.08 s, total: 15.7 s\n", + "Wall time: 18 s\n" + ] + } + ], + "source": [ + "%%time\n", + "for i in range(100):\n", + " fg2 = fg.gfql([n({'id': 0}), e_forward(hops=2)])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "JFKIBa8mJCvJ", + "outputId": "c22022f0-b33d-483a-db64-29992c5161e8" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 35 - }, - "id": "S5_y0CbLkjft", - "outputId": "a68a9c4b-c9c5-4b8b-ea4f-7bf1e4ddf315" - }, - "outputs": [ - { - "data": { - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "string" - }, - "text/plain": [ - "'0.32.0+12.g72e778c'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "import graphistry\n", - "\n", - "from graphistry import (\n", - "\n", - " # graph operators\n", - " n, e_undirected, e_forward, e_reverse,\n", - "\n", - " # attribute predicates\n", - " is_in, ge, startswith, contains, match as match_re\n", - ")\n", - "graphistry.__version__" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(4039, 1) (88234, 2)\n", + "(1519, 1) (4060, 2)\n", + "CPU times: user 11.8 s, sys: 28.1 ms, total: 11.8 s\n", + "Wall time: 11.9 s\n" + ] + } + ], + "source": [ + "%%time\n", + "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "for i in range(100):\n", + " fg2 = fg_gdf.gfql([n({'id': 0}), e_forward(hops=2)])\n", + "print(fg._nodes.shape, fg._edges.shape)\n", + "print(fg2._nodes.shape, fg2._edges.shape)\n", + "del fg_gdf\n", + "del fg2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "-KBGLexek5tS", + "outputId": "2f462e6c-578a-4fa1-ec29-91bae753f4c5" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "I7Fg75jsG4co" - }, - "outputs": [], - "source": [ - "import cudf" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(4039, 1) (88234, 2)\n", + "(3829, 1) (86074, 2)\n", + "CPU times: user 15.4 s, sys: 808 ms, total: 16.2 s\n", + "Wall time: 16.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "for i in range(50):\n", + " fg2 = fg.gfql([n({'id': 0}), e_forward(hops=5)])\n", + "print(fg._nodes.shape, fg._edges.shape)\n", + "print(fg2._nodes.shape, fg2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "CVpcbhpdHFEF", + "outputId": "aba04ee1-781e-4226-b593-b42415a55fc4" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "uLZKph2-a5M4" - }, - "outputs": [], - "source": [ - "#work around google colab shell encoding bugs\n", - "\n", - "import locale\n", - "locale.getpreferredencoding = lambda: \"UTF-8\"" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(4039, 1) (88234, 2)\n", + "(3829, 1) (86074, 2)\n", + "CPU times: user 9.82 s, sys: 133 ms, total: 9.95 s\n", + "Wall time: 10.1 s\n" + ] + } + ], + "source": [ + "%%time\n", + "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "for i in range(50):\n", + " fg2 = fg_gdf.gfql([n({'id': 0}), e_forward(hops=5)])\n", + "print(fg._nodes.shape, fg._edges.shape)\n", + "print(fg2._nodes.shape, fg2._edges.shape)\n", + "del fg_gdf\n", + "del fg2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "1cFIyJF9pLjE", + "outputId": "107329af-8e4b-428c-8b03-77ed00bdf5bf" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "eU9SyauNUHtR" - }, - "source": [ - "## 2. Perf benchmarks" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 11.8 s, sys: 377 ms, total: 12.1 s\n", + "Wall time: 13.1 s\n" + ] + } + ], + "source": [ + "%%time\n", + "for i in range(100):\n", + " fg2 = fg.gfql([e_forward(source_node_match={'id': 0}, hops=5)])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "M5uRiD6uJVNW", + "outputId": "5e938a19-2992-4280-80c2-784382d40113" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "NA0Ym11fkB8j" - }, - "source": [ - "### Facebook: 88K edges" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(4039, 1) (88234, 2)\n", + "(348, 1) (347, 2)\n", + "CPU times: user 14.1 s, sys: 48.5 ms, total: 14.2 s\n", + "Wall time: 14.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "for i in range(100):\n", + " fg2 = fg_gdf.gfql([e_forward(source_node_match={'id': 0}, hops=5)])\n", + "print(fg._nodes.shape, fg._edges.shape)\n", + "print(fg2._nodes.shape, fg2._edges.shape)\n", + "del fg_gdf\n", + "del fg2\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Y9vgzfT69x41", + "outputId": "6882c1ce-0df8-4087-dda4-0a105a8617e1" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 224 - }, - "id": "vXuQogHekClJ", - "outputId": "64db92c0-2704-438b-d0e4-25865acbb5e9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(88234, 2)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sd
001
102
203
304
405
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " s d\n", - "0 0 1\n", - "1 0 2\n", - "2 0 3\n", - "3 0 4\n", - "4 0 5" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.read_csv('https://raw.githubusercontent.com/graphistry/pygraphistry/master/demos/data/facebook_combined.txt', sep=' ', names=['s', 'd'])\n", - "print(df.shape)\n", - "df.head(5)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(1519, 1) (4060, 2)\n", + "CPU times: user 4.5 s, sys: 1.35 s, total: 5.85 s\n", + "Wall time: 6.09 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({fg._node: [0]})\n", + "for i in range(100):\n", + " fg2 = fg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "print(fg2._nodes.shape, fg2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "c7ybJqjc-T31", + "outputId": "37ccc1fb-6460-4193-8aa7-22837ff06d0a" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 224 - }, - "id": "jEma7hvvkzkN", - "outputId": "dbf21342-6b80-429c-bd3f-b1494c6854c7" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(4039, 1) (88234, 2)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
id
00
11
22
33
44
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " id\n", - "0 0\n", - "1 1\n", - "2 2\n", - "3 3\n", - "4 4" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fg = graphistry.edges(df, 's', 'd').materialize_nodes()\n", - "print(fg._nodes.shape, fg._edges.shape)\n", - "fg._nodes.head(5)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(1519, 1) (4060, 2)\n", + "CPU times: user 2.58 s, sys: 6.75 ms, total: 2.59 s\n", + "Wall time: 2.58 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({fg._node: [0]})\n", + "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n", + "for i in range(100):\n", + " fg2 = fg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "print(fg2._nodes.shape, fg2._edges.shape)\n", + "del start_nodes\n", + "del fg_gdf\n", + "del fg2" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Dy7a4zDZ-7_G", + "outputId": "077b5d9c-c9ae-411a-8228-3c026b07a910" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5lEdCBw9lzd7", - "outputId": "ed7451e0-401e-4edc-c8de-79c5afd0c95b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.6 s, sys: 2.08 s, total: 15.7 s\n", - "Wall time: 18 s\n" - ] - } - ], - "source": [ - "%%time\n", - "for i in range(100):\n", - " fg2 = fg.chain([n({'id': 0}), e_forward(hops=2)])" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(3829, 1) (86074, 2)\n", + "CPU times: user 13.2 s, sys: 2 s, total: 15.2 s\n", + "Wall time: 18.3 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({fg._node: [0]})\n", + "for i in range(100):\n", + " fg2 = fg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=5)\n", + "print(fg2._nodes.shape, fg2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "N5aUtF1a--ML", + "outputId": "0c2b67b8-fac6-45b3-dfbe-8002b5506e91" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "JFKIBa8mJCvJ", - "outputId": "c22022f0-b33d-483a-db64-29992c5161e8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(4039, 1) (88234, 2)\n", - "(1519, 1) (4060, 2)\n", - "CPU times: user 11.8 s, sys: 28.1 ms, total: 11.8 s\n", - "Wall time: 11.9 s\n" - ] - } - ], - "source": [ - "%%time\n", - "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "for i in range(100):\n", - " fg2 = fg_gdf.chain([n({'id': 0}), e_forward(hops=2)])\n", - "print(fg._nodes.shape, fg._edges.shape)\n", - "print(fg2._nodes.shape, fg2._edges.shape)\n", - "del fg_gdf\n", - "del fg2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(3829, 1) (86074, 2)\n", + "CPU times: user 5.72 s, sys: 159 ms, total: 5.88 s\n", + "Wall time: 5.86 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({fg._node: [0]})\n", + "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n", + "for i in range(100):\n", + " fg2 = fg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=5)\n", + "print(fg2._nodes.shape, fg2._edges.shape)\n", + "del start_nodes\n", + "del fg_gdf\n", + "del fg2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KrJKjXy2KLos" + }, + "source": [ + "## Twitter\n", + "\n", + "- edges: 2420766\n", + "- nodes: 81306" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "fO2qasGqpubr", + "outputId": "d41a110e-9f7c-4710-9ce3-3f4906ab02ae" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-KBGLexek5tS", - "outputId": "2f462e6c-578a-4fa1-ec29-91bae753f4c5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(4039, 1) (88234, 2)\n", - "(3829, 1) (86074, 2)\n", - "CPU times: user 15.4 s, sys: 808 ms, total: 16.2 s\n", - "Wall time: 16.2 s\n" - ] - } - ], - "source": [ - "%%time\n", - "for i in range(50):\n", - " fg2 = fg.chain([n({'id': 0}), e_forward(hops=5)])\n", - "print(fg._nodes.shape, fg._edges.shape)\n", - "print(fg2._nodes.shape, fg2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "--2023-12-25 21:58:27-- https://snap.stanford.edu/data/twitter_combined.txt.gz\n", + "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n", + "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 10621918 (10M) [application/x-gzip]\n", + "Saving to: ‘twitter_combined.txt.gz’\n", + "\n", + "twitter_combined.tx 100%[===================>] 10.13M 3.00MB/s in 4.0s \n", + "\n", + "2023-12-25 21:58:32 (2.52 MB/s) - ‘twitter_combined.txt.gz’ saved [10621918/10621918]\n", + "\n" + ] + } + ], + "source": [ + "#! wget 'https://snap.stanford.edu/data/twitter_combined.txt.gz'" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "id": "fn7zeA3SGlEo" + }, + "outputs": [], + "source": [ + "#! gunzip twitter_combined.txt.gz" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "68TAZkhLGz9g", + "outputId": "8ba7c23d-267f-4b59-d6c6-b3f66caec9cf" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "CVpcbhpdHFEF", - "outputId": "aba04ee1-781e-4226-b593-b42415a55fc4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(4039, 1) (88234, 2)\n", - "(3829, 1) (86074, 2)\n", - "CPU times: user 9.82 s, sys: 133 ms, total: 9.95 s\n", - "Wall time: 10.1 s\n" - ] - } - ], - "source": [ - "%%time\n", - "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "for i in range(50):\n", - " fg2 = fg_gdf.chain([n({'id': 0}), e_forward(hops=5)])\n", - "print(fg._nodes.shape, fg._edges.shape)\n", - "print(fg2._nodes.shape, fg2._edges.shape)\n", - "del fg_gdf\n", - "del fg2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "214328887 34428380\n", + "17116707 28465635\n", + "380580781 18996905\n", + "221036078 153460275\n", + "107830991 17868918\n" + ] + } + ], + "source": [ + "#! head -n 5 twitter_combined.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "QU2wNeGXG2GC", + "outputId": "349ac9c0-6f6c-4ce6-fec0-8bae75fca635" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "1cFIyJF9pLjE", - "outputId": "107329af-8e4b-428c-8b03-77ed00bdf5bf" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 11.8 s, sys: 377 ms, total: 12.1 s\n", - "Wall time: 13.1 s\n" - ] - } - ], - "source": [ - "%%time\n", - "for i in range(100):\n", - " fg2 = fg.chain([e_forward(source_node_match={'id': 0}, hops=5)])" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 474 ms, sys: 61.9 ms, total: 536 ms\n", + "Wall time: 534 ms\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "M5uRiD6uJVNW", - "outputId": "5e938a19-2992-4280-80c2-784382d40113" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(4039, 1) (88234, 2)\n", - "(348, 1) (347, 2)\n", - "CPU times: user 14.1 s, sys: 48.5 ms, total: 14.2 s\n", - "Wall time: 14.2 s\n" - ] - } - ], - "source": [ - "%%time\n", - "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "for i in range(100):\n", - " fg2 = fg_gdf.chain([e_forward(source_node_match={'id': 0}, hops=5)])\n", - "print(fg._nodes.shape, fg._edges.shape)\n", - "print(fg2._nodes.shape, fg2._edges.shape)\n", - "del fg_gdf\n", - "del fg2\n", - "\n" + "data": { + "text/plain": [ + "(2420766, 2)" ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "te_df = pd.read_csv('twitter_combined.txt', sep=' ', names=['s', 'd'])\n", + "te_df.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "id": "EK5gQH2iG5UU" + }, + "outputs": [], + "source": [ + "import graphistry" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ZtIW-eFGG_R4", + "outputId": "0686e9b3-b684-4b93-da03-289244394338" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Y9vgzfT69x41", - "outputId": "6882c1ce-0df8-4087-dda4-0a105a8617e1" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1519, 1) (4060, 2)\n", - "CPU times: user 4.5 s, sys: 1.35 s, total: 5.85 s\n", - "Wall time: 6.09 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({fg._node: [0]})\n", - "for i in range(100):\n", - " fg2 = fg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "print(fg2._nodes.shape, fg2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 86.4 ms, sys: 106 ms, total: 193 ms\n", + "Wall time: 191 ms\n" + ] }, { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "c7ybJqjc-T31", - "outputId": "37ccc1fb-6460-4193-8aa7-22837ff06d0a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1519, 1) (4060, 2)\n", - "CPU times: user 2.58 s, sys: 6.75 ms, total: 2.59 s\n", - "Wall time: 2.58 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({fg._node: [0]})\n", - "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n", - "for i in range(100):\n", - " fg2 = fg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "print(fg2._nodes.shape, fg2._edges.shape)\n", - "del start_nodes\n", - "del fg_gdf\n", - "del fg2" + "data": { + "text/plain": [ + "(81306, 1)" ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "g = graphistry.edges(te_df, 's', 'd').materialize_nodes()\n", + "g._nodes.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "yUaRfw4FHGMb", + "outputId": "3945cc5a-c36c-451b-ac95-8af992a3546f" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Dy7a4zDZ-7_G", - "outputId": "077b5d9c-c9ae-411a-8228-3c026b07a910" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3829, 1) (86074, 2)\n", - "CPU times: user 13.2 s, sys: 2 s, total: 15.2 s\n", - "Wall time: 18.3 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({fg._node: [0]})\n", - "for i in range(100):\n", - " fg2 = fg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=5)\n", - "print(fg2._nodes.shape, fg2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 11.8 s, sys: 8.4 s, total: 20.2 s\n", + "Wall time: 23 s\n" + ] }, { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "N5aUtF1a--ML", - "outputId": "0c2b67b8-fac6-45b3-dfbe-8002b5506e91" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3829, 1) (86074, 2)\n", - "CPU times: user 5.72 s, sys: 159 ms, total: 5.88 s\n", - "Wall time: 5.86 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({fg._node: [0]})\n", - "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n", - "for i in range(100):\n", - " fg2 = fg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=5)\n", - "print(fg2._nodes.shape, fg2._edges.shape)\n", - "del start_nodes\n", - "del fg_gdf\n", - "del fg2" + "data": { + "text/plain": [ + "((140, 1), (615, 2))" ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "for i in range(10):\n", + " g2 = g.gfql([n({'id': 17116707}), e_forward(hops=1)])\n", + "g2._nodes.shape, g2._edges.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "5hM4NBu2_eks", + "outputId": "54505262-4871-44ee-e5e4-ad7ab32c13c2" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "KrJKjXy2KLos" - }, - "source": [ - "## Twitter\n", - "\n", - "- edges: 2420766\n", - "- nodes: 81306" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(140, 1)\n", + "CPU times: user 1.33 s, sys: 46.6 ms, total: 1.38 s\n", + "Wall time: 1.63 s\n" + ] + } + ], + "source": [ + "%%time\n", + "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "for i in range(10):\n", + " out = g_gdf.gfql([n({'id': 17116707}), e_forward(hops=1)])._nodes\n", + "print(out.shape)\n", + "del g_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "m2-MxD5lHX6u", + "outputId": "e89b9d4b-6c04-45c7-9e7f-cbdbbe0a4730" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "fO2qasGqpubr", - "outputId": "d41a110e-9f7c-4710-9ce3-3f4906ab02ae" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2023-12-25 21:58:27-- https://snap.stanford.edu/data/twitter_combined.txt.gz\n", - "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n", - "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 10621918 (10M) [application/x-gzip]\n", - "Saving to: ‘twitter_combined.txt.gz’\n", - "\n", - "twitter_combined.tx 100%[===================>] 10.13M 3.00MB/s in 4.0s \n", - "\n", - "2023-12-25 21:58:32 (2.52 MB/s) - ‘twitter_combined.txt.gz’ saved [10621918/10621918]\n", - "\n" - ] - } - ], - "source": [ - "#! wget 'https://snap.stanford.edu/data/twitter_combined.txt.gz'" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(2345, 1) (68536, 2)\n", + "CPU times: user 13.3 s, sys: 8.05 s, total: 21.4 s\n", + "Wall time: 21.6 s\n" + ] + } + ], + "source": [ + "%%time\n", + "for i in range(10):\n", + " out = g.gfql([n({'id': 17116707}), e_forward(hops=2)])\n", + "print(out._nodes.shape, out._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "7EQSRbIqLaGw", + "outputId": "60c00a03-9e7b-46b5-fce3-f4f567a09430" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "id": "fn7zeA3SGlEo" - }, - "outputs": [], - "source": [ - "#! gunzip twitter_combined.txt.gz" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(2345, 1)\n", + "CPU times: user 1.67 s, sys: 55.8 ms, total: 1.72 s\n", + "Wall time: 1.75 s\n" + ] + } + ], + "source": [ + "%%time\n", + "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "for i in range(10):\n", + " out = g_gdf.gfql([n({'id': 17116707}), e_forward(hops=2)])._nodes\n", + "print(out.shape)\n", + "del g_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "hh6WnjI3ITpB", + "outputId": "33138efe-a581-49ed-b2b4-247f8e9bdc09" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "68TAZkhLGz9g", - "outputId": "8ba7c23d-267f-4b59-d6c6-b3f66caec9cf" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "214328887 34428380\n", - "17116707 28465635\n", - "380580781 18996905\n", - "221036078 153460275\n", - "107830991 17868918\n" - ] - } - ], - "source": [ - "#! head -n 5 twitter_combined.txt" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(81304, 1) (2417796, 2)\n", + "CPU times: user 1min 56s, sys: 17.1 s, total: 2min 13s\n", + "Wall time: 2min 22s\n" + ] + } + ], + "source": [ + "%%time\n", + "for i in range(10):\n", + " out = g.gfql([n({'id': 17116707}), e_forward(hops=8)])\n", + "print(out._nodes.shape, out._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "7jFFVUenM87j", + "outputId": "2cceb720-9de3-488e-8b74-b820fd06e6c1" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QU2wNeGXG2GC", - "outputId": "349ac9c0-6f6c-4ce6-fec0-8bae75fca635" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 474 ms, sys: 61.9 ms, total: 536 ms\n", - "Wall time: 534 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "(2420766, 2)" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "te_df = pd.read_csv('twitter_combined.txt', sep=' ', names=['s', 'd'])\n", - "te_df.shape" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(81304, 1)\n", + "CPU times: user 5.3 s, sys: 1.48 s, total: 6.78 s\n", + "Wall time: 7.89 s\n" + ] + } + ], + "source": [ + "%%time\n", + "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "for i in range(10):\n", + " out = g_gdf.gfql([n({'id': 17116707}), e_forward(hops=8)])._nodes\n", + "print(out.shape)\n", + "del g_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "_5LD0bZB_lU4", + "outputId": "bc31bd03-e79f-46d2-ea8f-3b01d9ef39a2" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "id": "EK5gQH2iG5UU" - }, - "outputs": [], - "source": [ - "import graphistry" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 1) (0, 2)\n", + "CPU times: user 2.58 s, sys: 1.59 s, total: 4.17 s\n", + "Wall time: 6.02 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({g._node: [17116707]})\n", + "for i in range(10):\n", + " g2 = g.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=1)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "M_rHjqtvACQw", + "outputId": "8d3e308e-b1e2-452b-f402-573be0dd5b58" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ZtIW-eFGG_R4", - "outputId": "0686e9b3-b684-4b93-da03-289244394338" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 86.4 ms, sys: 106 ms, total: 193 ms\n", - "Wall time: 191 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "(81306, 1)" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "g = graphistry.edges(te_df, 's', 'd').materialize_nodes()\n", - "g._nodes.shape" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(61827, 1) (1473599, 2)\n", + "CPU times: user 822 ms, sys: 179 ms, total: 1 s\n", + "Wall time: 997 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({g._node: [17116707]})\n", + "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n", + "for i in range(10):\n", + " g2 = g_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=5)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del g_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "0zEIucaCAbj_", + "outputId": "83e64b0f-2b3a-4e4b-d189-3e6a8ef78f53" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "yUaRfw4FHGMb", - "outputId": "3945cc5a-c36c-451b-ac95-8af992a3546f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 11.8 s, sys: 8.4 s, total: 20.2 s\n", - "Wall time: 23 s\n" - ] - }, - { - "data": { - "text/plain": [ - "((140, 1), (615, 2))" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "for i in range(10):\n", - " g2 = g.chain([n({'id': 17116707}), e_forward(hops=1)])\n", - "g2._nodes.shape, g2._edges.shape" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(2345, 1) (68536, 2)\n", + "CPU times: user 8.93 s, sys: 5.92 s, total: 14.9 s\n", + "Wall time: 15.8 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({g._node: [17116707]})\n", + "for i in range(10):\n", + " g2 = g.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "LKJh5gRtAdIj", + "outputId": "e3c7883d-74c0-4d55-b238-88457296c6bc" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5hM4NBu2_eks", - "outputId": "54505262-4871-44ee-e5e4-ad7ab32c13c2" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(140, 1)\n", - "CPU times: user 1.33 s, sys: 46.6 ms, total: 1.38 s\n", - "Wall time: 1.63 s\n" - ] - } - ], - "source": [ - "%%time\n", - "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "for i in range(10):\n", - " out = g_gdf.chain([n({'id': 17116707}), e_forward(hops=1)])._nodes\n", - "print(out.shape)\n", - "del g_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(2345, 1) (68536, 2)\n", + "CPU times: user 374 ms, sys: 6.92 ms, total: 381 ms\n", + "Wall time: 379 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({g._node: [17116707]})\n", + "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n", + "for i in range(10):\n", + " g2 = g_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del g_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "JZwxdofNAfmb", + "outputId": "2731be4c-75d9-47f4-8602-4f2d6cb2ddac" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "m2-MxD5lHX6u", - "outputId": "e89b9d4b-6c04-45c7-9e7f-cbdbbe0a4730" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(2345, 1) (68536, 2)\n", - "CPU times: user 13.3 s, sys: 8.05 s, total: 21.4 s\n", - "Wall time: 21.6 s\n" - ] - } - ], - "source": [ - "%%time\n", - "for i in range(10):\n", - " out = g.chain([n({'id': 17116707}), e_forward(hops=2)])\n", - "print(out._nodes.shape, out._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(81304, 1) (2417796, 2)\n", + "CPU times: user 38.8 s, sys: 8.7 s, total: 47.5 s\n", + "Wall time: 48.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({g._node: [17116707]})\n", + "for i in range(10):\n", + " g2 = g.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=8)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "9o_og8bSAhe3", + "outputId": "dd3e4f8f-f426-4705-98c4-60f1912ba28a" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 36, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "7EQSRbIqLaGw", - "outputId": "60c00a03-9e7b-46b5-fce3-f4f567a09430" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(2345, 1)\n", - "CPU times: user 1.67 s, sys: 55.8 ms, total: 1.72 s\n", - "Wall time: 1.75 s\n" - ] - } - ], - "source": [ - "%%time\n", - "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "for i in range(10):\n", - " out = g_gdf.chain([n({'id': 17116707}), e_forward(hops=2)])._nodes\n", - "print(out.shape)\n", - "del g_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(81304, 1) (2417796, 2)\n", + "CPU times: user 1.8 s, sys: 506 ms, total: 2.3 s\n", + "Wall time: 2.3 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({g._node: [17116707]})\n", + "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n", + "for i in range(10):\n", + " g2 = g_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=8)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del g_gdf\n", + "del g2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9dZzAAVONCD2" + }, + "source": [ + "## GPlus\n", + "\n", + "- edges: 30494866\n", + "- nodes: 107614" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "-nhWGNekKpcZ", + "outputId": "e2175290-337c-4faa-e5d8-4bc401583326" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 37, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "hh6WnjI3ITpB", - "outputId": "33138efe-a581-49ed-b2b4-247f8e9bdc09" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(81304, 1) (2417796, 2)\n", - "CPU times: user 1min 56s, sys: 17.1 s, total: 2min 13s\n", - "Wall time: 2min 22s\n" - ] - } - ], - "source": [ - "%%time\n", - "for i in range(10):\n", - " out = g.chain([n({'id': 17116707}), e_forward(hops=8)])\n", - "print(out._nodes.shape, out._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "--2023-12-26 18:36:29-- https://snap.stanford.edu/data/gplus_combined.txt.gz\n", + "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n", + "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 398930514 (380M) [application/x-gzip]\n", + "Saving to: ‘gplus_combined.txt.gz’\n", + "\n", + "gplus_combined.txt. 100%[===================>] 380.45M 34.7MB/s in 9.6s \n", + "\n", + "2023-12-26 18:36:39 (39.7 MB/s) - ‘gplus_combined.txt.gz’ saved [398930514/398930514]\n", + "\n" + ] + } + ], + "source": [ + "#! wget https://snap.stanford.edu/data/gplus_combined.txt.gz" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "g5wgA_c2KqwJ" + }, + "outputs": [], + "source": [ + "#! gunzip gplus_combined.txt.gz" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 }, + "id": "52hgDbr0Kti6", + "outputId": "217203fc-7095-4784-c4c4-d46ee9c78808" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 38, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "7jFFVUenM87j", - "outputId": "2cceb720-9de3-488e-8b74-b820fd06e6c1" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(81304, 1)\n", - "CPU times: user 5.3 s, sys: 1.48 s, total: 6.78 s\n", - "Wall time: 7.89 s\n" - ] - } - ], - "source": [ - "%%time\n", - "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "for i in range(10):\n", - " out = g_gdf.chain([n({'id': 17116707}), e_forward(hops=8)])._nodes\n", - "print(out.shape)\n", - "del g_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(30494866, 2)\n", + "CPU times: user 16 s, sys: 1.45 s, total: 17.5 s\n", + "Wall time: 22.5 s\n" + ] }, { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "_5LD0bZB_lU4", - "outputId": "bc31bd03-e79f-46d2-ea8f-3b01d9ef39a2" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(0, 1) (0, 2)\n", - "CPU times: user 2.58 s, sys: 1.59 s, total: 4.17 s\n", - "Wall time: 6.02 s\n" - ] - } + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sd
0116374117927631468606101765416973555767821
1112188647432305746617107727150903234299458
2116719211656774388392100432456209427807893
3117421021456205115327101096322838605097368
4116407635616074189669113556266482860931616
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({g._node: [17116707]})\n", - "for i in range(10):\n", - " g2 = g.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=1)\n", - "print(g2._nodes.shape, g2._edges.shape)" + "text/plain": [ + " s d\n", + "0 116374117927631468606 101765416973555767821\n", + "1 112188647432305746617 107727150903234299458\n", + "2 116719211656774388392 100432456209427807893\n", + "3 117421021456205115327 101096322838605097368\n", + "4 116407635616074189669 113556266482860931616" ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "ge_df = pd.read_csv('gplus_combined.txt', sep=' ', names=['s', 'd'])\n", + "print(ge_df.shape)\n", + "ge_df.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 }, + "id": "w5YkN-nLK6UV", + "outputId": "dc98380d-54c2-4b36-c56e-5e8401c4ffa4" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 44, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "M_rHjqtvACQw", - "outputId": "8d3e308e-b1e2-452b-f402-573be0dd5b58" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(61827, 1) (1473599, 2)\n", - "CPU times: user 822 ms, sys: 179 ms, total: 1 s\n", - "Wall time: 997 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({g._node: [17116707]})\n", - "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n", - "for i in range(10):\n", - " g2 = g_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=5)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del g_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(30494866, 2) (107614, 1)\n", + "CPU times: user 4.49 s, sys: 1.25 s, total: 5.74 s\n", + "Wall time: 5.97 s\n" + ] }, { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0zEIucaCAbj_", - "outputId": "83e64b0f-2b3a-4e4b-d189-3e6a8ef78f53" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(2345, 1) (68536, 2)\n", - "CPU times: user 8.93 s, sys: 5.92 s, total: 14.9 s\n", - "Wall time: 15.8 s\n" - ] - } + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
id
0116374117927631468606
1112188647432305746617
2116719211656774388392
3117421021456205115327
4116407635616074189669
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({g._node: [17116707]})\n", - "for i in range(10):\n", - " g2 = g.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "print(g2._nodes.shape, g2._edges.shape)" + "text/plain": [ + " id\n", + "0 116374117927631468606\n", + "1 112188647432305746617\n", + "2 116719211656774388392\n", + "3 117421021456205115327\n", + "4 116407635616074189669" ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "gg = graphistry.edges(ge_df, 's', 'd').materialize_nodes()\n", + "gg = graphistry.edges(ge_df, 's', 'd').nodes(gg._nodes, 'id')\n", + "print(gg._edges.shape, gg._nodes.shape)\n", + "gg._nodes.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 115 }, + "id": "NKtz54uELX-8", + "outputId": "5d8f3eef-893d-47cc-e7a9-c5cbfec8270c" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 41, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "LKJh5gRtAdIj", - "outputId": "e3c7883d-74c0-4d55-b238-88457296c6bc" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(2345, 1) (68536, 2)\n", - "CPU times: user 374 ms, sys: 6.92 ms, total: 381 ms\n", - "Wall time: 379 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({g._node: [17116707]})\n", - "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n", - "for i in range(10):\n", - " g2 = g_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del g_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 534 ms, sys: 598 ms, total: 1.13 s\n", + "Wall time: 1.65 s\n" + ] }, { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "JZwxdofNAfmb", - "outputId": "2731be4c-75d9-47f4-8602-4f2d6cb2ddac" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(81304, 1) (2417796, 2)\n", - "CPU times: user 38.8 s, sys: 8.7 s, total: 47.5 s\n", - "Wall time: 48.2 s\n" - ] - } + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
id
0116374117927631468606
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({g._node: [17116707]})\n", - "for i in range(10):\n", - " g2 = g.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=8)\n", - "print(g2._nodes.shape, g2._edges.shape)" + "text/plain": [ + " id\n", + "0 116374117927631468606" ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "gg.gfql([ n({'id': '116374117927631468606'})])._nodes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "iNWdi00VLmZG", + "outputId": "ecfb56a6-c564-4bf6-f43f-2c95a103f4be" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "9o_og8bSAhe3", - "outputId": "dd3e4f8f-f426-4705-98c4-60f1912ba28a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(81304, 1) (2417796, 2)\n", - "CPU times: user 1.8 s, sys: 506 ms, total: 2.3 s\n", - "Wall time: 2.3 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({g._node: [17116707]})\n", - "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n", - "for i in range(10):\n", - " g2 = g_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=8)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del g_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 27.5 s, sys: 11.1 s, total: 38.5 s\n", + "Wall time: 39.5 s\n" + ] }, { - "cell_type": "markdown", - "metadata": { - "id": "9dZzAAVONCD2" - }, - "source": [ - "## GPlus\n", - "\n", - "- edges: 30494866\n", - "- nodes: 107614" + "data": { + "text/plain": [ + "(1473, 1)" ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=1)])._nodes\n", + "out.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Q6p3h6uCOABh", + "outputId": "817fc80f-ef5d-4070-eb48-a12344be709c" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-nhWGNekKpcZ", - "outputId": "e2175290-337c-4faa-e5d8-4bc401583326" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2023-12-26 18:36:29-- https://snap.stanford.edu/data/gplus_combined.txt.gz\n", - "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n", - "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 398930514 (380M) [application/x-gzip]\n", - "Saving to: ‘gplus_combined.txt.gz’\n", - "\n", - "gplus_combined.txt. 100%[===================>] 380.45M 34.7MB/s in 9.6s \n", - "\n", - "2023-12-26 18:36:39 (39.7 MB/s) - ‘gplus_combined.txt.gz’ saved [398930514/398930514]\n", - "\n" - ] - } - ], - "source": [ - "#! wget https://snap.stanford.edu/data/gplus_combined.txt.gz" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(1473, 1) (13375, 2)\n", + "CPU times: user 4.57 s, sys: 2.11 s, total: 6.68 s\n", + "Wall time: 7.63 s\n" + ] + } + ], + "source": [ + "%%time\n", + "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=1)])\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del gg_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "6UdCcMdqLw-P", + "outputId": "70742c79-b22b-4db2-c548-cb1e25d572eb" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "g5wgA_c2KqwJ" - }, - "outputs": [], - "source": [ - "#! gunzip gplus_combined.txt.gz" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 45.8 s, sys: 17 s, total: 1min 2s\n", + "Wall time: 1min 5s\n" + ] }, { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 258 - }, - "id": "52hgDbr0Kti6", - "outputId": "217203fc-7095-4784-c4c4-d46ee9c78808" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(30494866, 2)\n", - "CPU times: user 16 s, sys: 1.45 s, total: 17.5 s\n", - "Wall time: 22.5 s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sd
0116374117927631468606101765416973555767821
1112188647432305746617107727150903234299458
2116719211656774388392100432456209427807893
3117421021456205115327101096322838605097368
4116407635616074189669113556266482860931616
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " s d\n", - "0 116374117927631468606 101765416973555767821\n", - "1 112188647432305746617 107727150903234299458\n", - "2 116719211656774388392 100432456209427807893\n", - "3 117421021456205115327 101096322838605097368\n", - "4 116407635616074189669 113556266482860931616" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "ge_df = pd.read_csv('gplus_combined.txt', sep=' ', names=['s', 'd'])\n", - "print(ge_df.shape)\n", - "ge_df.head(5)" + "data": { + "text/plain": [ + "(44073, 1)" ] + }, + "execution_count": 77, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=2)])._nodes\n", + "out.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "QElqatDyNYCS", + "outputId": "0e15bd3e-d2d9-4965-df7d-c8856d036680" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 258 - }, - "id": "w5YkN-nLK6UV", - "outputId": "dc98380d-54c2-4b36-c56e-5e8401c4ffa4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(30494866, 2) (107614, 1)\n", - "CPU times: user 4.49 s, sys: 1.25 s, total: 5.74 s\n", - "Wall time: 5.97 s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
id
0116374117927631468606
1112188647432305746617
2116719211656774388392
3117421021456205115327
4116407635616074189669
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " id\n", - "0 116374117927631468606\n", - "1 112188647432305746617\n", - "2 116719211656774388392\n", - "3 117421021456205115327\n", - "4 116407635616074189669" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "gg = graphistry.edges(ge_df, 's', 'd').materialize_nodes()\n", - "gg = graphistry.edges(ge_df, 's', 'd').nodes(gg._nodes, 'id')\n", - "print(gg._edges.shape, gg._nodes.shape)\n", - "gg._nodes.head(5)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(44073, 1) (2069325, 2)\n", + "CPU times: user 4.97 s, sys: 2.36 s, total: 7.34 s\n", + "Wall time: 10.6 s\n" + ] + } + ], + "source": [ + "%%time\n", + "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=2)])\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del gg_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "3HJOItZ4MQMG", + "outputId": "f5be7bb4-7f09-4f80-c549-e703e99f5067" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 115 - }, - "id": "NKtz54uELX-8", - "outputId": "5d8f3eef-893d-47cc-e7a9-c5cbfec8270c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 534 ms, sys: 598 ms, total: 1.13 s\n", - "Wall time: 1.65 s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
id
0116374117927631468606
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "
\n", - "
\n" - ], - "text/plain": [ - " id\n", - "0 116374117927631468606" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "gg.chain([ n({'id': '116374117927631468606'})])._nodes" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 3min 45s, sys: 1min 5s, total: 4min 50s\n", + "Wall time: 4min 52s\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "iNWdi00VLmZG", - "outputId": "ecfb56a6-c564-4bf6-f43f-2c95a103f4be" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 27.5 s, sys: 11.1 s, total: 38.5 s\n", - "Wall time: 39.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "(1473, 1)" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=1)])._nodes\n", - "out.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Q6p3h6uCOABh", - "outputId": "817fc80f-ef5d-4070-eb48-a12344be709c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1473, 1) (13375, 2)\n", - "CPU times: user 4.57 s, sys: 2.11 s, total: 6.68 s\n", - "Wall time: 7.63 s\n" - ] - } - ], - "source": [ - "%%time\n", - "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=1)])\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del gg_gdf\n", - "del out" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "6UdCcMdqLw-P", - "outputId": "70742c79-b22b-4db2-c548-cb1e25d572eb" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 45.8 s, sys: 17 s, total: 1min 2s\n", - "Wall time: 1min 5s\n" - ] - }, - { - "data": { - "text/plain": [ - "(44073, 1)" - ] - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=2)])._nodes\n", - "out.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QElqatDyNYCS", - "outputId": "0e15bd3e-d2d9-4965-df7d-c8856d036680" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(44073, 1) (2069325, 2)\n", - "CPU times: user 4.97 s, sys: 2.36 s, total: 7.34 s\n", - "Wall time: 10.6 s\n" - ] - } - ], - "source": [ - "%%time\n", - "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=2)])\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del gg_gdf\n", - "del out" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3HJOItZ4MQMG", - "outputId": "f5be7bb4-7f09-4f80-c549-e703e99f5067" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 45s, sys: 1min 5s, total: 4min 50s\n", - "Wall time: 4min 52s\n" - ] - }, - { - "data": { - "text/plain": [ - "(102414, 1)" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=3)])._nodes\n", - "out.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "G32t_xthOUle", - "outputId": "7721741f-9c86-41aa-eb0b-2c8f0db2ed54" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(102414, 1) (24851333, 2)\n", - "CPU times: user 6.95 s, sys: 2.63 s, total: 9.57 s\n", - "Wall time: 9.84 s\n" - ] - } - ], - "source": [ - "%%time\n", - "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=3)])\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del gg_gdf\n", - "del out" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "bXy2yyJsMsEG", - "outputId": "911f2680-067c-44f2-9ba2-7f27d3c9bc6b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105479, 1) (30450354, 2)\n", - "CPU times: user 4min 36s, sys: 1min 25s, total: 6min 2s\n", - "Wall time: 6min 4s\n" - ] - } - ], - "source": [ - "%%time\n", - "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n", - "print(out._nodes.shape, out._edges.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Vt8hhjWDP_W_", - "outputId": "824ae644-e1cf-4239-bda9-84aecde52ad8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105479, 1) (30450354, 2)\n", - "CPU times: user 7.44 s, sys: 2.45 s, total: 9.88 s\n", - "Wall time: 9.9 s\n" - ] - } - ], - "source": [ - "%%time\n", - "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del gg_gdf\n", - "del out" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "_z4KpNZaOH8t", - "outputId": "2417f78b-e1b7-452d-8e26-7df259620c88" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105604, 1) (30468335, 2)\n", - "CPU times: user 5min 36s, sys: 1min 39s, total: 7min 16s\n", - "Wall time: 7min 15s\n" - ] - } - ], - "source": [ - "%%time\n", - "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n", - "print(out._nodes.shape, out._edges.shape)" + "data": { + "text/plain": [ + "(102414, 1)" ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=3)])._nodes\n", + "out.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "G32t_xthOUle", + "outputId": "7721741f-9c86-41aa-eb0b-2c8f0db2ed54" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "spUBH9EHSz2O", - "outputId": "22340ce3-e8d4-4a72-b485-9839c667b965" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105604, 1) (30468335, 2)\n", - "CPU times: user 8.82 s, sys: 2.71 s, total: 11.5 s\n", - "Wall time: 11.9 s\n" - ] - } - ], - "source": [ - "%%time\n", - "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del gg_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(102414, 1) (24851333, 2)\n", + "CPU times: user 6.95 s, sys: 2.63 s, total: 9.57 s\n", + "Wall time: 9.84 s\n" + ] + } + ], + "source": [ + "%%time\n", + "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=3)])\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del gg_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "bXy2yyJsMsEG", + "outputId": "911f2680-067c-44f2-9ba2-7f27d3c9bc6b" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "vCsdmc62A7OM", - "outputId": "adc05d29-c628-49ed-cd6d-8921c6dcd206" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1473, 1) (13375, 2)\n", - "CPU times: user 19.9 s, sys: 9.36 s, total: 29.2 s\n", - "Wall time: 41.8 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", - "for i in range(1):\n", - " g2 = gg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=1)\n", - "print(g2._nodes.shape, g2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105479, 1) (30450354, 2)\n", + "CPU times: user 4min 36s, sys: 1min 25s, total: 6min 2s\n", + "Wall time: 6min 4s\n" + ] + } + ], + "source": [ + "%%time\n", + "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n", + "print(out._nodes.shape, out._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Vt8hhjWDP_W_", + "outputId": "824ae644-e1cf-4239-bda9-84aecde52ad8" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "J3kV8NBYBQdW", - "outputId": "76073248-43e1-4c3c-c004-67324cc1d312" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1473, 1) (13375, 2)\n", - "CPU times: user 3.71 s, sys: 2.09 s, total: 5.8 s\n", - "Wall time: 6.05 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", - "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", - "for i in range(1):\n", - " g2 = gg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=1)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del gg_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105479, 1) (30450354, 2)\n", + "CPU times: user 7.44 s, sys: 2.45 s, total: 9.88 s\n", + "Wall time: 9.9 s\n" + ] + } + ], + "source": [ + "%%time\n", + "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del gg_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "_z4KpNZaOH8t", + "outputId": "2417f78b-e1b7-452d-8e26-7df259620c88" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 53, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ONv1RQeWBeeK", - "outputId": "58d57fa4-be72-45bc-abfa-5de9d1102f55" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(44073, 1) (2069325, 2)\n", - "CPU times: user 27.8 s, sys: 13.2 s, total: 41 s\n", - "Wall time: 43.9 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", - "for i in range(1):\n", - " g2 = gg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "print(g2._nodes.shape, g2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105604, 1) (30468335, 2)\n", + "CPU times: user 5min 36s, sys: 1min 39s, total: 7min 16s\n", + "Wall time: 7min 15s\n" + ] + } + ], + "source": [ + "%%time\n", + "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n", + "print(out._nodes.shape, out._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "spUBH9EHSz2O", + "outputId": "22340ce3-e8d4-4a72-b485-9839c667b965" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ke5SZZ01BgqR", - "outputId": "4173fd28-a11b-4300-d28b-6fdb87e8e9f3" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(44073, 1) (2069325, 2)\n", - "CPU times: user 4.26 s, sys: 2.37 s, total: 6.63 s\n", - "Wall time: 7.91 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", - "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", - "for i in range(1):\n", - " g2 = gg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del gg_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105604, 1) (30468335, 2)\n", + "CPU times: user 8.82 s, sys: 2.71 s, total: 11.5 s\n", + "Wall time: 11.9 s\n" + ] + } + ], + "source": [ + "%%time\n", + "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del gg_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "vCsdmc62A7OM", + "outputId": "adc05d29-c628-49ed-cd6d-8921c6dcd206" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "U795pIBUBiZV", - "outputId": "d499433c-cc0c-4bbf-c69f-36b5d55402d9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(102414, 1) (24851333, 2)\n", - "CPU times: user 1min 3s, sys: 22.7 s, total: 1min 26s\n", - "Wall time: 1min 35s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", - "for i in range(1):\n", - " g2 = gg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=3)\n", - "print(g2._nodes.shape, g2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(1473, 1) (13375, 2)\n", + "CPU times: user 19.9 s, sys: 9.36 s, total: 29.2 s\n", + "Wall time: 41.8 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", + "for i in range(1):\n", + " g2 = gg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=1)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "J3kV8NBYBQdW", + "outputId": "76073248-43e1-4c3c-c004-67324cc1d312" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 56, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "kIZYwSe1Bj2e", - "outputId": "b7e1ed9f-47d1-412e-9593-ecc436ac1486" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(102414, 1) (24851333, 2)\n", - "CPU times: user 3.96 s, sys: 2.11 s, total: 6.07 s\n", - "Wall time: 6.05 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", - "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", - "for i in range(1):\n", - " g2 = gg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=3)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del gg_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(1473, 1) (13375, 2)\n", + "CPU times: user 3.71 s, sys: 2.09 s, total: 5.8 s\n", + "Wall time: 6.05 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", + "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", + "for i in range(1):\n", + " g2 = gg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=1)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del gg_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ONv1RQeWBeeK", + "outputId": "58d57fa4-be72-45bc-abfa-5de9d1102f55" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 57, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "YTI5sD6YBpYL", - "outputId": "b37bf2df-07dc-404c-8a83-a83f28e38bf6" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105479, 1) (30450354, 2)\n", - "CPU times: user 1min 34s, sys: 30.6 s, total: 2min 5s\n", - "Wall time: 2min 5s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", - "for i in range(1):\n", - " g2 = gg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=4)\n", - "print(g2._nodes.shape, g2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(44073, 1) (2069325, 2)\n", + "CPU times: user 27.8 s, sys: 13.2 s, total: 41 s\n", + "Wall time: 43.9 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", + "for i in range(1):\n", + " g2 = gg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ke5SZZ01BgqR", + "outputId": "4173fd28-a11b-4300-d28b-6fdb87e8e9f3" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 58, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "d5WBazICBrSz", - "outputId": "ef95e893-3a0f-4d47-ede4-bd8a6faebf98" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105479, 1) (30450354, 2)\n", - "CPU times: user 5.25 s, sys: 2.41 s, total: 7.67 s\n", - "Wall time: 7.69 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", - "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", - "for i in range(1):\n", - " g2 = gg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=4)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del gg_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(44073, 1) (2069325, 2)\n", + "CPU times: user 4.26 s, sys: 2.37 s, total: 6.63 s\n", + "Wall time: 7.91 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", + "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", + "for i in range(1):\n", + " g2 = gg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del gg_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "U795pIBUBiZV", + "outputId": "d499433c-cc0c-4bbf-c69f-36b5d55402d9" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ozQlRPaFBtPD", - "outputId": "4f1655c4-38fd-47f9-942d-836585e0d866" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105604, 1) (30468335, 2)\n", - "CPU times: user 2min 16s, sys: 39.1 s, total: 2min 55s\n", - "Wall time: 2min 58s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", - "for i in range(1):\n", - " g2 = gg.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=5)\n", - "print(g2._nodes.shape, g2._edges.shape)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(102414, 1) (24851333, 2)\n", + "CPU times: user 1min 3s, sys: 22.7 s, total: 1min 26s\n", + "Wall time: 1min 35s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", + "for i in range(1):\n", + " g2 = gg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=3)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "kIZYwSe1Bj2e", + "outputId": "b7e1ed9f-47d1-412e-9593-ecc436ac1486" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-ACkMG20B6HM", - "outputId": "f26c03a9-9f25-4f93-c7d3-0e8676694040" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(105604, 1) (30468335, 2)\n", - "CPU times: user 5.79 s, sys: 2.51 s, total: 8.3 s\n", - "Wall time: 8.29 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", - "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", - "for i in range(1):\n", - " g2 = gg_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=5)\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del gg_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(102414, 1) (24851333, 2)\n", + "CPU times: user 3.96 s, sys: 2.11 s, total: 6.07 s\n", + "Wall time: 6.05 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", + "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", + "for i in range(1):\n", + " g2 = gg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=3)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del gg_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "YTI5sD6YBpYL", + "outputId": "b37bf2df-07dc-404c-8a83-a83f28e38bf6" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "R03M_swxarKC" - }, - "source": [ - "## Orkut\n", - "- 117M edges\n", - "- 3M nodes" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105479, 1) (30450354, 2)\n", + "CPU times: user 1min 34s, sys: 30.6 s, total: 2min 5s\n", + "Wall time: 2min 5s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", + "for i in range(1):\n", + " g2 = gg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=4)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "d5WBazICBrSz", + "outputId": "ef95e893-3a0f-4d47-ede4-bd8a6faebf98" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QoabYR2maxPo", - "outputId": "2bb6275d-46bb-42da-ec05-d0e5a58b1f77" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2023-12-26 00:55:52-- https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz\n", - "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n", - "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 447251958 (427M) [application/x-gzip]\n", - "Saving to: ‘com-orkut.ungraph.txt.gz’\n", - "\n", - "com-orkut.ungraph.t 100%[===================>] 426.53M 45.1MB/s in 9.7s \n", - "\n", - "2023-12-26 00:56:02 (44.0 MB/s) - ‘com-orkut.ungraph.txt.gz’ saved [447251958/447251958]\n", - "\n" - ] - } - ], - "source": [ - "#! wget https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105479, 1) (30450354, 2)\n", + "CPU times: user 5.25 s, sys: 2.41 s, total: 7.67 s\n", + "Wall time: 7.69 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", + "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", + "for i in range(1):\n", + " g2 = gg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=4)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del gg_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ozQlRPaFBtPD", + "outputId": "4f1655c4-38fd-47f9-942d-836585e0d866" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "BvvfFPKWbAVJ" - }, - "outputs": [], - "source": [ - "#! gunzip com-orkut.ungraph.txt.gz" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105604, 1) (30468335, 2)\n", + "CPU times: user 2min 16s, sys: 39.1 s, total: 2min 55s\n", + "Wall time: 2min 58s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n", + "for i in range(1):\n", + " g2 = gg.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=5)\n", + "print(g2._nodes.shape, g2._edges.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "-ACkMG20B6HM", + "outputId": "f26c03a9-9f25-4f93-c7d3-0e8676694040" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "YsWwRoPqbPIb", - "outputId": "2eb4f862-b4e1-42bf-ff5d-eec10b27cedc" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Undirected graph: ../../data/output/orkut.txt\n", - "# Orkut\n", - "# Nodes: 3072441 Edges: 117185083\n", - "# FromNodeId\tToNodeId\n", - "1\t2\n", - "1\t3\n", - "1\t4\n" - ] - } - ], - "source": [ - "#! head -n 7 com-orkut.ungraph.txt" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(105604, 1) (30468335, 2)\n", + "CPU times: user 5.79 s, sys: 2.51 s, total: 8.3 s\n", + "Wall time: 8.29 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n", + "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n", + "for i in range(1):\n", + " g2 = gg_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=5)\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del gg_gdf\n", + "del g2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R03M_swxarKC" + }, + "source": [ + "## Orkut\n", + "- 117M edges\n", + "- 3M nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "QoabYR2maxPo", + "outputId": "2bb6275d-46bb-42da-ec05-d0e5a58b1f77" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cbMC8r2ldjbW", - "outputId": "82688d53-7d56-4563-d65e-7c5cd32ac14e" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "('23.12.01', '0.32.0+12.g72e778c')" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "import graphistry\n", - "\n", - "from graphistry import (\n", - "\n", - " # graph operators\n", - " n, e_undirected, e_forward, e_reverse,\n", - "\n", - " # attribute predicates\n", - " is_in, ge, startswith, contains, match as match_re\n", - ")\n", - "\n", - "import cudf\n", - "\n", - "#work around google colab shell encoding bugs\n", - "import locale\n", - "locale.getpreferredencoding = lambda: \"UTF-8\"\n", - "\n", - "cudf.__version__, graphistry.__version__" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "--2023-12-26 00:55:52-- https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz\n", + "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n", + "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 447251958 (427M) [application/x-gzip]\n", + "Saving to: ‘com-orkut.ungraph.txt.gz’\n", + "\n", + "com-orkut.ungraph.t 100%[===================>] 426.53M 45.1MB/s in 9.7s \n", + "\n", + "2023-12-26 00:56:02 (44.0 MB/s) - ‘com-orkut.ungraph.txt.gz’ saved [447251958/447251958]\n", + "\n" + ] + } + ], + "source": [ + "#! wget https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "BvvfFPKWbAVJ" + }, + "outputs": [], + "source": [ + "#! gunzip com-orkut.ungraph.txt.gz" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "YsWwRoPqbPIb", + "outputId": "2eb4f862-b4e1-42bf-ff5d-eec10b27cedc" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "TopFxAvnh_Cv", - "outputId": "cc9d9dc9-e594-4190-fe84-3f1b6dce8a1a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:27 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 47C P0 27W / 70W | 103MiB / 15360MiB | 0% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "#! nvidia-smi" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "# Undirected graph: ../../data/output/orkut.txt\n", + "# Orkut\n", + "# Nodes: 3072441 Edges: 117185083\n", + "# FromNodeId\tToNodeId\n", + "1\t2\n", + "1\t3\n", + "1\t4\n" + ] + } + ], + "source": [ + "#! head -n 7 com-orkut.ungraph.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "cbMC8r2ldjbW", + "outputId": "82688d53-7d56-4563-d65e-7c5cd32ac14e" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Oczs87ITbJgw", - "outputId": "ac203ddd-e684-4eb9-a586-f6a49fd1625d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(117185082, 2)\n", - " s d\n", - "0 1 3\n", - "1 1 4\n", - "2 1 5\n", - "3 1 6\n", - "4 1 7\n", - "s int64\n", - "d int64\n", - "dtype: object\n", - "CPU times: user 2.56 s, sys: 4.2 s, total: 6.76 s\n", - "Wall time: 6.76 s\n" - ] - } - ], - "source": [ - "%%time\n", - "co_df = cudf.read_csv('com-orkut.ungraph.txt', sep='\\t', names=['s', 'd'], skiprows=5).to_pandas()\n", - "print(co_df.shape)\n", - "print(co_df.head(5))\n", - "print(co_df.dtypes)\n", - "#del co_df" + "data": { + "text/plain": [ + "('23.12.01', '0.32.0+12.g72e778c')" ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "import graphistry\n", + "\n", + "from graphistry import (\n", + "\n", + " # graph operators\n", + " n, e_undirected, e_forward, e_reverse,\n", + "\n", + " # attribute predicates\n", + " is_in, ge, startswith, contains, match as match_re\n", + ")\n", + "\n", + "import cudf\n", + "\n", + "#work around google colab shell encoding bugs\n", + "import locale\n", + "locale.getpreferredencoding = lambda: \"UTF-8\"\n", + "\n", + "cudf.__version__, graphistry.__version__" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "TopFxAvnh_Cv", + "outputId": "cc9d9dc9-e594-4190-fe84-3f1b6dce8a1a" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 258 - }, - "id": "gGSDjTtveFAT", - "outputId": "e7b38f4f-dc07-4f35-9bab-9c80a80bbf0b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3072441, 1) (117185082, 2)\n", - "CPU times: user 1.96 s, sys: 2.95 s, total: 4.91 s\n", - "Wall time: 4.92 s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
id
01
12
23
34
45
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " id\n", - "0 1\n", - "1 2\n", - "2 3\n", - "3 4\n", - "4 5" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "co_g = graphistry.edges(cudf.DataFrame(co_df), 's', 'd').materialize_nodes(engine='cudf')\n", - "co_g = co_g.nodes(lambda g: g._nodes.to_pandas()).edges(lambda g: g._edges.to_pandas())\n", - "print(co_g._nodes.shape, co_g._edges.shape)\n", - "co_g._nodes.head(5)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:27 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 47C P0 27W / 70W | 103MiB / 15360MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "#! nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Oczs87ITbJgw", + "outputId": "ac203ddd-e684-4eb9-a586-f6a49fd1625d" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "V5qL8K7-dqIZ", - "outputId": "e08319fc-74d3-4f33-df0f-f98950dc8c99" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:39 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 49C P0 27W / 70W | 2819MiB / 15360MiB | 0% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "#! nvidia-smi" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(117185082, 2)\n", + " s d\n", + "0 1 3\n", + "1 1 4\n", + "2 1 5\n", + "3 1 6\n", + "4 1 7\n", + "s int64\n", + "d int64\n", + "dtype: object\n", + "CPU times: user 2.56 s, sys: 4.2 s, total: 6.76 s\n", + "Wall time: 6.76 s\n" + ] + } + ], + "source": [ + "%%time\n", + "co_df = cudf.read_csv('com-orkut.ungraph.txt', sep='\\t', names=['s', 'd'], skiprows=5).to_pandas()\n", + "print(co_df.shape)\n", + "print(co_df.head(5))\n", + "print(co_df.dtypes)\n", + "#del co_df" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 }, + "id": "gGSDjTtveFAT", + "outputId": "e7b38f4f-dc07-4f35-9bab-9c80a80bbf0b" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "hCbxZ8UmhRLp", - "outputId": "519aed6c-733d-41f4-d462-e57f5e32b131" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n", - "Wall time: 47.7 µs\n" - ] - } - ], - "source": [ - "%%time\n", - "# crashes\n", - "if False:\n", - " out = co_g.chain([ n({'id': 1}), e_forward(hops=1)])._nodes\n", - " print(out.shape)\n", - " del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "(3072441, 1) (117185082, 2)\n", + "CPU times: user 1.96 s, sys: 2.95 s, total: 4.91 s\n", + "Wall time: 4.92 s\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Q682scC_eC-S", - "outputId": "7ff5f829-0de7-4a6c-a77d-e2857896a8a5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Dec 25 06:23:46 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 63C P0 30W / 70W | 1925MiB / 15360MiB | 35% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Mon Dec 25 06:23:49 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 66C P0 72W / 70W | 2845MiB / 15360MiB | 84% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(12, 1) (11, 2)\n", - "CPU times: user 4.42 s, sys: 131 ms, total: 4.55 s\n", - "Wall time: 4.42 s\n" - ] - } + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
id
01
12
23
34
45
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], - "source": [ - "%%time\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=1)])\n", - "#! nvidia-smi\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del co_gdf\n", - "del out" + "text/plain": [ + " id\n", + "0 1\n", + "1 2\n", + "2 3\n", + "3 4\n", + "4 5" ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "co_g = graphistry.edges(cudf.DataFrame(co_df), 's', 'd').materialize_nodes(engine='cudf')\n", + "co_g = co_g.nodes(lambda g: g._nodes.to_pandas()).edges(lambda g: g._edges.to_pandas())\n", + "print(co_g._nodes.shape, co_g._edges.shape)\n", + "co_g._nodes.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "V5qL8K7-dqIZ", + "outputId": "e08319fc-74d3-4f33-df0f-f98950dc8c99" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "i0AXhfqVbVsm", - "outputId": "8271f469-a73f-48e3-e1a9-3077026ab8ec" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Dec 25 06:24:52 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 22% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Mon Dec 25 06:24:58 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 64C P0 71W / 70W | 2845MiB / 15360MiB | 57% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(391, 1) (461, 2)\n", - "CPU times: user 5.34 s, sys: 132 ms, total: 5.47 s\n", - "Wall time: 6.13 s\n" - ] - } - ], - "source": [ - "%%time\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=2)])\n", - "#! nvidia-smi\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del co_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:39 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 49C P0 27W / 70W | 2819MiB / 15360MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "#! nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "hCbxZ8UmhRLp", + "outputId": "519aed6c-733d-41f4-d462-e57f5e32b131" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Hid0-iPKhpOd", - "outputId": "ecaeb534-d4d7-48fa-d4e1-c80b22626afe" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Dec 25 06:25:25 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 31% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Mon Dec 25 06:25:31 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 65C P0 71W / 70W | 2849MiB / 15360MiB | 58% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(21767, 1) (28480, 2)\n", - "CPU times: user 6.25 s, sys: 100 ms, total: 6.35 s\n", - "Wall time: 6.37 s\n" - ] - } - ], - "source": [ - "%%time\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=3)])\n", - "#! nvidia-smi\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del co_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n", + "Wall time: 47.7 µs\n" + ] + } + ], + "source": [ + "%%time\n", + "# crashes\n", + "if False:\n", + " out = co_g.gfql([ n({'id': 1}), e_forward(hops=1)])._nodes\n", + " print(out.shape)\n", + " del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Q682scC_eC-S", + "outputId": "7ff5f829-0de7-4a6c-a77d-e2857896a8a5" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "buutj-ZjhrEe", - "outputId": "ae11addd-6bea-44e9-81c0-b431e1db8089" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Dec 25 06:26:04 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 61C P0 29W / 70W | 1927MiB / 15360MiB | 36% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Mon Dec 25 06:26:13 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 65C P0 71W / 70W | 2931MiB / 15360MiB | 90% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(718640, 1) (2210961, 2)\n", - "CPU times: user 9.01 s, sys: 1.03 s, total: 10 s\n", - "Wall time: 9.84 s\n" - ] - } - ], - "source": [ - "%%time\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=4)])\n", - "#! nvidia-smi\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del co_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Mon Dec 25 06:23:46 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 63C P0 30W / 70W | 1925MiB / 15360MiB | 35% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Mon Dec 25 06:23:49 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 66C P0 72W / 70W | 2845MiB / 15360MiB | 84% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(12, 1) (11, 2)\n", + "CPU times: user 4.42 s, sys: 131 ms, total: 4.55 s\n", + "Wall time: 4.42 s\n" + ] + } + ], + "source": [ + "%%time\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=1)])\n", + "#! nvidia-smi\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del co_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "i0AXhfqVbVsm", + "outputId": "8271f469-a73f-48e3-e1a9-3077026ab8ec" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "bK4C9Ly0hso-", - "outputId": "8a9a32ab-03e2-42b4-8b71-2bcf797b31b1" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Dec 25 06:27:18 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 60C P0 29W / 70W | 1927MiB / 15360MiB | 28% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Mon Dec 25 06:27:57 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 72C P0 43W / 70W | 4351MiB / 15360MiB | 100% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(3041556, 1) (47622917, 2)\n", - "CPU times: user 34.9 s, sys: 4.76 s, total: 39.6 s\n", - "Wall time: 39.2 s\n" - ] - } - ], - "source": [ - "%%time\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=5)])\n", - "#! nvidia-smi\n", - "print(out._nodes.shape, out._edges.shape)\n", - "del co_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Mon Dec 25 06:24:52 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 22% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Mon Dec 25 06:24:58 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 64C P0 71W / 70W | 2845MiB / 15360MiB | 57% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(391, 1) (461, 2)\n", + "CPU times: user 5.34 s, sys: 132 ms, total: 5.47 s\n", + "Wall time: 6.13 s\n" + ] + } + ], + "source": [ + "%%time\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=2)])\n", + "#! nvidia-smi\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del co_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Hid0-iPKhpOd", + "outputId": "ecaeb534-d4d7-48fa-d4e1-c80b22626afe" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qrga-la0hwhh" - }, - "outputs": [], - "source": [ - "%%time\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "out = co_gdf.chain([ n({'id': 1}), e_forward(hops=6)])._nodes\n", - "print(out.shape)\n", - "del co_gdf\n", - "del out" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Mon Dec 25 06:25:25 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 31% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Mon Dec 25 06:25:31 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 65C P0 71W / 70W | 2849MiB / 15360MiB | 58% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(21767, 1) (28480, 2)\n", + "CPU times: user 6.25 s, sys: 100 ms, total: 6.35 s\n", + "Wall time: 6.37 s\n" + ] + } + ], + "source": [ + "%%time\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=3)])\n", + "#! nvidia-smi\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del co_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "buutj-ZjhrEe", + "outputId": "ae11addd-6bea-44e9-81c0-b431e1db8089" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "eiXFImxF-rzw", - "outputId": "b807cc3d-ed1a-4bef-c6e0-bfc2df7356ff" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Architecture: x86_64\n", - " CPU op-mode(s): 32-bit, 64-bit\n", - " Address sizes: 46 bits physical, 48 bits virtual\n", - " Byte Order: Little Endian\n", - "CPU(s): 2\n", - " On-line CPU(s) list: 0,1\n", - "Vendor ID: GenuineIntel\n", - " Model name: Intel(R) Xeon(R) CPU @ 2.20GHz\n", - " CPU family: 6\n", - " Model: 79\n", - " Thread(s) per core: 2\n", - " Core(s) per socket: 1\n", - " Socket(s): 1\n", - " Stepping: 0\n", - " BogoMIPS: 4399.99\n", - " Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clf\n", - " lush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_\n", - " good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fm\n", - " a cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hyp\n", - " ervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp fsgsb\n", - " ase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap xsa\n", - " veopt arat md_clear arch_capabilities\n", - "Virtualization features: \n", - " Hypervisor vendor: KVM\n", - " Virtualization type: full\n", - "Caches (sum of all): \n", - " L1d: 32 KiB (1 instance)\n", - " L1i: 32 KiB (1 instance)\n", - " L2: 256 KiB (1 instance)\n", - " L3: 55 MiB (1 instance)\n", - "NUMA: \n", - " NUMA node(s): 1\n", - " NUMA node0 CPU(s): 0,1\n", - "Vulnerabilities: \n", - " Gather data sampling: Not affected\n", - " Itlb multihit: Not affected\n", - " L1tf: Mitigation; PTE Inversion\n", - " Mds: Vulnerable; SMT Host state unknown\n", - " Meltdown: Vulnerable\n", - " Mmio stale data: Vulnerable\n", - " Retbleed: Vulnerable\n", - " Spec rstack overflow: Not affected\n", - " Spec store bypass: Vulnerable\n", - " Spectre v1: Vulnerable: __user pointer sanitization and usercopy barriers only; no swap\n", - " gs barriers\n", - " Spectre v2: Vulnerable, IBPB: disabled, STIBP: disabled, PBRSB-eIBRS: Not affected\n", - " Srbds: Not affected\n", - " Tsx async abort: Vulnerable\n" - ] - } - ], - "source": [ - "#!lscpu\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Mon Dec 25 06:26:04 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 61C P0 29W / 70W | 1927MiB / 15360MiB | 36% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Mon Dec 25 06:26:13 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 65C P0 71W / 70W | 2931MiB / 15360MiB | 90% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(718640, 1) (2210961, 2)\n", + "CPU times: user 9.01 s, sys: 1.03 s, total: 10 s\n", + "Wall time: 9.84 s\n" + ] + } + ], + "source": [ + "%%time\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=4)])\n", + "#! nvidia-smi\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del co_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "bK4C9Ly0hso-", + "outputId": "8a9a32ab-03e2-42b4-8b71-2bcf797b31b1" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "wJohLi58-sN5", - "outputId": "c3e144f6-c19a-4c68-e867-f5e7fa2e9df4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " total used free shared buff/cache available\n", - "Mem: 12Gi 717Mi 8.0Gi 1.0Mi 3.9Gi 11Gi\n", - "Swap: 0B 0B 0B\n" - ] - } - ], - "source": [ - "#!free -h\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Mon Dec 25 06:27:18 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 60C P0 29W / 70W | 1927MiB / 15360MiB | 28% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Mon Dec 25 06:27:57 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 72C P0 43W / 70W | 4351MiB / 15360MiB | 100% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(3041556, 1) (47622917, 2)\n", + "CPU times: user 34.9 s, sys: 4.76 s, total: 39.6 s\n", + "Wall time: 39.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=5)])\n", + "#! nvidia-smi\n", + "print(out._nodes.shape, out._edges.shape)\n", + "del co_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qrga-la0hwhh" + }, + "outputs": [], + "source": [ + "%%time\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=6)])._nodes\n", + "print(out.shape)\n", + "del co_gdf\n", + "del out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "eiXFImxF-rzw", + "outputId": "b807cc3d-ed1a-4bef-c6e0-bfc2df7356ff" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zak4Inhco5il", - "outputId": "30bcf2bc-853e-4e5e-8c57-ba0cd9429554" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 01:01:43 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 64C P0 30W / 70W | 2821MiB / 15360MiB | 0% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = pd.DataFrame({'id': [1]})\n", - "#! nvidia-smi\n", - "for i in range(1):\n", - " g2 = co_g.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=1)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "#del start_nodes\n", - "#del co_gdf\n", - "#del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Architecture: x86_64\n", + " CPU op-mode(s): 32-bit, 64-bit\n", + " Address sizes: 46 bits physical, 48 bits virtual\n", + " Byte Order: Little Endian\n", + "CPU(s): 2\n", + " On-line CPU(s) list: 0,1\n", + "Vendor ID: GenuineIntel\n", + " Model name: Intel(R) Xeon(R) CPU @ 2.20GHz\n", + " CPU family: 6\n", + " Model: 79\n", + " Thread(s) per core: 2\n", + " Core(s) per socket: 1\n", + " Socket(s): 1\n", + " Stepping: 0\n", + " BogoMIPS: 4399.99\n", + " Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clf\n", + " lush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_\n", + " good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fm\n", + " a cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hyp\n", + " ervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp fsgsb\n", + " ase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap xsa\n", + " veopt arat md_clear arch_capabilities\n", + "Virtualization features: \n", + " Hypervisor vendor: KVM\n", + " Virtualization type: full\n", + "Caches (sum of all): \n", + " L1d: 32 KiB (1 instance)\n", + " L1i: 32 KiB (1 instance)\n", + " L2: 256 KiB (1 instance)\n", + " L3: 55 MiB (1 instance)\n", + "NUMA: \n", + " NUMA node(s): 1\n", + " NUMA node0 CPU(s): 0,1\n", + "Vulnerabilities: \n", + " Gather data sampling: Not affected\n", + " Itlb multihit: Not affected\n", + " L1tf: Mitigation; PTE Inversion\n", + " Mds: Vulnerable; SMT Host state unknown\n", + " Meltdown: Vulnerable\n", + " Mmio stale data: Vulnerable\n", + " Retbleed: Vulnerable\n", + " Spec rstack overflow: Not affected\n", + " Spec store bypass: Vulnerable\n", + " Spectre v1: Vulnerable: __user pointer sanitization and usercopy barriers only; no swap\n", + " gs barriers\n", + " Spectre v2: Vulnerable, IBPB: disabled, STIBP: disabled, PBRSB-eIBRS: Not affected\n", + " Srbds: Not affected\n", + " Tsx async abort: Vulnerable\n" + ] + } + ], + "source": [ + "#!lscpu\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "wJohLi58-sN5", + "outputId": "c3e144f6-c19a-4c68-e867-f5e7fa2e9df4" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-SmFlCBS_Bgx", - "outputId": "d2326cf7-3ea6-4f99-9548-f2e98ece59a4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:45 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 49C P0 28W / 70W | 1923MiB / 15360MiB | 37% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Tue Dec 26 00:56:47 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 52C P0 70W / 70W | 2819MiB / 15360MiB | 79% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(12, 1) (11, 2)\n", - "CPU times: user 1.6 s, sys: 37.3 ms, total: 1.64 s\n", - "Wall time: 1.84 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({'id': [1]})\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " g2 = co_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=1)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del co_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + " total used free shared buff/cache available\n", + "Mem: 12Gi 717Mi 8.0Gi 1.0Mi 3.9Gi 11Gi\n", + "Swap: 0B 0B 0B\n" + ] + } + ], + "source": [ + "#!free -h\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "zak4Inhco5il", + "outputId": "30bcf2bc-853e-4e5e-8c57-ba0cd9429554" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "fjjt3YnYnabv", - "outputId": "05762f50-bfe1-4d23-9153-31431418c8e5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:47 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 51C P0 35W / 70W | 1923MiB / 15360MiB | 59% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Tue Dec 26 00:56:49 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 53C P0 59W / 70W | 2821MiB / 15360MiB | 86% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(391, 1) (461, 2)\n", - "CPU times: user 2.32 s, sys: 58.5 ms, total: 2.38 s\n", - "Wall time: 2.51 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({'id': [1]})\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " g2 = co_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=2)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del co_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 01:01:43 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 64C P0 30W / 70W | 2821MiB / 15360MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = pd.DataFrame({'id': [1]})\n", + "#! nvidia-smi\n", + "for i in range(1):\n", + " g2 = co_g.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=1)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "#del start_nodes\n", + "#del co_gdf\n", + "#del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "-SmFlCBS_Bgx", + "outputId": "d2326cf7-3ea6-4f99-9548-f2e98ece59a4" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "oIouuORgnbcY", - "outputId": "f07abe4c-5137-4ee3-935a-afbb2c5eaa1e" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:50 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 52C P0 36W / 70W | 1925MiB / 15360MiB | 55% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Tue Dec 26 00:56:53 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 54C P0 75W / 70W | 2825MiB / 15360MiB | 74% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(21767, 1) (28480, 2)\n", - "CPU times: user 3.04 s, sys: 63.6 ms, total: 3.1 s\n", - "Wall time: 3.25 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({'id': [1]})\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " g2 = co_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=3)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del co_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:45 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 49C P0 28W / 70W | 1923MiB / 15360MiB | 37% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Tue Dec 26 00:56:47 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 52C P0 70W / 70W | 2819MiB / 15360MiB | 79% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(12, 1) (11, 2)\n", + "CPU times: user 1.6 s, sys: 37.3 ms, total: 1.64 s\n", + "Wall time: 1.84 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({'id': [1]})\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " g2 = co_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=1)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del co_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "fjjt3YnYnabv", + "outputId": "05762f50-bfe1-4d23-9153-31431418c8e5" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "oNLZGjwInc85", - "outputId": "534097cf-4022-48cc-9419-a00c135f69e1" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:53 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 54C P0 36W / 70W | 1927MiB / 15360MiB | 54% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Tue Dec 26 00:56:58 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 56C P0 38W / 70W | 2907MiB / 15360MiB | 89% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(718640, 1) (2210961, 2)\n", - "CPU times: user 4.58 s, sys: 309 ms, total: 4.89 s\n", - "Wall time: 5.02 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({'id': [1]})\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " g2 = co_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=4)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del co_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:47 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 51C P0 35W / 70W | 1923MiB / 15360MiB | 59% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Tue Dec 26 00:56:49 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 53C P0 59W / 70W | 2821MiB / 15360MiB | 86% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(391, 1) (461, 2)\n", + "CPU times: user 2.32 s, sys: 58.5 ms, total: 2.38 s\n", + "Wall time: 2.51 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({'id': [1]})\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " g2 = co_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=2)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del co_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "oIouuORgnbcY", + "outputId": "f07abe4c-5137-4ee3-935a-afbb2c5eaa1e" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ePqaeujMneX8", - "outputId": "ffd88fff-016e-4ac0-ecb9-fa06baca60f8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:56:58 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 55C P0 37W / 70W | 1925MiB / 15360MiB | 59% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Tue Dec 26 00:57:10 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 60C P0 48W / 70W | 4325MiB / 15360MiB | 99% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(3041556, 1) (47622917, 2)\n", - "CPU times: user 10.8 s, sys: 1.29 s, total: 12.1 s\n", - "Wall time: 12 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({'id': [1]})\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " g2 = co_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=5)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del co_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:50 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 52C P0 36W / 70W | 1925MiB / 15360MiB | 55% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Tue Dec 26 00:56:53 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 54C P0 75W / 70W | 2825MiB / 15360MiB | 74% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(21767, 1) (28480, 2)\n", + "CPU times: user 3.04 s, sys: 63.6 ms, total: 3.1 s\n", + "Wall time: 3.25 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({'id': [1]})\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " g2 = co_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=3)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del co_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "oNLZGjwInc85", + "outputId": "534097cf-4022-48cc-9419-a00c135f69e1" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "PTBkoIVHnfzK", - "outputId": "5615ecd7-47ea-46ab-fd36-13bce4b3c787" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue Dec 26 00:57:10 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 59C P0 38W / 70W | 1925MiB / 15360MiB | 44% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "Tue Dec 26 00:57:38 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", - "| N/A 68C P0 55W / 70W | 6445MiB / 15360MiB | 95% Default |\n", - "| | | N/A |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "+---------------------------------------------------------------------------------------+\n", - "(3071927, 1) (117032738, 2)\n", - "CPU times: user 23.5 s, sys: 2.68 s, total: 26.2 s\n", - "Wall time: 28.2 s\n" - ] - } - ], - "source": [ - "%%time\n", - "start_nodes = cudf.DataFrame({'id': [1]})\n", - "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", - "#! nvidia-smi\n", - "for i in range(10):\n", - " g2 = co_gdf.hop(\n", - " nodes=start_nodes,\n", - " direction='forward',\n", - " hops=6)\n", - "#! nvidia-smi\n", - "print(g2._nodes.shape, g2._edges.shape)\n", - "del start_nodes\n", - "del co_gdf\n", - "del g2" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:53 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 54C P0 36W / 70W | 1927MiB / 15360MiB | 54% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Tue Dec 26 00:56:58 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 56C P0 38W / 70W | 2907MiB / 15360MiB | 89% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(718640, 1) (2210961, 2)\n", + "CPU times: user 4.58 s, sys: 309 ms, total: 4.89 s\n", + "Wall time: 5.02 s\n" + ] + } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({'id': [1]})\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " g2 = co_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=4)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del co_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ePqaeujMneX8", + "outputId": "ffd88fff-016e-4ac0-ecb9-fa06baca60f8" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Ygc2nrkznlCu" - }, - "outputs": [], - "source": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:56:58 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 55C P0 37W / 70W | 1925MiB / 15360MiB | 59% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Tue Dec 26 00:57:10 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 60C P0 48W / 70W | 4325MiB / 15360MiB | 99% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(3041556, 1) (47622917, 2)\n", + "CPU times: user 10.8 s, sys: 1.29 s, total: 12.1 s\n", + "Wall time: 12 s\n" + ] } - ], - "metadata": { - "accelerator": "GPU", + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({'id': [1]})\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " g2 = co_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=5)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del co_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { "colab": { - "gpuType": "T4", - "provenance": [] + "base_uri": "https://localhost:8080/" }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" + "id": "PTBkoIVHnfzK", + "outputId": "5615ecd7-47ea-46ab-fd36-13bce4b3c787" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Dec 26 00:57:10 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 59C P0 38W / 70W | 1925MiB / 15360MiB | 44% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "Tue Dec 26 00:57:38 2023 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 68C P0 55W / 70W | 6445MiB / 15360MiB | 95% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "+---------------------------------------------------------------------------------------+\n", + "(3071927, 1) (117032738, 2)\n", + "CPU times: user 23.5 s, sys: 2.68 s, total: 26.2 s\n", + "Wall time: 28.2 s\n" + ] } + ], + "source": [ + "%%time\n", + "start_nodes = cudf.DataFrame({'id': [1]})\n", + "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n", + "#! nvidia-smi\n", + "for i in range(10):\n", + " g2 = co_gdf.hop(\n", + " nodes=start_nodes,\n", + " direction='forward',\n", + " hops=6)\n", + "#! nvidia-smi\n", + "print(g2._nodes.shape, g2._edges.shape)\n", + "del start_nodes\n", + "del co_gdf\n", + "del g2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ygc2nrkznlCu" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 0 -} + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/demos/gfql/gfql_remote.ipynb b/demos/gfql/gfql_remote.ipynb index 60be1125b7..1b207df8cb 100644 --- a/demos/gfql/gfql_remote.ipynb +++ b/demos/gfql/gfql_remote.ipynb @@ -275,7 +275,7 @@ "source": [ "%%time\n", "\n", - "two_hop_g = g2.chain_remote(two_hop_query)" + "two_hop_g = g2.gfql_remote(two_hop_query)" ] }, { @@ -439,7 +439,7 @@ }, "outputs": [], "source": [ - "assert len(two_hop_g._edges) == len(g.chain(two_hop_query)._edges), \"Remote result should match local results\"" + "assert len(two_hop_g._edges) == len(g.gfql(two_hop_query)._edges), \"Remote result should match local results\"" ] }, { @@ -477,7 +477,7 @@ ], "source": [ "%%time\n", - "two_hop_g_gpu1 = g2.chain_remote(two_hop_query, engine='cudf')" + "two_hop_g_gpu1 = g2.gfql_remote(two_hop_query, engine='cudf')" ] }, { @@ -505,7 +505,7 @@ ], "source": [ "%%time\n", - "two_hop_g_cpu1 = g2.chain_remote(two_hop_query, engine='pandas')" + "two_hop_g_cpu1 = g2.gfql_remote(two_hop_query, engine='pandas')" ] }, { @@ -564,7 +564,7 @@ "source": [ "### Shape: Check result counts without downloading the graph\n", "\n", - "Often the important aspect is whether or not a search had hits, and you rather not pay the performance penalty of transfering all the hits. In these cases, switch to `chain_remote_shape()`:" + "Often the important aspect is whether or not a search had hits, and you rather not pay the performance penalty of transfering all the hits. In these cases, switch to `gfql_remote_shape()`:" ] }, { @@ -725,7 +725,7 @@ "source": [ "%%time\n", "\n", - "two_hops_nodes = g2.chain_remote(two_hop_query, output_type=\"nodes\")\n", + "two_hops_nodes = g2.gfql_remote(two_hop_query, output_type=\"nodes\")\n", "\n", "assert two_hops_nodes._edges is None, \"No edges returned\"\n", "\n", @@ -822,7 +822,7 @@ "source": [ "%%time\n", "\n", - "two_hops_edges = g2.chain_remote(two_hop_query, output_type=\"edges\")\n", + "two_hops_edges = g2.gfql_remote(two_hop_query, output_type=\"edges\")\n", "\n", "assert two_hops_edges._nodes is None, \"No nodes returned\"\n", "\n", @@ -866,7 +866,7 @@ "source": [ "%%time\n", "\n", - "two_hops_IDs_g = g2.chain_remote(two_hop_query, node_col_subset=['id'], edge_col_subset=['s', 'd'])" + "two_hops_IDs_g = g2.gfql_remote(two_hop_query, node_col_subset=['id'], edge_col_subset=['s', 'd'])" ] }, { @@ -1083,7 +1083,7 @@ "source": [ "### Remotely query remote data\n", "\n", - "Use `chain_remote()` and `chain_remote_shape()` as usual:" + "Use `chain_remote()` and `gfql_remote_shape()` as usual:" ] }, { @@ -1165,7 +1165,7 @@ "source": [ "### Fetch remote data\n", "\n", - "Use `chain_remote()` to fetch the nodes and edges table. Note that the below takes care to fetch nodes that are not connected to any edges." + "Use `gfql_remote()` to fetch the nodes and edges table. Note that the below takes care to fetch nodes that are not connected to any edges." ] }, { @@ -1194,8 +1194,8 @@ "source": [ "%%time\n", "\n", - "remote_g_nodes = g3_bound.chain_remote([n()], output_type='nodes')\n", - "remote_g_edges = g3_bound.chain_remote([e_undirected()], output_type='edges')\n", + "remote_g_nodes = g3_bound.gfql_remote([n()], output_type='nodes')\n", + "remote_g_edges = g3_bound.gfql_remote([e_undirected()], output_type='edges')\n", "\n", "g3_fetched_g = (graphistry\n", " .nodes(remote_g_nodes._nodes, 'id')\n", @@ -1409,7 +1409,7 @@ { "cell_type": "code", "id": "mwr3948llv", - "source": "%%time\n\n# Define a complex query using Python API for cleaner syntax\npagerank_analysis_query = let({\n # Step 1: Compute PageRank scores\n 'with_pagerank': ASTCall('compute_pagerank'),\n \n # Step 2: Filter nodes with high PageRank scores\n 'important_nodes': ref('with_pagerank', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ]),\n \n # Step 3: Get 1-hop neighborhoods of important nodes\n 'important_neighborhoods': ref('important_nodes', [\n e_undirected({'hops': 1}),\n n()\n ])\n})\n\n# Note: The 'in' clause is automatically the last binding when using Python let()\n# To specify a different output, pass it as second argument: let(bindings, 'output_name')\n\n# Execute the query remotely - chain_remote accepts Python objects directly!\nresult = g_complex.chain_remote([pagerank_analysis_query])\n\nprint(f\"Result has {len(result._nodes)} nodes and {len(result._edges)} edges\")\nprint(\"\\nNodes with PageRank scores:\")\nprint(result._nodes)", + "source": "%%time\n\n# Define a complex query using Python API for cleaner syntax\npagerank_analysis_query = let({\n # Step 1: Compute PageRank scores\n 'with_pagerank': ASTCall('compute_pagerank'),\n \n # Step 2: Filter nodes with high PageRank scores\n 'important_nodes': ref('with_pagerank', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ]),\n \n # Step 3: Get 1-hop neighborhoods of important nodes\n 'important_neighborhoods': ref('important_nodes', [\n e_undirected({'hops': 1}),\n n()\n ])\n})\n\n# Note: The 'in' clause is automatically the last binding when using Python let()\n# To specify a different output, pass it as second argument: let(bindings, 'output_name')\n\n# Execute the query remotely - chain_remote accepts Python objects directly!\nresult = g_complex.gfql_remote([pagerank_analysis_query])\n\nprint(f\"Result has {len(result._nodes)} nodes and {len(result._edges)} edges\")\nprint(\"\\nNodes with PageRank scores:\")\nprint(result._nodes)", "metadata": {}, "execution_count": null, "outputs": [] @@ -1429,7 +1429,7 @@ { "cell_type": "code", "id": "sc00048dhan", - "source": "# Comparison: Python API vs JSON format\n\n# Clean Python API (what we use above):\npython_query = let({\n 'data': ASTCall('compute_pagerank'),\n 'filtered': ref('data', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ])\n})\n\n# Equivalent verbose JSON format:\njson_query = {\n 'let': {\n 'data': {\n 'type': 'Call',\n 'function': 'compute_pagerank',\n 'params': {}\n },\n 'filtered': {\n 'type': 'Ref',\n 'ref': 'data',\n 'chain': [{\n 'type': 'Node',\n 'filter_dict': {\n 'filter': {'gte': [{'col': 'pagerank'}, 0.15]}\n }\n }]\n }\n },\n 'in': {'type': 'Ref', 'ref': 'filtered', 'chain': []}\n}\n\n# Both work with chain_remote:\n# result = g.chain_remote([python_query]) # Clean!\n# result = g.chain_remote([json_query]) # Verbose but equivalent\n\nprint(\"Python object converts to JSON:\")\nprint(python_query.to_json())", + "source": "# Comparison: Python API vs JSON format\n\n# Clean Python API (what we use above):\npython_query = let({\n 'data': ASTCall('compute_pagerank'),\n 'filtered': ref('data', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ])\n})\n\n# Equivalent verbose JSON format:\njson_query = {\n 'let': {\n 'data': {\n 'type': 'Call',\n 'function': 'compute_pagerank',\n 'params': {}\n },\n 'filtered': {\n 'type': 'Ref',\n 'ref': 'data',\n 'chain': [{\n 'type': 'Node',\n 'filter_dict': {\n 'filter': {'gte': [{'col': 'pagerank'}, 0.15]}\n }\n }]\n }\n },\n 'in': {'type': 'Ref', 'ref': 'filtered', 'chain': []}\n}\n\n# Both work with chain_remote:\n# result = g.gfql_remote([python_query]) # Clean!\n# result = g.gfql_remote([json_query]) # Verbose but equivalent\n\nprint(\"Python object converts to JSON:\")\nprint(python_query.to_json())", "metadata": {}, "execution_count": null, "outputs": [] @@ -1437,7 +1437,7 @@ { "cell_type": "code", "id": "40m2wl0yn3m", - "source": "%%time\n\n# Analyze different edge types using clean Python API\nedge_type_analysis = let({\n # Analyze follow edges\n 'follow_network': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'follow']}\n }),\n \n # Compute centrality on follow network \n 'follow_centrality': ref('follow_network', [\n n(),\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find mention patterns\n 'mention_edges': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'mention']}\n }),\n \n # Get nodes that are both highly connected and frequently mentioned\n 'influential_nodes': ref('follow_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.5]}}),\n ref('mention_edges', []),\n n()\n ])\n})\n\n# Execute remotely\ninfluential_result = g_complex.chain_remote([edge_type_analysis])\n\nprint(f\"Found {len(influential_result._nodes)} influential nodes\")\nprint(f\"Connected by {len(influential_result._edges)} edges\")\nprint(\"\\nInfluential nodes with centrality scores:\")\nprint(influential_result._nodes)", + "source": "%%time\n\n# Analyze different edge types using clean Python API\nedge_type_analysis = let({\n # Analyze follow edges\n 'follow_network': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'follow']}\n }),\n \n # Compute centrality on follow network \n 'follow_centrality': ref('follow_network', [\n n(),\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find mention patterns\n 'mention_edges': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'mention']}\n }),\n \n # Get nodes that are both highly connected and frequently mentioned\n 'influential_nodes': ref('follow_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.5]}}),\n ref('mention_edges', []),\n n()\n ])\n})\n\n# Execute remotely\ninfluential_result = g_complex.gfql_remote([edge_type_analysis])\n\nprint(f\"Found {len(influential_result._nodes)} influential nodes\")\nprint(f\"Connected by {len(influential_result._edges)} edges\")\nprint(\"\\nInfluential nodes with centrality scores:\")\nprint(influential_result._nodes)", "metadata": {}, "execution_count": null, "outputs": [] @@ -1451,7 +1451,7 @@ { "cell_type": "code", "id": "gmy30drvbts", - "source": "%%time\n\n# Complex analysis with multiple algorithms using Python API\ncomprehensive_analysis = let({\n # Base graph with PageRank computation\n 'enriched_graph': ASTCall('compute_pagerank'),\n \n # Add centrality metrics\n 'with_centrality': ref('enriched_graph', [\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find bridge nodes (high PageRank, low-medium centrality)\n 'bridge_nodes': ref('with_centrality', [\n n({\n 'filter': {\n 'and': [\n {'gte': [{'col': 'pagerank'}, 0.1]},\n {'lte': [{'col': 'degree_centrality'}, 0.7]}\n ]\n }\n })\n ]),\n \n # Find hub nodes (high degree centrality)\n 'hub_nodes': ref('with_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.7]}})\n ]),\n \n # Get connections between bridges and hubs\n 'critical_paths': ref('bridge_nodes', [\n e_forward({'to_nodes': ref('hub_nodes', [])}),\n n()\n ])\n})\n\n# Execute remotely with GPU acceleration\ncritical_paths_result = g_complex.chain_remote([comprehensive_analysis], engine='cudf')\n\nprint(f\"Critical paths network: {len(critical_paths_result._nodes)} nodes, {len(critical_paths_result._edges)} edges\")\n\n# Check if we got results\nif len(critical_paths_result._nodes) > 0:\n print(\"\\nCritical path nodes:\")\n print(critical_paths_result._nodes)\nelse:\n print(\"\\nNo critical paths found with current thresholds\")", + "source": "%%time\n\n# Complex analysis with multiple algorithms using Python API\ncomprehensive_analysis = let({\n # Base graph with PageRank computation\n 'enriched_graph': ASTCall('compute_pagerank'),\n \n # Add centrality metrics\n 'with_centrality': ref('enriched_graph', [\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find bridge nodes (high PageRank, low-medium centrality)\n 'bridge_nodes': ref('with_centrality', [\n n({\n 'filter': {\n 'and': [\n {'gte': [{'col': 'pagerank'}, 0.1]},\n {'lte': [{'col': 'degree_centrality'}, 0.7]}\n ]\n }\n })\n ]),\n \n # Find hub nodes (high degree centrality)\n 'hub_nodes': ref('with_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.7]}})\n ]),\n \n # Get connections between bridges and hubs\n 'critical_paths': ref('bridge_nodes', [\n e_forward({'to_nodes': ref('hub_nodes', [])}),\n n()\n ])\n})\n\n# Execute remotely with GPU acceleration\ncritical_paths_result = g_complex.gfql_remote([comprehensive_analysis], engine='cudf')\n\nprint(f\"Critical paths network: {len(critical_paths_result._nodes)} nodes, {len(critical_paths_result._edges)} edges\")\n\n# Check if we got results\nif len(critical_paths_result._nodes) > 0:\n print(\"\\nCritical path nodes:\")\n print(critical_paths_result._nodes)\nelse:\n print(\"\\nNo critical paths found with current thresholds\")", "metadata": {}, "execution_count": null, "outputs": [] @@ -1465,7 +1465,7 @@ { "cell_type": "code", "id": "4glzmgi1u3s", - "source": "%%time\n\n# Prepare visualization-ready data with all enrichments\nviz_prep_query = {\n 'let': {\n # Compute all metrics - sequential operations\n 'with_pagerank': {\n 'call': {'method': 'compute_pagerank', 'args': [], 'kwargs': {}}\n },\n \n 'with_metrics': {\n 'type': 'Ref',\n 'ref': 'with_pagerank',\n 'chain': [\n {'call': {'method': 'compute_degree_centrality', 'args': [], 'kwargs': {}}},\n # Add node colors based on PageRank\n {\n 'call': {\n 'method': 'nodes',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'node_color': {\n 'case': [\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.2]},\n 'then': 65280 # Green for high PageRank\n },\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.15]},\n 'then': 16776960 # Yellow for medium\n }\n ],\n 'else': 16711680 # Red for low\n },\n 'node_size': {\n 'mul': [\n {'col': 'degree_centrality'},\n 50 # Scale factor\n ]\n }\n }\n }\n }\n }\n ]\n },\n \n # Add edge styling based on type and weight\n 'styled_graph': {\n 'type': 'Ref',\n 'ref': 'with_metrics',\n 'chain': [\n {\n 'call': {\n 'method': 'edges',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'edge_color': {\n 'case': [\n {\n 'when': {'eq': [{'col': 'type'}, 'follow']},\n 'then': 255 # Blue for follows\n }\n ],\n 'else': 16711935 # Magenta for mentions\n },\n 'edge_weight': {\n 'col': 'weight'\n }\n }\n }\n }\n }\n ]\n },\n \n # Focus on top nodes and their connections\n 'viz_subgraph': {\n 'type': 'Ref',\n 'ref': 'styled_graph',\n 'chain': [\n {\n 'n': {\n 'filter': {\n 'or': [\n {'gte': [{'col': 'pagerank'}, 0.15]},\n {'gte': [{'col': 'degree_centrality'}, 0.6]}\n ]\n }\n }\n },\n {'e_undirected': {'hops': 1}},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'type': 'Ref', 'ref': 'viz_subgraph', 'chain': []}\n}\n\n# Get visualization-ready data\nviz_result = g_complex.chain_remote([viz_prep_query])\n\nprint(f\"Visualization subgraph: {len(viz_result._nodes)} nodes, {len(viz_result._edges)} edges\")\nprint(\"\\nNodes with visualization attributes:\")\nprint(viz_result._nodes)\nprint(\"\\nEdges with styling:\")\nprint(viz_result._edges)\n\n# Ready to visualize\n# viz_result.plot() # Uncomment to create visualization", + "source": "%%time\n\n# Prepare visualization-ready data with all enrichments\nviz_prep_query = {\n 'let': {\n # Compute all metrics - sequential operations\n 'with_pagerank': {\n 'call': {'method': 'compute_pagerank', 'args': [], 'kwargs': {}}\n },\n \n 'with_metrics': {\n 'type': 'Ref',\n 'ref': 'with_pagerank',\n 'chain': [\n {'call': {'method': 'compute_degree_centrality', 'args': [], 'kwargs': {}}},\n # Add node colors based on PageRank\n {\n 'call': {\n 'method': 'nodes',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'node_color': {\n 'case': [\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.2]},\n 'then': 65280 # Green for high PageRank\n },\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.15]},\n 'then': 16776960 # Yellow for medium\n }\n ],\n 'else': 16711680 # Red for low\n },\n 'node_size': {\n 'mul': [\n {'col': 'degree_centrality'},\n 50 # Scale factor\n ]\n }\n }\n }\n }\n }\n ]\n },\n \n # Add edge styling based on type and weight\n 'styled_graph': {\n 'type': 'Ref',\n 'ref': 'with_metrics',\n 'chain': [\n {\n 'call': {\n 'method': 'edges',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'edge_color': {\n 'case': [\n {\n 'when': {'eq': [{'col': 'type'}, 'follow']},\n 'then': 255 # Blue for follows\n }\n ],\n 'else': 16711935 # Magenta for mentions\n },\n 'edge_weight': {\n 'col': 'weight'\n }\n }\n }\n }\n }\n ]\n },\n \n # Focus on top nodes and their connections\n 'viz_subgraph': {\n 'type': 'Ref',\n 'ref': 'styled_graph',\n 'chain': [\n {\n 'n': {\n 'filter': {\n 'or': [\n {'gte': [{'col': 'pagerank'}, 0.15]},\n {'gte': [{'col': 'degree_centrality'}, 0.6]}\n ]\n }\n }\n },\n {'e_undirected': {'hops': 1}},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'type': 'Ref', 'ref': 'viz_subgraph', 'chain': []}\n}\n\n# Get visualization-ready data\nviz_result = g_complex.gfql_remote([viz_prep_query])\n\nprint(f\"Visualization subgraph: {len(viz_result._nodes)} nodes, {len(viz_result._edges)} edges\")\nprint(\"\\nNodes with visualization attributes:\")\nprint(viz_result._nodes)\nprint(\"\\nEdges with styling:\")\nprint(viz_result._edges)\n\n# Ready to visualize\n# viz_result.plot() # Uncomment to create visualization", "metadata": {}, "execution_count": null, "outputs": [] diff --git a/demos/gfql/gfql_validation_fundamentals.ipynb b/demos/gfql/gfql_validation_fundamentals.ipynb index 9736f6d08b..6699a7e89d 100644 --- a/demos/gfql/gfql_validation_fundamentals.ipynb +++ b/demos/gfql/gfql_validation_fundamentals.ipynb @@ -104,14 +104,14 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Valid query - columns exist\ntry:\n result = g.chain([\n n({'type': 'customer'}),\n e_forward({'edge_type': 'buys'}),\n n({'type': 'product'})\n ])\n print(f\"Query executed successfully!\")\n print(f\" Found {len(result._nodes)} nodes\")\n print(f\" Found {len(result._edges)} edges\")\nexcept GFQLSchemaError as e:\n print(f\"Schema error: {e}\")" + "source": "# Valid query - columns exist\ntry:\n result = g.gfql([\n n({'type': 'customer'}),\n e_forward({'edge_type': 'buys'}),\n n({'type': 'product'})\n ])\n print(f\"Query executed successfully!\")\n print(f\" Found {len(result._nodes)} nodes\")\n print(f\" Found {len(result._edges)} edges\")\nexcept GFQLSchemaError as e:\n print(f\"Schema error: {e}\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Invalid query - column doesn't exist\ntry:\n result = g.chain([\n n({'category': 'VIP'}) # 'category' column doesn't exist\n ])\nexcept GFQLSchemaError as e:\n print(f\"Schema validation caught the error!\")\n print(f\" Error code: {e.code}\")\n print(f\" Message: {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")" + "source": "# Invalid query - column doesn't exist\ntry:\n result = g.gfql([\n n({'category': 'VIP'}) # 'category' column doesn't exist\n ])\nexcept GFQLSchemaError as e:\n print(f\"Schema validation caught the error!\")\n print(f\" Error code: {e.code}\")\n print(f\" Message: {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")" }, { "cell_type": "markdown", @@ -127,14 +127,14 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Type mismatch: string value on numeric column\ntry:\n result = g.chain([\n n({'score': 'high'}) # 'score' is numeric, not string\n ])\nexcept GFQLSchemaError as e:\n print(f\"Type mismatch detected!\")\n print(f\" {e}\")\n print(f\"\\n Column type: {e.context.get('column_type')}\")" + "source": "# Type mismatch: string value on numeric column\ntry:\n result = g.gfql([\n n({'score': 'high'}) # 'score' is numeric, not string\n ])\nexcept GFQLSchemaError as e:\n print(f\"Type mismatch detected!\")\n print(f\" {e}\")\n print(f\"\\n Column type: {e.context.get('column_type')}\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Using predicates\nfrom graphistry.compute.predicates.numeric import gt\nfrom graphistry.compute.predicates.str import contains\n\n# Correct: numeric predicate on numeric column\ntry:\n result = g.chain([n({'score': gt(90)})])\n print(f\"Valid: Found {len(result._nodes)} high-scoring nodes\")\nexcept GFQLSchemaError as e:\n print(f\"Error: {e}\")\n\n# Wrong: string predicate on numeric column\ntry:\n result = g.chain([n({'score': contains('9')})])\nexcept GFQLSchemaError as e:\n print(f\"\\nPredicate type mismatch caught!\")\n print(f\" {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")" + "source": "# Using predicates\nfrom graphistry.compute.predicates.numeric import gt\nfrom graphistry.compute.predicates.str import contains\n\n# Correct: numeric predicate on numeric column\ntry:\n result = g.gfql([n({'score': gt(90)})])\n print(f\"Valid: Found {len(result._nodes)} high-scoring nodes\")\nexcept GFQLSchemaError as e:\n print(f\"Error: {e}\")\n\n# Wrong: string predicate on numeric column\ntry:\n result = g.gfql([n({'score': contains('9')})])\nexcept GFQLSchemaError as e:\n print(f\"\\nPredicate type mismatch caught!\")\n print(f\" {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")" }, { "cell_type": "markdown", @@ -150,7 +150,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Use validate_schema parameter\ntry:\n result = g.chain(chain_to_test.chain, validate_schema=True)\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (No graph operations were performed)\")" + "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Use validate_schema parameter\ntry:\n result = g.gfql(chain_to_test.chain, validate_schema=True)\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (No graph operations were performed)\")" }, { "cell_type": "code", @@ -201,7 +201,7 @@ " # print(f\"Warning: {len(errors)} schema issues found\")\n", " \n", " # Execute\n", - " result = g.chain(operations)\n", + " result = g.gfql(operations)\n", " return result\n", " \n", " except GFQLSyntaxError as e:\n", @@ -242,7 +242,7 @@ }, { "cell_type": "markdown", - "source": "## Summary\n\n### Key Takeaways\n\n1. **Automatic Validation**: GFQL validates automatically - no separate validation calls needed\n2. **Structured Errors**: Error codes (E1xx, E2xx, E3xx) help with programmatic handling\n3. **Helpful Messages**: Errors include suggestions for fixing issues\n4. **Two Validation Stages**:\n - Syntax/Type: During chain construction\n - Schema: During execution (or pre-execution)\n5. **Flexible Modes**: Choose between fail-fast or collect-all errors\n\n### Quick Reference\n\n```python\n# Automatic syntax validation\nchain = Chain([...]) # Validates syntax/types\n\n# Runtime schema validation \nresult = g.chain([...]) # Validates against data\n\n# Pre-execution schema validation\nresult = g.chain([...], validate_schema=True)\n\n# Validate chain against graph schema\nfrom graphistry.compute.validate_schema import validate_chain_schema\nvalidate_chain_schema(g, chain) # Throws GFQLSchemaError if invalid\n\n# Collect all syntax errors\nerrors = chain.validate(collect_all=True)\n\n# Collect all schema errors\nschema_errors = validate_chain_schema(g, chain, collect_all=True)\n```\n\n### Next Steps\n\n- Explore more complex query patterns\n- Learn about GFQL predicates for advanced filtering\n- Use validation in production applications", + "source": "## Summary\n\n### Key Takeaways\n\n1. **Automatic Validation**: GFQL validates automatically - no separate validation calls needed\n2. **Structured Errors**: Error codes (E1xx, E2xx, E3xx) help with programmatic handling\n3. **Helpful Messages**: Errors include suggestions for fixing issues\n4. **Two Validation Stages**:\n - Syntax/Type: During chain construction\n - Schema: During execution (or pre-execution)\n5. **Flexible Modes**: Choose between fail-fast or collect-all errors\n\n### Quick Reference\n\n```python\n# Automatic syntax validation\nchain = Chain([...]) # Validates syntax/types\n\n# Runtime schema validation \nresult = g.gfql([...]) # Validates against data\n\n# Pre-execution schema validation\nresult = g.gfql([...], validate_schema=True)\n\n# Validate chain against graph schema\nfrom graphistry.compute.validate_schema import validate_chain_schema\nvalidate_chain_schema(g, chain) # Throws GFQLSchemaError if invalid\n\n# Collect all syntax errors\nerrors = chain.validate(collect_all=True)\n\n# Collect all schema errors\nschema_errors = validate_chain_schema(g, chain, collect_all=True)\n```\n\n### Next Steps\n\n- Explore more complex query patterns\n- Learn about GFQL predicates for advanced filtering\n- Use validation in production applications", "metadata": {} } ], diff --git a/demos/gfql/temporal_predicates.ipynb b/demos/gfql/temporal_predicates.ipynb index 877306dd09..4ecd7350b4 100644 --- a/demos/gfql/temporal_predicates.ipynb +++ b/demos/gfql/temporal_predicates.ipynb @@ -107,7 +107,7 @@ "source": [ "# Alternative: Use chain with edge operations\n", "# Start from all nodes, then follow edges with temporal filter\n", - "recent_chain = g.chain([\n", + "recent_chain = g.gfql([\n", " n(), # Start with all nodes\n", " e_forward({\n", " \"timestamp\": gt(pd.Timestamp(cutoff_date))\n", @@ -152,7 +152,7 @@ "outputs": [], "source": [ "# Filter accounts created after a specific date\n", - "new_accounts = g.chain([\n", + "new_accounts = g.gfql([\n", " n(filter_dict={\n", " \"created_date\": ge(date(2021, 1, 1))\n", " })\n", @@ -170,7 +170,7 @@ "source": [ "# Find accounts active in the last 90 days\n", "ninety_days_ago = datetime.now().date() - timedelta(days=90)\n", - "active_accounts = g.chain([\n", + "active_accounts = g.gfql([\n", " n(filter_dict={\n", " \"last_active\": gt(pd.Timestamp(ninety_days_ago))\n", " })\n", @@ -258,7 +258,7 @@ "cell_type": "code", "metadata": {}, "outputs": [], - "source": "# Multi-hop query: Find accounts that received money recently\n# and then sent money to business accounts\nthirty_days_ago = datetime.now() - timedelta(days=30)\n\n# First, find recent transactions\nrecent_edges = g._edges[gt(pd.Timestamp(thirty_days_ago))(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\n# Use chain to find money flow pattern\nmoney_flow = recent_g.chain([\n # Start with any node\n n(),\n # Follow incoming edges (as destination)\n e_reverse(),\n # Go to source nodes\n n(),\n # Follow outgoing edges\n e_forward(),\n # To business accounts\n n(filter_dict={\"account_type\": \"business\"})\n])\n\nprint(f\"Money flow pattern found: {len(money_flow._nodes)} business accounts\")", + "source": "# Multi-hop query: Find accounts that received money recently\n# and then sent money to business accounts\nthirty_days_ago = datetime.now() - timedelta(days=30)\n\n# First, find recent transactions\nrecent_edges = g._edges[gt(pd.Timestamp(thirty_days_ago))(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\n# Use chain to find money flow pattern\nmoney_flow = recent_g.gfql([\n # Start with any node\n n(),\n # Follow incoming edges (as destination)\n e_reverse(),\n # Go to source nodes\n n(),\n # Follow outgoing edges\n e_forward(),\n # To business accounts\n n(filter_dict={\"account_type\": \"business\"})\n])\n\nprint(f\"Money flow pattern found: {len(money_flow._nodes)} business accounts\")", "execution_count": 12 }, { @@ -338,7 +338,7 @@ "outputs": [], "source": [ "# Find paths through recent high-value transactions\n", - "recent_high_value = g.chain([\n", + "recent_high_value = g.gfql([\n", " # Start from all nodes\n", " n(),\n", " # Follow edges with temporal and amount filters\n", @@ -369,7 +369,7 @@ }, { "cell_type": "code", - "source": "# Build predicates programmatically with wire protocol dicts\ndef create_date_filter(year, month, day, comparison=\"gt\"):\n \"\"\"Create a date filter using wire protocol format\"\"\"\n date_dict = {\n \"type\": \"date\",\n \"value\": f\"{year:04d}-{month:02d}-{day:02d}\"\n }\n \n if comparison == \"gt\":\n return gt(date_dict)\n elif comparison == \"lt\":\n return lt(date_dict)\n elif comparison == \"ge\":\n return ge(date_dict)\n elif comparison == \"le\":\n return le(date_dict)\n else:\n raise ValueError(f\"Unknown comparison: {comparison}\")\n\n# Use the programmatic filter\nfilter_2023 = create_date_filter(2023, 1, 1, \"ge\")\naccounts_2023 = g.chain([\n n(filter_dict={\n \"created_date\": filter_2023\n })\n])\n\nprint(f\"Accounts created in 2023 or later: {len(accounts_2023._nodes)}\")", + "source": "# Build predicates programmatically with wire protocol dicts\ndef create_date_filter(year, month, day, comparison=\"gt\"):\n \"\"\"Create a date filter using wire protocol format\"\"\"\n date_dict = {\n \"type\": \"date\",\n \"value\": f\"{year:04d}-{month:02d}-{day:02d}\"\n }\n \n if comparison == \"gt\":\n return gt(date_dict)\n elif comparison == \"lt\":\n return lt(date_dict)\n elif comparison == \"ge\":\n return ge(date_dict)\n elif comparison == \"le\":\n return le(date_dict)\n else:\n raise ValueError(f\"Unknown comparison: {comparison}\")\n\n# Use the programmatic filter\nfilter_2023 = create_date_filter(2023, 1, 1, \"ge\")\naccounts_2023 = g.gfql([\n n(filter_dict={\n \"created_date\": filter_2023\n })\n])\n\nprint(f\"Accounts created in 2023 or later: {len(accounts_2023._nodes)}\")", "metadata": {}, "outputs": [], "execution_count": null @@ -402,7 +402,7 @@ "cell_type": "code", "metadata": {}, "outputs": [], - "source": "# Complex multi-hop with temporal constraints\n# Find 2-hop paths through recent transactions\ntwo_hop_recent = g.chain([\n # Start from business accounts\n n(filter_dict={\"account_type\": \"business\"}),\n # First hop: recent outgoing transactions\n e_forward({\n \"timestamp\": gt(datetime.now() - timedelta(days=30))\n }, name=\"hop1\"),\n # Intermediate nodes\n n(),\n # Second hop: any transaction\n e_forward(name=\"hop2\"),\n # Final nodes\n n()\n])\n\nprint(f\"2-hop paths from business accounts through recent transactions:\")\nprint(f\" Total edges: {len(two_hop_recent._edges)}\")\nprint(f\" Hop 1 edges: {two_hop_recent._edges['hop1'].sum()}\")\nprint(f\" Hop 2 edges: {two_hop_recent._edges['hop2'].sum()}\")", + "source": "# Complex multi-hop with temporal constraints\n# Find 2-hop paths through recent transactions\ntwo_hop_recent = g.gfql([\n # Start from business accounts\n n(filter_dict={\"account_type\": \"business\"}),\n # First hop: recent outgoing transactions\n e_forward({\n \"timestamp\": gt(datetime.now() - timedelta(days=30))\n }, name=\"hop1\"),\n # Intermediate nodes\n n(),\n # Second hop: any transaction\n e_forward(name=\"hop2\"),\n # Final nodes\n n()\n])\n\nprint(f\"2-hop paths from business accounts through recent transactions:\")\nprint(f\" Total edges: {len(two_hop_recent._edges)}\")\nprint(f\" Hop 1 edges: {two_hop_recent._edges['hop1'].sum()}\")\nprint(f\" Hop 2 edges: {two_hop_recent._edges['hop2'].sum()}\")", "execution_count": 16 }, {