diff --git a/demos/gfql/GPU_memory_consumption_tutorial.ipynb b/demos/gfql/GPU_memory_consumption_tutorial.ipynb
index c79fe460f6..b3312345ef 100644
--- a/demos/gfql/GPU_memory_consumption_tutorial.ipynb
+++ b/demos/gfql/GPU_memory_consumption_tutorial.ipynb
@@ -723,7 +723,7 @@
"# Step 4: Profile the GFQL 2-hop Traversal\n",
"g1 = graphistry.edges(gdf, 'src_ip', 'dst_ip') # Example edge specification for Graphistry\n",
"with rmm.statistics.profiler(name=\"GFQL 2-hop Traversal\"):\n",
- " g2 = g1.chain([\n",
+ " g2 = g1.gfql([\n",
" n(),\n",
" e(edge_match={'event_type': 'data_transfer'},\n",
" edge_query=\"bytes_transferred > 500\"),\n",
@@ -905,4 +905,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
+}
\ No newline at end of file
diff --git a/demos/gfql/benchmark_hops_cpu_gpu.ipynb b/demos/gfql/benchmark_hops_cpu_gpu.ipynb
index cafd90815a..a4b2d31f2d 100644
--- a/demos/gfql/benchmark_hops_cpu_gpu.ipynb
+++ b/demos/gfql/benchmark_hops_cpu_gpu.ipynb
@@ -1,4820 +1,4820 @@
{
- "cells": [
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "GZxoiU8sQDk_"
+ },
+ "source": [
+ "# GFQL CPU, GPU Benchmark\n",
+ "\n",
+ "This notebook examines GFQL progerty graph query performance on 1-8 hop queries using CPU + GPU modes on various real-world 100K - 100M edge graphs. The data comes from a variety of popular social networks. The single-threaded CPU mode benefits from GFQL's novel dataframe engine, and the GPU mode further adds single-GPU acceleration. Both the `chain()` and `hop()` methods are examined.\n",
+ "\n",
+ "The benchmark does not examine bigger-than-memory and distributed scenarios. The provided results here are from running on a free Google Colab T4 runtime, with a 2.2GHz Intel CPU (12 GB CPU RAM) and T4 Nvidia GPU (16 GB GPU RAM).\n",
+ "\n",
+ "## Data\n",
+ "From [SNAP](https://snap.stanford.edu/data/)\n",
+ "\n",
+ "| Network | Nodes | Edges |\n",
+ "|-------------|-----------|--------------|\n",
+ "| **Facebook**| 4,039 | 88,234 |\n",
+ "| **Twitter** | 81,306 | 2,420,766 |\n",
+ "| **GPlus** | 107,614 | 30,494,866 |\n",
+ "| **Orkut** | 3,072,441 | 117,185,082 |\n",
+ "\n",
+ "## Results\n",
+ "\n",
+ "Definitions:\n",
+ "\n",
+ "* GTEPS: Giga (billion) edges traversed per second\n",
+ "\n",
+ "* T edges / \\$: Estimated trillion edges traversed for 1\\$ USD based on observed GTEPS and a 3yr AWS reservation (as of 12/2023)\n",
+ "\n",
+ "Tasks:\n",
+ "\n",
+ "1. `chain()` - includes complex pre/post processing\n",
+ "\n",
+ " **Task**: `g.gfql([n({'id': some_id}), e_forward(hops=some_n)])`\n",
+ "\n",
+ "\n",
+ "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n",
+ "|-------------|--------------|-------------|-------------|----------------------------|--------------------------------|\n",
+ "| **Facebook**| 1.1X | 0.66 | 0.61 | 65.7 | 10.4 |\n",
+ "| **Twitter** | 17.4X | 0.17 | 2.81 | 16.7 | 48.1 |\n",
+ "| **GPlus** | 43.8X | 0.09 | 2.87 | 8.5 | 49.2 |\n",
+ "| **Orkut** | N/A | N/A | 12.15 | N/A | 208.3 |\n",
+ "| **AVG** | 20.7X | 0.30 | 4.61 | 30.3 | 79.0\n",
+ "| **MAX** | 43.8X | 0.66 | 12.15 | 65.7 | 208.3\n",
+ "\n",
+ "\n",
+ "2. `hop()` - core property search primitive similar to BFS\n",
+ "\n",
+ " **Task**: `g.hop(nodes=[some_id], direction='forward', hops=some_n)`\n",
+ "\n",
+ "\n",
+ "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n",
+ "|-------------|-------------|-----------|-----------|--------------------|--------------------------------|\n",
+ "| **Facebook**| 3X | 0.47 | 1.47 | 47.0 | 25.2 |\n",
+ "| **Twitter** | 42X | 0.50 | 10.51 | 50.2 | 180.2 |\n",
+ "| **GPlus** | 21X | 0.26 | 4.11 | 26.2 | 70.4 |\n",
+ "| **Orkut** | N/A | N/A | 41.50 | N/A | 711.4 |\n",
+ "| **AVG** | 22X | 0.41 | 14.4 | 41.1 | 246.8\n",
+ "| **MAX** | 42X | 0.50 | 41.50 | 50.2 | 711.4\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SAj8lhREEOwS"
+ },
+ "source": [
+ "## Optional: GPU setup - Google Colab"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4hrEEAAm7DTO"
+ },
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "W2MF6ZsjDv3B",
+ "outputId": "46088cbc-2db9-4529-f724-dc57ed85dfb7"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "GZxoiU8sQDk_"
- },
- "source": [
- "# GFQL CPU, GPU Benchmark\n",
- "\n",
- "This notebook examines GFQL progerty graph query performance on 1-8 hop queries using CPU + GPU modes on various real-world 100K - 100M edge graphs. The data comes from a variety of popular social networks. The single-threaded CPU mode benefits from GFQL's novel dataframe engine, and the GPU mode further adds single-GPU acceleration. Both the `chain()` and `hop()` methods are examined.\n",
- "\n",
- "The benchmark does not examine bigger-than-memory and distributed scenarios. The provided results here are from running on a free Google Colab T4 runtime, with a 2.2GHz Intel CPU (12 GB CPU RAM) and T4 Nvidia GPU (16 GB GPU RAM).\n",
- "\n",
- "## Data\n",
- "From [SNAP](https://snap.stanford.edu/data/)\n",
- "\n",
- "| Network | Nodes | Edges |\n",
- "|-------------|-----------|--------------|\n",
- "| **Facebook**| 4,039 | 88,234 |\n",
- "| **Twitter** | 81,306 | 2,420,766 |\n",
- "| **GPlus** | 107,614 | 30,494,866 |\n",
- "| **Orkut** | 3,072,441 | 117,185,082 |\n",
- "\n",
- "## Results\n",
- "\n",
- "Definitions:\n",
- "\n",
- "* GTEPS: Giga (billion) edges traversed per second\n",
- "\n",
- "* T edges / \\$: Estimated trillion edges traversed for 1\\$ USD based on observed GTEPS and a 3yr AWS reservation (as of 12/2023)\n",
- "\n",
- "Tasks:\n",
- "\n",
- "1. `chain()` - includes complex pre/post processing\n",
- "\n",
- " **Task**: `g.chain([n({'id': some_id}), e_forward(hops=some_n)])`\n",
- "\n",
- "\n",
- "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n",
- "|-------------|--------------|-------------|-------------|----------------------------|--------------------------------|\n",
- "| **Facebook**| 1.1X | 0.66 | 0.61 | 65.7 | 10.4 |\n",
- "| **Twitter** | 17.4X | 0.17 | 2.81 | 16.7 | 48.1 |\n",
- "| **GPlus** | 43.8X | 0.09 | 2.87 | 8.5 | 49.2 |\n",
- "| **Orkut** | N/A | N/A | 12.15 | N/A | 208.3 |\n",
- "| **AVG** | 20.7X | 0.30 | 4.61 | 30.3 | 79.0\n",
- "| **MAX** | 43.8X | 0.66 | 12.15 | 65.7 | 208.3\n",
- "\n",
- "\n",
- "2. `hop()` - core property search primitive similar to BFS\n",
- "\n",
- " **Task**: `g.hop(nodes=[some_id], direction='forward', hops=some_n)`\n",
- "\n",
- "\n",
- "| **Dataset** | Max GPU Speedup | CPU GTEPS | GPU GTEPS | T CPU edges / \\$ (t3.l) | T GPU edges / \\$ (g4dn.xl) |\n",
- "|-------------|-------------|-----------|-----------|--------------------|--------------------------------|\n",
- "| **Facebook**| 3X | 0.47 | 1.47 | 47.0 | 25.2 |\n",
- "| **Twitter** | 42X | 0.50 | 10.51 | 50.2 | 180.2 |\n",
- "| **GPlus** | 21X | 0.26 | 4.11 | 26.2 | 70.4 |\n",
- "| **Orkut** | N/A | N/A | 41.50 | N/A | 711.4 |\n",
- "| **AVG** | 22X | 0.41 | 14.4 | 41.1 | 246.8\n",
- "| **MAX** | 42X | 0.50 | 41.50 | 50.2 | 711.4\n"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:50:30 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 54C P8 10W / 70W | 0MiB / 15360MiB | 0% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "| No running processes found |\n",
+ "+---------------------------------------------------------------------------------------+\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Report GPU used when GPU benchmarking\n",
+ "# ! nvidia-smi"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "Aikh0x4ID_wK"
+ },
+ "outputs": [],
+ "source": [
+ "# if in google colab\n",
+ "#!git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n",
+ "#!python rapidsai-csp-utils/colab/pip-install.py"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 35
},
+ "id": "Lwekdei1dH3N",
+ "outputId": "71f5b01d-7917-4283-8338-969167d6e1e8"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "SAj8lhREEOwS"
+ "data": {
+ "application/vnd.google.colaboratory.intrinsic+json": {
+ "type": "string"
},
- "source": [
- "## Optional: GPU setup - Google Colab"
+ "text/plain": [
+ "'23.12.01'"
]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import cudf\n",
+ "cudf.__version__"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "QQpsrtwBT7sa"
+ },
+ "source": [
+ "## 1. Install & configure"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "cYjRbgkU9Sx8",
+ "outputId": "2cf25531-9b8b-4715-ccc7-e79094d84ebd"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "4hrEEAAm7DTO"
- },
- "source": []
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! pip install graphistry[igraph]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Ff6Tt9DhkePl"
+ },
+ "source": [
+ "### Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 35
},
+ "id": "S5_y0CbLkjft",
+ "outputId": "a68a9c4b-c9c5-4b8b-ea4f-7bf1e4ddf315"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "W2MF6ZsjDv3B",
- "outputId": "46088cbc-2db9-4529-f724-dc57ed85dfb7"
+ "data": {
+ "application/vnd.google.colaboratory.intrinsic+json": {
+ "type": "string"
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:50:30 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 54C P8 10W / 70W | 0MiB / 15360MiB | 0% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "| No running processes found |\n",
- "+---------------------------------------------------------------------------------------+\n"
- ]
- }
- ],
- "source": [
- "# Report GPU used when GPU benchmarking\n",
- "# ! nvidia-smi"
+ "text/plain": [
+ "'0.32.0+12.g72e778c'"
]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "import graphistry\n",
+ "\n",
+ "from graphistry import (\n",
+ "\n",
+ " # graph operators\n",
+ " n, e_undirected, e_forward, e_reverse,\n",
+ "\n",
+ " # attribute predicates\n",
+ " is_in, ge, startswith, contains, match as match_re\n",
+ ")\n",
+ "graphistry.__version__"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "I7Fg75jsG4co"
+ },
+ "outputs": [],
+ "source": [
+ "import cudf"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "uLZKph2-a5M4"
+ },
+ "outputs": [],
+ "source": [
+ "#work around google colab shell encoding bugs\n",
+ "\n",
+ "import locale\n",
+ "locale.getpreferredencoding = lambda: \"UTF-8\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eU9SyauNUHtR"
+ },
+ "source": [
+ "## 2. Perf benchmarks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "NA0Ym11fkB8j"
+ },
+ "source": [
+ "### Facebook: 88K edges"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 224
},
+ "id": "vXuQogHekClJ",
+ "outputId": "64db92c0-2704-438b-d0e4-25865acbb5e9"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "id": "Aikh0x4ID_wK"
- },
- "outputs": [],
- "source": [
- "# if in google colab\n",
- "#!git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n",
- "#!python rapidsai-csp-utils/colab/pip-install.py"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(88234, 2)\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 35
- },
- "id": "Lwekdei1dH3N",
- "outputId": "71f5b01d-7917-4283-8338-969167d6e1e8"
- },
- "outputs": [
- {
- "data": {
- "application/vnd.google.colaboratory.intrinsic+json": {
- "type": "string"
- },
- "text/plain": [
- "'23.12.01'"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
+ "data": {
+ "text/html": [
+ "\n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " s | \n",
+ " d | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 0 | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0 | \n",
+ " 3 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0 | \n",
+ " 4 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 0 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
],
- "source": [
- "import cudf\n",
- "cudf.__version__"
+ "text/plain": [
+ " s d\n",
+ "0 0 1\n",
+ "1 0 2\n",
+ "2 0 3\n",
+ "3 0 4\n",
+ "4 0 5"
]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df = pd.read_csv('https://raw.githubusercontent.com/graphistry/pygraphistry/master/demos/data/facebook_combined.txt', sep=' ', names=['s', 'd'])\n",
+ "print(df.shape)\n",
+ "df.head(5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 224
},
+ "id": "jEma7hvvkzkN",
+ "outputId": "dbf21342-6b80-429c-bd3f-b1494c6854c7"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "QQpsrtwBT7sa"
- },
- "source": [
- "## 1. Install & configure"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4039, 1) (88234, 2)\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "cYjRbgkU9Sx8",
- "outputId": "2cf25531-9b8b-4715-ccc7-e79094d84ebd"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
- ]
- }
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 3 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 4 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
],
- "source": [
- "#! pip install graphistry[igraph]"
+ "text/plain": [
+ " id\n",
+ "0 0\n",
+ "1 1\n",
+ "2 2\n",
+ "3 3\n",
+ "4 4"
]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "fg = graphistry.edges(df, 's', 'd').materialize_nodes()\n",
+ "print(fg._nodes.shape, fg._edges.shape)\n",
+ "fg._nodes.head(5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "5lEdCBw9lzd7",
+ "outputId": "ed7451e0-401e-4edc-c8de-79c5afd0c95b"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "Ff6Tt9DhkePl"
- },
- "source": [
- "### Imports"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 13.6 s, sys: 2.08 s, total: 15.7 s\n",
+ "Wall time: 18 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "for i in range(100):\n",
+ " fg2 = fg.gfql([n({'id': 0}), e_forward(hops=2)])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "JFKIBa8mJCvJ",
+ "outputId": "c22022f0-b33d-483a-db64-29992c5161e8"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 35
- },
- "id": "S5_y0CbLkjft",
- "outputId": "a68a9c4b-c9c5-4b8b-ea4f-7bf1e4ddf315"
- },
- "outputs": [
- {
- "data": {
- "application/vnd.google.colaboratory.intrinsic+json": {
- "type": "string"
- },
- "text/plain": [
- "'0.32.0+12.g72e778c'"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import pandas as pd\n",
- "\n",
- "import graphistry\n",
- "\n",
- "from graphistry import (\n",
- "\n",
- " # graph operators\n",
- " n, e_undirected, e_forward, e_reverse,\n",
- "\n",
- " # attribute predicates\n",
- " is_in, ge, startswith, contains, match as match_re\n",
- ")\n",
- "graphistry.__version__"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4039, 1) (88234, 2)\n",
+ "(1519, 1) (4060, 2)\n",
+ "CPU times: user 11.8 s, sys: 28.1 ms, total: 11.8 s\n",
+ "Wall time: 11.9 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "for i in range(100):\n",
+ " fg2 = fg_gdf.gfql([n({'id': 0}), e_forward(hops=2)])\n",
+ "print(fg._nodes.shape, fg._edges.shape)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)\n",
+ "del fg_gdf\n",
+ "del fg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "-KBGLexek5tS",
+ "outputId": "2f462e6c-578a-4fa1-ec29-91bae753f4c5"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "id": "I7Fg75jsG4co"
- },
- "outputs": [],
- "source": [
- "import cudf"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4039, 1) (88234, 2)\n",
+ "(3829, 1) (86074, 2)\n",
+ "CPU times: user 15.4 s, sys: 808 ms, total: 16.2 s\n",
+ "Wall time: 16.2 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "for i in range(50):\n",
+ " fg2 = fg.gfql([n({'id': 0}), e_forward(hops=5)])\n",
+ "print(fg._nodes.shape, fg._edges.shape)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "CVpcbhpdHFEF",
+ "outputId": "aba04ee1-781e-4226-b593-b42415a55fc4"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {
- "id": "uLZKph2-a5M4"
- },
- "outputs": [],
- "source": [
- "#work around google colab shell encoding bugs\n",
- "\n",
- "import locale\n",
- "locale.getpreferredencoding = lambda: \"UTF-8\""
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4039, 1) (88234, 2)\n",
+ "(3829, 1) (86074, 2)\n",
+ "CPU times: user 9.82 s, sys: 133 ms, total: 9.95 s\n",
+ "Wall time: 10.1 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "for i in range(50):\n",
+ " fg2 = fg_gdf.gfql([n({'id': 0}), e_forward(hops=5)])\n",
+ "print(fg._nodes.shape, fg._edges.shape)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)\n",
+ "del fg_gdf\n",
+ "del fg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "1cFIyJF9pLjE",
+ "outputId": "107329af-8e4b-428c-8b03-77ed00bdf5bf"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "eU9SyauNUHtR"
- },
- "source": [
- "## 2. Perf benchmarks"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 11.8 s, sys: 377 ms, total: 12.1 s\n",
+ "Wall time: 13.1 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "for i in range(100):\n",
+ " fg2 = fg.gfql([e_forward(source_node_match={'id': 0}, hops=5)])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "M5uRiD6uJVNW",
+ "outputId": "5e938a19-2992-4280-80c2-784382d40113"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "NA0Ym11fkB8j"
- },
- "source": [
- "### Facebook: 88K edges"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4039, 1) (88234, 2)\n",
+ "(348, 1) (347, 2)\n",
+ "CPU times: user 14.1 s, sys: 48.5 ms, total: 14.2 s\n",
+ "Wall time: 14.2 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "for i in range(100):\n",
+ " fg2 = fg_gdf.gfql([e_forward(source_node_match={'id': 0}, hops=5)])\n",
+ "print(fg._nodes.shape, fg._edges.shape)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)\n",
+ "del fg_gdf\n",
+ "del fg2\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Y9vgzfT69x41",
+ "outputId": "6882c1ce-0df8-4087-dda4-0a105a8617e1"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 224
- },
- "id": "vXuQogHekClJ",
- "outputId": "64db92c0-2704-438b-d0e4-25865acbb5e9"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(88234, 2)\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " s | \n",
- " d | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 0 | \n",
- " 1 | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 0 | \n",
- " 2 | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 0 | \n",
- " 3 | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 0 | \n",
- " 4 | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 0 | \n",
- " 5 | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
\n",
- "
\n"
- ],
- "text/plain": [
- " s d\n",
- "0 0 1\n",
- "1 0 2\n",
- "2 0 3\n",
- "3 0 4\n",
- "4 0 5"
- ]
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "df = pd.read_csv('https://raw.githubusercontent.com/graphistry/pygraphistry/master/demos/data/facebook_combined.txt', sep=' ', names=['s', 'd'])\n",
- "print(df.shape)\n",
- "df.head(5)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(1519, 1) (4060, 2)\n",
+ "CPU times: user 4.5 s, sys: 1.35 s, total: 5.85 s\n",
+ "Wall time: 6.09 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({fg._node: [0]})\n",
+ "for i in range(100):\n",
+ " fg2 = fg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "c7ybJqjc-T31",
+ "outputId": "37ccc1fb-6460-4193-8aa7-22837ff06d0a"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 224
- },
- "id": "jEma7hvvkzkN",
- "outputId": "dbf21342-6b80-429c-bd3f-b1494c6854c7"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(4039, 1) (88234, 2)\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 0 | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 1 | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 2 | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 3 | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 4 | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
\n",
- "
\n"
- ],
- "text/plain": [
- " id\n",
- "0 0\n",
- "1 1\n",
- "2 2\n",
- "3 3\n",
- "4 4"
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "fg = graphistry.edges(df, 's', 'd').materialize_nodes()\n",
- "print(fg._nodes.shape, fg._edges.shape)\n",
- "fg._nodes.head(5)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(1519, 1) (4060, 2)\n",
+ "CPU times: user 2.58 s, sys: 6.75 ms, total: 2.59 s\n",
+ "Wall time: 2.58 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({fg._node: [0]})\n",
+ "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n",
+ "for i in range(100):\n",
+ " fg2 = fg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)\n",
+ "del start_nodes\n",
+ "del fg_gdf\n",
+ "del fg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Dy7a4zDZ-7_G",
+ "outputId": "077b5d9c-c9ae-411a-8228-3c026b07a910"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "5lEdCBw9lzd7",
- "outputId": "ed7451e0-401e-4edc-c8de-79c5afd0c95b"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 13.6 s, sys: 2.08 s, total: 15.7 s\n",
- "Wall time: 18 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "for i in range(100):\n",
- " fg2 = fg.chain([n({'id': 0}), e_forward(hops=2)])"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(3829, 1) (86074, 2)\n",
+ "CPU times: user 13.2 s, sys: 2 s, total: 15.2 s\n",
+ "Wall time: 18.3 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({fg._node: [0]})\n",
+ "for i in range(100):\n",
+ " fg2 = fg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=5)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "N5aUtF1a--ML",
+ "outputId": "0c2b67b8-fac6-45b3-dfbe-8002b5506e91"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "JFKIBa8mJCvJ",
- "outputId": "c22022f0-b33d-483a-db64-29992c5161e8"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(4039, 1) (88234, 2)\n",
- "(1519, 1) (4060, 2)\n",
- "CPU times: user 11.8 s, sys: 28.1 ms, total: 11.8 s\n",
- "Wall time: 11.9 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "for i in range(100):\n",
- " fg2 = fg_gdf.chain([n({'id': 0}), e_forward(hops=2)])\n",
- "print(fg._nodes.shape, fg._edges.shape)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)\n",
- "del fg_gdf\n",
- "del fg2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(3829, 1) (86074, 2)\n",
+ "CPU times: user 5.72 s, sys: 159 ms, total: 5.88 s\n",
+ "Wall time: 5.86 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({fg._node: [0]})\n",
+ "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n",
+ "for i in range(100):\n",
+ " fg2 = fg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=5)\n",
+ "print(fg2._nodes.shape, fg2._edges.shape)\n",
+ "del start_nodes\n",
+ "del fg_gdf\n",
+ "del fg2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "KrJKjXy2KLos"
+ },
+ "source": [
+ "## Twitter\n",
+ "\n",
+ "- edges: 2420766\n",
+ "- nodes: 81306"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "fO2qasGqpubr",
+ "outputId": "d41a110e-9f7c-4710-9ce3-3f4906ab02ae"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "-KBGLexek5tS",
- "outputId": "2f462e6c-578a-4fa1-ec29-91bae753f4c5"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(4039, 1) (88234, 2)\n",
- "(3829, 1) (86074, 2)\n",
- "CPU times: user 15.4 s, sys: 808 ms, total: 16.2 s\n",
- "Wall time: 16.2 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "for i in range(50):\n",
- " fg2 = fg.chain([n({'id': 0}), e_forward(hops=5)])\n",
- "print(fg._nodes.shape, fg._edges.shape)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "--2023-12-25 21:58:27-- https://snap.stanford.edu/data/twitter_combined.txt.gz\n",
+ "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n",
+ "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n",
+ "HTTP request sent, awaiting response... 200 OK\n",
+ "Length: 10621918 (10M) [application/x-gzip]\n",
+ "Saving to: ‘twitter_combined.txt.gz’\n",
+ "\n",
+ "twitter_combined.tx 100%[===================>] 10.13M 3.00MB/s in 4.0s \n",
+ "\n",
+ "2023-12-25 21:58:32 (2.52 MB/s) - ‘twitter_combined.txt.gz’ saved [10621918/10621918]\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! wget 'https://snap.stanford.edu/data/twitter_combined.txt.gz'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "id": "fn7zeA3SGlEo"
+ },
+ "outputs": [],
+ "source": [
+ "#! gunzip twitter_combined.txt.gz"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "68TAZkhLGz9g",
+ "outputId": "8ba7c23d-267f-4b59-d6c6-b3f66caec9cf"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "CVpcbhpdHFEF",
- "outputId": "aba04ee1-781e-4226-b593-b42415a55fc4"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(4039, 1) (88234, 2)\n",
- "(3829, 1) (86074, 2)\n",
- "CPU times: user 9.82 s, sys: 133 ms, total: 9.95 s\n",
- "Wall time: 10.1 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "for i in range(50):\n",
- " fg2 = fg_gdf.chain([n({'id': 0}), e_forward(hops=5)])\n",
- "print(fg._nodes.shape, fg._edges.shape)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)\n",
- "del fg_gdf\n",
- "del fg2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "214328887 34428380\n",
+ "17116707 28465635\n",
+ "380580781 18996905\n",
+ "221036078 153460275\n",
+ "107830991 17868918\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! head -n 5 twitter_combined.txt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "QU2wNeGXG2GC",
+ "outputId": "349ac9c0-6f6c-4ce6-fec0-8bae75fca635"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "1cFIyJF9pLjE",
- "outputId": "107329af-8e4b-428c-8b03-77ed00bdf5bf"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 11.8 s, sys: 377 ms, total: 12.1 s\n",
- "Wall time: 13.1 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "for i in range(100):\n",
- " fg2 = fg.chain([e_forward(source_node_match={'id': 0}, hops=5)])"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 474 ms, sys: 61.9 ms, total: 536 ms\n",
+ "Wall time: 534 ms\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "M5uRiD6uJVNW",
- "outputId": "5e938a19-2992-4280-80c2-784382d40113"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(4039, 1) (88234, 2)\n",
- "(348, 1) (347, 2)\n",
- "CPU times: user 14.1 s, sys: 48.5 ms, total: 14.2 s\n",
- "Wall time: 14.2 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "fg_gdf = fg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "for i in range(100):\n",
- " fg2 = fg_gdf.chain([e_forward(source_node_match={'id': 0}, hops=5)])\n",
- "print(fg._nodes.shape, fg._edges.shape)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)\n",
- "del fg_gdf\n",
- "del fg2\n",
- "\n"
+ "data": {
+ "text/plain": [
+ "(2420766, 2)"
]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "te_df = pd.read_csv('twitter_combined.txt', sep=' ', names=['s', 'd'])\n",
+ "te_df.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {
+ "id": "EK5gQH2iG5UU"
+ },
+ "outputs": [],
+ "source": [
+ "import graphistry"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "ZtIW-eFGG_R4",
+ "outputId": "0686e9b3-b684-4b93-da03-289244394338"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Y9vgzfT69x41",
- "outputId": "6882c1ce-0df8-4087-dda4-0a105a8617e1"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(1519, 1) (4060, 2)\n",
- "CPU times: user 4.5 s, sys: 1.35 s, total: 5.85 s\n",
- "Wall time: 6.09 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({fg._node: [0]})\n",
- "for i in range(100):\n",
- " fg2 = fg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 86.4 ms, sys: 106 ms, total: 193 ms\n",
+ "Wall time: 191 ms\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 18,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "c7ybJqjc-T31",
- "outputId": "37ccc1fb-6460-4193-8aa7-22837ff06d0a"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(1519, 1) (4060, 2)\n",
- "CPU times: user 2.58 s, sys: 6.75 ms, total: 2.59 s\n",
- "Wall time: 2.58 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({fg._node: [0]})\n",
- "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n",
- "for i in range(100):\n",
- " fg2 = fg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)\n",
- "del start_nodes\n",
- "del fg_gdf\n",
- "del fg2"
+ "data": {
+ "text/plain": [
+ "(81306, 1)"
]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "g = graphistry.edges(te_df, 's', 'd').materialize_nodes()\n",
+ "g._nodes.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "yUaRfw4FHGMb",
+ "outputId": "3945cc5a-c36c-451b-ac95-8af992a3546f"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Dy7a4zDZ-7_G",
- "outputId": "077b5d9c-c9ae-411a-8228-3c026b07a910"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(3829, 1) (86074, 2)\n",
- "CPU times: user 13.2 s, sys: 2 s, total: 15.2 s\n",
- "Wall time: 18.3 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({fg._node: [0]})\n",
- "for i in range(100):\n",
- " fg2 = fg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=5)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 11.8 s, sys: 8.4 s, total: 20.2 s\n",
+ "Wall time: 23 s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "N5aUtF1a--ML",
- "outputId": "0c2b67b8-fac6-45b3-dfbe-8002b5506e91"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(3829, 1) (86074, 2)\n",
- "CPU times: user 5.72 s, sys: 159 ms, total: 5.88 s\n",
- "Wall time: 5.86 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({fg._node: [0]})\n",
- "fg_gdf = fg.nodes(cudf.from_pandas(fg._nodes)).edges(cudf.from_pandas(fg._edges))\n",
- "for i in range(100):\n",
- " fg2 = fg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=5)\n",
- "print(fg2._nodes.shape, fg2._edges.shape)\n",
- "del start_nodes\n",
- "del fg_gdf\n",
- "del fg2"
+ "data": {
+ "text/plain": [
+ "((140, 1), (615, 2))"
]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "for i in range(10):\n",
+ " g2 = g.gfql([n({'id': 17116707}), e_forward(hops=1)])\n",
+ "g2._nodes.shape, g2._edges.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "5hM4NBu2_eks",
+ "outputId": "54505262-4871-44ee-e5e4-ad7ab32c13c2"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "KrJKjXy2KLos"
- },
- "source": [
- "## Twitter\n",
- "\n",
- "- edges: 2420766\n",
- "- nodes: 81306"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(140, 1)\n",
+ "CPU times: user 1.33 s, sys: 46.6 ms, total: 1.38 s\n",
+ "Wall time: 1.63 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "for i in range(10):\n",
+ " out = g_gdf.gfql([n({'id': 17116707}), e_forward(hops=1)])._nodes\n",
+ "print(out.shape)\n",
+ "del g_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "m2-MxD5lHX6u",
+ "outputId": "e89b9d4b-6c04-45c7-9e7f-cbdbbe0a4730"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 21,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "fO2qasGqpubr",
- "outputId": "d41a110e-9f7c-4710-9ce3-3f4906ab02ae"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--2023-12-25 21:58:27-- https://snap.stanford.edu/data/twitter_combined.txt.gz\n",
- "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n",
- "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n",
- "HTTP request sent, awaiting response... 200 OK\n",
- "Length: 10621918 (10M) [application/x-gzip]\n",
- "Saving to: ‘twitter_combined.txt.gz’\n",
- "\n",
- "twitter_combined.tx 100%[===================>] 10.13M 3.00MB/s in 4.0s \n",
- "\n",
- "2023-12-25 21:58:32 (2.52 MB/s) - ‘twitter_combined.txt.gz’ saved [10621918/10621918]\n",
- "\n"
- ]
- }
- ],
- "source": [
- "#! wget 'https://snap.stanford.edu/data/twitter_combined.txt.gz'"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(2345, 1) (68536, 2)\n",
+ "CPU times: user 13.3 s, sys: 8.05 s, total: 21.4 s\n",
+ "Wall time: 21.6 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "for i in range(10):\n",
+ " out = g.gfql([n({'id': 17116707}), e_forward(hops=2)])\n",
+ "print(out._nodes.shape, out._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "7EQSRbIqLaGw",
+ "outputId": "60c00a03-9e7b-46b5-fce3-f4f567a09430"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 22,
- "metadata": {
- "id": "fn7zeA3SGlEo"
- },
- "outputs": [],
- "source": [
- "#! gunzip twitter_combined.txt.gz"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(2345, 1)\n",
+ "CPU times: user 1.67 s, sys: 55.8 ms, total: 1.72 s\n",
+ "Wall time: 1.75 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "for i in range(10):\n",
+ " out = g_gdf.gfql([n({'id': 17116707}), e_forward(hops=2)])._nodes\n",
+ "print(out.shape)\n",
+ "del g_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "hh6WnjI3ITpB",
+ "outputId": "33138efe-a581-49ed-b2b4-247f8e9bdc09"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 24,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "68TAZkhLGz9g",
- "outputId": "8ba7c23d-267f-4b59-d6c6-b3f66caec9cf"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "214328887 34428380\n",
- "17116707 28465635\n",
- "380580781 18996905\n",
- "221036078 153460275\n",
- "107830991 17868918\n"
- ]
- }
- ],
- "source": [
- "#! head -n 5 twitter_combined.txt"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(81304, 1) (2417796, 2)\n",
+ "CPU times: user 1min 56s, sys: 17.1 s, total: 2min 13s\n",
+ "Wall time: 2min 22s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "for i in range(10):\n",
+ " out = g.gfql([n({'id': 17116707}), e_forward(hops=8)])\n",
+ "print(out._nodes.shape, out._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "7jFFVUenM87j",
+ "outputId": "2cceb720-9de3-488e-8b74-b820fd06e6c1"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 25,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "QU2wNeGXG2GC",
- "outputId": "349ac9c0-6f6c-4ce6-fec0-8bae75fca635"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 474 ms, sys: 61.9 ms, total: 536 ms\n",
- "Wall time: 534 ms\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "(2420766, 2)"
- ]
- },
- "execution_count": 25,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "te_df = pd.read_csv('twitter_combined.txt', sep=' ', names=['s', 'd'])\n",
- "te_df.shape"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(81304, 1)\n",
+ "CPU times: user 5.3 s, sys: 1.48 s, total: 6.78 s\n",
+ "Wall time: 7.89 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "for i in range(10):\n",
+ " out = g_gdf.gfql([n({'id': 17116707}), e_forward(hops=8)])._nodes\n",
+ "print(out.shape)\n",
+ "del g_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "_5LD0bZB_lU4",
+ "outputId": "bc31bd03-e79f-46d2-ea8f-3b01d9ef39a2"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 26,
- "metadata": {
- "id": "EK5gQH2iG5UU"
- },
- "outputs": [],
- "source": [
- "import graphistry"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(0, 1) (0, 2)\n",
+ "CPU times: user 2.58 s, sys: 1.59 s, total: 4.17 s\n",
+ "Wall time: 6.02 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({g._node: [17116707]})\n",
+ "for i in range(10):\n",
+ " g2 = g.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=1)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "M_rHjqtvACQw",
+ "outputId": "8d3e308e-b1e2-452b-f402-573be0dd5b58"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 27,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "ZtIW-eFGG_R4",
- "outputId": "0686e9b3-b684-4b93-da03-289244394338"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 86.4 ms, sys: 106 ms, total: 193 ms\n",
- "Wall time: 191 ms\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "(81306, 1)"
- ]
- },
- "execution_count": 27,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "g = graphistry.edges(te_df, 's', 'd').materialize_nodes()\n",
- "g._nodes.shape"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(61827, 1) (1473599, 2)\n",
+ "CPU times: user 822 ms, sys: 179 ms, total: 1 s\n",
+ "Wall time: 997 ms\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({g._node: [17116707]})\n",
+ "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n",
+ "for i in range(10):\n",
+ " g2 = g_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=5)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del g_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "0zEIucaCAbj_",
+ "outputId": "83e64b0f-2b3a-4e4b-d189-3e6a8ef78f53"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 29,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "yUaRfw4FHGMb",
- "outputId": "3945cc5a-c36c-451b-ac95-8af992a3546f"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 11.8 s, sys: 8.4 s, total: 20.2 s\n",
- "Wall time: 23 s\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "((140, 1), (615, 2))"
- ]
- },
- "execution_count": 29,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "for i in range(10):\n",
- " g2 = g.chain([n({'id': 17116707}), e_forward(hops=1)])\n",
- "g2._nodes.shape, g2._edges.shape"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(2345, 1) (68536, 2)\n",
+ "CPU times: user 8.93 s, sys: 5.92 s, total: 14.9 s\n",
+ "Wall time: 15.8 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({g._node: [17116707]})\n",
+ "for i in range(10):\n",
+ " g2 = g.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "LKJh5gRtAdIj",
+ "outputId": "e3c7883d-74c0-4d55-b238-88457296c6bc"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 30,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "5hM4NBu2_eks",
- "outputId": "54505262-4871-44ee-e5e4-ad7ab32c13c2"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(140, 1)\n",
- "CPU times: user 1.33 s, sys: 46.6 ms, total: 1.38 s\n",
- "Wall time: 1.63 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "for i in range(10):\n",
- " out = g_gdf.chain([n({'id': 17116707}), e_forward(hops=1)])._nodes\n",
- "print(out.shape)\n",
- "del g_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(2345, 1) (68536, 2)\n",
+ "CPU times: user 374 ms, sys: 6.92 ms, total: 381 ms\n",
+ "Wall time: 379 ms\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({g._node: [17116707]})\n",
+ "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n",
+ "for i in range(10):\n",
+ " g2 = g_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del g_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "JZwxdofNAfmb",
+ "outputId": "2731be4c-75d9-47f4-8602-4f2d6cb2ddac"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 31,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "m2-MxD5lHX6u",
- "outputId": "e89b9d4b-6c04-45c7-9e7f-cbdbbe0a4730"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(2345, 1) (68536, 2)\n",
- "CPU times: user 13.3 s, sys: 8.05 s, total: 21.4 s\n",
- "Wall time: 21.6 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "for i in range(10):\n",
- " out = g.chain([n({'id': 17116707}), e_forward(hops=2)])\n",
- "print(out._nodes.shape, out._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(81304, 1) (2417796, 2)\n",
+ "CPU times: user 38.8 s, sys: 8.7 s, total: 47.5 s\n",
+ "Wall time: 48.2 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({g._node: [17116707]})\n",
+ "for i in range(10):\n",
+ " g2 = g.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=8)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "9o_og8bSAhe3",
+ "outputId": "dd3e4f8f-f426-4705-98c4-60f1912ba28a"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 36,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "7EQSRbIqLaGw",
- "outputId": "60c00a03-9e7b-46b5-fce3-f4f567a09430"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(2345, 1)\n",
- "CPU times: user 1.67 s, sys: 55.8 ms, total: 1.72 s\n",
- "Wall time: 1.75 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "for i in range(10):\n",
- " out = g_gdf.chain([n({'id': 17116707}), e_forward(hops=2)])._nodes\n",
- "print(out.shape)\n",
- "del g_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(81304, 1) (2417796, 2)\n",
+ "CPU times: user 1.8 s, sys: 506 ms, total: 2.3 s\n",
+ "Wall time: 2.3 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({g._node: [17116707]})\n",
+ "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n",
+ "for i in range(10):\n",
+ " g2 = g_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=8)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del g_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9dZzAAVONCD2"
+ },
+ "source": [
+ "## GPlus\n",
+ "\n",
+ "- edges: 30494866\n",
+ "- nodes: 107614"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "-nhWGNekKpcZ",
+ "outputId": "e2175290-337c-4faa-e5d8-4bc401583326"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 37,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "hh6WnjI3ITpB",
- "outputId": "33138efe-a581-49ed-b2b4-247f8e9bdc09"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(81304, 1) (2417796, 2)\n",
- "CPU times: user 1min 56s, sys: 17.1 s, total: 2min 13s\n",
- "Wall time: 2min 22s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "for i in range(10):\n",
- " out = g.chain([n({'id': 17116707}), e_forward(hops=8)])\n",
- "print(out._nodes.shape, out._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "--2023-12-26 18:36:29-- https://snap.stanford.edu/data/gplus_combined.txt.gz\n",
+ "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n",
+ "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n",
+ "HTTP request sent, awaiting response... 200 OK\n",
+ "Length: 398930514 (380M) [application/x-gzip]\n",
+ "Saving to: ‘gplus_combined.txt.gz’\n",
+ "\n",
+ "gplus_combined.txt. 100%[===================>] 380.45M 34.7MB/s in 9.6s \n",
+ "\n",
+ "2023-12-26 18:36:39 (39.7 MB/s) - ‘gplus_combined.txt.gz’ saved [398930514/398930514]\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! wget https://snap.stanford.edu/data/gplus_combined.txt.gz"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "id": "g5wgA_c2KqwJ"
+ },
+ "outputs": [],
+ "source": [
+ "#! gunzip gplus_combined.txt.gz"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 258
},
+ "id": "52hgDbr0Kti6",
+ "outputId": "217203fc-7095-4784-c4c4-d46ee9c78808"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 38,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "7jFFVUenM87j",
- "outputId": "2cceb720-9de3-488e-8b74-b820fd06e6c1"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(81304, 1)\n",
- "CPU times: user 5.3 s, sys: 1.48 s, total: 6.78 s\n",
- "Wall time: 7.89 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "g_gdf = g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "for i in range(10):\n",
- " out = g_gdf.chain([n({'id': 17116707}), e_forward(hops=8)])._nodes\n",
- "print(out.shape)\n",
- "del g_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(30494866, 2)\n",
+ "CPU times: user 16 s, sys: 1.45 s, total: 17.5 s\n",
+ "Wall time: 22.5 s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 39,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "_5LD0bZB_lU4",
- "outputId": "bc31bd03-e79f-46d2-ea8f-3b01d9ef39a2"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(0, 1) (0, 2)\n",
- "CPU times: user 2.58 s, sys: 1.59 s, total: 4.17 s\n",
- "Wall time: 6.02 s\n"
- ]
- }
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " s | \n",
+ " d | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 116374117927631468606 | \n",
+ " 101765416973555767821 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 112188647432305746617 | \n",
+ " 107727150903234299458 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 116719211656774388392 | \n",
+ " 100432456209427807893 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 117421021456205115327 | \n",
+ " 101096322838605097368 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 116407635616074189669 | \n",
+ " 113556266482860931616 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({g._node: [17116707]})\n",
- "for i in range(10):\n",
- " g2 = g.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=1)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
+ "text/plain": [
+ " s d\n",
+ "0 116374117927631468606 101765416973555767821\n",
+ "1 112188647432305746617 107727150903234299458\n",
+ "2 116719211656774388392 100432456209427807893\n",
+ "3 117421021456205115327 101096322838605097368\n",
+ "4 116407635616074189669 113556266482860931616"
]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "ge_df = pd.read_csv('gplus_combined.txt', sep=' ', names=['s', 'd'])\n",
+ "print(ge_df.shape)\n",
+ "ge_df.head(5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 258
},
+ "id": "w5YkN-nLK6UV",
+ "outputId": "dc98380d-54c2-4b36-c56e-5e8401c4ffa4"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 44,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "M_rHjqtvACQw",
- "outputId": "8d3e308e-b1e2-452b-f402-573be0dd5b58"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(61827, 1) (1473599, 2)\n",
- "CPU times: user 822 ms, sys: 179 ms, total: 1 s\n",
- "Wall time: 997 ms\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({g._node: [17116707]})\n",
- "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n",
- "for i in range(10):\n",
- " g2 = g_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=5)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del g_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(30494866, 2) (107614, 1)\n",
+ "CPU times: user 4.49 s, sys: 1.25 s, total: 5.74 s\n",
+ "Wall time: 5.97 s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 40,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "0zEIucaCAbj_",
- "outputId": "83e64b0f-2b3a-4e4b-d189-3e6a8ef78f53"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(2345, 1) (68536, 2)\n",
- "CPU times: user 8.93 s, sys: 5.92 s, total: 14.9 s\n",
- "Wall time: 15.8 s\n"
- ]
- }
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 116374117927631468606 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 112188647432305746617 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 116719211656774388392 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 117421021456205115327 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 116407635616074189669 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({g._node: [17116707]})\n",
- "for i in range(10):\n",
- " g2 = g.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
+ "text/plain": [
+ " id\n",
+ "0 116374117927631468606\n",
+ "1 112188647432305746617\n",
+ "2 116719211656774388392\n",
+ "3 117421021456205115327\n",
+ "4 116407635616074189669"
]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg = graphistry.edges(ge_df, 's', 'd').materialize_nodes()\n",
+ "gg = graphistry.edges(ge_df, 's', 'd').nodes(gg._nodes, 'id')\n",
+ "print(gg._edges.shape, gg._nodes.shape)\n",
+ "gg._nodes.head(5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 115
},
+ "id": "NKtz54uELX-8",
+ "outputId": "5d8f3eef-893d-47cc-e7a9-c5cbfec8270c"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 41,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "LKJh5gRtAdIj",
- "outputId": "e3c7883d-74c0-4d55-b238-88457296c6bc"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(2345, 1) (68536, 2)\n",
- "CPU times: user 374 ms, sys: 6.92 ms, total: 381 ms\n",
- "Wall time: 379 ms\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({g._node: [17116707]})\n",
- "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n",
- "for i in range(10):\n",
- " g2 = g_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del g_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 534 ms, sys: 598 ms, total: 1.13 s\n",
+ "Wall time: 1.65 s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 42,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "JZwxdofNAfmb",
- "outputId": "2731be4c-75d9-47f4-8602-4f2d6cb2ddac"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(81304, 1) (2417796, 2)\n",
- "CPU times: user 38.8 s, sys: 8.7 s, total: 47.5 s\n",
- "Wall time: 48.2 s\n"
- ]
- }
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 116374117927631468606 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({g._node: [17116707]})\n",
- "for i in range(10):\n",
- " g2 = g.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=8)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
+ "text/plain": [
+ " id\n",
+ "0 116374117927631468606"
]
+ },
+ "execution_count": 49,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg.gfql([ n({'id': '116374117927631468606'})])._nodes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "iNWdi00VLmZG",
+ "outputId": "ecfb56a6-c564-4bf6-f43f-2c95a103f4be"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 43,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "9o_og8bSAhe3",
- "outputId": "dd3e4f8f-f426-4705-98c4-60f1912ba28a"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(81304, 1) (2417796, 2)\n",
- "CPU times: user 1.8 s, sys: 506 ms, total: 2.3 s\n",
- "Wall time: 2.3 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({g._node: [17116707]})\n",
- "g_gdf = g.nodes(cudf.from_pandas(g._nodes)).edges(cudf.from_pandas(g._edges))\n",
- "for i in range(10):\n",
- " g2 = g_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=8)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del g_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 27.5 s, sys: 11.1 s, total: 38.5 s\n",
+ "Wall time: 39.5 s\n"
+ ]
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "9dZzAAVONCD2"
- },
- "source": [
- "## GPlus\n",
- "\n",
- "- edges: 30494866\n",
- "- nodes: 107614"
+ "data": {
+ "text/plain": [
+ "(1473, 1)"
]
+ },
+ "execution_count": 75,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=1)])._nodes\n",
+ "out.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Q6p3h6uCOABh",
+ "outputId": "817fc80f-ef5d-4070-eb48-a12344be709c"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "-nhWGNekKpcZ",
- "outputId": "e2175290-337c-4faa-e5d8-4bc401583326"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--2023-12-26 18:36:29-- https://snap.stanford.edu/data/gplus_combined.txt.gz\n",
- "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n",
- "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n",
- "HTTP request sent, awaiting response... 200 OK\n",
- "Length: 398930514 (380M) [application/x-gzip]\n",
- "Saving to: ‘gplus_combined.txt.gz’\n",
- "\n",
- "gplus_combined.txt. 100%[===================>] 380.45M 34.7MB/s in 9.6s \n",
- "\n",
- "2023-12-26 18:36:39 (39.7 MB/s) - ‘gplus_combined.txt.gz’ saved [398930514/398930514]\n",
- "\n"
- ]
- }
- ],
- "source": [
- "#! wget https://snap.stanford.edu/data/gplus_combined.txt.gz"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(1473, 1) (13375, 2)\n",
+ "CPU times: user 4.57 s, sys: 2.11 s, total: 6.68 s\n",
+ "Wall time: 7.63 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=1)])\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del gg_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "6UdCcMdqLw-P",
+ "outputId": "70742c79-b22b-4db2-c548-cb1e25d572eb"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "id": "g5wgA_c2KqwJ"
- },
- "outputs": [],
- "source": [
- "#! gunzip gplus_combined.txt.gz"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 45.8 s, sys: 17 s, total: 1min 2s\n",
+ "Wall time: 1min 5s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 258
- },
- "id": "52hgDbr0Kti6",
- "outputId": "217203fc-7095-4784-c4c4-d46ee9c78808"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(30494866, 2)\n",
- "CPU times: user 16 s, sys: 1.45 s, total: 17.5 s\n",
- "Wall time: 22.5 s\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " s | \n",
- " d | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 116374117927631468606 | \n",
- " 101765416973555767821 | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 112188647432305746617 | \n",
- " 107727150903234299458 | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 116719211656774388392 | \n",
- " 100432456209427807893 | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 117421021456205115327 | \n",
- " 101096322838605097368 | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 116407635616074189669 | \n",
- " 113556266482860931616 | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
\n",
- "
\n"
- ],
- "text/plain": [
- " s d\n",
- "0 116374117927631468606 101765416973555767821\n",
- "1 112188647432305746617 107727150903234299458\n",
- "2 116719211656774388392 100432456209427807893\n",
- "3 117421021456205115327 101096322838605097368\n",
- "4 116407635616074189669 113556266482860931616"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "ge_df = pd.read_csv('gplus_combined.txt', sep=' ', names=['s', 'd'])\n",
- "print(ge_df.shape)\n",
- "ge_df.head(5)"
+ "data": {
+ "text/plain": [
+ "(44073, 1)"
]
+ },
+ "execution_count": 77,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=2)])._nodes\n",
+ "out.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "QElqatDyNYCS",
+ "outputId": "0e15bd3e-d2d9-4965-df7d-c8856d036680"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 258
- },
- "id": "w5YkN-nLK6UV",
- "outputId": "dc98380d-54c2-4b36-c56e-5e8401c4ffa4"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(30494866, 2) (107614, 1)\n",
- "CPU times: user 4.49 s, sys: 1.25 s, total: 5.74 s\n",
- "Wall time: 5.97 s\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 116374117927631468606 | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 112188647432305746617 | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 116719211656774388392 | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 117421021456205115327 | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 116407635616074189669 | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
\n",
- "
\n"
- ],
- "text/plain": [
- " id\n",
- "0 116374117927631468606\n",
- "1 112188647432305746617\n",
- "2 116719211656774388392\n",
- "3 117421021456205115327\n",
- "4 116407635616074189669"
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "gg = graphistry.edges(ge_df, 's', 'd').materialize_nodes()\n",
- "gg = graphistry.edges(ge_df, 's', 'd').nodes(gg._nodes, 'id')\n",
- "print(gg._edges.shape, gg._nodes.shape)\n",
- "gg._nodes.head(5)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(44073, 1) (2069325, 2)\n",
+ "CPU times: user 4.97 s, sys: 2.36 s, total: 7.34 s\n",
+ "Wall time: 10.6 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=2)])\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del gg_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "3HJOItZ4MQMG",
+ "outputId": "f5be7bb4-7f09-4f80-c549-e703e99f5067"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 49,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 115
- },
- "id": "NKtz54uELX-8",
- "outputId": "5d8f3eef-893d-47cc-e7a9-c5cbfec8270c"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 534 ms, sys: 598 ms, total: 1.13 s\n",
- "Wall time: 1.65 s\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 116374117927631468606 | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
\n",
- "
\n"
- ],
- "text/plain": [
- " id\n",
- "0 116374117927631468606"
- ]
- },
- "execution_count": 49,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "gg.chain([ n({'id': '116374117927631468606'})])._nodes"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 3min 45s, sys: 1min 5s, total: 4min 50s\n",
+ "Wall time: 4min 52s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "iNWdi00VLmZG",
- "outputId": "ecfb56a6-c564-4bf6-f43f-2c95a103f4be"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 27.5 s, sys: 11.1 s, total: 38.5 s\n",
- "Wall time: 39.5 s\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "(1473, 1)"
- ]
- },
- "execution_count": 75,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=1)])._nodes\n",
- "out.shape"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Q6p3h6uCOABh",
- "outputId": "817fc80f-ef5d-4070-eb48-a12344be709c"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(1473, 1) (13375, 2)\n",
- "CPU times: user 4.57 s, sys: 2.11 s, total: 6.68 s\n",
- "Wall time: 7.63 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=1)])\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del gg_gdf\n",
- "del out"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "6UdCcMdqLw-P",
- "outputId": "70742c79-b22b-4db2-c548-cb1e25d572eb"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 45.8 s, sys: 17 s, total: 1min 2s\n",
- "Wall time: 1min 5s\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "(44073, 1)"
- ]
- },
- "execution_count": 77,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=2)])._nodes\n",
- "out.shape"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "QElqatDyNYCS",
- "outputId": "0e15bd3e-d2d9-4965-df7d-c8856d036680"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(44073, 1) (2069325, 2)\n",
- "CPU times: user 4.97 s, sys: 2.36 s, total: 7.34 s\n",
- "Wall time: 10.6 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=2)])\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del gg_gdf\n",
- "del out"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "3HJOItZ4MQMG",
- "outputId": "f5be7bb4-7f09-4f80-c549-e703e99f5067"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 3min 45s, sys: 1min 5s, total: 4min 50s\n",
- "Wall time: 4min 52s\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "(102414, 1)"
- ]
- },
- "execution_count": 79,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=3)])._nodes\n",
- "out.shape"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "G32t_xthOUle",
- "outputId": "7721741f-9c86-41aa-eb0b-2c8f0db2ed54"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(102414, 1) (24851333, 2)\n",
- "CPU times: user 6.95 s, sys: 2.63 s, total: 9.57 s\n",
- "Wall time: 9.84 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=3)])\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del gg_gdf\n",
- "del out"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "bXy2yyJsMsEG",
- "outputId": "911f2680-067c-44f2-9ba2-7f27d3c9bc6b"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105479, 1) (30450354, 2)\n",
- "CPU times: user 4min 36s, sys: 1min 25s, total: 6min 2s\n",
- "Wall time: 6min 4s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n",
- "print(out._nodes.shape, out._edges.shape)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Vt8hhjWDP_W_",
- "outputId": "824ae644-e1cf-4239-bda9-84aecde52ad8"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105479, 1) (30450354, 2)\n",
- "CPU times: user 7.44 s, sys: 2.45 s, total: 9.88 s\n",
- "Wall time: 9.9 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del gg_gdf\n",
- "del out"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "_z4KpNZaOH8t",
- "outputId": "2417f78b-e1b7-452d-8e26-7df259620c88"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105604, 1) (30468335, 2)\n",
- "CPU times: user 5min 36s, sys: 1min 39s, total: 7min 16s\n",
- "Wall time: 7min 15s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "out = gg.chain([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n",
- "print(out._nodes.shape, out._edges.shape)"
+ "data": {
+ "text/plain": [
+ "(102414, 1)"
]
+ },
+ "execution_count": 79,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=3)])._nodes\n",
+ "out.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "G32t_xthOUle",
+ "outputId": "7721741f-9c86-41aa-eb0b-2c8f0db2ed54"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "spUBH9EHSz2O",
- "outputId": "22340ce3-e8d4-4a72-b485-9839c667b965"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105604, 1) (30468335, 2)\n",
- "CPU times: user 8.82 s, sys: 2.71 s, total: 11.5 s\n",
- "Wall time: 11.9 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "out = gg_gdf.chain([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del gg_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(102414, 1) (24851333, 2)\n",
+ "CPU times: user 6.95 s, sys: 2.63 s, total: 9.57 s\n",
+ "Wall time: 9.84 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=3)])\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del gg_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "bXy2yyJsMsEG",
+ "outputId": "911f2680-067c-44f2-9ba2-7f27d3c9bc6b"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 50,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "vCsdmc62A7OM",
- "outputId": "adc05d29-c628-49ed-cd6d-8921c6dcd206"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(1473, 1) (13375, 2)\n",
- "CPU times: user 19.9 s, sys: 9.36 s, total: 29.2 s\n",
- "Wall time: 41.8 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
- "for i in range(1):\n",
- " g2 = gg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=1)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105479, 1) (30450354, 2)\n",
+ "CPU times: user 4min 36s, sys: 1min 25s, total: 6min 2s\n",
+ "Wall time: 6min 4s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n",
+ "print(out._nodes.shape, out._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Vt8hhjWDP_W_",
+ "outputId": "824ae644-e1cf-4239-bda9-84aecde52ad8"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 52,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "J3kV8NBYBQdW",
- "outputId": "76073248-43e1-4c3c-c004-67324cc1d312"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(1473, 1) (13375, 2)\n",
- "CPU times: user 3.71 s, sys: 2.09 s, total: 5.8 s\n",
- "Wall time: 6.05 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
- "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
- "for i in range(1):\n",
- " g2 = gg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=1)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del gg_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105479, 1) (30450354, 2)\n",
+ "CPU times: user 7.44 s, sys: 2.45 s, total: 9.88 s\n",
+ "Wall time: 9.9 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=4)])\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del gg_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "_z4KpNZaOH8t",
+ "outputId": "2417f78b-e1b7-452d-8e26-7df259620c88"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 53,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "ONv1RQeWBeeK",
- "outputId": "58d57fa4-be72-45bc-abfa-5de9d1102f55"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(44073, 1) (2069325, 2)\n",
- "CPU times: user 27.8 s, sys: 13.2 s, total: 41 s\n",
- "Wall time: 43.9 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
- "for i in range(1):\n",
- " g2 = gg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105604, 1) (30468335, 2)\n",
+ "CPU times: user 5min 36s, sys: 1min 39s, total: 7min 16s\n",
+ "Wall time: 7min 15s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "out = gg.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n",
+ "print(out._nodes.shape, out._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "spUBH9EHSz2O",
+ "outputId": "22340ce3-e8d4-4a72-b485-9839c667b965"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 54,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "ke5SZZ01BgqR",
- "outputId": "4173fd28-a11b-4300-d28b-6fdb87e8e9f3"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(44073, 1) (2069325, 2)\n",
- "CPU times: user 4.26 s, sys: 2.37 s, total: 6.63 s\n",
- "Wall time: 7.91 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
- "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
- "for i in range(1):\n",
- " g2 = gg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del gg_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105604, 1) (30468335, 2)\n",
+ "CPU times: user 8.82 s, sys: 2.71 s, total: 11.5 s\n",
+ "Wall time: 11.9 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "gg_gdf = gg.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "out = gg_gdf.gfql([ n({'id': '116374117927631468606'}), e_forward(hops=5)])\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del gg_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "vCsdmc62A7OM",
+ "outputId": "adc05d29-c628-49ed-cd6d-8921c6dcd206"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 55,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "U795pIBUBiZV",
- "outputId": "d499433c-cc0c-4bbf-c69f-36b5d55402d9"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(102414, 1) (24851333, 2)\n",
- "CPU times: user 1min 3s, sys: 22.7 s, total: 1min 26s\n",
- "Wall time: 1min 35s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
- "for i in range(1):\n",
- " g2 = gg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=3)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(1473, 1) (13375, 2)\n",
+ "CPU times: user 19.9 s, sys: 9.36 s, total: 29.2 s\n",
+ "Wall time: 41.8 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "for i in range(1):\n",
+ " g2 = gg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=1)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "J3kV8NBYBQdW",
+ "outputId": "76073248-43e1-4c3c-c004-67324cc1d312"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 56,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "kIZYwSe1Bj2e",
- "outputId": "b7e1ed9f-47d1-412e-9593-ecc436ac1486"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(102414, 1) (24851333, 2)\n",
- "CPU times: user 3.96 s, sys: 2.11 s, total: 6.07 s\n",
- "Wall time: 6.05 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
- "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
- "for i in range(1):\n",
- " g2 = gg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=3)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del gg_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(1473, 1) (13375, 2)\n",
+ "CPU times: user 3.71 s, sys: 2.09 s, total: 5.8 s\n",
+ "Wall time: 6.05 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
+ "for i in range(1):\n",
+ " g2 = gg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=1)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del gg_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "ONv1RQeWBeeK",
+ "outputId": "58d57fa4-be72-45bc-abfa-5de9d1102f55"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 57,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "YTI5sD6YBpYL",
- "outputId": "b37bf2df-07dc-404c-8a83-a83f28e38bf6"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105479, 1) (30450354, 2)\n",
- "CPU times: user 1min 34s, sys: 30.6 s, total: 2min 5s\n",
- "Wall time: 2min 5s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
- "for i in range(1):\n",
- " g2 = gg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=4)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(44073, 1) (2069325, 2)\n",
+ "CPU times: user 27.8 s, sys: 13.2 s, total: 41 s\n",
+ "Wall time: 43.9 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "for i in range(1):\n",
+ " g2 = gg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "ke5SZZ01BgqR",
+ "outputId": "4173fd28-a11b-4300-d28b-6fdb87e8e9f3"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 58,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "d5WBazICBrSz",
- "outputId": "ef95e893-3a0f-4d47-ede4-bd8a6faebf98"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105479, 1) (30450354, 2)\n",
- "CPU times: user 5.25 s, sys: 2.41 s, total: 7.67 s\n",
- "Wall time: 7.69 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
- "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
- "for i in range(1):\n",
- " g2 = gg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=4)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del gg_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(44073, 1) (2069325, 2)\n",
+ "CPU times: user 4.26 s, sys: 2.37 s, total: 6.63 s\n",
+ "Wall time: 7.91 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
+ "for i in range(1):\n",
+ " g2 = gg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del gg_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "U795pIBUBiZV",
+ "outputId": "d499433c-cc0c-4bbf-c69f-36b5d55402d9"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 59,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "ozQlRPaFBtPD",
- "outputId": "4f1655c4-38fd-47f9-942d-836585e0d866"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105604, 1) (30468335, 2)\n",
- "CPU times: user 2min 16s, sys: 39.1 s, total: 2min 55s\n",
- "Wall time: 2min 58s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
- "for i in range(1):\n",
- " g2 = gg.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=5)\n",
- "print(g2._nodes.shape, g2._edges.shape)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(102414, 1) (24851333, 2)\n",
+ "CPU times: user 1min 3s, sys: 22.7 s, total: 1min 26s\n",
+ "Wall time: 1min 35s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "for i in range(1):\n",
+ " g2 = gg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=3)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "kIZYwSe1Bj2e",
+ "outputId": "b7e1ed9f-47d1-412e-9593-ecc436ac1486"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 60,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "-ACkMG20B6HM",
- "outputId": "f26c03a9-9f25-4f93-c7d3-0e8676694040"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(105604, 1) (30468335, 2)\n",
- "CPU times: user 5.79 s, sys: 2.51 s, total: 8.3 s\n",
- "Wall time: 8.29 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
- "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
- "for i in range(1):\n",
- " g2 = gg_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=5)\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del gg_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(102414, 1) (24851333, 2)\n",
+ "CPU times: user 3.96 s, sys: 2.11 s, total: 6.07 s\n",
+ "Wall time: 6.05 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
+ "for i in range(1):\n",
+ " g2 = gg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=3)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del gg_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "YTI5sD6YBpYL",
+ "outputId": "b37bf2df-07dc-404c-8a83-a83f28e38bf6"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "R03M_swxarKC"
- },
- "source": [
- "## Orkut\n",
- "- 117M edges\n",
- "- 3M nodes"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105479, 1) (30450354, 2)\n",
+ "CPU times: user 1min 34s, sys: 30.6 s, total: 2min 5s\n",
+ "Wall time: 2min 5s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "for i in range(1):\n",
+ " g2 = gg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=4)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "d5WBazICBrSz",
+ "outputId": "ef95e893-3a0f-4d47-ede4-bd8a6faebf98"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "QoabYR2maxPo",
- "outputId": "2bb6275d-46bb-42da-ec05-d0e5a58b1f77"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--2023-12-26 00:55:52-- https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz\n",
- "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n",
- "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n",
- "HTTP request sent, awaiting response... 200 OK\n",
- "Length: 447251958 (427M) [application/x-gzip]\n",
- "Saving to: ‘com-orkut.ungraph.txt.gz’\n",
- "\n",
- "com-orkut.ungraph.t 100%[===================>] 426.53M 45.1MB/s in 9.7s \n",
- "\n",
- "2023-12-26 00:56:02 (44.0 MB/s) - ‘com-orkut.ungraph.txt.gz’ saved [447251958/447251958]\n",
- "\n"
- ]
- }
- ],
- "source": [
- "#! wget https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105479, 1) (30450354, 2)\n",
+ "CPU times: user 5.25 s, sys: 2.41 s, total: 7.67 s\n",
+ "Wall time: 7.69 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
+ "for i in range(1):\n",
+ " g2 = gg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=4)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del gg_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "ozQlRPaFBtPD",
+ "outputId": "4f1655c4-38fd-47f9-942d-836585e0d866"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "id": "BvvfFPKWbAVJ"
- },
- "outputs": [],
- "source": [
- "#! gunzip com-orkut.ungraph.txt.gz"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105604, 1) (30468335, 2)\n",
+ "CPU times: user 2min 16s, sys: 39.1 s, total: 2min 55s\n",
+ "Wall time: 2min 58s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "for i in range(1):\n",
+ " g2 = gg.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=5)\n",
+ "print(g2._nodes.shape, g2._edges.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "-ACkMG20B6HM",
+ "outputId": "f26c03a9-9f25-4f93-c7d3-0e8676694040"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "YsWwRoPqbPIb",
- "outputId": "2eb4f862-b4e1-42bf-ff5d-eec10b27cedc"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "# Undirected graph: ../../data/output/orkut.txt\n",
- "# Orkut\n",
- "# Nodes: 3072441 Edges: 117185083\n",
- "# FromNodeId\tToNodeId\n",
- "1\t2\n",
- "1\t3\n",
- "1\t4\n"
- ]
- }
- ],
- "source": [
- "#! head -n 7 com-orkut.ungraph.txt"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(105604, 1) (30468335, 2)\n",
+ "CPU times: user 5.79 s, sys: 2.51 s, total: 8.3 s\n",
+ "Wall time: 8.29 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({gg._node: ['116374117927631468606']})\n",
+ "gg_gdf = gg.nodes(cudf.from_pandas(gg._nodes)).edges(cudf.from_pandas(gg._edges))\n",
+ "for i in range(1):\n",
+ " g2 = gg_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=5)\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del gg_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "R03M_swxarKC"
+ },
+ "source": [
+ "## Orkut\n",
+ "- 117M edges\n",
+ "- 3M nodes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "QoabYR2maxPo",
+ "outputId": "2bb6275d-46bb-42da-ec05-d0e5a58b1f77"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "cbMC8r2ldjbW",
- "outputId": "82688d53-7d56-4563-d65e-7c5cd32ac14e"
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "('23.12.01', '0.32.0+12.g72e778c')"
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import pandas as pd\n",
- "\n",
- "import graphistry\n",
- "\n",
- "from graphistry import (\n",
- "\n",
- " # graph operators\n",
- " n, e_undirected, e_forward, e_reverse,\n",
- "\n",
- " # attribute predicates\n",
- " is_in, ge, startswith, contains, match as match_re\n",
- ")\n",
- "\n",
- "import cudf\n",
- "\n",
- "#work around google colab shell encoding bugs\n",
- "import locale\n",
- "locale.getpreferredencoding = lambda: \"UTF-8\"\n",
- "\n",
- "cudf.__version__, graphistry.__version__"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "--2023-12-26 00:55:52-- https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz\n",
+ "Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80\n",
+ "Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:443... connected.\n",
+ "HTTP request sent, awaiting response... 200 OK\n",
+ "Length: 447251958 (427M) [application/x-gzip]\n",
+ "Saving to: ‘com-orkut.ungraph.txt.gz’\n",
+ "\n",
+ "com-orkut.ungraph.t 100%[===================>] 426.53M 45.1MB/s in 9.7s \n",
+ "\n",
+ "2023-12-26 00:56:02 (44.0 MB/s) - ‘com-orkut.ungraph.txt.gz’ saved [447251958/447251958]\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! wget https://snap.stanford.edu/data/bigdata/communities/com-orkut.ungraph.txt.gz"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "BvvfFPKWbAVJ"
+ },
+ "outputs": [],
+ "source": [
+ "#! gunzip com-orkut.ungraph.txt.gz"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "YsWwRoPqbPIb",
+ "outputId": "2eb4f862-b4e1-42bf-ff5d-eec10b27cedc"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "TopFxAvnh_Cv",
- "outputId": "cc9d9dc9-e594-4190-fe84-3f1b6dce8a1a"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:27 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 47C P0 27W / 70W | 103MiB / 15360MiB | 0% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n"
- ]
- }
- ],
- "source": [
- "#! nvidia-smi"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "# Undirected graph: ../../data/output/orkut.txt\n",
+ "# Orkut\n",
+ "# Nodes: 3072441 Edges: 117185083\n",
+ "# FromNodeId\tToNodeId\n",
+ "1\t2\n",
+ "1\t3\n",
+ "1\t4\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! head -n 7 com-orkut.ungraph.txt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "cbMC8r2ldjbW",
+ "outputId": "82688d53-7d56-4563-d65e-7c5cd32ac14e"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Oczs87ITbJgw",
- "outputId": "ac203ddd-e684-4eb9-a586-f6a49fd1625d"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(117185082, 2)\n",
- " s d\n",
- "0 1 3\n",
- "1 1 4\n",
- "2 1 5\n",
- "3 1 6\n",
- "4 1 7\n",
- "s int64\n",
- "d int64\n",
- "dtype: object\n",
- "CPU times: user 2.56 s, sys: 4.2 s, total: 6.76 s\n",
- "Wall time: 6.76 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "co_df = cudf.read_csv('com-orkut.ungraph.txt', sep='\\t', names=['s', 'd'], skiprows=5).to_pandas()\n",
- "print(co_df.shape)\n",
- "print(co_df.head(5))\n",
- "print(co_df.dtypes)\n",
- "#del co_df"
+ "data": {
+ "text/plain": [
+ "('23.12.01', '0.32.0+12.g72e778c')"
]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "import graphistry\n",
+ "\n",
+ "from graphistry import (\n",
+ "\n",
+ " # graph operators\n",
+ " n, e_undirected, e_forward, e_reverse,\n",
+ "\n",
+ " # attribute predicates\n",
+ " is_in, ge, startswith, contains, match as match_re\n",
+ ")\n",
+ "\n",
+ "import cudf\n",
+ "\n",
+ "#work around google colab shell encoding bugs\n",
+ "import locale\n",
+ "locale.getpreferredencoding = lambda: \"UTF-8\"\n",
+ "\n",
+ "cudf.__version__, graphistry.__version__"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "TopFxAvnh_Cv",
+ "outputId": "cc9d9dc9-e594-4190-fe84-3f1b6dce8a1a"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 258
- },
- "id": "gGSDjTtveFAT",
- "outputId": "e7b38f4f-dc07-4f35-9bab-9c80a80bbf0b"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(3072441, 1) (117185082, 2)\n",
- "CPU times: user 1.96 s, sys: 2.95 s, total: 4.91 s\n",
- "Wall time: 4.92 s\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 1 | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 2 | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 3 | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 4 | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 5 | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
\n",
- "
\n"
- ],
- "text/plain": [
- " id\n",
- "0 1\n",
- "1 2\n",
- "2 3\n",
- "3 4\n",
- "4 5"
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "co_g = graphistry.edges(cudf.DataFrame(co_df), 's', 'd').materialize_nodes(engine='cudf')\n",
- "co_g = co_g.nodes(lambda g: g._nodes.to_pandas()).edges(lambda g: g._edges.to_pandas())\n",
- "print(co_g._nodes.shape, co_g._edges.shape)\n",
- "co_g._nodes.head(5)"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:27 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 47C P0 27W / 70W | 103MiB / 15360MiB | 0% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! nvidia-smi"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Oczs87ITbJgw",
+ "outputId": "ac203ddd-e684-4eb9-a586-f6a49fd1625d"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "V5qL8K7-dqIZ",
- "outputId": "e08319fc-74d3-4f33-df0f-f98950dc8c99"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:39 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 49C P0 27W / 70W | 2819MiB / 15360MiB | 0% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n"
- ]
- }
- ],
- "source": [
- "#! nvidia-smi"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(117185082, 2)\n",
+ " s d\n",
+ "0 1 3\n",
+ "1 1 4\n",
+ "2 1 5\n",
+ "3 1 6\n",
+ "4 1 7\n",
+ "s int64\n",
+ "d int64\n",
+ "dtype: object\n",
+ "CPU times: user 2.56 s, sys: 4.2 s, total: 6.76 s\n",
+ "Wall time: 6.76 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_df = cudf.read_csv('com-orkut.ungraph.txt', sep='\\t', names=['s', 'd'], skiprows=5).to_pandas()\n",
+ "print(co_df.shape)\n",
+ "print(co_df.head(5))\n",
+ "print(co_df.dtypes)\n",
+ "#del co_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 258
},
+ "id": "gGSDjTtveFAT",
+ "outputId": "e7b38f4f-dc07-4f35-9bab-9c80a80bbf0b"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "hCbxZ8UmhRLp",
- "outputId": "519aed6c-733d-41f4-d462-e57f5e32b131"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n",
- "Wall time: 47.7 µs\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "# crashes\n",
- "if False:\n",
- " out = co_g.chain([ n({'id': 1}), e_forward(hops=1)])._nodes\n",
- " print(out.shape)\n",
- " del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(3072441, 1) (117185082, 2)\n",
+ "CPU times: user 1.96 s, sys: 2.95 s, total: 4.91 s\n",
+ "Wall time: 4.92 s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Q682scC_eC-S",
- "outputId": "7ff5f829-0de7-4a6c-a77d-e2857896a8a5"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Mon Dec 25 06:23:46 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 63C P0 30W / 70W | 1925MiB / 15360MiB | 35% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Mon Dec 25 06:23:49 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 66C P0 72W / 70W | 2845MiB / 15360MiB | 84% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(12, 1) (11, 2)\n",
- "CPU times: user 4.42 s, sys: 131 ms, total: 4.55 s\n",
- "Wall time: 4.42 s\n"
- ]
- }
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 3 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 4 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
],
- "source": [
- "%%time\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=1)])\n",
- "#! nvidia-smi\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del co_gdf\n",
- "del out"
+ "text/plain": [
+ " id\n",
+ "0 1\n",
+ "1 2\n",
+ "2 3\n",
+ "3 4\n",
+ "4 5"
]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_g = graphistry.edges(cudf.DataFrame(co_df), 's', 'd').materialize_nodes(engine='cudf')\n",
+ "co_g = co_g.nodes(lambda g: g._nodes.to_pandas()).edges(lambda g: g._edges.to_pandas())\n",
+ "print(co_g._nodes.shape, co_g._edges.shape)\n",
+ "co_g._nodes.head(5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "V5qL8K7-dqIZ",
+ "outputId": "e08319fc-74d3-4f33-df0f-f98950dc8c99"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "i0AXhfqVbVsm",
- "outputId": "8271f469-a73f-48e3-e1a9-3077026ab8ec"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Mon Dec 25 06:24:52 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 22% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Mon Dec 25 06:24:58 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 64C P0 71W / 70W | 2845MiB / 15360MiB | 57% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(391, 1) (461, 2)\n",
- "CPU times: user 5.34 s, sys: 132 ms, total: 5.47 s\n",
- "Wall time: 6.13 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=2)])\n",
- "#! nvidia-smi\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del co_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:39 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 49C P0 27W / 70W | 2819MiB / 15360MiB | 0% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n"
+ ]
+ }
+ ],
+ "source": [
+ "#! nvidia-smi"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "hCbxZ8UmhRLp",
+ "outputId": "519aed6c-733d-41f4-d462-e57f5e32b131"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "Hid0-iPKhpOd",
- "outputId": "ecaeb534-d4d7-48fa-d4e1-c80b22626afe"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Mon Dec 25 06:25:25 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 31% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Mon Dec 25 06:25:31 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 65C P0 71W / 70W | 2849MiB / 15360MiB | 58% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(21767, 1) (28480, 2)\n",
- "CPU times: user 6.25 s, sys: 100 ms, total: 6.35 s\n",
- "Wall time: 6.37 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=3)])\n",
- "#! nvidia-smi\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del co_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n",
+ "Wall time: 47.7 µs\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "# crashes\n",
+ "if False:\n",
+ " out = co_g.gfql([ n({'id': 1}), e_forward(hops=1)])._nodes\n",
+ " print(out.shape)\n",
+ " del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Q682scC_eC-S",
+ "outputId": "7ff5f829-0de7-4a6c-a77d-e2857896a8a5"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "buutj-ZjhrEe",
- "outputId": "ae11addd-6bea-44e9-81c0-b431e1db8089"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Mon Dec 25 06:26:04 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 61C P0 29W / 70W | 1927MiB / 15360MiB | 36% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Mon Dec 25 06:26:13 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 65C P0 71W / 70W | 2931MiB / 15360MiB | 90% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(718640, 1) (2210961, 2)\n",
- "CPU times: user 9.01 s, sys: 1.03 s, total: 10 s\n",
- "Wall time: 9.84 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=4)])\n",
- "#! nvidia-smi\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del co_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Mon Dec 25 06:23:46 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 63C P0 30W / 70W | 1925MiB / 15360MiB | 35% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Mon Dec 25 06:23:49 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 66C P0 72W / 70W | 2845MiB / 15360MiB | 84% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(12, 1) (11, 2)\n",
+ "CPU times: user 4.42 s, sys: 131 ms, total: 4.55 s\n",
+ "Wall time: 4.42 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=1)])\n",
+ "#! nvidia-smi\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del co_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "i0AXhfqVbVsm",
+ "outputId": "8271f469-a73f-48e3-e1a9-3077026ab8ec"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "bK4C9Ly0hso-",
- "outputId": "8a9a32ab-03e2-42b4-8b71-2bcf797b31b1"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Mon Dec 25 06:27:18 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 60C P0 29W / 70W | 1927MiB / 15360MiB | 28% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Mon Dec 25 06:27:57 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 72C P0 43W / 70W | 4351MiB / 15360MiB | 100% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(3041556, 1) (47622917, 2)\n",
- "CPU times: user 34.9 s, sys: 4.76 s, total: 39.6 s\n",
- "Wall time: 39.2 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " out = co_gdf.chain([ n({'id': 1}), e_forward(hops=5)])\n",
- "#! nvidia-smi\n",
- "print(out._nodes.shape, out._edges.shape)\n",
- "del co_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Mon Dec 25 06:24:52 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 22% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Mon Dec 25 06:24:58 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 64C P0 71W / 70W | 2845MiB / 15360MiB | 57% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(391, 1) (461, 2)\n",
+ "CPU times: user 5.34 s, sys: 132 ms, total: 5.47 s\n",
+ "Wall time: 6.13 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=2)])\n",
+ "#! nvidia-smi\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del co_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "Hid0-iPKhpOd",
+ "outputId": "ecaeb534-d4d7-48fa-d4e1-c80b22626afe"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "qrga-la0hwhh"
- },
- "outputs": [],
- "source": [
- "%%time\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "out = co_gdf.chain([ n({'id': 1}), e_forward(hops=6)])._nodes\n",
- "print(out.shape)\n",
- "del co_gdf\n",
- "del out"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Mon Dec 25 06:25:25 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 61C P0 29W / 70W | 1925MiB / 15360MiB | 31% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Mon Dec 25 06:25:31 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 65C P0 71W / 70W | 2849MiB / 15360MiB | 58% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(21767, 1) (28480, 2)\n",
+ "CPU times: user 6.25 s, sys: 100 ms, total: 6.35 s\n",
+ "Wall time: 6.37 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=3)])\n",
+ "#! nvidia-smi\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del co_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "buutj-ZjhrEe",
+ "outputId": "ae11addd-6bea-44e9-81c0-b431e1db8089"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "eiXFImxF-rzw",
- "outputId": "b807cc3d-ed1a-4bef-c6e0-bfc2df7356ff"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Architecture: x86_64\n",
- " CPU op-mode(s): 32-bit, 64-bit\n",
- " Address sizes: 46 bits physical, 48 bits virtual\n",
- " Byte Order: Little Endian\n",
- "CPU(s): 2\n",
- " On-line CPU(s) list: 0,1\n",
- "Vendor ID: GenuineIntel\n",
- " Model name: Intel(R) Xeon(R) CPU @ 2.20GHz\n",
- " CPU family: 6\n",
- " Model: 79\n",
- " Thread(s) per core: 2\n",
- " Core(s) per socket: 1\n",
- " Socket(s): 1\n",
- " Stepping: 0\n",
- " BogoMIPS: 4399.99\n",
- " Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clf\n",
- " lush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_\n",
- " good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fm\n",
- " a cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hyp\n",
- " ervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp fsgsb\n",
- " ase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap xsa\n",
- " veopt arat md_clear arch_capabilities\n",
- "Virtualization features: \n",
- " Hypervisor vendor: KVM\n",
- " Virtualization type: full\n",
- "Caches (sum of all): \n",
- " L1d: 32 KiB (1 instance)\n",
- " L1i: 32 KiB (1 instance)\n",
- " L2: 256 KiB (1 instance)\n",
- " L3: 55 MiB (1 instance)\n",
- "NUMA: \n",
- " NUMA node(s): 1\n",
- " NUMA node0 CPU(s): 0,1\n",
- "Vulnerabilities: \n",
- " Gather data sampling: Not affected\n",
- " Itlb multihit: Not affected\n",
- " L1tf: Mitigation; PTE Inversion\n",
- " Mds: Vulnerable; SMT Host state unknown\n",
- " Meltdown: Vulnerable\n",
- " Mmio stale data: Vulnerable\n",
- " Retbleed: Vulnerable\n",
- " Spec rstack overflow: Not affected\n",
- " Spec store bypass: Vulnerable\n",
- " Spectre v1: Vulnerable: __user pointer sanitization and usercopy barriers only; no swap\n",
- " gs barriers\n",
- " Spectre v2: Vulnerable, IBPB: disabled, STIBP: disabled, PBRSB-eIBRS: Not affected\n",
- " Srbds: Not affected\n",
- " Tsx async abort: Vulnerable\n"
- ]
- }
- ],
- "source": [
- "#!lscpu\n"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Mon Dec 25 06:26:04 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 61C P0 29W / 70W | 1927MiB / 15360MiB | 36% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Mon Dec 25 06:26:13 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 65C P0 71W / 70W | 2931MiB / 15360MiB | 90% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(718640, 1) (2210961, 2)\n",
+ "CPU times: user 9.01 s, sys: 1.03 s, total: 10 s\n",
+ "Wall time: 9.84 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=4)])\n",
+ "#! nvidia-smi\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del co_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "bK4C9Ly0hso-",
+ "outputId": "8a9a32ab-03e2-42b4-8b71-2bcf797b31b1"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "wJohLi58-sN5",
- "outputId": "c3e144f6-c19a-4c68-e867-f5e7fa2e9df4"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " total used free shared buff/cache available\n",
- "Mem: 12Gi 717Mi 8.0Gi 1.0Mi 3.9Gi 11Gi\n",
- "Swap: 0B 0B 0B\n"
- ]
- }
- ],
- "source": [
- "#!free -h\n"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Mon Dec 25 06:27:18 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 60C P0 29W / 70W | 1927MiB / 15360MiB | 28% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Mon Dec 25 06:27:57 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 72C P0 43W / 70W | 4351MiB / 15360MiB | 100% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(3041556, 1) (47622917, 2)\n",
+ "CPU times: user 34.9 s, sys: 4.76 s, total: 39.6 s\n",
+ "Wall time: 39.2 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=5)])\n",
+ "#! nvidia-smi\n",
+ "print(out._nodes.shape, out._edges.shape)\n",
+ "del co_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "qrga-la0hwhh"
+ },
+ "outputs": [],
+ "source": [
+ "%%time\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "out = co_gdf.gfql([ n({'id': 1}), e_forward(hops=6)])._nodes\n",
+ "print(out.shape)\n",
+ "del co_gdf\n",
+ "del out"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "eiXFImxF-rzw",
+ "outputId": "b807cc3d-ed1a-4bef-c6e0-bfc2df7356ff"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "zak4Inhco5il",
- "outputId": "30bcf2bc-853e-4e5e-8c57-ba0cd9429554"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 01:01:43 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 64C P0 30W / 70W | 2821MiB / 15360MiB | 0% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = pd.DataFrame({'id': [1]})\n",
- "#! nvidia-smi\n",
- "for i in range(1):\n",
- " g2 = co_g.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=1)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "#del start_nodes\n",
- "#del co_gdf\n",
- "#del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Architecture: x86_64\n",
+ " CPU op-mode(s): 32-bit, 64-bit\n",
+ " Address sizes: 46 bits physical, 48 bits virtual\n",
+ " Byte Order: Little Endian\n",
+ "CPU(s): 2\n",
+ " On-line CPU(s) list: 0,1\n",
+ "Vendor ID: GenuineIntel\n",
+ " Model name: Intel(R) Xeon(R) CPU @ 2.20GHz\n",
+ " CPU family: 6\n",
+ " Model: 79\n",
+ " Thread(s) per core: 2\n",
+ " Core(s) per socket: 1\n",
+ " Socket(s): 1\n",
+ " Stepping: 0\n",
+ " BogoMIPS: 4399.99\n",
+ " Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clf\n",
+ " lush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_\n",
+ " good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fm\n",
+ " a cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hyp\n",
+ " ervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp fsgsb\n",
+ " ase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap xsa\n",
+ " veopt arat md_clear arch_capabilities\n",
+ "Virtualization features: \n",
+ " Hypervisor vendor: KVM\n",
+ " Virtualization type: full\n",
+ "Caches (sum of all): \n",
+ " L1d: 32 KiB (1 instance)\n",
+ " L1i: 32 KiB (1 instance)\n",
+ " L2: 256 KiB (1 instance)\n",
+ " L3: 55 MiB (1 instance)\n",
+ "NUMA: \n",
+ " NUMA node(s): 1\n",
+ " NUMA node0 CPU(s): 0,1\n",
+ "Vulnerabilities: \n",
+ " Gather data sampling: Not affected\n",
+ " Itlb multihit: Not affected\n",
+ " L1tf: Mitigation; PTE Inversion\n",
+ " Mds: Vulnerable; SMT Host state unknown\n",
+ " Meltdown: Vulnerable\n",
+ " Mmio stale data: Vulnerable\n",
+ " Retbleed: Vulnerable\n",
+ " Spec rstack overflow: Not affected\n",
+ " Spec store bypass: Vulnerable\n",
+ " Spectre v1: Vulnerable: __user pointer sanitization and usercopy barriers only; no swap\n",
+ " gs barriers\n",
+ " Spectre v2: Vulnerable, IBPB: disabled, STIBP: disabled, PBRSB-eIBRS: Not affected\n",
+ " Srbds: Not affected\n",
+ " Tsx async abort: Vulnerable\n"
+ ]
+ }
+ ],
+ "source": [
+ "#!lscpu\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "wJohLi58-sN5",
+ "outputId": "c3e144f6-c19a-4c68-e867-f5e7fa2e9df4"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "-SmFlCBS_Bgx",
- "outputId": "d2326cf7-3ea6-4f99-9548-f2e98ece59a4"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:45 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 49C P0 28W / 70W | 1923MiB / 15360MiB | 37% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Tue Dec 26 00:56:47 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 52C P0 70W / 70W | 2819MiB / 15360MiB | 79% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(12, 1) (11, 2)\n",
- "CPU times: user 1.6 s, sys: 37.3 ms, total: 1.64 s\n",
- "Wall time: 1.84 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({'id': [1]})\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " g2 = co_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=1)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del co_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " total used free shared buff/cache available\n",
+ "Mem: 12Gi 717Mi 8.0Gi 1.0Mi 3.9Gi 11Gi\n",
+ "Swap: 0B 0B 0B\n"
+ ]
+ }
+ ],
+ "source": [
+ "#!free -h\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "zak4Inhco5il",
+ "outputId": "30bcf2bc-853e-4e5e-8c57-ba0cd9429554"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "fjjt3YnYnabv",
- "outputId": "05762f50-bfe1-4d23-9153-31431418c8e5"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:47 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 51C P0 35W / 70W | 1923MiB / 15360MiB | 59% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Tue Dec 26 00:56:49 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 53C P0 59W / 70W | 2821MiB / 15360MiB | 86% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(391, 1) (461, 2)\n",
- "CPU times: user 2.32 s, sys: 58.5 ms, total: 2.38 s\n",
- "Wall time: 2.51 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({'id': [1]})\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " g2 = co_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=2)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del co_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 01:01:43 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 64C P0 30W / 70W | 2821MiB / 15360MiB | 0% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = pd.DataFrame({'id': [1]})\n",
+ "#! nvidia-smi\n",
+ "for i in range(1):\n",
+ " g2 = co_g.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=1)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "#del start_nodes\n",
+ "#del co_gdf\n",
+ "#del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "-SmFlCBS_Bgx",
+ "outputId": "d2326cf7-3ea6-4f99-9548-f2e98ece59a4"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 18,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "oIouuORgnbcY",
- "outputId": "f07abe4c-5137-4ee3-935a-afbb2c5eaa1e"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:50 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 52C P0 36W / 70W | 1925MiB / 15360MiB | 55% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Tue Dec 26 00:56:53 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 54C P0 75W / 70W | 2825MiB / 15360MiB | 74% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(21767, 1) (28480, 2)\n",
- "CPU times: user 3.04 s, sys: 63.6 ms, total: 3.1 s\n",
- "Wall time: 3.25 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({'id': [1]})\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " g2 = co_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=3)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del co_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:45 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 49C P0 28W / 70W | 1923MiB / 15360MiB | 37% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Tue Dec 26 00:56:47 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 52C P0 70W / 70W | 2819MiB / 15360MiB | 79% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(12, 1) (11, 2)\n",
+ "CPU times: user 1.6 s, sys: 37.3 ms, total: 1.64 s\n",
+ "Wall time: 1.84 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({'id': [1]})\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " g2 = co_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=1)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del co_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "fjjt3YnYnabv",
+ "outputId": "05762f50-bfe1-4d23-9153-31431418c8e5"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "oNLZGjwInc85",
- "outputId": "534097cf-4022-48cc-9419-a00c135f69e1"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:53 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 54C P0 36W / 70W | 1927MiB / 15360MiB | 54% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Tue Dec 26 00:56:58 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 56C P0 38W / 70W | 2907MiB / 15360MiB | 89% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(718640, 1) (2210961, 2)\n",
- "CPU times: user 4.58 s, sys: 309 ms, total: 4.89 s\n",
- "Wall time: 5.02 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({'id': [1]})\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " g2 = co_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=4)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del co_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:47 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 51C P0 35W / 70W | 1923MiB / 15360MiB | 59% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Tue Dec 26 00:56:49 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 53C P0 59W / 70W | 2821MiB / 15360MiB | 86% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(391, 1) (461, 2)\n",
+ "CPU times: user 2.32 s, sys: 58.5 ms, total: 2.38 s\n",
+ "Wall time: 2.51 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({'id': [1]})\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " g2 = co_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=2)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del co_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "oIouuORgnbcY",
+ "outputId": "f07abe4c-5137-4ee3-935a-afbb2c5eaa1e"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "ePqaeujMneX8",
- "outputId": "ffd88fff-016e-4ac0-ecb9-fa06baca60f8"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:56:58 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 55C P0 37W / 70W | 1925MiB / 15360MiB | 59% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Tue Dec 26 00:57:10 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 60C P0 48W / 70W | 4325MiB / 15360MiB | 99% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(3041556, 1) (47622917, 2)\n",
- "CPU times: user 10.8 s, sys: 1.29 s, total: 12.1 s\n",
- "Wall time: 12 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({'id': [1]})\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " g2 = co_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=5)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del co_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:50 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 52C P0 36W / 70W | 1925MiB / 15360MiB | 55% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Tue Dec 26 00:56:53 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 54C P0 75W / 70W | 2825MiB / 15360MiB | 74% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(21767, 1) (28480, 2)\n",
+ "CPU times: user 3.04 s, sys: 63.6 ms, total: 3.1 s\n",
+ "Wall time: 3.25 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({'id': [1]})\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " g2 = co_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=3)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del co_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "oNLZGjwInc85",
+ "outputId": "534097cf-4022-48cc-9419-a00c135f69e1"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 21,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "PTBkoIVHnfzK",
- "outputId": "5615ecd7-47ea-46ab-fd36-13bce4b3c787"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Tue Dec 26 00:57:10 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 59C P0 38W / 70W | 1925MiB / 15360MiB | 44% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "Tue Dec 26 00:57:38 2023 \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
- "|-----------------------------------------+----------------------+----------------------+\n",
- "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
- "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
- "| | | MIG M. |\n",
- "|=========================================+======================+======================|\n",
- "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
- "| N/A 68C P0 55W / 70W | 6445MiB / 15360MiB | 95% Default |\n",
- "| | | N/A |\n",
- "+-----------------------------------------+----------------------+----------------------+\n",
- " \n",
- "+---------------------------------------------------------------------------------------+\n",
- "| Processes: |\n",
- "| GPU GI CI PID Type Process name GPU Memory |\n",
- "| ID ID Usage |\n",
- "|=======================================================================================|\n",
- "+---------------------------------------------------------------------------------------+\n",
- "(3071927, 1) (117032738, 2)\n",
- "CPU times: user 23.5 s, sys: 2.68 s, total: 26.2 s\n",
- "Wall time: 28.2 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "start_nodes = cudf.DataFrame({'id': [1]})\n",
- "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
- "#! nvidia-smi\n",
- "for i in range(10):\n",
- " g2 = co_gdf.hop(\n",
- " nodes=start_nodes,\n",
- " direction='forward',\n",
- " hops=6)\n",
- "#! nvidia-smi\n",
- "print(g2._nodes.shape, g2._edges.shape)\n",
- "del start_nodes\n",
- "del co_gdf\n",
- "del g2"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:53 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 54C P0 36W / 70W | 1927MiB / 15360MiB | 54% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Tue Dec 26 00:56:58 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 56C P0 38W / 70W | 2907MiB / 15360MiB | 89% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(718640, 1) (2210961, 2)\n",
+ "CPU times: user 4.58 s, sys: 309 ms, total: 4.89 s\n",
+ "Wall time: 5.02 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({'id': [1]})\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " g2 = co_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=4)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del co_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "ePqaeujMneX8",
+ "outputId": "ffd88fff-016e-4ac0-ecb9-fa06baca60f8"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "Ygc2nrkznlCu"
- },
- "outputs": [],
- "source": []
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:56:58 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 55C P0 37W / 70W | 1925MiB / 15360MiB | 59% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Tue Dec 26 00:57:10 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 60C P0 48W / 70W | 4325MiB / 15360MiB | 99% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(3041556, 1) (47622917, 2)\n",
+ "CPU times: user 10.8 s, sys: 1.29 s, total: 12.1 s\n",
+ "Wall time: 12 s\n"
+ ]
}
- ],
- "metadata": {
- "accelerator": "GPU",
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({'id': [1]})\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " g2 = co_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=5)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del co_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
"colab": {
- "gpuType": "T4",
- "provenance": []
+ "base_uri": "https://localhost:8080/"
},
- "kernelspec": {
- "display_name": "Python 3",
- "name": "python3"
- },
- "language_info": {
- "name": "python"
+ "id": "PTBkoIVHnfzK",
+ "outputId": "5615ecd7-47ea-46ab-fd36-13bce4b3c787"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Tue Dec 26 00:57:10 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 59C P0 38W / 70W | 1925MiB / 15360MiB | 44% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "Tue Dec 26 00:57:38 2023 \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n",
+ "|-----------------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|=========================================+======================+======================|\n",
+ "| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 68C P0 55W / 70W | 6445MiB / 15360MiB | 95% Default |\n",
+ "| | | N/A |\n",
+ "+-----------------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=======================================================================================|\n",
+ "+---------------------------------------------------------------------------------------+\n",
+ "(3071927, 1) (117032738, 2)\n",
+ "CPU times: user 23.5 s, sys: 2.68 s, total: 26.2 s\n",
+ "Wall time: 28.2 s\n"
+ ]
}
+ ],
+ "source": [
+ "%%time\n",
+ "start_nodes = cudf.DataFrame({'id': [1]})\n",
+ "co_gdf = co_g.nodes(lambda g: cudf.DataFrame(g._nodes)).edges(lambda g: cudf.DataFrame(g._edges))\n",
+ "#! nvidia-smi\n",
+ "for i in range(10):\n",
+ " g2 = co_gdf.hop(\n",
+ " nodes=start_nodes,\n",
+ " direction='forward',\n",
+ " hops=6)\n",
+ "#! nvidia-smi\n",
+ "print(g2._nodes.shape, g2._edges.shape)\n",
+ "del start_nodes\n",
+ "del co_gdf\n",
+ "del g2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Ygc2nrkznlCu"
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "gpuType": "T4",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
},
- "nbformat": 4,
- "nbformat_minor": 0
-}
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/demos/gfql/gfql_remote.ipynb b/demos/gfql/gfql_remote.ipynb
index 60be1125b7..1b207df8cb 100644
--- a/demos/gfql/gfql_remote.ipynb
+++ b/demos/gfql/gfql_remote.ipynb
@@ -275,7 +275,7 @@
"source": [
"%%time\n",
"\n",
- "two_hop_g = g2.chain_remote(two_hop_query)"
+ "two_hop_g = g2.gfql_remote(two_hop_query)"
]
},
{
@@ -439,7 +439,7 @@
},
"outputs": [],
"source": [
- "assert len(two_hop_g._edges) == len(g.chain(two_hop_query)._edges), \"Remote result should match local results\""
+ "assert len(two_hop_g._edges) == len(g.gfql(two_hop_query)._edges), \"Remote result should match local results\""
]
},
{
@@ -477,7 +477,7 @@
],
"source": [
"%%time\n",
- "two_hop_g_gpu1 = g2.chain_remote(two_hop_query, engine='cudf')"
+ "two_hop_g_gpu1 = g2.gfql_remote(two_hop_query, engine='cudf')"
]
},
{
@@ -505,7 +505,7 @@
],
"source": [
"%%time\n",
- "two_hop_g_cpu1 = g2.chain_remote(two_hop_query, engine='pandas')"
+ "two_hop_g_cpu1 = g2.gfql_remote(two_hop_query, engine='pandas')"
]
},
{
@@ -564,7 +564,7 @@
"source": [
"### Shape: Check result counts without downloading the graph\n",
"\n",
- "Often the important aspect is whether or not a search had hits, and you rather not pay the performance penalty of transfering all the hits. In these cases, switch to `chain_remote_shape()`:"
+ "Often the important aspect is whether or not a search had hits, and you rather not pay the performance penalty of transfering all the hits. In these cases, switch to `gfql_remote_shape()`:"
]
},
{
@@ -725,7 +725,7 @@
"source": [
"%%time\n",
"\n",
- "two_hops_nodes = g2.chain_remote(two_hop_query, output_type=\"nodes\")\n",
+ "two_hops_nodes = g2.gfql_remote(two_hop_query, output_type=\"nodes\")\n",
"\n",
"assert two_hops_nodes._edges is None, \"No edges returned\"\n",
"\n",
@@ -822,7 +822,7 @@
"source": [
"%%time\n",
"\n",
- "two_hops_edges = g2.chain_remote(two_hop_query, output_type=\"edges\")\n",
+ "two_hops_edges = g2.gfql_remote(two_hop_query, output_type=\"edges\")\n",
"\n",
"assert two_hops_edges._nodes is None, \"No nodes returned\"\n",
"\n",
@@ -866,7 +866,7 @@
"source": [
"%%time\n",
"\n",
- "two_hops_IDs_g = g2.chain_remote(two_hop_query, node_col_subset=['id'], edge_col_subset=['s', 'd'])"
+ "two_hops_IDs_g = g2.gfql_remote(two_hop_query, node_col_subset=['id'], edge_col_subset=['s', 'd'])"
]
},
{
@@ -1083,7 +1083,7 @@
"source": [
"### Remotely query remote data\n",
"\n",
- "Use `chain_remote()` and `chain_remote_shape()` as usual:"
+ "Use `chain_remote()` and `gfql_remote_shape()` as usual:"
]
},
{
@@ -1165,7 +1165,7 @@
"source": [
"### Fetch remote data\n",
"\n",
- "Use `chain_remote()` to fetch the nodes and edges table. Note that the below takes care to fetch nodes that are not connected to any edges."
+ "Use `gfql_remote()` to fetch the nodes and edges table. Note that the below takes care to fetch nodes that are not connected to any edges."
]
},
{
@@ -1194,8 +1194,8 @@
"source": [
"%%time\n",
"\n",
- "remote_g_nodes = g3_bound.chain_remote([n()], output_type='nodes')\n",
- "remote_g_edges = g3_bound.chain_remote([e_undirected()], output_type='edges')\n",
+ "remote_g_nodes = g3_bound.gfql_remote([n()], output_type='nodes')\n",
+ "remote_g_edges = g3_bound.gfql_remote([e_undirected()], output_type='edges')\n",
"\n",
"g3_fetched_g = (graphistry\n",
" .nodes(remote_g_nodes._nodes, 'id')\n",
@@ -1409,7 +1409,7 @@
{
"cell_type": "code",
"id": "mwr3948llv",
- "source": "%%time\n\n# Define a complex query using Python API for cleaner syntax\npagerank_analysis_query = let({\n # Step 1: Compute PageRank scores\n 'with_pagerank': ASTCall('compute_pagerank'),\n \n # Step 2: Filter nodes with high PageRank scores\n 'important_nodes': ref('with_pagerank', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ]),\n \n # Step 3: Get 1-hop neighborhoods of important nodes\n 'important_neighborhoods': ref('important_nodes', [\n e_undirected({'hops': 1}),\n n()\n ])\n})\n\n# Note: The 'in' clause is automatically the last binding when using Python let()\n# To specify a different output, pass it as second argument: let(bindings, 'output_name')\n\n# Execute the query remotely - chain_remote accepts Python objects directly!\nresult = g_complex.chain_remote([pagerank_analysis_query])\n\nprint(f\"Result has {len(result._nodes)} nodes and {len(result._edges)} edges\")\nprint(\"\\nNodes with PageRank scores:\")\nprint(result._nodes)",
+ "source": "%%time\n\n# Define a complex query using Python API for cleaner syntax\npagerank_analysis_query = let({\n # Step 1: Compute PageRank scores\n 'with_pagerank': ASTCall('compute_pagerank'),\n \n # Step 2: Filter nodes with high PageRank scores\n 'important_nodes': ref('with_pagerank', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ]),\n \n # Step 3: Get 1-hop neighborhoods of important nodes\n 'important_neighborhoods': ref('important_nodes', [\n e_undirected({'hops': 1}),\n n()\n ])\n})\n\n# Note: The 'in' clause is automatically the last binding when using Python let()\n# To specify a different output, pass it as second argument: let(bindings, 'output_name')\n\n# Execute the query remotely - chain_remote accepts Python objects directly!\nresult = g_complex.gfql_remote([pagerank_analysis_query])\n\nprint(f\"Result has {len(result._nodes)} nodes and {len(result._edges)} edges\")\nprint(\"\\nNodes with PageRank scores:\")\nprint(result._nodes)",
"metadata": {},
"execution_count": null,
"outputs": []
@@ -1429,7 +1429,7 @@
{
"cell_type": "code",
"id": "sc00048dhan",
- "source": "# Comparison: Python API vs JSON format\n\n# Clean Python API (what we use above):\npython_query = let({\n 'data': ASTCall('compute_pagerank'),\n 'filtered': ref('data', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ])\n})\n\n# Equivalent verbose JSON format:\njson_query = {\n 'let': {\n 'data': {\n 'type': 'Call',\n 'function': 'compute_pagerank',\n 'params': {}\n },\n 'filtered': {\n 'type': 'Ref',\n 'ref': 'data',\n 'chain': [{\n 'type': 'Node',\n 'filter_dict': {\n 'filter': {'gte': [{'col': 'pagerank'}, 0.15]}\n }\n }]\n }\n },\n 'in': {'type': 'Ref', 'ref': 'filtered', 'chain': []}\n}\n\n# Both work with chain_remote:\n# result = g.chain_remote([python_query]) # Clean!\n# result = g.chain_remote([json_query]) # Verbose but equivalent\n\nprint(\"Python object converts to JSON:\")\nprint(python_query.to_json())",
+ "source": "# Comparison: Python API vs JSON format\n\n# Clean Python API (what we use above):\npython_query = let({\n 'data': ASTCall('compute_pagerank'),\n 'filtered': ref('data', [\n n({'filter': {'gte': [{'col': 'pagerank'}, 0.15]}})\n ])\n})\n\n# Equivalent verbose JSON format:\njson_query = {\n 'let': {\n 'data': {\n 'type': 'Call',\n 'function': 'compute_pagerank',\n 'params': {}\n },\n 'filtered': {\n 'type': 'Ref',\n 'ref': 'data',\n 'chain': [{\n 'type': 'Node',\n 'filter_dict': {\n 'filter': {'gte': [{'col': 'pagerank'}, 0.15]}\n }\n }]\n }\n },\n 'in': {'type': 'Ref', 'ref': 'filtered', 'chain': []}\n}\n\n# Both work with chain_remote:\n# result = g.gfql_remote([python_query]) # Clean!\n# result = g.gfql_remote([json_query]) # Verbose but equivalent\n\nprint(\"Python object converts to JSON:\")\nprint(python_query.to_json())",
"metadata": {},
"execution_count": null,
"outputs": []
@@ -1437,7 +1437,7 @@
{
"cell_type": "code",
"id": "40m2wl0yn3m",
- "source": "%%time\n\n# Analyze different edge types using clean Python API\nedge_type_analysis = let({\n # Analyze follow edges\n 'follow_network': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'follow']}\n }),\n \n # Compute centrality on follow network \n 'follow_centrality': ref('follow_network', [\n n(),\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find mention patterns\n 'mention_edges': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'mention']}\n }),\n \n # Get nodes that are both highly connected and frequently mentioned\n 'influential_nodes': ref('follow_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.5]}}),\n ref('mention_edges', []),\n n()\n ])\n})\n\n# Execute remotely\ninfluential_result = g_complex.chain_remote([edge_type_analysis])\n\nprint(f\"Found {len(influential_result._nodes)} influential nodes\")\nprint(f\"Connected by {len(influential_result._edges)} edges\")\nprint(\"\\nInfluential nodes with centrality scores:\")\nprint(influential_result._nodes)",
+ "source": "%%time\n\n# Analyze different edge types using clean Python API\nedge_type_analysis = let({\n # Analyze follow edges\n 'follow_network': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'follow']}\n }),\n \n # Compute centrality on follow network \n 'follow_centrality': ref('follow_network', [\n n(),\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find mention patterns\n 'mention_edges': e_undirected({\n 'filter': {'eq': [{'col': 'type'}, 'mention']}\n }),\n \n # Get nodes that are both highly connected and frequently mentioned\n 'influential_nodes': ref('follow_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.5]}}),\n ref('mention_edges', []),\n n()\n ])\n})\n\n# Execute remotely\ninfluential_result = g_complex.gfql_remote([edge_type_analysis])\n\nprint(f\"Found {len(influential_result._nodes)} influential nodes\")\nprint(f\"Connected by {len(influential_result._edges)} edges\")\nprint(\"\\nInfluential nodes with centrality scores:\")\nprint(influential_result._nodes)",
"metadata": {},
"execution_count": null,
"outputs": []
@@ -1451,7 +1451,7 @@
{
"cell_type": "code",
"id": "gmy30drvbts",
- "source": "%%time\n\n# Complex analysis with multiple algorithms using Python API\ncomprehensive_analysis = let({\n # Base graph with PageRank computation\n 'enriched_graph': ASTCall('compute_pagerank'),\n \n # Add centrality metrics\n 'with_centrality': ref('enriched_graph', [\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find bridge nodes (high PageRank, low-medium centrality)\n 'bridge_nodes': ref('with_centrality', [\n n({\n 'filter': {\n 'and': [\n {'gte': [{'col': 'pagerank'}, 0.1]},\n {'lte': [{'col': 'degree_centrality'}, 0.7]}\n ]\n }\n })\n ]),\n \n # Find hub nodes (high degree centrality)\n 'hub_nodes': ref('with_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.7]}})\n ]),\n \n # Get connections between bridges and hubs\n 'critical_paths': ref('bridge_nodes', [\n e_forward({'to_nodes': ref('hub_nodes', [])}),\n n()\n ])\n})\n\n# Execute remotely with GPU acceleration\ncritical_paths_result = g_complex.chain_remote([comprehensive_analysis], engine='cudf')\n\nprint(f\"Critical paths network: {len(critical_paths_result._nodes)} nodes, {len(critical_paths_result._edges)} edges\")\n\n# Check if we got results\nif len(critical_paths_result._nodes) > 0:\n print(\"\\nCritical path nodes:\")\n print(critical_paths_result._nodes)\nelse:\n print(\"\\nNo critical paths found with current thresholds\")",
+ "source": "%%time\n\n# Complex analysis with multiple algorithms using Python API\ncomprehensive_analysis = let({\n # Base graph with PageRank computation\n 'enriched_graph': ASTCall('compute_pagerank'),\n \n # Add centrality metrics\n 'with_centrality': ref('enriched_graph', [\n ASTCall('compute_degree_centrality')\n ]),\n \n # Find bridge nodes (high PageRank, low-medium centrality)\n 'bridge_nodes': ref('with_centrality', [\n n({\n 'filter': {\n 'and': [\n {'gte': [{'col': 'pagerank'}, 0.1]},\n {'lte': [{'col': 'degree_centrality'}, 0.7]}\n ]\n }\n })\n ]),\n \n # Find hub nodes (high degree centrality)\n 'hub_nodes': ref('with_centrality', [\n n({'filter': {'gte': [{'col': 'degree_centrality'}, 0.7]}})\n ]),\n \n # Get connections between bridges and hubs\n 'critical_paths': ref('bridge_nodes', [\n e_forward({'to_nodes': ref('hub_nodes', [])}),\n n()\n ])\n})\n\n# Execute remotely with GPU acceleration\ncritical_paths_result = g_complex.gfql_remote([comprehensive_analysis], engine='cudf')\n\nprint(f\"Critical paths network: {len(critical_paths_result._nodes)} nodes, {len(critical_paths_result._edges)} edges\")\n\n# Check if we got results\nif len(critical_paths_result._nodes) > 0:\n print(\"\\nCritical path nodes:\")\n print(critical_paths_result._nodes)\nelse:\n print(\"\\nNo critical paths found with current thresholds\")",
"metadata": {},
"execution_count": null,
"outputs": []
@@ -1465,7 +1465,7 @@
{
"cell_type": "code",
"id": "4glzmgi1u3s",
- "source": "%%time\n\n# Prepare visualization-ready data with all enrichments\nviz_prep_query = {\n 'let': {\n # Compute all metrics - sequential operations\n 'with_pagerank': {\n 'call': {'method': 'compute_pagerank', 'args': [], 'kwargs': {}}\n },\n \n 'with_metrics': {\n 'type': 'Ref',\n 'ref': 'with_pagerank',\n 'chain': [\n {'call': {'method': 'compute_degree_centrality', 'args': [], 'kwargs': {}}},\n # Add node colors based on PageRank\n {\n 'call': {\n 'method': 'nodes',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'node_color': {\n 'case': [\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.2]},\n 'then': 65280 # Green for high PageRank\n },\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.15]},\n 'then': 16776960 # Yellow for medium\n }\n ],\n 'else': 16711680 # Red for low\n },\n 'node_size': {\n 'mul': [\n {'col': 'degree_centrality'},\n 50 # Scale factor\n ]\n }\n }\n }\n }\n }\n ]\n },\n \n # Add edge styling based on type and weight\n 'styled_graph': {\n 'type': 'Ref',\n 'ref': 'with_metrics',\n 'chain': [\n {\n 'call': {\n 'method': 'edges',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'edge_color': {\n 'case': [\n {\n 'when': {'eq': [{'col': 'type'}, 'follow']},\n 'then': 255 # Blue for follows\n }\n ],\n 'else': 16711935 # Magenta for mentions\n },\n 'edge_weight': {\n 'col': 'weight'\n }\n }\n }\n }\n }\n ]\n },\n \n # Focus on top nodes and their connections\n 'viz_subgraph': {\n 'type': 'Ref',\n 'ref': 'styled_graph',\n 'chain': [\n {\n 'n': {\n 'filter': {\n 'or': [\n {'gte': [{'col': 'pagerank'}, 0.15]},\n {'gte': [{'col': 'degree_centrality'}, 0.6]}\n ]\n }\n }\n },\n {'e_undirected': {'hops': 1}},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'type': 'Ref', 'ref': 'viz_subgraph', 'chain': []}\n}\n\n# Get visualization-ready data\nviz_result = g_complex.chain_remote([viz_prep_query])\n\nprint(f\"Visualization subgraph: {len(viz_result._nodes)} nodes, {len(viz_result._edges)} edges\")\nprint(\"\\nNodes with visualization attributes:\")\nprint(viz_result._nodes)\nprint(\"\\nEdges with styling:\")\nprint(viz_result._edges)\n\n# Ready to visualize\n# viz_result.plot() # Uncomment to create visualization",
+ "source": "%%time\n\n# Prepare visualization-ready data with all enrichments\nviz_prep_query = {\n 'let': {\n # Compute all metrics - sequential operations\n 'with_pagerank': {\n 'call': {'method': 'compute_pagerank', 'args': [], 'kwargs': {}}\n },\n \n 'with_metrics': {\n 'type': 'Ref',\n 'ref': 'with_pagerank',\n 'chain': [\n {'call': {'method': 'compute_degree_centrality', 'args': [], 'kwargs': {}}},\n # Add node colors based on PageRank\n {\n 'call': {\n 'method': 'nodes',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'node_color': {\n 'case': [\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.2]},\n 'then': 65280 # Green for high PageRank\n },\n {\n 'when': {'gte': [{'col': 'pagerank'}, 0.15]},\n 'then': 16776960 # Yellow for medium\n }\n ],\n 'else': 16711680 # Red for low\n },\n 'node_size': {\n 'mul': [\n {'col': 'degree_centrality'},\n 50 # Scale factor\n ]\n }\n }\n }\n }\n }\n ]\n },\n \n # Add edge styling based on type and weight\n 'styled_graph': {\n 'type': 'Ref',\n 'ref': 'with_metrics',\n 'chain': [\n {\n 'call': {\n 'method': 'edges',\n 'args': [],\n 'kwargs': {\n 'assign': {\n 'edge_color': {\n 'case': [\n {\n 'when': {'eq': [{'col': 'type'}, 'follow']},\n 'then': 255 # Blue for follows\n }\n ],\n 'else': 16711935 # Magenta for mentions\n },\n 'edge_weight': {\n 'col': 'weight'\n }\n }\n }\n }\n }\n ]\n },\n \n # Focus on top nodes and their connections\n 'viz_subgraph': {\n 'type': 'Ref',\n 'ref': 'styled_graph',\n 'chain': [\n {\n 'n': {\n 'filter': {\n 'or': [\n {'gte': [{'col': 'pagerank'}, 0.15]},\n {'gte': [{'col': 'degree_centrality'}, 0.6]}\n ]\n }\n }\n },\n {'e_undirected': {'hops': 1}},\n {'n': {}}\n ]\n }\n },\n \n 'in': {'type': 'Ref', 'ref': 'viz_subgraph', 'chain': []}\n}\n\n# Get visualization-ready data\nviz_result = g_complex.gfql_remote([viz_prep_query])\n\nprint(f\"Visualization subgraph: {len(viz_result._nodes)} nodes, {len(viz_result._edges)} edges\")\nprint(\"\\nNodes with visualization attributes:\")\nprint(viz_result._nodes)\nprint(\"\\nEdges with styling:\")\nprint(viz_result._edges)\n\n# Ready to visualize\n# viz_result.plot() # Uncomment to create visualization",
"metadata": {},
"execution_count": null,
"outputs": []
diff --git a/demos/gfql/gfql_validation_fundamentals.ipynb b/demos/gfql/gfql_validation_fundamentals.ipynb
index 9736f6d08b..6699a7e89d 100644
--- a/demos/gfql/gfql_validation_fundamentals.ipynb
+++ b/demos/gfql/gfql_validation_fundamentals.ipynb
@@ -104,14 +104,14 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": "# Valid query - columns exist\ntry:\n result = g.chain([\n n({'type': 'customer'}),\n e_forward({'edge_type': 'buys'}),\n n({'type': 'product'})\n ])\n print(f\"Query executed successfully!\")\n print(f\" Found {len(result._nodes)} nodes\")\n print(f\" Found {len(result._edges)} edges\")\nexcept GFQLSchemaError as e:\n print(f\"Schema error: {e}\")"
+ "source": "# Valid query - columns exist\ntry:\n result = g.gfql([\n n({'type': 'customer'}),\n e_forward({'edge_type': 'buys'}),\n n({'type': 'product'})\n ])\n print(f\"Query executed successfully!\")\n print(f\" Found {len(result._nodes)} nodes\")\n print(f\" Found {len(result._edges)} edges\")\nexcept GFQLSchemaError as e:\n print(f\"Schema error: {e}\")"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": "# Invalid query - column doesn't exist\ntry:\n result = g.chain([\n n({'category': 'VIP'}) # 'category' column doesn't exist\n ])\nexcept GFQLSchemaError as e:\n print(f\"Schema validation caught the error!\")\n print(f\" Error code: {e.code}\")\n print(f\" Message: {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")"
+ "source": "# Invalid query - column doesn't exist\ntry:\n result = g.gfql([\n n({'category': 'VIP'}) # 'category' column doesn't exist\n ])\nexcept GFQLSchemaError as e:\n print(f\"Schema validation caught the error!\")\n print(f\" Error code: {e.code}\")\n print(f\" Message: {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")"
},
{
"cell_type": "markdown",
@@ -127,14 +127,14 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": "# Type mismatch: string value on numeric column\ntry:\n result = g.chain([\n n({'score': 'high'}) # 'score' is numeric, not string\n ])\nexcept GFQLSchemaError as e:\n print(f\"Type mismatch detected!\")\n print(f\" {e}\")\n print(f\"\\n Column type: {e.context.get('column_type')}\")"
+ "source": "# Type mismatch: string value on numeric column\ntry:\n result = g.gfql([\n n({'score': 'high'}) # 'score' is numeric, not string\n ])\nexcept GFQLSchemaError as e:\n print(f\"Type mismatch detected!\")\n print(f\" {e}\")\n print(f\"\\n Column type: {e.context.get('column_type')}\")"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": "# Using predicates\nfrom graphistry.compute.predicates.numeric import gt\nfrom graphistry.compute.predicates.str import contains\n\n# Correct: numeric predicate on numeric column\ntry:\n result = g.chain([n({'score': gt(90)})])\n print(f\"Valid: Found {len(result._nodes)} high-scoring nodes\")\nexcept GFQLSchemaError as e:\n print(f\"Error: {e}\")\n\n# Wrong: string predicate on numeric column\ntry:\n result = g.chain([n({'score': contains('9')})])\nexcept GFQLSchemaError as e:\n print(f\"\\nPredicate type mismatch caught!\")\n print(f\" {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")"
+ "source": "# Using predicates\nfrom graphistry.compute.predicates.numeric import gt\nfrom graphistry.compute.predicates.str import contains\n\n# Correct: numeric predicate on numeric column\ntry:\n result = g.gfql([n({'score': gt(90)})])\n print(f\"Valid: Found {len(result._nodes)} high-scoring nodes\")\nexcept GFQLSchemaError as e:\n print(f\"Error: {e}\")\n\n# Wrong: string predicate on numeric column\ntry:\n result = g.gfql([n({'score': contains('9')})])\nexcept GFQLSchemaError as e:\n print(f\"\\nPredicate type mismatch caught!\")\n print(f\" {e.message}\")\n print(f\" Suggestion: {e.context.get('suggestion')}\")"
},
{
"cell_type": "markdown",
@@ -150,7 +150,7 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Use validate_schema parameter\ntry:\n result = g.chain(chain_to_test.chain, validate_schema=True)\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (No graph operations were performed)\")"
+ "source": "# Pre-validate to catch errors early\nchain_to_test = Chain([\n n({'missing_col': 'value'}),\n e_forward({'also_missing': 'value'})\n])\n\n# Method 1: Use validate_schema parameter\ntry:\n result = g.gfql(chain_to_test.chain, validate_schema=True)\nexcept GFQLSchemaError as e:\n print(\"Pre-execution validation caught error!\")\n print(f\" Error: {e}\")\n print(\" (No graph operations were performed)\")"
},
{
"cell_type": "code",
@@ -201,7 +201,7 @@
" # print(f\"Warning: {len(errors)} schema issues found\")\n",
" \n",
" # Execute\n",
- " result = g.chain(operations)\n",
+ " result = g.gfql(operations)\n",
" return result\n",
" \n",
" except GFQLSyntaxError as e:\n",
@@ -242,7 +242,7 @@
},
{
"cell_type": "markdown",
- "source": "## Summary\n\n### Key Takeaways\n\n1. **Automatic Validation**: GFQL validates automatically - no separate validation calls needed\n2. **Structured Errors**: Error codes (E1xx, E2xx, E3xx) help with programmatic handling\n3. **Helpful Messages**: Errors include suggestions for fixing issues\n4. **Two Validation Stages**:\n - Syntax/Type: During chain construction\n - Schema: During execution (or pre-execution)\n5. **Flexible Modes**: Choose between fail-fast or collect-all errors\n\n### Quick Reference\n\n```python\n# Automatic syntax validation\nchain = Chain([...]) # Validates syntax/types\n\n# Runtime schema validation \nresult = g.chain([...]) # Validates against data\n\n# Pre-execution schema validation\nresult = g.chain([...], validate_schema=True)\n\n# Validate chain against graph schema\nfrom graphistry.compute.validate_schema import validate_chain_schema\nvalidate_chain_schema(g, chain) # Throws GFQLSchemaError if invalid\n\n# Collect all syntax errors\nerrors = chain.validate(collect_all=True)\n\n# Collect all schema errors\nschema_errors = validate_chain_schema(g, chain, collect_all=True)\n```\n\n### Next Steps\n\n- Explore more complex query patterns\n- Learn about GFQL predicates for advanced filtering\n- Use validation in production applications",
+ "source": "## Summary\n\n### Key Takeaways\n\n1. **Automatic Validation**: GFQL validates automatically - no separate validation calls needed\n2. **Structured Errors**: Error codes (E1xx, E2xx, E3xx) help with programmatic handling\n3. **Helpful Messages**: Errors include suggestions for fixing issues\n4. **Two Validation Stages**:\n - Syntax/Type: During chain construction\n - Schema: During execution (or pre-execution)\n5. **Flexible Modes**: Choose between fail-fast or collect-all errors\n\n### Quick Reference\n\n```python\n# Automatic syntax validation\nchain = Chain([...]) # Validates syntax/types\n\n# Runtime schema validation \nresult = g.gfql([...]) # Validates against data\n\n# Pre-execution schema validation\nresult = g.gfql([...], validate_schema=True)\n\n# Validate chain against graph schema\nfrom graphistry.compute.validate_schema import validate_chain_schema\nvalidate_chain_schema(g, chain) # Throws GFQLSchemaError if invalid\n\n# Collect all syntax errors\nerrors = chain.validate(collect_all=True)\n\n# Collect all schema errors\nschema_errors = validate_chain_schema(g, chain, collect_all=True)\n```\n\n### Next Steps\n\n- Explore more complex query patterns\n- Learn about GFQL predicates for advanced filtering\n- Use validation in production applications",
"metadata": {}
}
],
diff --git a/demos/gfql/temporal_predicates.ipynb b/demos/gfql/temporal_predicates.ipynb
index 877306dd09..4ecd7350b4 100644
--- a/demos/gfql/temporal_predicates.ipynb
+++ b/demos/gfql/temporal_predicates.ipynb
@@ -107,7 +107,7 @@
"source": [
"# Alternative: Use chain with edge operations\n",
"# Start from all nodes, then follow edges with temporal filter\n",
- "recent_chain = g.chain([\n",
+ "recent_chain = g.gfql([\n",
" n(), # Start with all nodes\n",
" e_forward({\n",
" \"timestamp\": gt(pd.Timestamp(cutoff_date))\n",
@@ -152,7 +152,7 @@
"outputs": [],
"source": [
"# Filter accounts created after a specific date\n",
- "new_accounts = g.chain([\n",
+ "new_accounts = g.gfql([\n",
" n(filter_dict={\n",
" \"created_date\": ge(date(2021, 1, 1))\n",
" })\n",
@@ -170,7 +170,7 @@
"source": [
"# Find accounts active in the last 90 days\n",
"ninety_days_ago = datetime.now().date() - timedelta(days=90)\n",
- "active_accounts = g.chain([\n",
+ "active_accounts = g.gfql([\n",
" n(filter_dict={\n",
" \"last_active\": gt(pd.Timestamp(ninety_days_ago))\n",
" })\n",
@@ -258,7 +258,7 @@
"cell_type": "code",
"metadata": {},
"outputs": [],
- "source": "# Multi-hop query: Find accounts that received money recently\n# and then sent money to business accounts\nthirty_days_ago = datetime.now() - timedelta(days=30)\n\n# First, find recent transactions\nrecent_edges = g._edges[gt(pd.Timestamp(thirty_days_ago))(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\n# Use chain to find money flow pattern\nmoney_flow = recent_g.chain([\n # Start with any node\n n(),\n # Follow incoming edges (as destination)\n e_reverse(),\n # Go to source nodes\n n(),\n # Follow outgoing edges\n e_forward(),\n # To business accounts\n n(filter_dict={\"account_type\": \"business\"})\n])\n\nprint(f\"Money flow pattern found: {len(money_flow._nodes)} business accounts\")",
+ "source": "# Multi-hop query: Find accounts that received money recently\n# and then sent money to business accounts\nthirty_days_ago = datetime.now() - timedelta(days=30)\n\n# First, find recent transactions\nrecent_edges = g._edges[gt(pd.Timestamp(thirty_days_ago))(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\n# Use chain to find money flow pattern\nmoney_flow = recent_g.gfql([\n # Start with any node\n n(),\n # Follow incoming edges (as destination)\n e_reverse(),\n # Go to source nodes\n n(),\n # Follow outgoing edges\n e_forward(),\n # To business accounts\n n(filter_dict={\"account_type\": \"business\"})\n])\n\nprint(f\"Money flow pattern found: {len(money_flow._nodes)} business accounts\")",
"execution_count": 12
},
{
@@ -338,7 +338,7 @@
"outputs": [],
"source": [
"# Find paths through recent high-value transactions\n",
- "recent_high_value = g.chain([\n",
+ "recent_high_value = g.gfql([\n",
" # Start from all nodes\n",
" n(),\n",
" # Follow edges with temporal and amount filters\n",
@@ -369,7 +369,7 @@
},
{
"cell_type": "code",
- "source": "# Build predicates programmatically with wire protocol dicts\ndef create_date_filter(year, month, day, comparison=\"gt\"):\n \"\"\"Create a date filter using wire protocol format\"\"\"\n date_dict = {\n \"type\": \"date\",\n \"value\": f\"{year:04d}-{month:02d}-{day:02d}\"\n }\n \n if comparison == \"gt\":\n return gt(date_dict)\n elif comparison == \"lt\":\n return lt(date_dict)\n elif comparison == \"ge\":\n return ge(date_dict)\n elif comparison == \"le\":\n return le(date_dict)\n else:\n raise ValueError(f\"Unknown comparison: {comparison}\")\n\n# Use the programmatic filter\nfilter_2023 = create_date_filter(2023, 1, 1, \"ge\")\naccounts_2023 = g.chain([\n n(filter_dict={\n \"created_date\": filter_2023\n })\n])\n\nprint(f\"Accounts created in 2023 or later: {len(accounts_2023._nodes)}\")",
+ "source": "# Build predicates programmatically with wire protocol dicts\ndef create_date_filter(year, month, day, comparison=\"gt\"):\n \"\"\"Create a date filter using wire protocol format\"\"\"\n date_dict = {\n \"type\": \"date\",\n \"value\": f\"{year:04d}-{month:02d}-{day:02d}\"\n }\n \n if comparison == \"gt\":\n return gt(date_dict)\n elif comparison == \"lt\":\n return lt(date_dict)\n elif comparison == \"ge\":\n return ge(date_dict)\n elif comparison == \"le\":\n return le(date_dict)\n else:\n raise ValueError(f\"Unknown comparison: {comparison}\")\n\n# Use the programmatic filter\nfilter_2023 = create_date_filter(2023, 1, 1, \"ge\")\naccounts_2023 = g.gfql([\n n(filter_dict={\n \"created_date\": filter_2023\n })\n])\n\nprint(f\"Accounts created in 2023 or later: {len(accounts_2023._nodes)}\")",
"metadata": {},
"outputs": [],
"execution_count": null
@@ -402,7 +402,7 @@
"cell_type": "code",
"metadata": {},
"outputs": [],
- "source": "# Complex multi-hop with temporal constraints\n# Find 2-hop paths through recent transactions\ntwo_hop_recent = g.chain([\n # Start from business accounts\n n(filter_dict={\"account_type\": \"business\"}),\n # First hop: recent outgoing transactions\n e_forward({\n \"timestamp\": gt(datetime.now() - timedelta(days=30))\n }, name=\"hop1\"),\n # Intermediate nodes\n n(),\n # Second hop: any transaction\n e_forward(name=\"hop2\"),\n # Final nodes\n n()\n])\n\nprint(f\"2-hop paths from business accounts through recent transactions:\")\nprint(f\" Total edges: {len(two_hop_recent._edges)}\")\nprint(f\" Hop 1 edges: {two_hop_recent._edges['hop1'].sum()}\")\nprint(f\" Hop 2 edges: {two_hop_recent._edges['hop2'].sum()}\")",
+ "source": "# Complex multi-hop with temporal constraints\n# Find 2-hop paths through recent transactions\ntwo_hop_recent = g.gfql([\n # Start from business accounts\n n(filter_dict={\"account_type\": \"business\"}),\n # First hop: recent outgoing transactions\n e_forward({\n \"timestamp\": gt(datetime.now() - timedelta(days=30))\n }, name=\"hop1\"),\n # Intermediate nodes\n n(),\n # Second hop: any transaction\n e_forward(name=\"hop2\"),\n # Final nodes\n n()\n])\n\nprint(f\"2-hop paths from business accounts through recent transactions:\")\nprint(f\" Total edges: {len(two_hop_recent._edges)}\")\nprint(f\" Hop 1 edges: {two_hop_recent._edges['hop1'].sum()}\")\nprint(f\" Hop 2 edges: {two_hop_recent._edges['hop2'].sum()}\")",
"execution_count": 16
},
{