Add benchmarking notebook for ONNX Runtime on GPU and CPU

Analysis.md

@@ -1,27 +1,73 @@
 ### Metrics
 
-#### Stable Diffusion UNet on Nvidia Geforce GTX 1070 GPU
-
 see `results/benchmarks/benchmark_results.csv`
 
+GPU - Nvidia Geforce GTX 1070 GPU
+CPU - Intel Core i7-8750H
+
+#### Stable Diffusion UNet on GPU
+
 ``` bash
-INFO - 2025-09-01 16:26:26,203 - 3315694995.py - UNet Inference time: 0.10s
-INFO - 2025-09-01 16:26:26,448 - 3315694995.py - UNet Inference time: 0.24s
-INFO - 2025-09-01 16:26:26,711 - 3315694995.py - UNet Inference time: 0.26s
-INFO - 2025-09-01 16:26:26,974 - 3315694995.py - UNet Inference time: 0.26s
-INFO - 2025-09-01 16:26:27,238 - 3315694995.py - UNet Inference time: 0.26s
-INFO - 2025-09-01 16:26:27,494 - 3315694995.py - UNet Inference time: 0.25s
-INFO - 2025-09-01 16:26:27,760 - 3315694995.py - UNet Inference time: 0.27s
-INFO - 2025-09-01 16:26:28,017 - 3315694995.py - UNet Inference time: 0.26s
-INFO - 2025-09-01 16:26:28,285 - 3315694995.py - UNet Inference time: 0.27s
-INFO - 2025-09-01 16:26:28,538 - 3315694995.py - UNet Inference time: 0.25s
-INFO - 2025-09-01 16:26:28,538 - 3315694995.py -
-
-Average inference time: 0.24s ± 0.05s
-
-Average CPU memory usage: 1009.82MB ± 0.03MB
-
-Average GPU memory usage: 3817.92MB ± 0.00MB
+INFO - UNet Inference time: 0.52s
+INFO - UNet Inference time: 0.15s
+INFO - UNet Inference time: 0.26s
+INFO - UNet Inference time: 0.26s
+INFO - UNet Inference time: 0.26s
+INFO - UNet Inference time: 0.26s
+INFO - UNet Inference time: 0.24s
+INFO - UNet Inference time: 0.28s
+INFO - UNet Inference time: 0.24s
+INFO - UNet Inference time: 0.27s
+
+Average inference time: 0.27s ± 0.09s
+
+Average CPU memory usage: 889.63MB ± 0.00MB
+
+Average GPU memory usage: 2511.14MB ± 0.00MB
 ```
 
 ---
+
+#### Stable Diffusion UNet via ONNX Runtime on GPU
+
+```bash
+INFO - ONNXRuntime Inference time: 1.82s
+INFO - ONNXRuntime Inference time: 1.59s
+INFO - ONNXRuntime Inference time: 1.59s
+INFO - ONNXRuntime Inference time: 1.60s
+INFO - ONNXRuntime Inference time: 1.59s
+INFO - ONNXRuntime Inference time: 1.59s
+INFO - ONNXRuntime Inference time: 1.58s
+INFO - ONNXRuntime Inference time: 1.58s
+INFO - ONNXRuntime Inference time: 1.60s
+INFO - ONNXRuntime Inference time: 1.58s
+
+Average inference time: 1.61s ± 0.07s
+
+Average CPU memory usage: 1390.87MB ± 0.03MB
+
+Average GPU memory usage: 1355.65MB ± 0.00MB
+
+```
+
+---
+
+#### Stable Diffusion UNet via ONNX Runtime on CPU
+
+```bash
+INFO - ONNXRuntime Inference time: 8.94s
+INFO - ONNXRuntime Inference time: 9.48s
+INFO - ONNXRuntime Inference time: 7.13s
+INFO - ONNXRuntime Inference time: 7.42s
+INFO - ONNXRuntime Inference time: 7.06s
+INFO - ONNXRuntime Inference time: 6.98s
+INFO - ONNXRuntime Inference time: 6.91s
+INFO - ONNXRuntime Inference time: 7.57s
+INFO - ONNXRuntime Inference time: 6.90s
+INFO - ONNXRuntime Inference time: 6.98s
+
+Average inference time: 7.54s ± 0.92s
+
+Average CPU memory usage: 4968.51MB ± 344.71MB
+
+```

notebooks/baseline_generation.ipynb

@@ -160,6 +160,16 @@
    "id": "12",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "LOGGER.info(f\"Input channels: {pipe.unet.config.in_channels}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
    "source": [
     "batch_size = 1\n",
     "height = width = 64 # for 512x512 images\n",
@@ -180,7 +190,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "13",
+   "id": "14",
    "metadata": {},
    "source": [
     "Benchmarking"
@@ -189,7 +199,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "14",
+   "id": "15",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -202,7 +212,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "15",
+   "id": "16",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -258,29 +268,18 @@
   },
   {
    "cell_type": "markdown",
-   "id": "16",
+   "id": "17",
    "metadata": {},
    "source": [
     "The inference time would be really small here because we're running only one denoising step of the UNet as opposed to say 50 denoising steps. "
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "17",
-   "metadata": {},
-   "source": [
-    "Save benchmark details as CSV"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
    "id": "18",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "BENCHMARK_SAVE_PATH = os.path.join(os.path.dirname(os.getcwd()), \"results\", \"benchmarks\")\n",
-    "os.makedirs(BENCHMARK_SAVE_PATH, exist_ok=True)"
+    "Save benchmark details as CSV"
    ]
   },
   {
@@ -290,10 +289,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = pd.DataFrame(results)\n",
-    "csv_path = os.path.join(BENCHMARK_SAVE_PATH, \"benchmark_results.csv\")\n",
-    "df.to_csv(csv_path, index=False)\n",
-    "LOGGER.info(f\"Saved benchmark results to {csv_path}\")"
+    "from tinydiffusion.utils.csv_utils import save_results_to_csv\n",
+    "\n",
+    "save_results_to_csv(results)"
    ]
   },
   {