Add Jupyter notebook generation, integration tests, and standardize c…

.gitignore

@@ -37,3 +37,6 @@ haystack-test/strands-venv/
 __pycache__/
 *.pyc
 node_modules/
+
+cookbooks/*
+notebooks/*

.python-version

@@ -0,0 +1 @@
+3.14

README.md

@@ -134,6 +134,17 @@ node build.js      # builds all tracks
 node build.js semantic-caching   # build one track
 ```
 
+### Generating Jupyter notebooks (optional)
+
+Every cookbook can be exported as a `.ipynb` notebook for use in JupyterLab, Colab, or VS Code:
+
+```bash
+node build-notebooks.js                    # all tracks → notebooks/
+node build-notebooks.js semantic-caching   # one track
+```
+
+Notebooks are generated from the same markdown source in `content/`. Python blocks become code cells, bash blocks become `!`-prefixed code cells, and prose stays as markdown cells.
+
 ## Project structure
 
 ```

build-notebooks.js

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+/**
+ * build-notebooks.js - Converts Markdown cookbooks to Jupyter notebooks (.ipynb)
+ *
+ * Usage:
+ *   node build-notebooks.js                    # builds all tracks
+ *   node build-notebooks.js semantic-caching   # builds one track
+ *
+ * Output goes to notebooks/<track>/<notebook>.ipynb
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const CONTENT_DIR = path.join(__dirname, 'content');
+const OUTPUT_DIR = path.join(__dirname, 'notebooks');
+
+/**
+ * Split markdown into alternating prose / code blocks.
+ * Returns an array of { type: 'markdown' | 'code', lang?: string, content: string }
+ */
+function splitMarkdown(md) {
+  const blocks = [];
+  const fenceRe = /^```(\w*)\s*$/;
+  const lines = md.split('\n');
+  let i = 0;
+
+  let mdBuf = [];
+
+  function flushMd() {
+    const text = mdBuf.join('\n').trim();
+    if (text) blocks.push({ type: 'markdown', content: text });
+    mdBuf = [];
+  }
+
+  while (i < lines.length) {
+    const fenceMatch = lines[i].match(fenceRe);
+    if (fenceMatch) {
+      flushMd();
+      const lang = fenceMatch[1] || '';
+      i++; // skip opening fence
+      const codeBuf = [];
+      while (i < lines.length && !lines[i].match(/^```\s*$/)) {
+        codeBuf.push(lines[i]);
+        i++;
+      }
+      i++; // skip closing fence
+      blocks.push({ type: 'code', lang, content: codeBuf.join('\n') });
+    } else {
+      mdBuf.push(lines[i]);
+      i++;
+    }
+  }
+  flushMd();
+  return blocks;
+}
+
+/**
+ * Convert a list of blocks into Jupyter notebook cells.
+ * - python code → code cells
+ * - bash code → code cells prefixed with ! (so they run in notebook)
+ * - other fenced code (json, text, pseudo) → markdown cells wrapped in fences
+ * - prose → markdown cells
+ */
+function blocksToNotebookCells(blocks) {
+  const cells = [];
+
+  for (const block of blocks) {
+    if (block.type === 'markdown') {
+      cells.push(markdownCell(block.content));
+    } else if (block.type === 'code') {
+      const lang = block.lang.toLowerCase();
+      if (lang === 'python' || lang === 'py') {
+        cells.push(codeCell(block.content));
+      } else if (lang === 'mermaid') {
+        // Skip mermaid diagrams — not supported in Jupyter
+      } else if (lang === 'bash' || lang === 'sh' || lang === 'shell') {
+        // Convert each line to a !-prefixed shell command for notebooks
+        const shellLines = block.content
+          .split('\n')
+          .map(line => {
+            const trimmed = line.trim();
+            if (!trimmed || trimmed.startsWith('#')) return trimmed;
+            return trimmed.startsWith('!') ? trimmed : `!${trimmed}`;
+          })
+          .join('\n');
+        cells.push(codeCell(shellLines));
+      } else {
+        // Non-executable code (json, yaml, text, pseudo-code) → markdown fence
+        cells.push(markdownCell(`\`\`\`${block.lang}\n${block.content}\n\`\`\``));
+      }
+    }
+  }
+  return cells;
+}
+
+
+function markdownCell(source) {
+  return {
+    cell_type: 'markdown',
+    metadata: {},
+    source: sourceLines(source),
+  };
+}
+
+function codeCell(source) {
+  return {
+    cell_type: 'code',
+    execution_count: null,
+    metadata: {},
+    outputs: [],
+    source: sourceLines(source),
+  };
+}
+
+/** Jupyter expects source as an array of lines, each ending with \n except the last */
+function sourceLines(text) {
+  const lines = text.split('\n');
+  return lines.map((line, i) => (i < lines.length - 1 ? line + '\n' : line));
+}
+
+function buildNotebook(cells, title) {
+  return {
+    nbformat: 4,
+    nbformat_minor: 5,
+    metadata: {
+      kernelspec: {
+        display_name: 'Python 3',
+        language: 'python',
+        name: 'python3',
+      },
+      language_info: {
+        name: 'python',
+        version: '3.11.0',
+      },
+      title,
+    },
+    cells,
+  };
+}
+
+function loadMeta(trackDir) {
+  const metaPath = path.join(trackDir, 'meta.json');
+  if (!fs.existsSync(metaPath)) {
+    console.error(`  ⚠️  No meta.json in ${trackDir}`);
+    return null;
+  }
+  return JSON.parse(fs.readFileSync(metaPath, 'utf8'));
+}
+
+function buildTrack(trackName) {
+  const trackDir = path.join(CONTENT_DIR, trackName);
+  const outDir = path.join(OUTPUT_DIR, trackName);
+
+  if (!fs.existsSync(trackDir)) {
+    console.error(`❌ Track not found: ${trackDir}`);
+    return;
+  }
+
+  const meta = loadMeta(trackDir);
+  if (!meta) return;
+
+  fs.mkdirSync(outDir, { recursive: true });
+
+  console.log(`📓 Building notebooks: ${meta.trackName} (${meta.cookbooks.length} cookbooks)`);
+
+  for (const cookbook of meta.cookbooks) {
+    const mdPath = path.join(trackDir, cookbook.source);
+    if (!fs.existsSync(mdPath)) {
+      console.error(`  ⚠️  Missing: ${mdPath}`);
+      continue;
+    }
+
+    const md = fs.readFileSync(mdPath, 'utf8');
+
+    // Add a title cell at the top
+    const titleMd = `# ${cookbook.title}\n\n**${cookbook.difficulty}** · ~${cookbook.time} · ${meta.trackName}`;
+    const blocks = splitMarkdown(md);
+    const cells = [markdownCell(titleMd), ...blocksToNotebookCells(blocks)];
+
+    const nb = buildNotebook(cells, cookbook.title);
+    const outName = cookbook.source.replace(/\.md$/, '.ipynb');
+    const outPath = path.join(outDir, outName);
+    fs.writeFileSync(outPath, JSON.stringify(nb, null, 1));
+    console.log(`  ✅ ${outName}`);
+  }
+}
+
+// --- Main ---
+const targetTrack = process.argv[2];
+
+if (targetTrack) {
+  buildTrack(targetTrack);
+} else {
+  const tracks = fs.readdirSync(CONTENT_DIR).filter(d =>
+    fs.statSync(path.join(CONTENT_DIR, d)).isDirectory()
+  );
+  console.log(`🔨 Building notebooks for ${tracks.length} track(s)...\n`);
+  for (const track of tracks) {
+    buildTrack(track);
+  }
+}
+
+console.log('\n✨ Done!');

build.js

@@ -30,7 +30,14 @@ function loadMeta(trackDir) {
 }
 
 function buildPage(mdContent, cookbook, track, meta) {
-  const htmlContent = marked.parse(mdContent);
+  let htmlContent = marked.parse(mdContent);
+
+  // Convert <pre><code class="language-mermaid">...</code></pre> to <pre class="mermaid">...</pre>
+  // so mermaid.js can find and render them
+  htmlContent = htmlContent.replace(
+    /<pre><code class="language-mermaid">([\s\S]*?)<\/code><\/pre>/g,
+    (_, code) => `<pre class="mermaid">${code}</pre>`
+  );
 
   const diffClass = { 'Beginner': 'diff-easy', 'Intermediate': 'diff-medium', 'Advanced': 'diff-hard' }[cookbook.difficulty] || 'diff-medium';
 
@@ -83,6 +90,7 @@ ${nextHtml}
 <button class="theme-toggle" onclick="toggleTheme()" aria-label="Toggle theme"><svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="12" r="5"/><line x1="12" y1="1" x2="12" y2="3"/><line x1="12" y1="21" x2="12" y2="23"/><line x1="4.22" y1="4.22" x2="5.64" y2="5.64"/><line x1="18.36" y1="18.36" x2="19.78" y2="19.78"/><line x1="1" y1="12" x2="3" y2="12"/><line x1="21" y1="12" x2="23" y2="12"/></svg></button>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
 <script>hljs.highlightAll();</script>
+<script type="module">import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs';mermaid.initialize({startOnLoad:true,theme:document.body.classList.contains('light')?'default':'dark'});</script>
 <script>function toggleTheme(){var b=document.body,l=b.classList.toggle("light");localStorage.setItem("theme",l?"light":"dark")}(function(){if(localStorage.getItem("theme")==="light")document.body.classList.add("light")})()</script>
 </body>
 </html>`;

content/betterdb-agent-cache/01-getting-started.md

@@ -17,7 +17,7 @@ Unlike `langgraph-checkpoint-redis` or similar packages, it requires **no Valkey
 No special image needed - the standard Valkey image works:
 
 ```bash
-docker run -d --name valkey -p 6379:6379 valkey/valkey:latest
+docker run -d --name valkey -p 6379:6379 valkey/valkey-bundle:9-alpine
 ```
 
 ```bash

content/betterdb-semantic-cache/01-getting-started.md

@@ -13,7 +13,7 @@ Unlike other semantic cache libraries it is **Valkey-native** (handles `valkey-s
 The `valkey-bundle` image includes the `valkey-search` module required for vector indexing.
 
 ```bash
-docker run -d --name valkey -p 6379:6379 valkey/valkey-bundle:latest
+docker run -d --name valkey -p 6379:6379 valkey/valkey-bundle:9-alpine
 ```
 
 Verify the module is loaded:

content/context-engineering/01-fundamentals.md

@@ -23,15 +23,28 @@ Every LLM call needs context assembled from up to 5 sources. Valkey can serve as
 ## Prerequisites
 
 - Valkey with the **valkey-search** module (or ElastiCache for Valkey 8.2+)
-- Python 3.9+ with `valkey`
+- Python 3.12+ with `valkey`
+
+## Step 1: Start Valkey
 
 ```bash
-pip install valkey
+docker run -d --name valkey -p 6379:6379 valkey/valkey-bundle:9-alpine
 ```
 
-## Step 1: Store System Instructions
+## Step 2: Install Dependencies
+
+```bash
+uv pip install valkey python-dotenv
+```
+
+## Step 3: Store System Instructions
 
 ```python
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
 import valkey
 import json
 
@@ -47,7 +60,7 @@ client.hset("agent:config:support_bot", mapping={
 print("System instructions stored")
 ```
 
-## Step 2: Manage Conversation History
+## Step 4: Manage Conversation History
 
 ```python
 import time
@@ -76,7 +89,7 @@ for msg in history:
     print(f"  {msg['role']}: {msg['content'][:60]}...")
 ```
 
-## Step 3: Store Tool Outputs
+## Step 5: Store Tool Outputs
 
 ```python
 def store_tool_output(session_id: str, step: int, tool_name: str, result: dict):
@@ -109,7 +122,7 @@ store_tool_output("sess_001", 1, "check_order", {
 })
 ```
 
-## Step 4: Long-term User Memory
+## Step 6: Long-term User Memory
 
 ```python
 def remember(user_id: str, key: str, value: str):

content/context-engineering/02-assembly-pipeline.md

@@ -3,6 +3,11 @@
 Before every LLM call, an agent needs to assemble context from multiple sources. This function is the core of context engineering - it gathers everything the LLM needs into a structured prompt.
 
 ```python
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
 import valkey
 import json
 import struct

content/context-engineering/03-production.md

@@ -3,6 +3,11 @@
 The memory layer is the foundation of context engineering at scale. Valkey supports both memory types through TTL management:
 
 ```python
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
 import valkey
 import json
 import time

content/conversation-memory/01-getting-started.md

@@ -10,12 +10,12 @@ LLMs are stateless - every API call starts from scratch. Conversation memory bri
 ## Prerequisites
 
   * Docker installed (or a running Valkey instance)
-  * Python 3.9+
+  * Python 3.12+
 
 ## Step 1: Start Valkey
 
 ```bash
-docker run -d --name valkey -p 6379:6379 valkey/valkey:latest
+docker run -d --name valkey -p 6379:6379 valkey/valkey-bundle:9-alpine
 ```
 
 Verify it's running:
@@ -28,7 +28,7 @@ docker exec valkey valkey-cli PING
 ## Step 2: Install GLIDE
 
 ```bash
-pip install valkey-glide
+uv pip install valkey-glide python-dotenv
 ```
 
 GLIDE is the official Valkey client - Rust core with Python bindings. It works with both standalone Valkey and ElastiCache for Valkey clusters.
@@ -54,6 +54,11 @@ chat:sess_abc123 → [
 ## Step 4: Connect and Store Messages
 
 ```python
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
 import asyncio
 import json
 from glide import GlideClient, GlideClientConfiguration, NodeAddress