Enhance build process with cache generation, and add cache validation…

README.md

@@ -15,3 +15,47 @@
 > [MDN website](https://developer.mozilla.org/en-US/about)
 
 > The idea of using the sitemap file came from this [random-mdn twitter-bot](https://github.com/random-mdn/random-mdn-bot)
+
+## Development
+
+### Cache Generation
+
+This project uses a static cache system to improve performance. The cache includes:
+- All MDN Web documentation links (~11,865 links)
+- Pre-fetched metadata for the top 1,000 popular links
+
+**Cache Workflow:**
+- Cache file location: `src/assets/sitemap-cache.json` (committed to git)
+- Cache is conditionally regenerated during build (only if missing or older than 7 days)
+- Cache generation fetches the MDN sitemap and pre-loads metadata for popular links (~5 minutes)
+- Most builds skip cache regeneration for speed (~10 seconds vs ~5 minutes)
+
+**Build Process:**
+The build command intelligently manages cache:
+```bash
+# Standard build (fast if cache is fresh < 7 days old)
+npm run build  # Checks cache age -> skips if fresh -> TypeScript compile -> Vite build
+
+# Force cache regeneration
+npm run build:force-cache  # Always regenerates cache (~5 min)
+
+# Skip cache check entirely (uses existing cache no matter age)
+npm run build:skip-cache  # Fastest build, uses any existing cache
+```
+
+**Manual Cache Operations:**
+```bash
+# Generate/regenerate cache manually
+npm run generate-cache
+
+# Validate cache integrity
+npm run validate-cache
+```
+
+**Vercel Deployment:**
+- Cache is bundled with serverless functions during deployment
+- By default, cache only regenerates if older than 7 days (keeps builds fast)
+- To force fresh cache on each deployment, set `FORCE_CACHE_REGEN=true` in Vercel build settings
+- Cache size (currently ~1.5MB) is well within Vercel's 50MB serverless function limit
+
+For more details, see [STATIC_CACHE_PLAN.md](./STATIC_CACHE_PLAN.md).

api/getRandomLink.ts

@@ -1,60 +1,115 @@
 import type { VercelRequest, VercelResponse } from "@vercel/node";
+import sitemapCache from "../src/assets/sitemap-cache.json" with { type: "json" };
 
-const BASE_URL = "https://developer.mozilla.org";
-const SITEMAP_URL = `${BASE_URL}/sitemaps/en-us/sitemap.xml.gz`;
-const WEB_PATH = `${BASE_URL}/en-US/docs/Web`;
-const LINK_REGEX = /<loc>(.*?)<\/loc>/g;
 const TITLE_REGEX = /<h1>(.*?)<\/h1>/i;
 const DESCRIPTION_REGEX = /<meta name="description" content="([^"]*)"/i;
+const FIRST_PARAGRAPH_REGEX = /<p[^>]*>(.*?)<\/p>/is;
 const SECTION_REGEX = /Web\/(.*?)\//;
-
-export const getSitemapLinks = async (): Promise<string[]> => {
-	try {
-		const response = await fetch(SITEMAP_URL, {
-			headers: { "Accept-Encoding": "gzip" },
-		});
-		const sitemap = await response.text();
-		const allMatches = Array.from(sitemap.matchAll(LINK_REGEX));
-		const webDocsLinks = allMatches
-			.map((match) => match[1])
-			.filter((url) => url.startsWith(WEB_PATH));
-
-		console.log(`Found ${webDocsLinks.length} Web docs links`);
-		return webDocsLinks;
-	} catch (error) {
-		console.log(error);
-		return [];
-	}
-};
+const DEPRECATED_REGEX = /class="[^"]*notecard[^"]*deprecated[^"]*"|class="[^"]*deprecated[^"]*notecard[^"]*"/i;
 
 type LinkMetaData = {
 	tag: string;
 	title: string;
 	description: string;
 	url: string;
+	isDeprecated?: boolean;
+};
+
+const isDeprecated = (html: string): boolean => {
+	return DEPRECATED_REGEX.test(html);
+};
+
+const extractFirstParagraph = (html: string): string => {
+	const match = html.match(FIRST_PARAGRAPH_REGEX);
+	if (!match) return "";
+
+	const paragraph = match[1]
+		.replace(/<[^>]+>/g, "") // Remove HTML tags
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.trim();
+
+	return paragraph || "";
 };
 
 export const getLinkMetaData = async (link: string): Promise<LinkMetaData> => {
+	if (sitemapCache.popularMetadata[link]) {
+		console.log("Using cached metadata");
+		return sitemapCache.popularMetadata[link];
+	}
+
+	console.log("Fetching metadata from HTML");
+
 	try {
+		const response = await fetch(link);
+		const html = await response.text();
+
 		const tag = link.match(SECTION_REGEX)?.[1] || "";
-		const htmlDocument: string = await fetch(link).then((res) => res.text());
-		const title = htmlDocument.match(TITLE_REGEX)?.[1] || "Unknown reference";
-		const description =
-			htmlDocument.match(DESCRIPTION_REGEX)?.[1] || "Missing description";
-		return {
+		const title = html.match(TITLE_REGEX)?.[1] || "Unknown reference";
+
+		// Check for deprecated status
+		const deprecated = isDeprecated(html);
+
+		// Try meta description first, fallback to first paragraph
+		let description = html.match(DESCRIPTION_REGEX)?.[1] || "";
+		if (!description || description === "Missing description") {
+			const firstParagraph = extractFirstParagraph(html);
+			description = firstParagraph || "Missing description";
+		}
+
+		const metadata: LinkMetaData = {
 			tag: tag === "API" ? "WEB API" : tag,
-			title,
-			description,
+			title: title
+				.replace(/&lt;/g, "<")
+				.replace(/&gt;/g, ">")
+				.replace(/&amp;/g, "&"),
+			description: description
+				.replace(/&lt;/g, "<")
+				.replace(/&gt;/g, ">")
+				.replace(/&amp;/g, "&"),
 			url: link,
+			isDeprecated: deprecated,
 		};
+
+		return metadata;
 	} catch (error) {
-		console.log(error);
-		return { tag: "", title: "", description: "", url: link };
+		console.error("Metadata fetch failed:", error);
+		return { tag: "", title: "Unknown reference", description: "", url: link };
 	}
 };
 
-// TODO Filter deprecated class names =>  "notecard", "deprecated"
-//TODO if no meta description get it from html document first <p> tag
+const FALLBACK_LINKS = [
+	"https://developer.mozilla.org/en-US/docs/Web/JavaScript",
+	"https://developer.mozilla.org/en-US/docs/Web/HTML",
+	"https://developer.mozilla.org/en-US/docs/Web/CSS",
+];
+
+export const getSitemapLinks = async (): Promise<string[]> => {
+	try {
+		if (!sitemapCache?.links?.length) {
+			console.warn("Cache unavailable, using fallback links");
+			return FALLBACK_LINKS;
+		}
+
+		const cacheAge = Date.now() - sitemapCache.metadata.buildTime;
+		const maxAge = 24 * 60 * 60 * 1000; // 24 hours
+
+		if (cacheAge > maxAge) {
+			console.warn("Cache is stale, but using anyway");
+		}
+
+		console.log(
+			`📋 Using cached sitemap: ${sitemapCache.links.length} links (${sitemapCache.metadata.cachedMetadataCount} with metadata)`,
+		);
+		return sitemapCache.links;
+	} catch (error) {
+		console.error("❌ Cache read failed:", error);
+		return FALLBACK_LINKS;
+	}
+};
 
 export default async (request: VercelRequest, response: VercelResponse) => {
 	try {
@@ -87,10 +142,32 @@ export default async (request: VercelRequest, response: VercelResponse) => {
 			}
 		}
 
-		const randomLink =
-			filteredLinks[Math.floor(Math.random() * filteredLinks.length)];
-		const linkMetaData = await getLinkMetaData(randomLink);
-		return response.status(200).send(linkMetaData);
+		// Try up to 5 times to find a non-deprecated link
+		const maxRetries = 5;
+		let attempts = 0;
+		let linkMetaData: LinkMetaData | null = null;
+
+		while (attempts < maxRetries) {
+			const randomLink =
+				filteredLinks[Math.floor(Math.random() * filteredLinks.length)];
+			linkMetaData = await getLinkMetaData(randomLink);
+
+			// Skip deprecated links, but allow if all attempts exhausted
+			if (!linkMetaData.isDeprecated || attempts === maxRetries - 1) {
+				break;
+			}
+
+			attempts++;
+			console.log(`Skipping deprecated link, retry ${attempts}/${maxRetries}`);
+		}
+
+		if (!linkMetaData) {
+			return response.status(500).send("Failed to fetch link metadata");
+		}
+
+		// Remove isDeprecated from response (internal only)
+		const { isDeprecated: _deprecated, ...responseData } = linkMetaData;
+		return response.status(200).send(responseData);
 	} catch (_error) {
 		return response.status(500).send("Something went wrong");
 	}

index.html

@@ -14,13 +14,13 @@
     <meta property="og:type" content="website">
     <meta property="og:title" content="Random MDN - Discover MDN Documentation">
     <meta property="og:description" content="Discover random MDN Web Docs articles to expand your web development knowledge">
-    <meta property="og:image" content="/logo.png">
+    <meta property="og:image" content="/opengraph-image.png">
 
     <!-- Twitter -->
     <meta name="twitter:card" content="summary">
     <meta name="twitter:title" content="Random MDN - Discover MDN Documentation">
     <meta name="twitter:description" content="Discover random MDN Web Docs articles to expand your web development knowledge">
-    <meta name="twitter:image" content="/logo.png">
+    <meta name="twitter:image" content="/opengraph-image.png">
 
     <!-- Theme Color -->
     <meta name="theme-color" content="#3451b2">

package.json

@@ -6,9 +6,13 @@
 	"scripts": {
 		"dev": "vite",
 		"serverless-dev": "vercel dev",
-		"build": "tsc -b && vite build",
+		"build": "node scripts/ensure-cache.ts && tsc -b && vite build",
+		"build:force-cache": "FORCE_CACHE_REGEN=true npm run build",
+		"build:skip-cache": "SKIP_CACHE_CHECK=true npm run build",
 		"preview": "vite preview",
-		"lint": "biome check --write"
+		"lint": "biome check --write",
+		"generate-cache": "node scripts/generate-cache.ts",
+		"validate-cache": "node scripts/validate-cache.ts"
 	},
 	"dependencies": {
 		"@radix-ui/react-checkbox": "^1.3.3",

scripts/ensure-cache.ts

@@ -0,0 +1,71 @@
+/**
+ * Smart Cache Management Script
+ *
+ * Conditionally regenerates cache based on age, existence, and environment variables.
+ * This keeps builds fast by skipping regeneration when cache is fresh.
+ */
+
+import { existsSync, readFileSync } from "fs";
+import { execSync } from "child_process";
+import path from "path";
+
+const CACHE_PATH = path.join(process.cwd(), "src/assets/sitemap-cache.json");
+const MAX_CACHE_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+const FORCE_REGENERATE = process.env.FORCE_CACHE_REGEN === "true";
+const SKIP_CACHE_CHECK = process.env.SKIP_CACHE_CHECK === "true";
+
+function shouldRegenerateCache(): boolean {
+	// Force regeneration via env var
+	if (FORCE_REGENERATE) {
+		console.log("🔄 Force regenerating cache (FORCE_CACHE_REGEN=true)");
+		return true;
+	}
+
+	// Skip cache generation entirely
+	if (SKIP_CACHE_CHECK) {
+		console.log("⏭️  Skipping cache generation (SKIP_CACHE_CHECK=true)");
+		return false;
+	}
+
+	// Cache doesn't exist - must generate
+	if (!existsSync(CACHE_PATH)) {
+		console.log("📦 Cache missing - generating...");
+		return true;
+	}
+
+	// Check cache age
+	try {
+		const cache = JSON.parse(readFileSync(CACHE_PATH, "utf-8"));
+		const cacheAge = Date.now() - (cache.metadata?.buildTime || 0);
+		const ageDays = Math.round(cacheAge / (24 * 60 * 60 * 1000));
+
+		if (cacheAge > MAX_CACHE_AGE_MS) {
+			console.log(
+				`🔄 Cache is ${ageDays} days old (max: ${Math.round(MAX_CACHE_AGE_MS / (24 * 60 * 60 * 1000))} days) - regenerating...`,
+			);
+			return true;
+		}
+
+		console.log(
+			`✅ Cache is fresh (${ageDays} days old) - skipping regeneration`,
+		);
+		return false;
+	} catch (error) {
+		console.warn("⚠️  Error reading cache, regenerating...", error);
+		return true;
+	}
+}
+
+if (shouldRegenerateCache()) {
+	console.log("⏳ Generating cache (this takes ~5 minutes)...");
+	try {
+		execSync("node scripts/generate-cache.ts", { stdio: "inherit" });
+		console.log("✨ Cache generation completed");
+	} catch (error) {
+		console.error("❌ Cache generation failed:", error);
+		process.exit(1);
+	}
+} else {
+	console.log("✨ Using existing cache");
+}
+