feat(docs): add sitemap.xml + robots.txt for SEO

docsite/.lighthouserc.cjs

@@ -2,23 +2,10 @@ const fs = require("node:fs");
 const path = require("node:path");
 
 const SITE = "http://127.0.0.1:4173";
-const DOCS_ROOT = path.join(__dirname, "content/docs");
+const SITEMAP = path.join(__dirname, "dist/client/sitemap.xml");
 
-function collectDocUrls(dir, urlPrefix = "/docs") {
-  const urls = [];
-  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-    const full = path.join(dir, entry.name);
-    if (entry.isDirectory()) {
-      urls.push(...collectDocUrls(full, `${urlPrefix}/${entry.name}`));
-    } else if (entry.name.endsWith(".mdx")) {
-      const slug = entry.name.replace(/\.mdx$/, "");
-      urls.push(slug === "index" ? urlPrefix : `${urlPrefix}/${slug}`);
-    }
-  }
-  return urls;
-}
-
-const urls = ["/", ...collectDocUrls(DOCS_ROOT)].map((p) => `${SITE}${p}`);
+const xml = fs.readFileSync(SITEMAP, "utf8");
+const urls = Array.from(xml.matchAll(/<loc>([^<]+)<\/loc>/g), (m) => m[1].replace(/^https?:\/\/[^/]+/, SITE));
 
 module.exports = {
   ci: {

docsite/CLAUDE.md

@@ -19,6 +19,6 @@ The pinned `@lhci/cli@X.Y.Z` version lives in two places that **must be kept in
 
 When bumping the version, update both — otherwise CI keeps restoring the old `~/.cache/pnpm/dlx` and the new bin is re-downloaded every run, defeating the cache.
 
-URLs and budgets live in [`.lighthouserc.cjs`](./.lighthouserc.cjs). The URL list is auto-derived by walking `content/docs/**/*.mdx` — new docs are picked up automatically. Budgets: SEO and best-practices pinned at 100 (`error`); accessibility ≥90 and performance ≥50 (`warn`). Any regression in `meta-description`, `document-title`, `is-crawlable`, etc. fails the PR.
+URLs and budgets live in [`.lighthouserc.cjs`](./.lighthouserc.cjs). The URL list is parsed from `dist/client/sitemap.xml` (produced by prerendering [`src/routes/sitemap[.]xml.ts`](./src/routes/sitemap[.]xml.ts) during `vite build`) — new docs are picked up automatically. Budgets: SEO and best-practices pinned at 100 (`error`); accessibility ≥90 and performance ≥50 (`warn`). Any regression in `meta-description`, `document-title`, `is-crawlable`, etc. fails the PR.
 
 **Why Lighthouse CI instead of Unlighthouse:** an earlier version of this workflow used Unlighthouse. It worked but `unlighthouse-ci` leaks Chrome process handles after a successful scan and never exits cleanly — under GitHub Actions the step would hang several minutes after the audit finished. LHCI handles process cleanup correctly and is the more battle-tested CI primitive.

docsite/public/robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Allow: /
+Disallow: /api/
+
+Sitemap: https://env.oss.variable.land/sitemap.xml

docsite/src/components/landing/data.ts

@@ -123,3 +123,5 @@ export const LANDING_META = {
   docsHref: "/docs",
   publishDate: "2026",
 } as const;
+
+export const SITE_URL = "https://env.oss.variable.land";

docsite/src/routeTree.gen.ts

@@ -9,13 +9,19 @@
 // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
 
 import { Route as rootRouteImport } from './routes/__root'
+import { Route as SitemapDotxmlRouteImport } from './routes/sitemap[.]xml'
 import { Route as LlmsDottxtRouteImport } from './routes/llms[.]txt'
 import { Route as LlmsFullDottxtRouteImport } from './routes/llms-full[.]txt'
 import { Route as IndexRouteImport } from './routes/index'
 import { Route as RawSplatRouteImport } from './routes/raw/$'
 import { Route as DocsSplatRouteImport } from './routes/docs/$'
 import { Route as ApiSearchRouteImport } from './routes/api/search'
 
+const SitemapDotxmlRoute = SitemapDotxmlRouteImport.update({
+  id: '/sitemap.xml',
+  path: '/sitemap.xml',
+  getParentRoute: () => rootRouteImport,
+} as any)
 const LlmsDottxtRoute = LlmsDottxtRouteImport.update({
   id: '/llms.txt',
   path: '/llms.txt',
@@ -51,6 +57,7 @@ export interface FileRoutesByFullPath {
   '/': typeof IndexRoute
   '/llms-full.txt': typeof LlmsFullDottxtRoute
   '/llms.txt': typeof LlmsDottxtRoute
+  '/sitemap.xml': typeof SitemapDotxmlRoute
   '/api/search': typeof ApiSearchRoute
   '/docs/$': typeof DocsSplatRoute
   '/raw/$': typeof RawSplatRoute
@@ -59,6 +66,7 @@ export interface FileRoutesByTo {
   '/': typeof IndexRoute
   '/llms-full.txt': typeof LlmsFullDottxtRoute
   '/llms.txt': typeof LlmsDottxtRoute
+  '/sitemap.xml': typeof SitemapDotxmlRoute
   '/api/search': typeof ApiSearchRoute
   '/docs/$': typeof DocsSplatRoute
   '/raw/$': typeof RawSplatRoute
@@ -68,6 +76,7 @@ export interface FileRoutesById {
   '/': typeof IndexRoute
   '/llms-full.txt': typeof LlmsFullDottxtRoute
   '/llms.txt': typeof LlmsDottxtRoute
+  '/sitemap.xml': typeof SitemapDotxmlRoute
   '/api/search': typeof ApiSearchRoute
   '/docs/$': typeof DocsSplatRoute
   '/raw/$': typeof RawSplatRoute
@@ -78,6 +87,7 @@ export interface FileRouteTypes {
     | '/'
     | '/llms-full.txt'
     | '/llms.txt'
+    | '/sitemap.xml'
     | '/api/search'
     | '/docs/$'
     | '/raw/$'
@@ -86,6 +96,7 @@ export interface FileRouteTypes {
     | '/'
     | '/llms-full.txt'
     | '/llms.txt'
+    | '/sitemap.xml'
     | '/api/search'
     | '/docs/$'
     | '/raw/$'
@@ -94,6 +105,7 @@ export interface FileRouteTypes {
     | '/'
     | '/llms-full.txt'
     | '/llms.txt'
+    | '/sitemap.xml'
     | '/api/search'
     | '/docs/$'
     | '/raw/$'
@@ -103,13 +115,21 @@ export interface RootRouteChildren {
   IndexRoute: typeof IndexRoute
   LlmsFullDottxtRoute: typeof LlmsFullDottxtRoute
   LlmsDottxtRoute: typeof LlmsDottxtRoute
+  SitemapDotxmlRoute: typeof SitemapDotxmlRoute
   ApiSearchRoute: typeof ApiSearchRoute
   DocsSplatRoute: typeof DocsSplatRoute
   RawSplatRoute: typeof RawSplatRoute
 }
 
 declare module '@tanstack/react-router' {
   interface FileRoutesByPath {
+    '/sitemap.xml': {
+      id: '/sitemap.xml'
+      path: '/sitemap.xml'
+      fullPath: '/sitemap.xml'
+      preLoaderRoute: typeof SitemapDotxmlRouteImport
+      parentRoute: typeof rootRouteImport
+    }
     '/llms.txt': {
       id: '/llms.txt'
       path: '/llms.txt'
@@ -159,6 +179,7 @@ const rootRouteChildren: RootRouteChildren = {
   IndexRoute: IndexRoute,
   LlmsFullDottxtRoute: LlmsFullDottxtRoute,
   LlmsDottxtRoute: LlmsDottxtRoute,
+  SitemapDotxmlRoute: SitemapDotxmlRoute,
   ApiSearchRoute: ApiSearchRoute,
   DocsSplatRoute: DocsSplatRoute,
   RawSplatRoute: RawSplatRoute,

docsite/src/routes/__root.tsx

@@ -1,10 +1,9 @@
 import { createRootRoute, HeadContent, Outlet, Scripts } from "@tanstack/react-router";
 import { RootProvider } from "fumadocs-ui/provider/tanstack";
 import type * as React from "react";
+import { SITE_URL } from "#src/components/landing/data.ts";
 import appCss from "#src/styles/app.css?url";
 
-const SITE_URL = "https://env.oss.variable.land";
-
 export const Route = createRootRoute({
   head: () => ({
     meta: [

docsite/src/routes/sitemap[.]xml.ts

@@ -0,0 +1,22 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { SITE_URL } from "#src/components/landing/data.ts";
+import { source } from "#src/lib/source.ts";
+
+export const Route = createFileRoute("/sitemap.xml")({
+  server: {
+    handlers: {
+      GET: () => {
+        const paths = ["/", ...source.getPages().map((page) => page.url)];
+        const urls = paths.map((path) => `  <url><loc>${SITE_URL}${path}</loc></url>`).join("\n");
+        const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls}
+</urlset>
+`;
+        return new Response(xml, {
+          headers: { "Content-Type": "application/xml; charset=utf-8" },
+        });
+      },
+    },
+  },
+});

docsite/vite.config.ts

@@ -6,5 +6,18 @@ import mdx from "fumadocs-mdx/vite";
 import { defineConfig } from "vite";
 
 export default defineConfig({
-  plugins: [cloudflare({ viteEnvironment: { name: "ssr" } }), tailwindcss(), mdx(), tanstackStart(), react()],
+  plugins: [
+    cloudflare({ viteEnvironment: { name: "ssr" } }),
+    tailwindcss(),
+    mdx(),
+    tanstackStart({
+      prerender: {
+        enabled: true,
+        crawlLinks: true,
+        filter: (page) => !page.path.startsWith("/api/") && !page.path.includes("#"),
+      },
+      pages: [{ path: "/" }, { path: "/sitemap.xml" }, { path: "/llms.txt" }, { path: "/llms-full.txt" }],
+    }),
+    react(),
+  ],
 });