diff --git a/docsite/.lighthouserc.cjs b/docsite/.lighthouserc.cjs index 0ff5250..7952fc1 100644 --- a/docsite/.lighthouserc.cjs +++ b/docsite/.lighthouserc.cjs @@ -2,23 +2,10 @@ const fs = require("node:fs"); const path = require("node:path"); const SITE = "http://127.0.0.1:4173"; -const DOCS_ROOT = path.join(__dirname, "content/docs"); +const SITEMAP = path.join(__dirname, "dist/client/sitemap.xml"); -function collectDocUrls(dir, urlPrefix = "/docs") { - const urls = []; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - urls.push(...collectDocUrls(full, `${urlPrefix}/${entry.name}`)); - } else if (entry.name.endsWith(".mdx")) { - const slug = entry.name.replace(/\.mdx$/, ""); - urls.push(slug === "index" ? urlPrefix : `${urlPrefix}/${slug}`); - } - } - return urls; -} - -const urls = ["/", ...collectDocUrls(DOCS_ROOT)].map((p) => `${SITE}${p}`); +const xml = fs.readFileSync(SITEMAP, "utf8"); +const urls = Array.from(xml.matchAll(/([^<]+)<\/loc>/g), (m) => m[1].replace(/^https?:\/\/[^/]+/, SITE)); module.exports = { ci: { diff --git a/docsite/CLAUDE.md b/docsite/CLAUDE.md index ccde0ac..683eadb 100644 --- a/docsite/CLAUDE.md +++ b/docsite/CLAUDE.md @@ -19,6 +19,6 @@ The pinned `@lhci/cli@X.Y.Z` version lives in two places that **must be kept in When bumping the version, update both — otherwise CI keeps restoring the old `~/.cache/pnpm/dlx` and the new bin is re-downloaded every run, defeating the cache. -URLs and budgets live in [`.lighthouserc.cjs`](./.lighthouserc.cjs). The URL list is auto-derived by walking `content/docs/**/*.mdx` — new docs are picked up automatically. Budgets: SEO and best-practices pinned at 100 (`error`); accessibility ≥90 and performance ≥50 (`warn`). Any regression in `meta-description`, `document-title`, `is-crawlable`, etc. fails the PR. +URLs and budgets live in [`.lighthouserc.cjs`](./.lighthouserc.cjs). The URL list is parsed from `dist/client/sitemap.xml` (produced by prerendering [`src/routes/sitemap[.]xml.ts`](./src/routes/sitemap[.]xml.ts) during `vite build`) — new docs are picked up automatically. Budgets: SEO and best-practices pinned at 100 (`error`); accessibility ≥90 and performance ≥50 (`warn`). Any regression in `meta-description`, `document-title`, `is-crawlable`, etc. fails the PR. **Why Lighthouse CI instead of Unlighthouse:** an earlier version of this workflow used Unlighthouse. It worked but `unlighthouse-ci` leaks Chrome process handles after a successful scan and never exits cleanly — under GitHub Actions the step would hang several minutes after the audit finished. LHCI handles process cleanup correctly and is the more battle-tested CI primitive. diff --git a/docsite/public/robots.txt b/docsite/public/robots.txt new file mode 100644 index 0000000..4ca8b1e --- /dev/null +++ b/docsite/public/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Allow: / +Disallow: /api/ + +Sitemap: https://env.oss.variable.land/sitemap.xml diff --git a/docsite/src/components/landing/data.ts b/docsite/src/components/landing/data.ts index c91d05e..9e8f550 100644 --- a/docsite/src/components/landing/data.ts +++ b/docsite/src/components/landing/data.ts @@ -123,3 +123,5 @@ export const LANDING_META = { docsHref: "/docs", publishDate: "2026", } as const; + +export const SITE_URL = "https://env.oss.variable.land"; diff --git a/docsite/src/routeTree.gen.ts b/docsite/src/routeTree.gen.ts index 913f671..95a528d 100644 --- a/docsite/src/routeTree.gen.ts +++ b/docsite/src/routeTree.gen.ts @@ -9,6 +9,7 @@ // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. import { Route as rootRouteImport } from './routes/__root' +import { Route as SitemapDotxmlRouteImport } from './routes/sitemap[.]xml' import { Route as LlmsDottxtRouteImport } from './routes/llms[.]txt' import { Route as LlmsFullDottxtRouteImport } from './routes/llms-full[.]txt' import { Route as IndexRouteImport } from './routes/index' @@ -16,6 +17,11 @@ import { Route as RawSplatRouteImport } from './routes/raw/$' import { Route as DocsSplatRouteImport } from './routes/docs/$' import { Route as ApiSearchRouteImport } from './routes/api/search' +const SitemapDotxmlRoute = SitemapDotxmlRouteImport.update({ + id: '/sitemap.xml', + path: '/sitemap.xml', + getParentRoute: () => rootRouteImport, +} as any) const LlmsDottxtRoute = LlmsDottxtRouteImport.update({ id: '/llms.txt', path: '/llms.txt', @@ -51,6 +57,7 @@ export interface FileRoutesByFullPath { '/': typeof IndexRoute '/llms-full.txt': typeof LlmsFullDottxtRoute '/llms.txt': typeof LlmsDottxtRoute + '/sitemap.xml': typeof SitemapDotxmlRoute '/api/search': typeof ApiSearchRoute '/docs/$': typeof DocsSplatRoute '/raw/$': typeof RawSplatRoute @@ -59,6 +66,7 @@ export interface FileRoutesByTo { '/': typeof IndexRoute '/llms-full.txt': typeof LlmsFullDottxtRoute '/llms.txt': typeof LlmsDottxtRoute + '/sitemap.xml': typeof SitemapDotxmlRoute '/api/search': typeof ApiSearchRoute '/docs/$': typeof DocsSplatRoute '/raw/$': typeof RawSplatRoute @@ -68,6 +76,7 @@ export interface FileRoutesById { '/': typeof IndexRoute '/llms-full.txt': typeof LlmsFullDottxtRoute '/llms.txt': typeof LlmsDottxtRoute + '/sitemap.xml': typeof SitemapDotxmlRoute '/api/search': typeof ApiSearchRoute '/docs/$': typeof DocsSplatRoute '/raw/$': typeof RawSplatRoute @@ -78,6 +87,7 @@ export interface FileRouteTypes { | '/' | '/llms-full.txt' | '/llms.txt' + | '/sitemap.xml' | '/api/search' | '/docs/$' | '/raw/$' @@ -86,6 +96,7 @@ export interface FileRouteTypes { | '/' | '/llms-full.txt' | '/llms.txt' + | '/sitemap.xml' | '/api/search' | '/docs/$' | '/raw/$' @@ -94,6 +105,7 @@ export interface FileRouteTypes { | '/' | '/llms-full.txt' | '/llms.txt' + | '/sitemap.xml' | '/api/search' | '/docs/$' | '/raw/$' @@ -103,6 +115,7 @@ export interface RootRouteChildren { IndexRoute: typeof IndexRoute LlmsFullDottxtRoute: typeof LlmsFullDottxtRoute LlmsDottxtRoute: typeof LlmsDottxtRoute + SitemapDotxmlRoute: typeof SitemapDotxmlRoute ApiSearchRoute: typeof ApiSearchRoute DocsSplatRoute: typeof DocsSplatRoute RawSplatRoute: typeof RawSplatRoute @@ -110,6 +123,13 @@ export interface RootRouteChildren { declare module '@tanstack/react-router' { interface FileRoutesByPath { + '/sitemap.xml': { + id: '/sitemap.xml' + path: '/sitemap.xml' + fullPath: '/sitemap.xml' + preLoaderRoute: typeof SitemapDotxmlRouteImport + parentRoute: typeof rootRouteImport + } '/llms.txt': { id: '/llms.txt' path: '/llms.txt' @@ -159,6 +179,7 @@ const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, LlmsFullDottxtRoute: LlmsFullDottxtRoute, LlmsDottxtRoute: LlmsDottxtRoute, + SitemapDotxmlRoute: SitemapDotxmlRoute, ApiSearchRoute: ApiSearchRoute, DocsSplatRoute: DocsSplatRoute, RawSplatRoute: RawSplatRoute, diff --git a/docsite/src/routes/__root.tsx b/docsite/src/routes/__root.tsx index 6463552..748bde5 100644 --- a/docsite/src/routes/__root.tsx +++ b/docsite/src/routes/__root.tsx @@ -1,10 +1,9 @@ import { createRootRoute, HeadContent, Outlet, Scripts } from "@tanstack/react-router"; import { RootProvider } from "fumadocs-ui/provider/tanstack"; import type * as React from "react"; +import { SITE_URL } from "#src/components/landing/data.ts"; import appCss from "#src/styles/app.css?url"; -const SITE_URL = "https://env.oss.variable.land"; - export const Route = createRootRoute({ head: () => ({ meta: [ diff --git a/docsite/src/routes/sitemap[.]xml.ts b/docsite/src/routes/sitemap[.]xml.ts new file mode 100644 index 0000000..d3cabb9 --- /dev/null +++ b/docsite/src/routes/sitemap[.]xml.ts @@ -0,0 +1,22 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { SITE_URL } from "#src/components/landing/data.ts"; +import { source } from "#src/lib/source.ts"; + +export const Route = createFileRoute("/sitemap.xml")({ + server: { + handlers: { + GET: () => { + const paths = ["/", ...source.getPages().map((page) => page.url)]; + const urls = paths.map((path) => ` ${SITE_URL}${path}`).join("\n"); + const xml = ` + +${urls} + +`; + return new Response(xml, { + headers: { "Content-Type": "application/xml; charset=utf-8" }, + }); + }, + }, + }, +}); diff --git a/docsite/vite.config.ts b/docsite/vite.config.ts index ba3eba1..4ea0e9a 100644 --- a/docsite/vite.config.ts +++ b/docsite/vite.config.ts @@ -6,5 +6,18 @@ import mdx from "fumadocs-mdx/vite"; import { defineConfig } from "vite"; export default defineConfig({ - plugins: [cloudflare({ viteEnvironment: { name: "ssr" } }), tailwindcss(), mdx(), tanstackStart(), react()], + plugins: [ + cloudflare({ viteEnvironment: { name: "ssr" } }), + tailwindcss(), + mdx(), + tanstackStart({ + prerender: { + enabled: true, + crawlLinks: true, + filter: (page) => !page.path.startsWith("/api/") && !page.path.includes("#"), + }, + pages: [{ path: "/" }, { path: "/sitemap.xml" }, { path: "/llms.txt" }, { path: "/llms-full.txt" }], + }), + react(), + ], });