From 99d4307de5e956f9caba53ce1d8a08c7cd18d360 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Fri, 3 Jul 2026 13:08:36 +0200 Subject: [PATCH] =?UTF-8?q?feat(mcp):=20optional=20built-in=20MCP=20server?= =?UTF-8?q?=20=E2=80=94=20list/get/search=20docs=20over=20HTTP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Any docs-kit site can now turn on a read-only MCP endpoint with one gem + one route — every current and future site becomes agent-queryable for free. A reader adds `https://docs.example.com/mcp` to Claude Code / Claude.ai / Cursor once and the docs become first-class tools (search, page retrieval) instead of scraped text. Built on the official `mcp` Ruby SDK, exposed as a stateless Streamable HTTP JSON-RPC endpoint mounted by the host app, OFF unless BOTH the gem is present and the route is drawn. Three tools over the SAME registry the docs render from (so an agent queries live docs, never a stale scrape): - `list_pages` → every authored page {slug, title, group, url} - `get_page(slug:)` → the page's Markdown twin, or a graceful not-found listing valid slugs - `search_docs(query:)` → ranked SearchIndex hits {page_title, section_title, url, snippet} Layering mirrors the existing llms.txt/search split — a pure, Rails-free core (DocsKit::McpTools) wrapped into an MCP::Server (DocsKit::McpServer), threaded by a thin controller (DocsKit::McpController) that owns only the Rails view context. ## Optional-dependency gate docs-kit adds `mcp` to no gemspec list. `Configuration#mcp_enabled?` gates on the `c.mcp` knob (default true) AND the gem being loadable. The `mcp` gem lives in its own bundler group; a new CI leg installs `--without mcp` and runs the suite green, proving the feature no-ops (the MCP specs self-skip) when the gem is absent — a site that never bundles it is byte-identical to before. ## Also - `/llms.txt` grows a final `## MCP` line advertising the endpoint when enabled, so agents discover it. - Install generator draws the /mcp route COMMENTED (opt-in); `docs_kit.rb.erb` and `new_site.rb` document the `gem "mcp"` + `c.mcp` opt-in. - Dogfooded on the gem's own docs/ app (real /mcp route + gem): verified from curl — tools/list, search_docs, get_page, and GET/DELETE → 405 all work end-to-end through the real Rails render path. - README "Add your docs to an agent (MCP)" section with the `claude mcp add --transport http` one-liner. ## Test Coverage - spec/docs_kit/mcp_tools_spec.rb — pure core: authored-only listing, markdown twin + graceful unknown slug, ranked hits with stripped (runs WITHOUT the mcp gem) - spec/docs_kit/mcp_server_spec.rb — real SDK via handle_json: tools/list + each tools/call round-trips; .build no-ops when the gem is absent (self-skips without the gem) - spec/docs_kit/mcp_controller_spec.rb — source-wiring (Rails-only class), mirrors llms_controller_spec - configuration_spec / llms_text_spec / install_generator_spec — the knob, the advertisement line, the commented route ## Verification - [x] bundle exec rspec — 448 examples, 0 failures, 96% line coverage - [x] bundle exec rubocop — no offenses - [x] suite green WITHOUT the mcp gem (optional-dependency gate) - [x] dogfooded end-to-end against docs/ (curl → all tools answer) Closes #20 --- .github/workflows/ci.yml | 25 +++ Gemfile | 10 ++ README.md | 54 +++++++ app/controllers/docs_kit/mcp_controller.rb | 60 +++++++ docs/Gemfile | 5 + docs/config/routes.rb | 6 + lib/docs_kit/configuration.rb | 36 +++++ lib/docs_kit/llms_text.rb | 28 +++- lib/docs_kit/mcp_server.rb | 128 +++++++++++++++ lib/docs_kit/mcp_tools.rb | 118 ++++++++++++++ lib/docs_kit/templates/new_site.rb | 4 + .../docs_kit/install/install_generator.rb | 13 ++ .../install/templates/docs_kit.rb.erb | 9 ++ spec/docs_kit/configuration_spec.rb | 36 +++++ spec/docs_kit/llms_text_spec.rb | 48 +++++- spec/docs_kit/mcp_controller_spec.rb | 61 +++++++ spec/docs_kit/mcp_server_spec.rb | 127 +++++++++++++++ spec/docs_kit/mcp_tools_spec.rb | 151 ++++++++++++++++++ spec/generators/install_generator_spec.rb | 9 ++ spec/spec_helper.rb | 11 ++ 20 files changed, 930 insertions(+), 9 deletions(-) create mode 100644 app/controllers/docs_kit/mcp_controller.rb create mode 100644 lib/docs_kit/mcp_server.rb create mode 100644 lib/docs_kit/mcp_tools.rb create mode 100644 spec/docs_kit/mcp_controller_spec.rb create mode 100644 spec/docs_kit/mcp_server_spec.rb create mode 100644 spec/docs_kit/mcp_tools_spec.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f06fec..9de259b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,3 +36,28 @@ jobs: - name: Run the suite + lint run: bundle exec rake + + # The optional-dependency gate: docs-kit's MCP server is built on the `mcp` + # gem, which is NOT a runtime dependency (a consuming site adds it itself). This + # leg installs WITHOUT the `mcp` group and runs the suite — proving the feature + # no-ops cleanly (the MCP specs self-skip) when the gem is absent, so a site + # that never bundles `mcp` is byte-identical to before this feature. + without-mcp: + name: gate (no mcp gem) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4" + bundler-cache: true + + - name: Install without the optional mcp gem + run: | + bundle config set --local without mcp + bundle install + + - name: Run the suite (must stay green with mcp absent) + run: bundle exec rspec diff --git a/Gemfile b/Gemfile index 15041d4..77ecd8c 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,16 @@ gemspec # `daisyui >= 1.2` dependency. gem "daisyui", path: "../daisyui" if File.directory?(File.expand_path("../daisyui", __dir__)) +# The official MCP Ruby SDK. docs-kit's MCP server (DocsKit::McpServer / +# DocsKit::McpController) is an OPTIONAL, runtime-detected feature — the gem is +# NOT a runtime dependency (a consuming site adds it itself). It lives in its own +# group so the optional-dependency GATE can be exercised by excluding it: +# `bundle config set --local without mcp && bundle install` (a CI leg does exactly +# this). Without the gem the MCP specs self-skip and the feature must no-op. +group :mcp do + gem "mcp" +end + group :development, :test do gem "rake" gem "rspec" diff --git a/README.md b/README.md index bb41ab3..0f81e0c 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,60 @@ get "/llms-full.txt" => "docs_kit/llms#full", as: :llms_full **Existing sites:** re-run `bin/rails g docs_kit:install` (it adds the two routes idempotently), or paste the two lines above into `config/routes.rb`. +## Add your docs to an agent (MCP) + +`llms.txt` covers fetch-style consumption; **MCP** (the Model Context Protocol) is +the native one — a reader adds one URL and your docs become first-class agent +tools instead of scraped text. docs-kit ships a **read-only, stateless** MCP +server that any site can turn on with one gem + one route. It exposes three tools +over the SAME registry the docs render from (so an agent queries live docs, never +a stale copy): + +| Tool | Returns | +|------|---------| +| `list_pages` | every authored page — `slug`, `title`, `group`, `url` | +| `get_page(slug:)` | one page's Markdown twin (the same `.md` twin `/llms.txt` links) | +| `search_docs(query:)` | ranked hits — `page_title`, `section_title`, `url`, `snippet` | + +The `mcp` gem is **optional** — docs-kit depends on it in no gemspec list, and the +endpoint stays off (byte-identical to before) unless you opt in. Two steps: + +```ruby +# Gemfile +gem "mcp" +``` + +```ruby +# config/routes.rb — the install generator scaffolds these COMMENTED; uncomment. +post "/mcp" => "docs_kit/mcp#create", as: :mcp +match "/mcp" => "docs_kit/mcp#method_not_allowed", via: %i[get delete] +``` + +Then a reader connects — for Claude Code: + +```bash +claude mcp add --transport http docs https://your-docs.example/mcp +``` + +and can ask Claude to search or read your docs, which now appear as tools. The +JSON-RPC is stateless (each `POST` is independent — no SSE session), so it works +behind the existing Kamal/Cloudflare deploy unchanged; `GET`/`DELETE` return +`405`. When enabled, `/llms.txt` grows a final `## MCP` line advertising the +endpoint so agents discover it. + +`c.mcp` defaults to `true`, so once the gem + route are present the endpoint is +live. Set it `false` to keep it off even on a site that bundles the gem: + +```ruby +DocsKit.configure { |c| c.mcp = false } +``` + +The endpoint is read-only over already-public content — writing docs is still +git, and private-docs auth is a host concern (the route is yours to wrap). Rate +limiting is the host's responsibility too (e.g. `rate_limit` in your base +controller). The server ships in the gem (`DocsKit::McpServer` / +`DocsKit::McpController`); the **route lives in your app**, like `llms.txt`. + ## API docs — one request, every client tab An endpoint example is a request shown in several clients (curl, JavaScript, diff --git a/app/controllers/docs_kit/mcp_controller.rb b/app/controllers/docs_kit/mcp_controller.rb new file mode 100644 index 0000000..920ed5e --- /dev/null +++ b/app/controllers/docs_kit/mcp_controller.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module DocsKit + # The built-in read-only MCP endpoint — one gem controller, host-drawn route + # (same shape as DocsKit::LlmsController/SearchController; the engine adds no + # routes): + # + # # config/routes.rb + # post "/mcp" => "docs_kit/mcp#create" + # match "/mcp" => "docs_kit/mcp#method_not_allowed", via: %i[get delete] + # + # A user adds `https://docs.example.com/mcp` to Claude Code / Claude.ai / Cursor + # once and the docs become first-class agent tools (list_pages / get_page / + # search_docs) over the SAME registry the site renders from. See + # DocsKit::McpServer / DocsKit::McpTools. + # + # Stateless JSON-RPC: each POST is independent (no SSE session), so it works + # behind the existing Kamal/Cloudflare deploy unchanged. #create delegates the + # whole protocol to DocsKit::McpServer#handle_json — the SDK parses the request, + # dispatches the tool, and serializes the response (including JSON-RPC errors), + # so the controller never hand-rolls the protocol. + # + # OFF unless BOTH the optional `mcp` gem is present AND the site left c.mcp on + # (DocsKit.configuration#mcp_enabled?). A site without the gem, or with + # c.mcp = false, gets a 404 here and is byte-identical to before this feature. + class McpController < ActionController::Base + # A JSON-RPC POST carries no CSRF token to verify (there's no form, no + # session — an agent posts a raw JSON body). Unlike the GET-only text + # endpoints (which use :null_session so csrf_meta_tags resolves in a rendered + # ), this action renders JSON only and never a Shell, so drop forgery + # protection outright. + skip_forgery_protection + + def create + return head(:not_found) unless docs_config.mcp_enabled? + + server = DocsKit::McpServer.build(docs_config, base_url: request.base_url, view_context:) + return head(:not_found) unless server + + # #handle_json returns an already-serialized JSON string, so render it as the + # raw body with the JSON content type — `render json:` would re-encode the + # string (wrapping it in quotes), corrupting the JSON-RPC envelope. + render body: server.handle_json(request.body.read), content_type: "application/json" + end + + # Read-only + stateless: the endpoint speaks JSON-RPC over POST only. There is + # no standalone SSE stream (GET) and no session to terminate (DELETE), so both + # are 405 rather than the SDK's session machinery. + def method_not_allowed + head :method_not_allowed + end + + private + + # NOT named #config — ActionController::Base#config is the Rails config object + # and RequestForgeryProtection delegates to it; shadowing it breaks forgery + # handling (see LlmsController). The DocsKit config reader is #docs_config. + def docs_config = DocsKit.configuration + end +end diff --git a/docs/Gemfile b/docs/Gemfile index bccc773..b8bd723 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -7,6 +7,11 @@ gem "phlex-rails" gem "rails_icons", "~> 1.1" gem "rouge" +# Dogfood docs-kit's OPTIONAL MCP endpoint — the /mcp route below serves these +# docs to AI agents (search_docs / get_page / list_pages). Optional for a +# consuming site; enabled here so the gem's own docs prove the feature. +gem "mcp" + # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main" gem "rails", "~> 8.1.3" # The modern asset pipeline for Rails [https://github.com/rails/propshaft] diff --git a/docs/config/routes.rb b/docs/config/routes.rb index 2c21573..b41e9b9 100644 --- a/docs/config/routes.rb +++ b/docs/config/routes.rb @@ -12,6 +12,12 @@ # concatenates every page's Markdown twin. get "/llms.txt" => "docs_kit/llms#index", as: :llms get "/llms-full.txt" => "docs_kit/llms#full", as: :llms_full + + # Read-only MCP endpoint (DocsKit::McpController) — dogfooding docs-kit's own + # optional MCP server. POST speaks JSON-RPC (search_docs / get_page / + # list_pages); GET/DELETE are 405 (read-only, stateless — no SSE session). + post "/mcp" => "docs_kit/mcp#create", as: :mcp + match "/mcp" => "docs_kit/mcp#method_not_allowed", via: %i[get delete] # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500. diff --git a/lib/docs_kit/configuration.rb b/lib/docs_kit/configuration.rb index b3954c3..eeaae54 100644 --- a/lib/docs_kit/configuration.rb +++ b/lib/docs_kit/configuration.rb @@ -128,6 +128,14 @@ class Configuration # works). See DocsKit::MarkdownExport / DocsKit::Controller#render_page. attr_accessor :page_markdown_action + # Whether the built-in read-only MCP endpoint (DocsKit::McpController, a + # POST /mcp JSON-RPC server exposing list_pages / get_page / search_docs over + # the same registry the docs render from) is active. Defaults to true, but the + # endpoint only turns on when the optional `mcp` gem is ALSO present and the + # host draws the route — #mcp_enabled? gates on both. Set false to keep the + # endpoint off even on a site that bundles the gem. See DocsKit::McpServer. + attr_accessor :mcp + # Whether the topbar renders the docs-search form (and the docs-nav palette # markup). Defaults to true. Set false to hide search site-wide — the route # can stay drawn, but no affordance points at it. Gated together with a @@ -219,6 +227,7 @@ def initialize @code_lexer_fallback = "plaintext" @code_language_labels = {} @page_markdown_action = true + @mcp = true @search = true @search_path = "/docs/search" @search_shortcuts = DEFAULT_SEARCH_SHORTCUTS @@ -270,6 +279,24 @@ def normalize_on_page(value) private + # True when the optional `mcp` gem can be loaded. Memoized across both + # outcomes so a site without the gem doesn't pay a failed require per request. + # We attempt the require lazily (rather than only checking defined?(MCP)) so a + # bundled-but-not-yet-required gem still counts as present — the same + # degrade-gracefully-on-a-missing-optional-gem posture as DocsUI::Icon's + # rails_icons guard. + def mcp_gem_present? + return @mcp_gem_present if defined?(@mcp_gem_present) + + @mcp_gem_present = + begin + require "mcp" + defined?(::MCP::Server) ? true : false + rescue LoadError + false + end + end + # { heading => registry.nav_items }, dropping headings with no authored # pages so the sidebar never shows an empty group. def nav_groups_from_registries @@ -299,6 +326,15 @@ def title_suffix @title_suffix || @brand end + # Whether the built-in MCP endpoint is active: the #mcp toggle is on AND the + # optional `mcp` gem is loadable — the same "toggle AND capability" shape as + # #search_enabled?. Read by DocsKit::LlmsText (to advertise /mcp in llms.txt) + # and DocsKit::McpController (to 404 when off), so a site without the gem, or + # one that set c.mcp = false, is byte-identical to before this feature. + def mcp_enabled? + !!@mcp && mcp_gem_present? + end + # Whether the Shell renders the search affordance: search is on AND a path is # set to submit to. A site with @search_path blanked (or nil) gets no form # even if @search is true — there'd be nothing to submit to. diff --git a/lib/docs_kit/llms_text.rb b/lib/docs_kit/llms_text.rb index 4ba3281..5fb4602 100644 --- a/lib/docs_kit/llms_text.rb +++ b/lib/docs_kit/llms_text.rb @@ -36,15 +36,24 @@ def index(config, base_url: nil) blocks = ["# #{config.brand}"] tagline = config.tagline blocks << "> #{tagline}" if tagline && !tagline.to_s.empty? + blocks.concat(section_blocks(config, base_url)) - groups(config).each do |group, links| - section = ["## #{group}", *links.map { |link| link_line(link, base_url) }] - blocks << section.join("\n") - end + # Advertise the built-in MCP endpoint last, so an agent that reads llms.txt + # discovers it can also connect over the protocol (native tools vs fetching + # text). Only when the endpoint is actually live (gem present + c.mcp on). + blocks << mcp_block(base_url) if config.mcp_enabled? blocks.join("\n\n") end + # One `## {group}` block per nav group, each a tight bullet list of its + # authored pages' `.md` links, in registry order. + def section_blocks(config, base_url) + groups(config).map do |group, links| + ["## #{group}", *links.map { |link| link_line(link, base_url) }].join("\n") + end + end + # The authored pages across every registry, in config/registry order — each # responds to #title / #href / #view_class. The controller renders these to # Markdown for .full. @@ -67,6 +76,17 @@ def groups(config) end end + # The `## MCP` section pointing an agent at the read-only MCP endpoint. The + # `/mcp` URL is absolutized against base_url when available (agents connect to + # a portable URL); relative otherwise. + def mcp_block(base_url) + url = base_url ? "#{base_url.chomp('/')}/mcp" : "/mcp" + "## MCP\n" \ + "This documentation is also available over the Model Context Protocol " \ + "(search, page retrieval) at #{url} — add it to an MCP client " \ + "(Claude Code, Claude.ai, Cursor) to query these docs as tools." + end + # `- [label](absolute .md url)`. The `.md` suffix targets the page's Markdown # twin (DocsKit::Controller#render_page). def link_line(link, base_url) diff --git a/lib/docs_kit/mcp_server.rb b/lib/docs_kit/mcp_server.rb new file mode 100644 index 0000000..5dd62d3 --- /dev/null +++ b/lib/docs_kit/mcp_server.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require "json" + +module DocsKit + # Builds the read-only MCP::Server a docs-kit site exposes at POST /mcp — a + # stateless JSON-RPC skin over DocsKit::McpTools, so an agent (Claude Code, + # Claude.ai, Cursor) adds one URL and the docs become first-class tools: + # + # server = DocsKit::McpServer.build(DocsKit.configuration, base_url:, view_context:) + # server.handle_json(request.body.read) # → the JSON-RPC response string + # + # The `mcp` gem is OPTIONAL and runtime-detected (docs-kit depends on it in no + # gemspec list). .build returns nil when the gem is absent, and the controller + # only reaches here when DocsKit.configuration#mcp_enabled? — so a site without + # the gem, or with c.mcp = false, is byte-identical to before this feature. + # + # base_url + view_context ride in the server_context (the SDK threads it to + # every tool block), so the tools render each page's Markdown twin through the + # Rails view context and absolutize URLs — the same seam LlmsController#full + # uses. The three tools mirror DocsKit::McpTools one-to-one. + module McpServer + module_function + + # The MCP::Server for this config, or nil when the `mcp` gem isn't loadable. + # base_url/view_context flow to the tools via server_context. + def build(config, base_url: nil, view_context: nil) + return unless mcp_available? + + server = MCP::Server.new( + name: config.brand.to_s, + version: DocsKit::VERSION, + instructions: instructions_for(config, base_url), + server_context: { config:, base_url:, view_context: } + ) + define_tools(server) + server + end + + # Whether the official MCP SDK is loadable — the runtime-detection gate. Kept + # as a seam so specs can force the gem-absent branch without unloading it. + def mcp_available? + require "mcp" + defined?(::MCP::Server) ? true : false + rescue LoadError + false + end + + # Server instructions: the site's tagline (when set) plus a pointer to + # /llms.txt, so an agent knows what these docs cover and where the full index + # lives. base_url absolutizes the /llms.txt hint when available. + def instructions_for(config, base_url) + llms = base_url ? "#{base_url.chomp('/')}/llms.txt" : "/llms.txt" + lines = [] + tagline = config.tagline + lines << tagline.to_s if tagline && !tagline.to_s.empty? + lines << "Read-only documentation tools for #{config.brand}. " \ + "Use search_docs to find sections, get_page to read a page's Markdown, " \ + "and list_pages to enumerate the docs. Full index: #{llms}." + lines.join(" ") + end + + # Register the three read-only tools. Each block pulls config + render context + # from server_context, calls the matching DocsKit::McpTools function, and + # returns the result as pretty JSON text (agents consume structured data). + def define_tools(server) + define_list_pages(server) + define_get_page(server) + define_search_docs(server) + end + + def define_list_pages(server) + server.define_tool( + name: "list_pages", + description: "List every documentation page: its slug, title, group, and URL. " \ + "Use the slug with get_page.", + input_schema: { type: "object", properties: {}, required: [] } + ) do |server_context:| + cfg = server_context[:config] + McpServer.json_response(McpTools.list_pages(cfg, base_url: server_context[:base_url])) + end + end + + def define_get_page(server) + server.define_tool( + name: "get_page", + description: "Fetch one documentation page as Markdown, by its slug (see list_pages). " \ + "Returns the page's full Markdown twin.", + input_schema: { + type: "object", + properties: { slug: { type: "string", description: "The page slug, e.g. \"installation\"." } }, + required: ["slug"] + } + ) do |slug:, server_context:| + cfg = server_context[:config] + McpServer.json_response( + McpTools.get_page(cfg, slug:, base_url: server_context[:base_url], + view_context: server_context[:view_context]) + ) + end + end + + def define_search_docs(server) + server.define_tool( + name: "search_docs", + description: "Full-text search across all documentation pages. " \ + "Returns ranked hits with the page, section, URL, and a snippet.", + input_schema: { + type: "object", + properties: { query: { type: "string", description: "The search terms." } }, + required: ["query"] + } + ) do |query:, server_context:| + cfg = server_context[:config] + McpServer.json_response( + McpTools.search_docs(cfg, query:, base_url: server_context[:base_url], + view_context: server_context[:view_context]) + ) + end + end + + # Wrap a Ruby data payload as a single-text MCP tool response, the data as + # pretty JSON so an agent parses structured fields (not prose). + def json_response(payload) + MCP::Tool::Response.new([{ type: "text", text: JSON.pretty_generate(payload) }]) + end + end +end diff --git a/lib/docs_kit/mcp_tools.rb b/lib/docs_kit/mcp_tools.rb new file mode 100644 index 0000000..44f281b --- /dev/null +++ b/lib/docs_kit/mcp_tools.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require "cgi" + +module DocsKit + # The pure, HTTP-free core the built-in MCP server exposes — three plain-Ruby + # functions over the SAME registry, Markdown twins, and search index the docs + # already render from, so an agent queries live docs, never a stale scrape: + # + # list_pages(config) → [{ slug, title, group, url }] authored pages only + # get_page(config, slug:) → { found:, title:, url:, markdown: } | { found: false, message: } + # search_docs(config, query:)→ [{ page_title, section_title, url, snippet }] ranked + # + # Zero `mcp`-gem dependency and zero JSON-RPC: DocsKit::McpServer wraps these + # into MCP tools, and the controller only threads the Rails view context (for + # url helpers/CSRF, the same seam DocsKit::LlmsController#full uses). So the + # whole consumption story is unit-testable without booting Rails or the SDK. + # + # "Authored" means a resolvable #view_class — DocsKit::LlmsText.pages already + # flattens every registry to just those, so an unwritten page is never listed, + # fetched, or indexed (no dead links, no 404s over the protocol). + module McpTools + module_function + + # The authored pages across every registry, in config/registry order, as a + # flat list of { slug, title, group, url } — url absolutized against base_url + # when given (agents fetch a portable URL), else the root-relative href. + def list_pages(config, base_url: nil) + LlmsText.pages(config).map do |page| + { + slug: page.slug, + title: page.title, + group: page.group, + url: absolutize(page.href, base_url) + } + end + end + + # A single page's Markdown twin by slug. Renders the page's #view_class + # through view_context (nil off-Rails) exactly as LlmsController#full does. An + # unknown or unwritten slug returns { found: false } with a message listing + # the valid slugs, so an agent can correct itself instead of hitting an error. + def get_page(config, slug:, base_url: nil, view_context: nil) + page = find_page(config, slug) + return not_found(config, slug) unless page + + { + found: true, + slug: page.slug, + title: page.title, + url: absolutize(page.href, base_url), + markdown: render_markdown(page, base_url:, view_context:) + } + end + + # The top DocsKit::SearchIndex hits for query, as { page_title, section_title, + # url, snippet } — url absolutized, snippet reduced to plain text (the index's + # HTML highlight stripped, since MCP delivers text, not HTML). Blank + # query → []. Builds the index from the same twins search + llms-full serve. + def search_docs(config, query:, base_url: nil, view_context: nil) + index_for(config, base_url:, view_context:).search(query).map do |hit| + { + page_title: hit.page_title, + section_title: hit.section_title, + url: absolutize(hit.href, base_url), + snippet: strip_html(hit.snippet) + } + end + end + + # The authored page with this slug across every registry, or nil (an unwritten + # page has no resolvable view_class, so it's absent from LlmsText.pages). + def find_page(config, slug) + LlmsText.pages(config).find { |page| page.slug.to_s == slug.to_s } + end + + # A { found: false } result whose message names every valid slug, so an agent + # that guessed wrong can retry with a real one. + def not_found(config, slug) + valid = list_pages(config).map { |page| page[:slug] } + { + found: false, + slug: slug, + message: "No page with slug #{slug.inspect}. Valid slugs: #{valid.join(', ')}." + } + end + + # A page's GFM Markdown twin, rendered through the view context so url helpers + # and relative-link absolutization resolve — the LlmsController#full seam. + def render_markdown(page, base_url:, view_context:) + MarkdownExport.new(page.view_class.new, view_context:, base_url:).to_md + end + + # A DocsKit::SearchIndex over every authored page's twin — the same triples + # DocsKit::SearchController builds, so MCP search can't drift from the pages. + def index_for(config, base_url:, view_context:) + triples = LlmsText.pages(config).map do |page| + [page.title, page.href, render_markdown(page, base_url:, view_context:)] + end + SearchIndex.new(triples) + end + + # href absolutized against base_url (no .md suffix — MCP serves the page URL, + # not the twin). Relative href when base_url is nil. Mirrors LlmsText.md_url. + def absolutize(href, base_url) + return href unless base_url + + "#{base_url.chomp('/')}#{href}" + end + + # Reduce the search index's HTML-safe snippet (term wrapped in , rest + # CGI-escaped) to plain text: drop the tags and unescape entities, so + # the MCP snippet is human/agent-readable text rather than HTML. + def strip_html(snippet) + CGI.unescapeHTML(snippet.to_s.gsub(%r{}, "")) + end + end +end diff --git a/lib/docs_kit/templates/new_site.rb b/lib/docs_kit/templates/new_site.rb index 8d63fa3..21e52eb 100644 --- a/lib/docs_kit/templates/new_site.rb +++ b/lib/docs_kit/templates/new_site.rb @@ -34,6 +34,10 @@ gem "phlex-rails" gem "rails_icons", "~> 1.1" gem "rouge" + + # Optional: expose these docs to AI agents over MCP (a read-only /mcp endpoint). + # Uncomment this and the /mcp route in config/routes.rb. See the docs-kit README. + # gem "mcp" RUBY end diff --git a/lib/generators/docs_kit/install/install_generator.rb b/lib/generators/docs_kit/install/install_generator.rb index c88798c..4cee63f 100644 --- a/lib/generators/docs_kit/install/install_generator.rb +++ b/lib/generators/docs_kit/install/install_generator.rb @@ -84,6 +84,19 @@ def add_routes # skips a line already present, so re-running the generator is idempotent. route %(get "/llms.txt" => "docs_kit/llms#index", as: :llms) route %(get "/llms-full.txt" => "docs_kit/llms#full", as: :llms_full) + + add_mcp_route + end + + # The read-only MCP endpoint (DocsKit::McpController), drawn COMMENTED OUT + # because it needs the OPTIONAL `mcp` gem — the generator can't assume it's + # bundled. A site opts in by adding `gem "mcp"` and uncommenting these. POST + # speaks JSON-RPC; GET/DELETE 405 (read-only, stateless — no SSE session). + # `route` prepends, so drawing `match` before `post` leaves `post` on top. + def add_mcp_route + route %(# match "/mcp" => "docs_kit/mcp#method_not_allowed", via: %i[get delete]) + route %(# post "/mcp" => "docs_kit/mcp#create") + route %(# Add your docs to an agent over MCP (needs `gem "mcp"`):) end def create_css_build diff --git a/lib/generators/docs_kit/install/templates/docs_kit.rb.erb b/lib/generators/docs_kit/install/templates/docs_kit.rb.erb index 74021d1..c0e75e3 100644 --- a/lib/generators/docs_kit/install/templates/docs_kit.rb.erb +++ b/lib/generators/docs_kit/install/templates/docs_kit.rb.erb @@ -48,6 +48,15 @@ Rails.application.config.to_prepare do # c.search_shortcuts = ["/", "mod+k"] # default; "mod" = ⌘ on mac, Ctrl elsewhere. # Each is a bare key ("/", "s") or a chord ("mod+k", "ctrl+shift+f"); [] binds none. + # OPTIONAL: expose your docs to AI agents over MCP (Model Context Protocol) — + # a read-only endpoint (search_docs / get_page / list_pages) at POST /mcp that + # Claude Code / Claude.ai / Cursor can add as tools. It's OFF until you add + # `gem "mcp"` to your Gemfile AND uncomment the /mcp route in config/routes.rb + # (see the commented lines there). `c.mcp` defaults to true, so once the gem + + # route are present the endpoint is live; set it false to keep it off even + # then. README → "Add your docs to an agent (MCP)". + # c.mcp = false + # The sidebar nav derives from the registry — one heading → one registry. # Each registry's authored pages become NavItems automatically (an unwritten # page is skipped, so no dead links). For bespoke nav (interleaved diff --git a/spec/docs_kit/configuration_spec.rb b/spec/docs_kit/configuration_spec.rb index c70a9f2..bed6f3b 100644 --- a/spec/docs_kit/configuration_spec.rb +++ b/spec/docs_kit/configuration_spec.rb @@ -49,6 +49,42 @@ end end + describe "#mcp" do + it "defaults to true (the endpoint is on wherever the mcp gem + route are present)" do + expect(described_class.new.mcp).to be(true) + end + + it "is overridable so a site with the gem installed can still disable the endpoint" do + DocsKit.configure { |c| c.mcp = false } + + expect(DocsKit.configuration.mcp).to be(false) + end + end + + describe "#mcp_enabled?" do + # The endpoint requires BOTH the config toggle on AND the optional `mcp` gem + # present — the same "toggle AND capability" shape as #search_enabled?. The + # suite loads `mcp` (dev/test group), so defined?(MCP) is true here. + it "is true when #mcp is on and the mcp gem is loaded" do + skip "mcp gem not loaded in this run" unless defined?(MCP) + + expect(described_class.new.mcp_enabled?).to be(true) + end + + it "is false when #mcp is off, even with the gem present" do + DocsKit.configure { |c| c.mcp = false } + + expect(DocsKit.configuration.mcp_enabled?).to be(false) + end + + it "is false when the mcp gem is absent, even with #mcp on" do + config = described_class.new + allow(config).to receive(:mcp_gem_present?).and_return(false) + + expect(config.mcp_enabled?).to be(false) + end + end + describe "#search_path" do it "defaults to \"/docs/search\" (the route the generator draws)" do expect(described_class.new.search_path).to eq("/docs/search") diff --git a/spec/docs_kit/llms_text_spec.rb b/spec/docs_kit/llms_text_spec.rb index 197f9d4..08fee47 100644 --- a/spec/docs_kit/llms_text_spec.rb +++ b/spec/docs_kit/llms_text_spec.rb @@ -116,13 +116,14 @@ def configure(**opts) context "with an empty registry (all pages unwritten)" do subject(:index) do - described_class.index( - configure(nav_registries: { "Docs" => registry(nav_items: {}) }), - base_url: "https://acme.dev" - ) + config = configure(nav_registries: { "Docs" => registry(nav_items: {}) }) + # Isolate the page-group behavior from the (gem-dependent) MCP block, which + # legitimately adds its own `## MCP` section when the endpoint is live. + allow(config).to receive(:mcp_enabled?).and_return(false) + described_class.index(config, base_url: "https://acme.dev") end - it "renders a valid index with no sections (never an empty ## group)" do + it "renders a valid index with no page sections (never an empty ## group)" do expect(index).to start_with("# docs-kit") expect(index).not_to include("##") end @@ -133,6 +134,43 @@ def configure(**opts) expect(index).to include("- [Overview](/docs/overview.md)") end + + context "when the MCP endpoint is enabled" do + subject(:index) do + config = configure(tagline: nil) + allow(config).to receive(:mcp_enabled?).and_return(true) + described_class.index(config, base_url: "https://acme.dev") + end + + it "advertises the MCP endpoint so agents can discover it" do + expect(index).to include("## MCP") + expect(index).to include("https://acme.dev/mcp") + end + + it "puts the MCP block last (after the page sections)" do + expect(index.index("## Getting started")).to be < index.index("## MCP") + end + + it "uses a relative /mcp path when no base_url is given" do + config = configure(tagline: nil) + allow(config).to receive(:mcp_enabled?).and_return(true) + + expect(described_class.index(config)).to include("/mcp") + end + end + + context "when the MCP endpoint is disabled (default / no gem)" do + subject(:index) do + config = configure(tagline: nil) + allow(config).to receive(:mcp_enabled?).and_return(false) + described_class.index(config, base_url: "https://acme.dev") + end + + it "omits the MCP advertisement entirely (byte-identical to before)" do + expect(index).not_to include("## MCP") + expect(index).not_to include("/mcp") + end + end end describe ".pages" do diff --git a/spec/docs_kit/mcp_controller_spec.rb b/spec/docs_kit/mcp_controller_spec.rb new file mode 100644 index 0000000..87ede3a --- /dev/null +++ b/spec/docs_kit/mcp_controller_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# Like DocsKit::LlmsController/SearchController, DocsKit::McpController subclasses +# ActionController::Base, so it can't load in the standalone suite (no Rails +# request stack). Its real behavior — the JSON-RPC round-trip — is proven by +# spec/docs_kit/mcp_server_spec.rb (the server it delegates to) and dogfooded +# against the docs/ app. Here we prove the SHIPPED FILE is where Rails autoloads +# DocsKit::McpController from, and that the thin controller wires the server, the +# CSRF/optional-gem gates, and the read-only method policy correctly. +# rubocop:disable RSpec/DescribeClass -- the class is Rails-only, can't constantize here +RSpec.describe "DocsKit::McpController (source wiring)" do + # app/controllers/docs_kit/mcp_controller.rb → DocsKit::McpController under + # Rails' default inflector, the same path/loader story as LlmsController. + let(:path) do + File.expand_path("../../app/controllers/docs_kit/mcp_controller.rb", __dir__) + end + let(:source) { File.read(path) } + + it "ships at the path Rails autoloads DocsKit::McpController from" do + expect(File.exist?(path)).to be(true) + end + + it "declares DocsKit::McpController < ActionController::Base" do + expect(source).to include("module DocsKit") + expect(source).to include("class McpController < ActionController::Base") + end + + it "exposes the POST JSON-RPC action" do + expect(source).to match(/def create\b/) + end + + it "delegates the JSON-RPC to DocsKit::McpServer over the request body" do + expect(source).to include("DocsKit::McpServer.build") + expect(source).to include("handle_json(request.body.read)") + end + + it "renders application/json (the JSON-RPC content type)" do + expect(source).to include("application/json") + end + + it "skips CSRF (a JSON-RPC POST carries no forgery token)" do + expect(source).to include("skip_forgery_protection") + end + + it "gates the endpoint on mcp_enabled? (off → not found, byte-identical to no feature)" do + expect(source).to include("mcp_enabled?") + end + + it "returns 405 for the non-POST verbs (read-only, stateless — no SSE session)" do + expect(source).to match(/def method_not_allowed\b/) + expect(source).to include(":method_not_allowed") + end + + it "does not shadow ActionController::Base#config (forgery delegates to it)" do + # Same guard as LlmsController: a `def config` on a gem controller reroutes + # RequestForgeryProtection's delegations. The DocsKit config reader is #docs_config. + expect(source).not_to match(/^\s*def config\b/) + expect(source).to include("def docs_config = DocsKit.configuration") + end +end +# rubocop:enable RSpec/DescribeClass diff --git a/spec/docs_kit/mcp_server_spec.rb b/spec/docs_kit/mcp_server_spec.rb new file mode 100644 index 0000000..6150c27 --- /dev/null +++ b/spec/docs_kit/mcp_server_spec.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +# DocsKit::McpServer wraps the pure DocsKit::McpTools functions into an +# MCP::Server (the official SDK) exposing list_pages / get_page / search_docs +# over stateless JSON-RPC. These specs drive the REAL SDK via #handle_json (no +# HTTP, no controller) — tools/list returns the three tools, and a tools/call +# round-trips through the registry. The whole thing is guarded on the optional +# `mcp` gem: without it the suite skips these (the optional-dependency gate) and +# .build must no-op. +RSpec.describe DocsKit::McpServer do + def view_rendering(inner_html) + Class.new do + define_method(:call) { "
#{inner_html}
" } + end + end + + def entry(slug:, title:, group:, href:, view_class:) + Struct.new(:slug, :title, :group, :href, :view_class, keyword_init: true) + .new(slug:, title:, group:, href:, view_class:) + end + + def registry(all:) + Class.new do + define_singleton_method(:all) { all } + end + end + + let(:doc_registry) do + registry( + all: [ + entry(slug: "overview", title: "Overview", group: "Guide", href: "/docs/overview", + view_class: view_rendering("

Setup

Install the gem to get started.

")), + entry(slug: "unwritten", title: "Unwritten", group: "Guide", href: "/docs/unwritten", view_class: nil) + ] + ) + end + + def configure + DocsKit.configure do |c| + c.brand = "Acme Docs" + c.tagline = "Everything about Acme." + c.nav_registries = { "Docs" => doc_registry } + end + DocsKit.configuration + end + + # Parse a JSON-RPC response string (what #handle_json returns). + def rpc(server, method, params = {}) + require "json" + body = { jsonrpc: "2.0", id: 1, method:, params: }.to_json + JSON.parse(server.handle_json(body)) + end + + describe ".build" do + context "when the mcp gem is present" do + subject(:server) { described_class.build(configure, base_url: "https://acme.dev") } + + before { skip "mcp gem not loaded" unless defined?(MCP) } + + it "returns an MCP::Server" do + expect(server).to be_a(MCP::Server) + end + + it "names the server from the brand" do + expect(server.name).to include("Acme Docs") + end + + it "sets instructions from the tagline and points agents at /llms.txt" do + expect(server.instructions).to include("Everything about Acme.") + expect(server.instructions).to include("/llms.txt") + end + + describe "tools/list" do + subject(:tool_names) { rpc(server, "tools/list").dig("result", "tools").map { |t| t["name"] } } + + it "advertises exactly the three read-only tools" do + expect(tool_names).to contain_exactly("list_pages", "get_page", "search_docs") + end + end + + describe "tools/call list_pages" do + subject(:text) do + rpc(server, "tools/call", { name: "list_pages", arguments: {} }) + .dig("result", "content", 0, "text") + end + + it "returns the authored pages (unwritten excluded), with absolute urls" do + expect(text).to include("overview").and include("https://acme.dev/docs/overview") + expect(text).not_to include("unwritten") + end + end + + describe "tools/call get_page" do + it "returns the page's Markdown twin for a known slug" do + text = rpc(server, "tools/call", { name: "get_page", arguments: { slug: "overview" } }) + .dig("result", "content", 0, "text") + + expect(text).to include("Install the gem") + end + + it "reports not found (listing valid slugs) for an unknown slug" do + text = rpc(server, "tools/call", { name: "get_page", arguments: { slug: "nope" } }) + .dig("result", "content", 0, "text") + + expect(text).to include("overview") + end + end + + describe "tools/call search_docs" do + it "returns ranked hits for a query" do + text = rpc(server, "tools/call", { name: "search_docs", arguments: { query: "install" } }) + .dig("result", "content", 0, "text") + + expect(text).to include("Overview") + end + end + end + + context "when the mcp gem is absent" do + it "no-ops (returns nil) rather than raising" do + allow(described_class).to receive(:mcp_available?).and_return(false) + + expect(described_class.build(configure)).to be_nil + end + end + end +end diff --git a/spec/docs_kit/mcp_tools_spec.rb b/spec/docs_kit/mcp_tools_spec.rb new file mode 100644 index 0000000..40c0d95 --- /dev/null +++ b/spec/docs_kit/mcp_tools_spec.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +# DocsKit::McpTools is the pure, HTTP-free core the MCP server exposes: three +# plain-Ruby functions over the SAME registry + Markdown twin + search index the +# docs render from (DocsKit::LlmsText.pages / MarkdownExport / SearchIndex). No +# `mcp` gem, no JSON-RPC, no controller — so the whole consumption story is +# unit-testable without booting Rails or the SDK. DocsKit::McpServer wraps these +# into MCP tools; this spec proves the data. +RSpec.describe DocsKit::McpTools do + # A page-ish registry entry: #slug / #title / #group / #href / #view_class — + # exactly the duck type DocsKit::Registry::Entry (and LlmsText.pages) exposes. + # A nil view_class means an unwritten page, excluded everywhere. + def entry(slug:, title:, group:, href:, view_class:) + Struct.new(:slug, :title, :group, :href, :view_class, keyword_init: true) + .new(slug:, title:, group:, href:, view_class:) + end + + # A fake registry standing in for a DocsKit::Registry class: #all lists every + # entry (authored + unwritten) and #from_slug looks one up, in registry order. + def registry(all:) + Class.new do + define_singleton_method(:all) { all } + define_singleton_method(:from_slug) { |slug| all.find { |e| e.slug.to_s == slug.to_s } } + end + end + + # A view whose #call renders a #docs-content region MarkdownExport can convert. + # Built as a bare class (no Phlex host needed — the converter works on the + # rendered HTML string), matching the markdown_export spec's html fixture. + def view_rendering(inner_html) + Class.new do + define_method(:call) { "
#{inner_html}
" } + end + end + + let(:overview_view) { view_rendering("

Setup

Install the gem, then configure it.

") } + let(:install_view) { view_rendering("

Add the gem to your Gemfile.

") } + + let(:doc_registry) do + registry( + all: [ + entry(slug: "overview", title: "Overview", group: "Guide", href: "/docs/overview", view_class: overview_view), + entry(slug: "unwritten", title: "Unwritten", group: "Guide", href: "/docs/unwritten", view_class: nil), + entry(slug: "installation", title: "Installation", group: "Guide", href: "/docs/installation", + view_class: install_view) + ] + ) + end + + def configure(registries: { "Docs" => doc_registry }) + DocsKit.configure { |c| c.nav_registries = registries } + DocsKit.configuration + end + + describe ".list_pages" do + subject(:pages) { described_class.list_pages(configure, base_url: "https://acme.dev") } + + it "returns one entry per AUTHORED page (unwritten pages excluded)" do + expect(pages.map { |p| p[:slug] }).to eq(%w[overview installation]) + end + + it "carries slug, title, group, and an absolute url for each page" do + overview = pages.first + + expect(overview).to include( + slug: "overview", + title: "Overview", + group: "Guide", + url: "https://acme.dev/docs/overview" + ) + end + + it "falls back to a relative url when no base_url is given" do + relative = described_class.list_pages(configure) + + expect(relative.first[:url]).to eq("/docs/overview") + end + + it "spans every registry in config order" do + api = registry(all: [entry(slug: "users", title: "Users", group: "API", href: "/api/users", + view_class: view_rendering("

List users.

"))]) + pages = described_class.list_pages(configure(registries: { "Docs" => doc_registry, "API" => api })) + + expect(pages.map { |p| p[:title] }).to eq(%w[Overview Installation Users]) + end + end + + describe ".get_page" do + it "returns the page's Markdown twin for a known slug" do + result = described_class.get_page(configure, slug: "overview", base_url: "https://acme.dev") + + expect(result[:markdown]).to include("## Setup").and include("Install the gem") + expect(result[:found]).to be(true) + end + + it "carries the page title and absolute url" do + result = described_class.get_page(configure, slug: "installation", base_url: "https://acme.dev") + + expect(result).to include(title: "Installation", url: "https://acme.dev/docs/installation") + end + + context "when the slug is unknown" do + subject(:result) { described_class.get_page(configure, slug: "nope") } + + it "reports not found rather than raising" do + expect(result[:found]).to be(false) + end + + it "lists the valid slugs so an agent can retry" do + expect(result[:message]).to include("overview").and include("installation") + expect(result[:message]).not_to include("unwritten") + end + end + + context "when the slug names an unwritten page" do + it "is treated as not found (its view_class doesn't resolve)" do + result = described_class.get_page(configure, slug: "unwritten") + + expect(result[:found]).to be(false) + end + end + end + + describe ".search_docs" do + subject(:hits) { described_class.search_docs(configure, query: "install", base_url: "https://acme.dev") } + + it "returns ranked hits with page_title, section_title, url, and snippet" do + hit = hits.first + + expect(hit).to include(:page_title, :section_title, :url, :snippet) + end + + it "matches content across every authored page's Markdown twin" do + titles = hits.map { |h| h[:page_title] } + + expect(titles).to include("Installation") + end + + it "absolutizes each hit's url against the base_url" do + expect(hits.map { |h| h[:url] }).to all(start_with("https://acme.dev/")) + end + + it "strips the highlight so the snippet is plain text (not HTML)" do + expect(hits.map { |h| h[:snippet] }.join).not_to include("") + end + + it "returns an empty list for a blank query" do + expect(described_class.search_docs(configure, query: "")).to eq([]) + end + end +end diff --git a/spec/generators/install_generator_spec.rb b/spec/generators/install_generator_spec.rb index 1905893..004a261 100644 --- a/spec/generators/install_generator_spec.rb +++ b/spec/generators/install_generator_spec.rb @@ -164,6 +164,15 @@ def silence_stream expect(routes).to include(%(get "/docs/search" => "docs_kit/search#index")) end + it "adds the MCP route COMMENTED OUT (opt-in: needs the optional mcp gem)" do + routes = read("config/routes.rb") + + # The MCP endpoint is off by default (the `mcp` gem is optional). The + # generator draws the route commented so a site opts in by uncommenting. + expect(routes).to include(%(# post "/mcp" => "docs_kit/mcp#create")) + expect(routes).to include(%(# match "/mcp" => "docs_kit/mcp#method_not_allowed", via: %i[get delete])) + end + it "draws /docs/search ABOVE docs/:doc so it isn't swallowed as :doc" do routes = read("config/routes.rb") diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 60687c4..0f48352 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -34,6 +34,17 @@ require "docs_kit" +# The MCP server (DocsKit::McpServer) is an OPTIONAL, runtime-detected feature +# built on the `mcp` gem — docs-kit never depends on it at runtime. The suite +# loads it (dev/test group) so the MCP specs exercise the real SDK; MCP-integration +# examples guard on `defined?(MCP)`, so a `bundle install --without mcp` run +# (the optional-dependency gate) simply skips them and the rest stays green. +begin + require "mcp" +rescue LoadError + # mcp not bundled (the without-mcp gate leg) — the MCP specs self-skip. +end + RSpec.configure do |config| config.expect_with(:rspec) { |c| c.syntax = :expect } config.disable_monkey_patching!