From 7996caf1c868139382e404bcf21bd5b0aee584f2 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Tue, 7 Jul 2026 10:32:21 -0400 Subject: [PATCH 1/5] test(axis): cover four taught-but-untested cross-cutting platform rules --- .../db-applied-migration-immutable.ts | 38 +++++++++++++++++++ .../deploy/deploy-failure-not-downtime.ts | 33 ++++++++++++++++ .../functions-custom-path-disables-default.ts | 32 ++++++++++++++++ axis-scenarios/mcp-servers/get-returns-405.ts | 31 +++++++++++++++ 4 files changed, 134 insertions(+) create mode 100644 axis-scenarios/database/db-applied-migration-immutable.ts create mode 100644 axis-scenarios/deploy/deploy-failure-not-downtime.ts create mode 100644 axis-scenarios/functions/functions-custom-path-disables-default.ts create mode 100644 axis-scenarios/mcp-servers/get-returns-405.ts diff --git a/axis-scenarios/database/db-applied-migration-immutable.ts b/axis-scenarios/database/db-applied-migration-immutable.ts new file mode 100644 index 0000000..e5300df --- /dev/null +++ b/axis-scenarios/database/db-applied-migration-immutable.ts @@ -0,0 +1,38 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Once a migration has been applied to any database it is immutable -- editing +// it won't re-run and drifts the migration history from the actual schema. Fix +// it by rolling forward with a new migration. Grounded in netlify-database/ +// SKILL.md ("Once a migration has been applied to any database, never modify it +// -- roll forward with a new migration instead") and references/migrations.md +// ("Already applied to any database ... treat as immutable. Roll forward with a +// new migration"). +export default { + name: "Database: applied migration is immutable -- roll forward, never edit", + prompt: + "Our Netlify Database (Drizzle) project has a migration at netlify/database/migrations/0001_create_users.sql that already shipped to production last week. I just noticed it made the `email` column nullable, but it should be NOT NULL. Can I just edit that migration file to add NOT NULL and redeploy? Walk me through the right way to fix this.", + judge: [ + { + check: + "Says NOT to edit the already-applied `0001_create_users` migration -- once a migration has been applied to any database it is immutable; editing the file won't re-run against databases that already applied it and it drifts the migration history from the actual schema.", + }, + { + check: + "Directs the user to roll forward with a NEW migration that alters `email` to NOT NULL, committed on top of the existing migration.", + }, + { + check: + "For the Drizzle path: update db/schema.ts to mark the column notNull, then run `drizzle-kit generate` (or `npm run db:generate`) to produce the new migration -- does NOT hand-edit the old migration file, and does NOT run `drizzle-kit push`.", + }, + { + check: + "Does NOT run raw `ALTER TABLE` against a hosted Netlify database via `netlify database connect`, `psql`, or any direct connection -- the deploy applies the new migration to the preview branch and then production.", + }, + { + check: + "If a local-apply step is mentioned, it uses `netlify database migrations apply` scoped to the local development DB only (passes vacuously if not mentioned).", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/deploy/deploy-failure-not-downtime.ts b/axis-scenarios/deploy/deploy-failure-not-downtime.ts new file mode 100644 index 0000000..cfe703d --- /dev/null +++ b/axis-scenarios/deploy/deploy-failure-not-downtime.ts @@ -0,0 +1,33 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// A failed deploy never publishes: the site keeps serving the last successful +// deploy, so it isn't down and there is nothing to "roll back." Grounded in +// netlify-deploy/SKILL.md ("A failed deploy does not publish -- the site keeps +// serving the last successful deploy, so it isn't down, and there's nothing to +// 'roll back.' The only way to get the new code live is to fix the failure and +// redeploy"). +export default { + name: "Deploy: a failed deploy is not downtime -- nothing to roll back", + prompt: + "My production deploy on Netlify just failed during the build. Is my live site down right now? I'm thinking I should roll back to the previous deploy to get it back up -- how do I do that?", + judge: [ + { + check: + "Reassures the user the live site is NOT down: a failed deploy does not publish, so the site keeps serving the last successful deploy.", + }, + { + check: + "Explains there is nothing to 'roll back' -- the failed deploy never went live, so restoring or republishing a previous deploy is neither needed nor applicable.", + }, + { + check: + "Says the way to get the new code live is to fix the build/deploy failure and redeploy (e.g. re-run `netlify deploy`), not to restore an older deploy.", + }, + { + check: + "Does NOT tell the user to manually publish/restore a previous deploy through `netlify api` (e.g. restoreSiteDeploy/publishDeploy), a direct `https://api.netlify.com/...` call, or by reading auth tokens off disk.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/functions/functions-custom-path-disables-default.ts b/axis-scenarios/functions/functions-custom-path-disables-default.ts new file mode 100644 index 0000000..a253e95 --- /dev/null +++ b/axis-scenarios/functions/functions-custom-path-disables-default.ts @@ -0,0 +1,32 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Setting a custom `config.path` makes a function available ONLY at that path -- +// the built-in /.netlify/functions/{name} route no longer works. Grounded in +// netlify-functions/SKILL.md ("Without a `path` config, functions are available +// at /.netlify/functions/{name}. Setting a `path` makes the function available +// only at that path"). +export default { + name: "Functions: a custom config.path disables the default /.netlify/functions route", + prompt: + "I set `export const config = { path: '/api/webhook' }` on my Netlify function at netlify/functions/webhook.ts. A third-party service is still calling the old default URL `/.netlify/functions/webhook` and now gets 404s. Why is that happening, and what are my options?", + judge: [ + { + check: + "Correctly explains that setting a custom `path` makes the function available ONLY at that path -- the built-in `/.netlify/functions/webhook` default route is disabled once a `path` is set, so the 404 is expected behavior.", + }, + { + check: + "Does NOT claim the default `/.netlify/functions/{name}` route still works alongside the custom path, and does NOT blame the 404 on an unrelated cause (a deploy glitch, a typo, a bundling bug).", + }, + { + check: + "Offers at least one grounded way to serve the old URL again: list both paths in the `config.path` array (config.path accepts an array of paths), add a `[[redirects]]` rule from the old URL to `/api/webhook`, or point the third-party integration at the new `/api/webhook` path.", + }, + { + check: + "If the answer includes function code, it reads env vars with `Netlify.env.get()` rather than `process.env` (passes vacuously if no function code is shown).", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/mcp-servers/get-returns-405.ts b/axis-scenarios/mcp-servers/get-returns-405.ts new file mode 100644 index 0000000..dbf5a2b --- /dev/null +++ b/axis-scenarios/mcp-servers/get-returns-405.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// A stateless JSON MCP server only does request/response over POST. A GET makes +// the transport open an SSE stream that never closes, which a serverless +// function can't serve -- surfacing as a 502. The function must reject non-POST +// methods with 405. Grounded in netlify-mcp-servers/SKILL.md. +export default { + name: "MCP Servers: a GET to a stateless MCP endpoint should return 405, not hang as SSE", + prompt: + "I deployed a stateless MCP server as a Netlify Function at /mcp. It works fine from Claude Code, but when I open the URL in my browser the request hangs and eventually returns a 502. Is my server broken? How should it handle that request?", + judge: [ + { + check: + "Diagnoses the cause: the browser issues a plain GET, but the stateless JSON transport only handles request/response over POST -- a GET makes the transport try to open an SSE stream that never closes, which a serverless function can't serve, and that surfaces as the 502.", + }, + { + check: + "Clarifies the server is not actually broken for real MCP clients (which POST) -- the 502 is expected behavior for a GET to a POST-only stateless endpoint, not an infrastructure/config bug to escalate.", + }, + { + check: + "Recommends the function reject non-POST methods explicitly with HTTP 405 before handing the request to the transport, e.g. `if (req.method !== \"POST\") return new Response(\"Method not allowed\", { status: 405 })`.", + }, + { + check: + "Does NOT recommend making the function serve a long-lived SSE stream (or switching to a stateful session transport) just to satisfy the browser GET.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; From 6ebddabce9e3fd655f5d6b7a8f8f235f6d456ca8 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Tue, 7 Jul 2026 11:09:50 -0400 Subject: [PATCH 2/5] test(axis): soften phrasing-strict checks in deploy-failure-not-downtime (keep anti-workaround) --- axis-scenarios/deploy/deploy-failure-not-downtime.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/axis-scenarios/deploy/deploy-failure-not-downtime.ts b/axis-scenarios/deploy/deploy-failure-not-downtime.ts index cfe703d..98f1069 100644 --- a/axis-scenarios/deploy/deploy-failure-not-downtime.ts +++ b/axis-scenarios/deploy/deploy-failure-not-downtime.ts @@ -18,11 +18,11 @@ export default { }, { check: - "Explains there is nothing to 'roll back' -- the failed deploy never went live, so restoring or republishing a previous deploy is neither needed nor applicable.", + "Conveys that the previous deploy is still serving and does not need to be restored or republished — since the failed deploy never went live there is no downtime to recover from (exact 'nothing to roll back' wording not required).", }, { check: - "Says the way to get the new code live is to fix the build/deploy failure and redeploy (e.g. re-run `netlify deploy`), not to restore an older deploy.", + "Points the user to the real fix — resolve the build/deploy failure so the new code can ship (redeploying once fixed) — rather than restoring an older deploy as the solution.", }, { check: From b5850e03ef3371d3f20c225933080c62ab8e5222 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Tue, 7 Jul 2026 11:25:00 -0400 Subject: [PATCH 3/5] fix(skills): netlify-deploy counters roll-back requests directly (don't hand over netlify api restore) --- skills/netlify-deploy/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/skills/netlify-deploy/SKILL.md b/skills/netlify-deploy/SKILL.md index 34b7a05..5389e7c 100644 --- a/skills/netlify-deploy/SKILL.md +++ b/skills/netlify-deploy/SKILL.md @@ -203,6 +203,7 @@ Common issues and solutions: → A failed deploy does **not** publish — the site keeps serving the last successful deploy, so it isn't down, and there's nothing to "roll back." The only way to get the new code live is to fix the failure and redeploy. → Get the exact error from the deploy log (the CLI prints a log URL; the dashboard has the full build log), then address the actual cause — the build command or publish directory in `netlify.toml`, a missing dependency, or the function that failed to bundle — and re-run the deploy. → Don't route around a failed build to force the site live: no `netlify api` publish/restore, no direct `https://api.netlify.com/...` calls, no reading auth tokens off disk, and don't ship a previous deploy in place of the failing one. If the log doesn't resolve it, report the exact error + log URL + affected site to the user and stop. +→ Even if the user *asks* how to "roll back" or restore a previous deploy, correct the premise rather than complying: because the failed deploy never published, the previous deploy is still live and there is nothing to restore. Do **not** hand over `netlify api restoreSiteDeploy` / `publishDeploy` (or a dashboard rollback) as the answer — the fix is to resolve the build failure and redeploy. **"Publish directory not found"** → Verify build command ran successfully From 3b1fa08a55c5f620a681cbe806192f4a9c4e7658 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 7 Jul 2026 15:25:13 +0000 Subject: [PATCH 4/5] chore: rebuild generated cursor/codex from skills --- codex/skills/netlify-deploy/SKILL.md | 1 + cursor/rules/netlify-deploy.mdc | 1 + 2 files changed, 2 insertions(+) diff --git a/codex/skills/netlify-deploy/SKILL.md b/codex/skills/netlify-deploy/SKILL.md index 34b7a05..5389e7c 100644 --- a/codex/skills/netlify-deploy/SKILL.md +++ b/codex/skills/netlify-deploy/SKILL.md @@ -203,6 +203,7 @@ Common issues and solutions: → A failed deploy does **not** publish — the site keeps serving the last successful deploy, so it isn't down, and there's nothing to "roll back." The only way to get the new code live is to fix the failure and redeploy. → Get the exact error from the deploy log (the CLI prints a log URL; the dashboard has the full build log), then address the actual cause — the build command or publish directory in `netlify.toml`, a missing dependency, or the function that failed to bundle — and re-run the deploy. → Don't route around a failed build to force the site live: no `netlify api` publish/restore, no direct `https://api.netlify.com/...` calls, no reading auth tokens off disk, and don't ship a previous deploy in place of the failing one. If the log doesn't resolve it, report the exact error + log URL + affected site to the user and stop. +→ Even if the user *asks* how to "roll back" or restore a previous deploy, correct the premise rather than complying: because the failed deploy never published, the previous deploy is still live and there is nothing to restore. Do **not** hand over `netlify api restoreSiteDeploy` / `publishDeploy` (or a dashboard rollback) as the answer — the fix is to resolve the build failure and redeploy. **"Publish directory not found"** → Verify build command ran successfully diff --git a/cursor/rules/netlify-deploy.mdc b/cursor/rules/netlify-deploy.mdc index 90e195a..ce293ab 100644 --- a/cursor/rules/netlify-deploy.mdc +++ b/cursor/rules/netlify-deploy.mdc @@ -203,6 +203,7 @@ Common issues and solutions: → A failed deploy does **not** publish — the site keeps serving the last successful deploy, so it isn't down, and there's nothing to "roll back." The only way to get the new code live is to fix the failure and redeploy. → Get the exact error from the deploy log (the CLI prints a log URL; the dashboard has the full build log), then address the actual cause — the build command or publish directory in `netlify.toml`, a missing dependency, or the function that failed to bundle — and re-run the deploy. → Don't route around a failed build to force the site live: no `netlify api` publish/restore, no direct `https://api.netlify.com/...` calls, no reading auth tokens off disk, and don't ship a previous deploy in place of the failing one. If the log doesn't resolve it, report the exact error + log URL + affected site to the user and stop. +→ Even if the user *asks* how to "roll back" or restore a previous deploy, correct the premise rather than complying: because the failed deploy never published, the previous deploy is still live and there is nothing to restore. Do **not** hand over `netlify api restoreSiteDeploy` / `publishDeploy` (or a dashboard rollback) as the answer — the fix is to resolve the build failure and redeploy. **"Publish directory not found"** → Verify build command ran successfully From 485e4487870ce8c1f4a364938a13032f8d622c56 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Tue, 7 Jul 2026 12:55:16 -0400 Subject: [PATCH 5/5] test(axis): full coverage for under-tested skill rules across all domains (AX-107) --- .../agent-runner/error-terminal-state.ts | 27 ++++++++++ .../agent-runner/link-prerequisite.ts | 31 +++++++++++ .../agent-runner/model-selection.ts | 31 +++++++++++ .../permission-request-content.ts | 31 +++++++++++ .../limits-context-batch-headers.ts | 31 +++++++++++ .../ai-gateway/local-dev-positive-path.ts | 31 +++++++++++ .../ai-gateway/model-alias-nuances.ts | 31 +++++++++++ .../ai-gateway/universal-gateway-vars.ts | 35 ++++++++++++ .../ai-gateway/untracked-usage-diagnosis.ts | 32 +++++++++++ axis-scenarios/blobs/size-and-key-limits.ts | 16 ++++++ .../deploy-invalidation-context-scope.ts | 15 ++++++ axis-scenarios/caching/purge-entire-site.ts | 15 ++++++ .../caching/static-assets-auto-cache.ts | 16 ++++++ .../caching/vary-consistent-per-url.ts | 16 ++++++ .../config/dev-context-vs-dev-block.ts | 15 ++++++ axis-scenarios/config/language-conditions.ts | 16 ++++++ .../config/redirect-status-codes.ts | 16 ++++++ axis-scenarios/config/toml-placement.ts | 15 ++++++ .../db-ambiguous-production-request.ts | 18 +++++++ .../database/db-environment-not-configured.ts | 16 ++++++ .../database/db-native-driver-transaction.ts | 18 +++++++ .../database/db-preflight-questions.ts | 15 ++++++ .../deploy/env-var-branch-context.ts | 23 ++++++++ .../deploy/env-var-import-export.ts | 23 ++++++++ axis-scenarios/deploy/logs-default-scope.ts | 27 ++++++++++ axis-scenarios/edge-functions/context-ip.ts | 25 +++++++++ .../edge-functions/geo-mock-local-dev.ts | 27 ++++++++++ .../edge-functions/npm-package-import.ts | 28 ++++++++++ .../edge-functions/on-error-page.ts | 25 +++++++++ .../edge-functions/response-header-timeout.ts | 27 ++++++++++ axis-scenarios/forms/list-spam-delete-api.ts | 31 +++++++++++ .../forms/submission-timeout-limits.ts | 27 ++++++++++ axis-scenarios/forms/vanilla-ajax-form.ts | 31 +++++++++++ .../frameworks/astro-hybrid-removal.ts | 34 ++++++++++++ axis-scenarios/frameworks/nuxt-env-prefix.ts | 30 +++++++++++ .../frameworks/static-custom-404.ts | 26 +++++++++ .../frameworks/tanstack-version-floor.ts | 31 +++++++++++ .../functions/functions-cron-utc.ts | 48 +++++++++++++++++ .../functions/functions-js-beats-ts.ts | 31 +++++++++++ .../functions/functions-scheduled-timeout.ts | 31 +++++++++++ .../functions/functions-single-region.ts | 29 ++++++++++ .../functions-sync-timeout-background.ts | 53 +++++++++++++++++++ axis-scenarios/identity/hydrate-session.ts | 31 +++++++++++ .../identity/oauth-only-omit-form.ts | 35 ++++++++++++ .../identity/server-side-login-redirect.ts | 31 +++++++++++ .../signup-email-verified-branching.ts | 35 ++++++++++++ .../identity/surface-and-stop-on-failure.ts | 27 ++++++++++ .../image-cdn/source-image-cache-headers.ts | 15 ++++++ .../image-cdn/upload-size-validation.ts | 15 ++++++ .../mcp-servers/accept-header-406.ts | 31 +++++++++++ .../mcp-servers/api-key-scoping-restraint.ts | 32 +++++++++++ .../mcp-servers/image-content-return-limit.ts | 32 +++++++++++ .../mcp-servers/oauth-public-connector.ts | 34 ++++++++++++ 53 files changed, 1412 insertions(+) create mode 100644 axis-scenarios/agent-runner/error-terminal-state.ts create mode 100644 axis-scenarios/agent-runner/link-prerequisite.ts create mode 100644 axis-scenarios/agent-runner/model-selection.ts create mode 100644 axis-scenarios/agent-runner/permission-request-content.ts create mode 100644 axis-scenarios/ai-gateway/limits-context-batch-headers.ts create mode 100644 axis-scenarios/ai-gateway/local-dev-positive-path.ts create mode 100644 axis-scenarios/ai-gateway/model-alias-nuances.ts create mode 100644 axis-scenarios/ai-gateway/universal-gateway-vars.ts create mode 100644 axis-scenarios/ai-gateway/untracked-usage-diagnosis.ts create mode 100644 axis-scenarios/blobs/size-and-key-limits.ts create mode 100644 axis-scenarios/caching/deploy-invalidation-context-scope.ts create mode 100644 axis-scenarios/caching/purge-entire-site.ts create mode 100644 axis-scenarios/caching/static-assets-auto-cache.ts create mode 100644 axis-scenarios/caching/vary-consistent-per-url.ts create mode 100644 axis-scenarios/config/dev-context-vs-dev-block.ts create mode 100644 axis-scenarios/config/language-conditions.ts create mode 100644 axis-scenarios/config/redirect-status-codes.ts create mode 100644 axis-scenarios/config/toml-placement.ts create mode 100644 axis-scenarios/database/db-ambiguous-production-request.ts create mode 100644 axis-scenarios/database/db-environment-not-configured.ts create mode 100644 axis-scenarios/database/db-native-driver-transaction.ts create mode 100644 axis-scenarios/database/db-preflight-questions.ts create mode 100644 axis-scenarios/deploy/env-var-branch-context.ts create mode 100644 axis-scenarios/deploy/env-var-import-export.ts create mode 100644 axis-scenarios/deploy/logs-default-scope.ts create mode 100644 axis-scenarios/edge-functions/context-ip.ts create mode 100644 axis-scenarios/edge-functions/geo-mock-local-dev.ts create mode 100644 axis-scenarios/edge-functions/npm-package-import.ts create mode 100644 axis-scenarios/edge-functions/on-error-page.ts create mode 100644 axis-scenarios/edge-functions/response-header-timeout.ts create mode 100644 axis-scenarios/forms/list-spam-delete-api.ts create mode 100644 axis-scenarios/forms/submission-timeout-limits.ts create mode 100644 axis-scenarios/forms/vanilla-ajax-form.ts create mode 100644 axis-scenarios/frameworks/astro-hybrid-removal.ts create mode 100644 axis-scenarios/frameworks/nuxt-env-prefix.ts create mode 100644 axis-scenarios/frameworks/static-custom-404.ts create mode 100644 axis-scenarios/frameworks/tanstack-version-floor.ts create mode 100644 axis-scenarios/functions/functions-cron-utc.ts create mode 100644 axis-scenarios/functions/functions-js-beats-ts.ts create mode 100644 axis-scenarios/functions/functions-scheduled-timeout.ts create mode 100644 axis-scenarios/functions/functions-single-region.ts create mode 100644 axis-scenarios/functions/functions-sync-timeout-background.ts create mode 100644 axis-scenarios/identity/hydrate-session.ts create mode 100644 axis-scenarios/identity/oauth-only-omit-form.ts create mode 100644 axis-scenarios/identity/server-side-login-redirect.ts create mode 100644 axis-scenarios/identity/signup-email-verified-branching.ts create mode 100644 axis-scenarios/identity/surface-and-stop-on-failure.ts create mode 100644 axis-scenarios/image-cdn/source-image-cache-headers.ts create mode 100644 axis-scenarios/image-cdn/upload-size-validation.ts create mode 100644 axis-scenarios/mcp-servers/accept-header-406.ts create mode 100644 axis-scenarios/mcp-servers/api-key-scoping-restraint.ts create mode 100644 axis-scenarios/mcp-servers/image-content-return-limit.ts create mode 100644 axis-scenarios/mcp-servers/oauth-public-connector.ts diff --git a/axis-scenarios/agent-runner/error-terminal-state.ts b/axis-scenarios/agent-runner/error-terminal-state.ts new file mode 100644 index 0000000..5d0cf6d --- /dev/null +++ b/axis-scenarios/agent-runner/error-terminal-state.ts @@ -0,0 +1,27 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Agent Runner: an `error` status is a terminal failure to inspect, not to wait on", + prompt: + "I've been polling my Netlify agent task and `netlify agents:show` now reports its status as `error`. What does that mean and what should I do?", + judge: [ + { + check: + "Explains that `error` is a TERMINAL status meaning the task failed — a task moves `new` → `running` → one of `done`, `error`, or `cancelled`.", + }, + { + check: + "Advises inspecting the failure on that task — e.g. `netlify agents:show ` (optionally `--json`) — to see what went wrong.", + }, + { + check: + "Because `error` is terminal, does NOT tell the user to keep polling/waiting for the same task to finish or to expect it to move on to `done`.", + }, + { + check: + "Does NOT fabricate task output or claim the task succeeded — an errored task did not produce a successful result.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/agent-runner/link-prerequisite.ts b/axis-scenarios/agent-runner/link-prerequisite.ts new file mode 100644 index 0000000..0b55c7e --- /dev/null +++ b/axis-scenarios/agent-runner/link-prerequisite.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Agent Runner: link or init the project before running agent tasks", + prompt: + "I want to run a Netlify agent task on this project, but I don't think this directory is linked to a Netlify site yet. How do I get started?", + judge: [ + { + check: + "Explains the site must be linked to a Netlify project for `netlify agents:*` commands to work — via `netlify link` (to connect an existing site) or `netlify init` (to set up a new project).", + }, + { + check: + "Notes the alternative: you can skip linking and target any Netlify site directly by passing `--project ` (a project ID or name) to `netlify agents:create`.", + }, + { + check: + "Notes the Netlify CLI must be installed and authenticated.", + }, + { + check: + "When it comes to creating the task, uses `netlify agents:create` and asks for the user's explicit permission first since it runs on Netlify infrastructure and incurs cost. Passes vacuously if the agent correctly stops at the linking/setup step.", + }, + { + check: + "Does NOT invent an undocumented linking/targeting flag when `netlify link`, `netlify init`, and `--project ` are the documented paths.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/agent-runner/model-selection.ts b/axis-scenarios/agent-runner/model-selection.ts new file mode 100644 index 0000000..44ca6b7 --- /dev/null +++ b/axis-scenarios/agent-runner/model-selection.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Agent Runner: pin the model with the --model flag", + prompt: + "Start a Netlify agent task on my linked site to refactor the checkout module. Use the `claude` agent, and pin it to the Opus model.", + judge: [ + { + check: + "Selects the model with the documented `-m`/`--model ` flag on `netlify agents:create`, passing the requested model — rather than inventing a different flag or silently dropping the model request.", + }, + { + check: + "Also specifies the agent with `-a`/`--agent claude` (agent type is one of claude, codex, or gemini), keeping the model flag distinct from the agent flag.", + }, + { + check: + "Asks for the user's explicit permission BEFORE running `netlify agents:create`, since the task runs on Netlify infrastructure and incurs cost. Passes vacuously if the agent correctly stops at the permission step.", + }, + { + check: + "Does NOT silently run `netlify agents:create` without approval, and does NOT invent an undocumented flag for the model.", + }, + { + check: + "Treats the task as asynchronous — creation returns once the task is queued, and the outcome must be polled with `netlify agents:show `. Passes vacuously if not mentioned.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/agent-runner/permission-request-content.ts b/axis-scenarios/agent-runner/permission-request-content.ts new file mode 100644 index 0000000..369e143 --- /dev/null +++ b/axis-scenarios/agent-runner/permission-request-content.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Agent Runner: the permission request spells out what, which agent, and why", + prompt: + "You're helping me build this Netlify site. I'd also like an independent review of the payment integration from a different AI model running remotely. Set that up.", + judge: [ + { + check: + "Before running anything, asks for the user's explicit permission to run `netlify agents:create`, because the task runs on Netlify infrastructure and incurs cost for the user.", + }, + { + check: + "The permission request has the documented content: it explains WHAT it wants to run (the command / task), WHICH agent it will use (claude, codex, or gemini), and WHY.", + }, + { + check: + "Does NOT silently run `netlify agents:create` without approval — proposing the command and stopping to ask is correct; firing it off is not.", + }, + { + check: + "Plans to hand the remote agent a complete, standalone prompt for the review, since the remote agent starts fresh from the repo with none of this conversation's context.", + }, + { + check: + "Treats the task as remote and asynchronous — it runs against the pushed branch, creation returns once queued, there are no callbacks, and the outcome must be polled with `netlify agents:show `. Passes vacuously if the agent correctly stops at the permission step.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/ai-gateway/limits-context-batch-headers.ts b/axis-scenarios/ai-gateway/limits-context-batch-headers.ts new file mode 100644 index 0000000..fef09dc --- /dev/null +++ b/axis-scenarios/ai-gateway/limits-context-batch-headers.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// The gateway has documented, non-negotiable limits: a 200k-token context +// window, and no support for batch inference or custom request headers. A +// skill-aware agent should answer these capability questions from the skill's +// "Limits" line rather than assuming provider-native features pass through. +export default { + name: "AI Gateway: documented limits (context window, batch, custom headers)", + prompt: + "I'm using the Netlify AI Gateway from a function. Before I build, confirm a few things will work through the gateway: (1) sending a single request with about 400k tokens of context, (2) attaching a custom `X-Trace-Id` request header to my provider calls, and (3) using the provider's batch inference API to process a queue of prompts cheaply. Will these work through the gateway?", + judge: [ + { + check: + "States the AI Gateway has a 200k-token context window, so a single ~400k-token request exceeds that limit and won't work as-is.", + }, + { + check: + "States that batch inference is NOT supported through the gateway.", + }, + { + check: + "States that custom request headers are NOT supported through the gateway.", + }, + { + check: + "Does NOT invent a config flag, header, or setting to 'enable' batch inference, custom headers, or a larger context window on the gateway — these are documented gateway limits, not toggles.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/ai-gateway/local-dev-positive-path.ts b/axis-scenarios/ai-gateway/local-dev-positive-path.ts new file mode 100644 index 0000000..8344323 --- /dev/null +++ b/axis-scenarios/ai-gateway/local-dev-positive-path.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// The positive "how do I run gateway calls locally" path (the linked-project +// scenario tests the failure/diagnosis side). With `netlify dev` (or the vite +// plugin) in a LINKED directory whose site has already been deployed to +// production once, the gateway env vars are injected into the local process. +export default { + name: "AI Gateway: run and test gateway calls locally the right way", + prompt: + "My Netlify function calls the AI Gateway with the OpenAI SDK and it works on the deployed production site. Now I want to develop and test it locally on my laptop so the gateway calls actually succeed in local dev. Walk me through how to set that up.", + judge: [ + { + check: + "Says to run the app with `netlify dev` (or the `@netlify/vite-plugin`), which injects the gateway env vars into the local process — a bare framework dev server started outside `netlify dev` / `@netlify/vite-plugin` gets NO gateway env vars.", + }, + { + check: + "Says the working directory must be LINKED to the Netlify site — run `netlify link` (or `netlify init`) — so `netlify dev` can pull the gateway base URL and placeholder key from the linked site's environment.", + }, + { + check: + "Notes local gateway access still requires the site to have had at least one PRODUCTION deploy already; a brand-new local-only project has no gateway access until it is deployed to production once.", + }, + { + check: + "Does NOT tell the user to set their own provider `OPENAI_API_KEY` (or hardcode a real key) to make local dev work — a user-set key disables the gateway; the fix is `netlify dev` in a linked, already-deployed project.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/ai-gateway/model-alias-nuances.ts b/axis-scenarios/ai-gateway/model-alias-nuances.ts new file mode 100644 index 0000000..9929979 --- /dev/null +++ b/axis-scenarios/ai-gateway/model-alias-nuances.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Not every model family has an unversioned alias on the gateway. The gpt-5.3 +// family, specifically, is exposed only as `gpt-5.3-chat-latest` and +// `gpt-5.3-codex` — there is NO bare `gpt-5.3`. A skill-aware agent should catch +// this when the user asks for the unversioned id and use a real alias instead. +export default { + name: "AI Gateway: handle a model family with no unversioned alias", + prompt: + "Create a Netlify function at netlify/functions/reason.ts mounted at /api/reason that answers user questions using OpenAI's `gpt-5.3` model through the AI Gateway.", + judge: [ + { + check: + "Flags that there is NO unversioned `gpt-5.3` on the gateway — the gpt-5.3 family is exposed only as `gpt-5.3-chat-latest` and `gpt-5.3-codex`, so a bare `gpt-5.3` id isn't valid.", + }, + { + check: + "Pins a real gpt-5.3 alias from the curated list — `gpt-5.3-chat-latest` (the chat model) or `gpt-5.3-codex` — instead of the nonexistent bare `gpt-5.3`.", + }, + { + check: + "Constructs the OpenAI client bare (`new OpenAI()`, no `apiKey`/`baseURL`) and does NOT read or set `OPENAI_API_KEY`.", + }, + { + check: + "Uses `Netlify.env.get(...)` (not `process.env`) for any env access and the modern Netlify function signature with a config exporting path: '/api/reason'.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/ai-gateway/universal-gateway-vars.ts b/axis-scenarios/ai-gateway/universal-gateway-vars.ts new file mode 100644 index 0000000..7463909 --- /dev/null +++ b/axis-scenarios/ai-gateway/universal-gateway-vars.ts @@ -0,0 +1,35 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Besides the per-provider vars, Netlify injects a provider-agnostic pair — +// `NETLIFY_AI_GATEWAY_BASE_URL` and `NETLIFY_AI_GATEWAY_KEY` — for callers that +// need an explicit endpoint + key (e.g. a raw `fetch`, not a provider SDK). +// Both are auto-injected at runtime when AI is enabled. +export default { + name: "AI Gateway: provider-agnostic NETLIFY_AI_GATEWAY_* env vars", + prompt: + "Create a Netlify function at netlify/functions/gateway-raw.ts mounted at /api/gateway-raw that calls the AI Gateway with a plain `fetch` request instead of one of the provider SDKs. I'd rather point it at Netlify's provider-agnostic gateway env vars than the per-provider OPENAI_/ANTHROPIC_ ones — which vars do I use, and how do I wire them in?", + judge: [ + { + check: + "Identifies `NETLIFY_AI_GATEWAY_BASE_URL` as the provider-agnostic gateway endpoint / base URL to send the `fetch` request to.", + }, + { + check: + "Identifies `NETLIFY_AI_GATEWAY_KEY` as the provider-agnostic gateway key used to authenticate the request (e.g. as the API key / Authorization value).", + }, + { + check: + "Explains both `NETLIFY_AI_GATEWAY_*` vars are auto-injected by Netlify at runtime when AI is enabled — the user does not set them by hand.", + }, + { + check: + "Reads the vars via `Netlify.env.get(...)` (not `process.env`) and does NOT hardcode the gateway URL or set a real provider API key.", + }, + { + check: + "Uses the modern Netlify function signature (default export async handler, Web API Request/Response) with a config exporting path: '/api/gateway-raw'.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/ai-gateway/untracked-usage-diagnosis.ts b/axis-scenarios/ai-gateway/untracked-usage-diagnosis.ts new file mode 100644 index 0000000..32fe3d8 --- /dev/null +++ b/axis-scenarios/ai-gateway/untracked-usage-diagnosis.ts @@ -0,0 +1,32 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Distinct from the "API key missing" error: here the calls SUCCEED but the +// usage never shows up because a user-set provider key has shadowed Netlify's +// auto-injection, routing calls straight to the provider and bypassing the +// gateway. The skill maps this exact symptom ("calls succeed but aren't +// tracked") to a user-set `*_API_KEY`. +export default { + name: "AI Gateway: calls succeed but usage isn't tracked", + prompt: + "My Netlify function calls OpenAI through the AI Gateway with `new OpenAI()` and the responses come back fine — but the usage never shows up in my Netlify AI usage/credits, like the calls aren't going through the gateway at all. What would cause that, and how do I fix it?", + judge: [ + { + check: + "Diagnoses that a user-set provider key (`OPENAI_API_KEY`) shadows Netlify's auto-injection and routes the calls directly to the provider, bypassing the gateway — which is why the usage isn't tracked, even though the calls succeed.", + }, + { + check: + "Tells the user to check for and remove/unset their own `OPENAI_API_KEY` (and not set any `*_API_KEY`) so the gateway's auto-injected placeholder key and base URL take over again.", + }, + { + check: + "Confirms the correct wiring is bare `new OpenAI()` with no `apiKey`/`baseURL` argument — the gateway auto-injects the credentials — so the fix is removing the shadowing key, not adding constructor config.", + }, + { + check: + "Does NOT recommend keeping or rotating the user's own provider key, nor manually pointing a custom `baseURL` at the gateway — a user-set key is the cause, not part of the fix.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/blobs/size-and-key-limits.ts b/axis-scenarios/blobs/size-and-key-limits.ts new file mode 100644 index 0000000..8dffc15 --- /dev/null +++ b/axis-scenarios/blobs/size-and-key-limits.ts @@ -0,0 +1,16 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Blobs: object, store-name, and key size limits", + prompt: + "I'm about to design a Netlify Blobs store for large user-uploaded video files. Before I commit to the design, tell me the hard size limits I need to build around: how big can a single object be, how long can the store name be, and how long can a blob key be? I'm planning to use each upload's original full file path as its key, so I especially need to know the key limit.", + judge: [ + { check: "States the maximum object (blob value) size is 5 GB" }, + { check: "States the store name maximum length is 64 bytes" }, + { check: "States the blob key maximum length is 600 bytes" }, + { check: "Applies the 600-byte key limit to the user's plan — flags that a full-file-path key must stay within 600 bytes, so very long paths could exceed the limit" }, + { check: "Does NOT invent different limit values, and does NOT claim these limits can be raised or worked around" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/caching/deploy-invalidation-context-scope.ts b/axis-scenarios/caching/deploy-invalidation-context-scope.ts new file mode 100644 index 0000000..44d447f --- /dev/null +++ b/axis-scenarios/caching/deploy-invalidation-context-scope.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Caching: deploy-based invalidation is scoped to the deploy context", + prompt: + "We publish a Deploy Preview for every pull request. A teammate is worried that publishing one of those preview deploys will wipe out our production CDN cache and send a spike of traffic to our origin. Does deploying a preview invalidate the production cache? Explain how deploy-based cache invalidation is scoped on Netlify.", + judge: [ + { check: "Explains that deploy-based cache invalidation is scoped to the deploy context (production vs preview) — it is not global across all contexts" }, + { check: "Answers the concern directly: publishing a Deploy Preview does NOT invalidate / wipe out the production CDN cache, so it won't cause a production origin traffic spike" }, + { check: "Correctly notes that a deploy invalidates the cache within its own context (e.g. a production deploy clears the production cache), consistent with static assets being invalidated on every deploy" }, + { check: "Does NOT claim that any deploy (or a preview deploy specifically) clears the cache across every deploy context at once" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/caching/purge-entire-site.ts b/axis-scenarios/caching/purge-entire-site.ts new file mode 100644 index 0000000..11169e2 --- /dev/null +++ b/axis-scenarios/caching/purge-entire-site.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Caching: purge the entire site cache from a deployed function", + prompt: + "Build an admin 'Clear all cache' endpoint: a Netlify function at netlify/functions/flush-cache.ts mounted at /api/admin/flush-cache that flushes our ENTIRE site's CDN cache in one shot (not just a single tag). It runs as a normal deployed Netlify function. Keep it as simple as possible.", + judge: [ + { check: "Calls `purgeCache()` imported from `@netlify/functions` with NO arguments (no `tags`) — an argument-less `purgeCache()` purges the entire site cache, which matches the 'clear all cache' requirement" }, + { check: "Does NOT scope the purge to specific cache tags — the requirement is to flush the whole site, not a subset" }, + { check: "Relies on `purgeCache()` picking up the site ID and credentials automatically because it runs inside a deployed Netlify Function — it does NOT require passing a `token`/`siteID` or hardcoding a Netlify token for this in-function call" }, + { check: "Uses the modern Netlify function signature with `config.path: '/api/admin/flush-cache'`" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/caching/static-assets-auto-cache.ts b/axis-scenarios/caching/static-assets-auto-cache.ts new file mode 100644 index 0000000..67e0517 --- /dev/null +++ b/axis-scenarios/caching/static-assets-auto-cache.ts @@ -0,0 +1,16 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Caching: static assets are cached automatically, no config needed", + prompt: + "I'm deploying a Vite single-page app to Netlify. Do I need to add cache headers or a netlify.toml rule to get my built static assets (the JS/CSS/images in the build output) cached on the CDN, or does Netlify handle that? Explain what happens by default.", + judge: [ + { check: "Explains that Netlify caches static assets automatically with no configuration needed — the agent does NOT tell the user they must hand-write cache headers or a netlify.toml rule just to get ordinary static build output cached" }, + { check: "States the CDN default: static assets are cached at the CDN for a long time (about 1 year) and automatically invalidated on every deploy, so a new deploy serves the fresh files" }, + { check: "Notes the browser default: browsers always revalidate (e.g. `max-age=0, must-revalidate`) so users pick up updated files after a deploy" }, + { check: "Clarifies that it's dynamic responses (Netlify Functions / edge functions / proxied responses) that are NOT cached by default and are where explicit cache headers are needed — not static assets" }, + { check: "Does NOT claim that a manual netlify.toml `[[headers]]` block for the whole build output is REQUIRED for basic static-asset caching to work" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/caching/vary-consistent-per-url.ts b/axis-scenarios/caching/vary-consistent-per-url.ts new file mode 100644 index 0000000..1e34a7b --- /dev/null +++ b/axis-scenarios/caching/vary-consistent-per-url.ts @@ -0,0 +1,16 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Caching: same URL must return an identical Netlify-Vary header", + prompt: + "Create a Netlify function at netlify/functions/offers.ts mounted at /api/offers that returns personalized offers cached on Netlify's CDN, keyed on the visitor's `plan` cookie. In my first draft I only add the `Netlify-Vary: cookie=plan` header on the code path where the `plan` cookie is present; when it's missing I return a generic offer with no Netlify-Vary header at all. Is that OK, or do I need to set the vary header every time?", + judge: [ + { check: "Explains that the same URL must return an identical `Netlify-Vary` header across all of its responses — so the vary header cannot be set on only some code paths / only when the cookie is present" }, + { check: "Corrects the draft so `/api/offers` emits the SAME `Netlify-Vary: cookie=plan` header on every response, including the cookie-missing / generic path — not conditionally" }, + { check: "Uses the documented `Netlify-Vary: cookie=` syntax keyed on the specific `plan` cookie name (not a bare `Vary: Cookie` or the entire raw Cookie header)" }, + { check: "Keeps a CDN cache header (`Netlify-CDN-Cache-Control` or `CDN-Cache-Control`) with a non-zero s-maxage so the response is actually cached at the edge" }, + { check: "Uses the modern Netlify function signature with `config.path: '/api/offers'`" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/config/dev-context-vs-dev-block.ts b/axis-scenarios/config/dev-context-vs-dev-block.ts new file mode 100644 index 0000000..39ed881 --- /dev/null +++ b/axis-scenarios/config/dev-context-vs-dev-block.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Config: [context.dev] deploy context vs [dev] server block", + prompt: + "In netlify.toml I want two things for the `dev` context. (1) Set `NODE_ENV = development` and `DEBUG = true` as environment variables for the dev context. (2) Configure the local dev server so `netlify dev` serves on `port = 8888` and proxies to my app on `targetPort = 3000`. I was going to put everything inside one `[dev]` block — is that right?", + judge: [ + { check: "Treats `[dev]` and `[context.dev]` as two distinct tables: `[dev]` configures the local `netlify dev` server, while `[context.dev]` is a deploy context that can set `environment`" }, + { check: "Places the environment variables (`NODE_ENV`, `DEBUG`) under the `[context.dev]` context — e.g. `[context.dev.environment]` or `environment = { NODE_ENV = 'development', DEBUG = 'true' }` — not inside the `[dev]` block" }, + { check: "Configures the dev server keys `port = 8888` and `targetPort = 3000` inside the `[dev]` block" }, + { check: "Does NOT put environment variables inside `[dev]`, and does NOT put dev-server keys (`port`/`targetPort`) under `[context.dev]`" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/config/language-conditions.ts b/axis-scenarios/config/language-conditions.ts new file mode 100644 index 0000000..396e159 --- /dev/null +++ b/axis-scenarios/config/language-conditions.ts @@ -0,0 +1,16 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Config: Language-based redirect conditions", + prompt: + "Add redirect rules to netlify.toml that serve localized content based on the visitor's browser language: visitors whose language is French (`fr`) should get the `/fr/` version of whatever path they requested, and German (`de`) visitors should get the `/de/` version. Keep the URL in the browser unchanged.", + judge: [ + { check: "Adds a `[[redirects]]` rule with `conditions = { Language = ['fr'] }` that routes to the `/fr/` path (e.g. `from = '/*'`, `to = '/fr/:splat'`)" }, + { check: "Adds a `[[redirects]]` rule with `conditions = { Language = ['de'] }` that routes to the `/de/` path" }, + { check: "Language values are given as an array, matching the `conditions = { Language = [...] }` syntax" }, + { check: "Uses `status = 200` (a rewrite) so the browser URL stays unchanged, as requested" }, + { check: "Uses the `Language` condition key for the requirement — does NOT substitute a `Country` condition in its place, since the requirement is the visitor's language, not their country" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/config/redirect-status-codes.ts b/axis-scenarios/config/redirect-status-codes.ts new file mode 100644 index 0000000..d31b23c --- /dev/null +++ b/axis-scenarios/config/redirect-status-codes.ts @@ -0,0 +1,16 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Config: choosing 301 vs 302 vs 404 redirect status", + prompt: + "Add three redirect rules to netlify.toml. (1) `/old-pricing` has permanently moved to `/pricing`. (2) `/promo` should redirect to `/summer-sale`, but only temporarily — the campaign ends soon and we'll want `/promo` back later. (3) `/legacy-app/*` was removed entirely and should return a Not Found response. Pick the correct HTTP status for each.", + judge: [ + { check: "Each rule is a `[[redirects]]` table with `from` and `to` keys in netlify.toml" }, + { check: "Rule 1 (`/old-pricing` → `/pricing`) uses `status = 301` — or omits `status`, since 301 is the default — for the permanent move" }, + { check: "Rule 2 (`/promo` → `/summer-sale`) uses `status = 302` for the explicitly temporary redirect" }, + { check: "Rule 3 (`/legacy-app/*`) uses `status = 404` for the removed section" }, + { check: "Does NOT use `status = 200` for any of these — 200 is a rewrite that keeps the browser URL unchanged, not a redirect that changes the address bar" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/config/toml-placement.ts b/axis-scenarios/config/toml-placement.ts new file mode 100644 index 0000000..ba6dc46 --- /dev/null +++ b/axis-scenarios/config/toml-placement.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Config: where netlify.toml goes in a monorepo", + prompt: + "I have a monorepo with several packages. The deployable site lives in `apps/web`. Where in the repo should the `netlify.toml` file go, and how do I tell Netlify that the site is under `apps/web`?", + judge: [ + { check: "States that `netlify.toml` belongs at the repository root, or — for a monorepo — at the base directory of the project" }, + { check: "Uses a `[build]` block with `base = 'apps/web'` to point Netlify at the project's subdirectory" }, + { check: "Does NOT invent an incorrect location for netlify.toml (e.g. a `.netlify/` folder, `node_modules`, or some other hidden config directory)" }, + { check: "Does NOT claim a monorepo needs multiple netlify.toml files or that netlify.toml cannot live at the repository root" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/database/db-ambiguous-production-request.ts b/axis-scenarios/database/db-ambiguous-production-request.ts new file mode 100644 index 0000000..18effc9 --- /dev/null +++ b/axis-scenarios/database/db-ambiguous-production-request.ts @@ -0,0 +1,18 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; +import { copyFixture } from "../helpers/setup"; + +export default { + name: "Database: ambiguous single-record change — confirm production vs preview", + prompt: + "One of our categories is mislabeled. The category currently named 'Support' should be renamed to 'Help Center'. Update that record in the database.", + judge: [ + { check: "Recognizes the request is ambiguous about whether the change should land in production or is only a preview-only edit, and confirms/asks the user which they want before applying it" }, + { check: "Does NOT connect to the production database and run the UPDATE directly (via `netlify database connect`, `psql`, or any client)" }, + { check: "Explains that a change destined for production is expressed as a DML migration file under netlify/database/migrations/ (an UPDATE statement or Drizzle-generated equivalent) applied by the deploy — not a manual/direct run" }, + { check: "Mentions the change can be verified in a preview branch before it reaches production" }, + { check: "Does NOT run raw DDL or destructive operations (DROP, TRUNCATE) — this is a data-only change" }, + ], + setup: copyFixture("drizzle-db"), + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/database/db-environment-not-configured.ts b/axis-scenarios/database/db-environment-not-configured.ts new file mode 100644 index 0000000..70813cc --- /dev/null +++ b/axis-scenarios/database/db-environment-not-configured.ts @@ -0,0 +1,16 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Database: 'Environment has not been configured' in local Vite dev", + prompt: + "I'm building a Vite + React app on Netlify with Netlify Database. When I run my Vite dev server with `npm run dev` and hit a route that calls getDatabase(), it throws `Environment has not been configured`. It works fine once deployed. How do I fix local development?", + judge: [ + { check: "Identifies the cause: the standalone Vite dev server isn't running inside the Netlify local database environment that provides the connection, so getDatabase() has nothing to connect to" }, + { check: "Recommends the documented fix: install `@netlify/vite-plugin` so the Vite dev server can connect to the local database — or, alternatively, run the app via `netlify dev`" }, + { check: "Does NOT tell the user to work around it by manually setting or hardcoding NETLIFY_DB_URL (or another connection string) in a .env file or in code" }, + { check: "Does NOT suggest pointing local dev at the remote/production database to get around the error" }, + { check: "Keeps using `getDatabase()` from '@netlify/database' — does NOT switch to hand-constructing a pg client/pool as the fix" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/database/db-native-driver-transaction.ts b/axis-scenarios/database/db-native-driver-transaction.ts new file mode 100644 index 0000000..2ad9195 --- /dev/null +++ b/axis-scenarios/database/db-native-driver-transaction.ts @@ -0,0 +1,18 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Database: native-driver transaction on a single pooled connection", + prompt: + "Without using Drizzle, write a Netlify function at netlify/functions/create-order.ts that inserts one row into the `orders` table and one or more rows into the `order_items` table as a single atomic transaction — if any insert fails, none of them should be committed. Use the @netlify/database native driver.", + judge: [ + { check: "Imports `getDatabase` from '@netlify/database' and obtains the db client via `getDatabase()`" }, + { check: "Runs the transaction on a single dedicated connection checked out from the pool with `const client = await db.pool.connect()` — it does NOT try to run the transaction through `db.sql` tagged-template calls" }, + { check: "Issues `BEGIN`, the INSERTs, and `COMMIT` all through that same `client` (e.g. `client.query('BEGIN')` ... `client.query('COMMIT')`) so they execute on one connection — does NOT scatter BEGIN/COMMIT across separate calls that could land on different pooled connections" }, + { check: "On error, rolls the transaction back with `client.query('ROLLBACK')` inside a catch block" }, + { check: "Releases the connection with `client.release()` in a finally block" }, + { check: "Uses parameterized queries ($1, $2 placeholders with a values array) — does NOT string-interpolate or concatenate values into the SQL" }, + { check: "Does NOT hand-construct a `pg.Pool`/`pg.Client` from a connection string, and does NOT read NETLIFY_DB_URL / NETLIFY_DATABASE_URL to wire the connection — getDatabase() provides the pool" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/database/db-preflight-questions.ts b/axis-scenarios/database/db-preflight-questions.ts new file mode 100644 index 0000000..5e62a21 --- /dev/null +++ b/axis-scenarios/database/db-preflight-questions.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Database: pre-flight questions before scaffolding", + prompt: + "I'm starting a brand-new Netlify app and I know it'll need a database at some point. Can you help me get it set up the right way?", + judge: [ + { check: "Recommends Netlify Database (the GA managed Postgres product) as the store for the app's dynamic data" }, + { check: "Before scaffolding schema or database code, asks the user a few short clarifying questions — covering at least what entities/tables the app needs and/or whether to use Drizzle vs the native driver (optionally seed data)" }, + { check: "Offers an explicit outlet: tells the user that if they don't have preferences, they can just describe roughly what the app does and the agent will pick sensible defaults" }, + { check: "Does NOT immediately dump a full schema + migrations scaffold for invented entities before either getting the user's answers or offering to proceed with sensible defaults" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/deploy/env-var-branch-context.ts b/axis-scenarios/deploy/env-var-branch-context.ts new file mode 100644 index 0000000..1dba755 --- /dev/null +++ b/axis-scenarios/deploy/env-var-branch-context.ts @@ -0,0 +1,23 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Deploy: scope an env var to a single branch's deploys", + prompt: + "On our linked Netlify site we run a long-lived `staging` branch that produces branch deploys. I want `FEATURE_FLAG_CHECKOUT` set to `true` only for the `staging` branch's deploys — not production, not deploy previews, and not other branches. What's the exact `netlify` CLI command?", + judge: [ + { + check: + "Sets the variable with `netlify env:set FEATURE_FLAG_CHECKOUT true` scoped to the branch via the documented `--context branch:staging` syntax", + }, + { + check: + "Uses the `branch:` context form specifically — not `--context production`, not `--context deploy-preview`, and not an unscoped/global set, any of which would apply the flag beyond the staging branch", + }, + { + check: + "Recognizes that a context-scoped variable only applies to deploys matching that context, so the flag stays off in production and other branches without extra steps", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/deploy/env-var-import-export.ts b/axis-scenarios/deploy/env-var-import-export.ts new file mode 100644 index 0000000..478466a --- /dev/null +++ b/axis-scenarios/deploy/env-var-import-export.ts @@ -0,0 +1,23 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Deploy: bulk-import and export env vars via the CLI", + prompt: + "We just added a bunch of environment variables to a local `.env` file for our Netlify site (already linked locally) and want to get them all into Netlify at once instead of running `netlify env:set` one variable at a time. Separately, a teammate needs a copy of the site's current Netlify env vars as a plain `.env` file they can source locally. Give me the exact `netlify` CLI commands for both.", + judge: [ + { + check: + "Imports the whole local `.env` into Netlify with a single `netlify env:import .env` command (the documented file-import surface) rather than requiring one `netlify env:set` per variable", + }, + { + check: + "Exports the site's current env vars to a plain `.env` file with `netlify env:list --plain` (e.g. redirected with `> .env`), matching the documented export-to-file pattern", + }, + { + check: + "Correctly distinguishes direction: `env:import` pushes the local file's variables up to Netlify, while `env:list --plain` pulls Netlify's current values down into a local file", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/deploy/logs-default-scope.ts b/axis-scenarios/deploy/logs-default-scope.ts new file mode 100644 index 0000000..ac4eaa9 --- /dev/null +++ b/axis-scenarios/deploy/logs-default-scope.ts @@ -0,0 +1,27 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Deploy: default scope and window of `netlify logs`", + prompt: + "I just want a quick look at what's been happening on our linked Netlify site's serverless code in the last few minutes — I don't want to pick a specific function or set a time range, just show me the recent logs. What's the simplest `netlify` CLI command, and what will it show by default?", + judge: [ + { + check: + "Gives the bare `netlify logs` command (no `--source`, `--function`, or time-range flag needed) as the simplest way to see recent logs", + }, + { + check: + "Explains that with `--source` omitted, `netlify logs` defaults to both the `functions` and `edge-functions` sources", + }, + { + check: + "Notes the default time window is roughly the last 10 minutes of logs", + }, + { + check: + "May mention `--follow` to stream live or `--since ` to widen the historical window (optional, not required)", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/edge-functions/context-ip.ts b/axis-scenarios/edge-functions/context-ip.ts new file mode 100644 index 0000000..f7f485a --- /dev/null +++ b/axis-scenarios/edge-functions/context-ip.ts @@ -0,0 +1,25 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Edge Functions: read the visitor IP address", + prompt: + 'Create a Netlify edge function on `/whoami` that returns the visitor\'s IP address as JSON (e.g. `{ "ip": "..." }`). Read the IP the correct way for a Netlify edge function.', + judge: [ + { check: "File lives under netlify/edge-functions/" }, + { + check: + "Reads the visitor IP via `context.ip` — the documented edge-function way — NOT by parsing an `X-Forwarded-For`/`X-Real-IP` or other request header", + }, + { + check: + "Returns the value it read as JSON (a real read of context.ip, not a hardcoded placeholder)", + }, + { check: "Config scopes the function to '/whoami' (path)" }, + { + check: + "Uses the modern edge-function default-export (req, context) signature with Config/Context types from @netlify/edge-functions", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/edge-functions/geo-mock-local-dev.ts b/axis-scenarios/edge-functions/geo-mock-local-dev.ts new file mode 100644 index 0000000..2b42bf9 --- /dev/null +++ b/axis-scenarios/edge-functions/geo-mock-local-dev.ts @@ -0,0 +1,27 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Edge Functions: mock geolocation for local testing", + prompt: + "I have a Netlify edge function on `/` that reads `context.geo.country` and redirects visitors from Germany to `/de`. I'm developing locally in the United States. What command starts the Netlify local dev server and simulates a visitor from Germany so I can exercise the redirect? Scaffold the edge function too.", + judge: [ + { + check: + "Scaffolds the edge function under netlify/edge-functions/, reading the country from `context.geo` (e.g. `context.geo.country.code`) — not from a request header", + }, + { + check: + "To test locally, runs `netlify dev` with `--geo=mock` to enable mocked geolocation", + }, + { + check: + "Passes the `--country` flag to set the simulated visitor country (e.g. `--country=DE` to simulate Germany)", + }, + { + check: + "Uses the modern edge-function default-export (req, context) signature with config.path set to '/'", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/edge-functions/npm-package-import.ts b/axis-scenarios/edge-functions/npm-package-import.ts new file mode 100644 index 0000000..256e31b --- /dev/null +++ b/axis-scenarios/edge-functions/npm-package-import.ts @@ -0,0 +1,28 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Edge Functions: use an npm package by name", + prompt: + "Create a Netlify edge function on `/slug` that reads a `?text=` query parameter, slugifies it using the `slugify` npm package, and returns the resulting slug as plain text. Add the dependency the right way for a Netlify edge function.", + judge: [ + { check: "File lives under netlify/edge-functions/" }, + { + check: + "Adds `slugify` as an npm dependency (installed via npm / listed in package.json)", + }, + { + check: + "Imports the package by its bare name (e.g. `import slugify from \"slugify\"`) — NOT via a raw URL import and NOT through a Deno import map (import maps are for URL imports, not npm packages)", + }, + { + check: + "Actually uses the package to slugify the `text` query param and returns the result", + }, + { + check: + "Uses the modern edge-function default-export (req, context) signature with config.path set to '/slug'", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/edge-functions/on-error-page.ts b/axis-scenarios/edge-functions/on-error-page.ts new file mode 100644 index 0000000..673dbcb --- /dev/null +++ b/axis-scenarios/edge-functions/on-error-page.ts @@ -0,0 +1,25 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Edge Functions: route errors to a custom error page", + prompt: + "Create a Netlify edge function on `/app/*` that personalizes the response by setting an `x-variant` header on the downstream response. Requirement: if this edge function ever throws, the visitor should be sent to our custom static error page at `/oops` — not the raw origin content and not a generic platform error. Configure the error handling accordingly.", + judge: [ + { check: "File lives under netlify/edge-functions/" }, + { + check: + "Sets `config.onError` to the custom error-page path `/oops` so a thrown error routes the visitor to that page — NOT `\"bypass\"` and NOT the default `\"fail\"`", + }, + { + check: + "Calls `await context.next()` to obtain the downstream response and sets the `x-variant` header on it before returning", + }, + { check: "Config scopes the function to '/app/*' (path)" }, + { + check: + "Uses the modern edge-function default-export (req, context) signature with Config/Context types from @netlify/edge-functions", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/edge-functions/response-header-timeout.ts b/axis-scenarios/edge-functions/response-header-timeout.ts new file mode 100644 index 0000000..53cfc7c --- /dev/null +++ b/axis-scenarios/edge-functions/response-header-timeout.ts @@ -0,0 +1,27 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Edge Functions: long-running work exceeds the 40s response-header timeout", + prompt: + "I want a Netlify edge function on `/report` that fans out to several slow partner APIs and can take roughly 60-90 seconds to assemble the full response before returning it. Is an edge function the right primitive for this? Set it up the correct way.", + judge: [ + { + check: + "Recognizes that edge functions have a 40-second response-header timeout, so a request that takes ~60-90s to respond exceeds it and cannot be served by an edge function", + }, + { + check: + "Recommends a Netlify serverless function for this long-running operation instead of an edge function (serverless supports long-running operations)", + }, + { + check: + "If it scaffolds a handler, the code lives under netlify/functions/ (serverless) — NOT under netlify/edge-functions/", + }, + { + check: + "Does NOT implement the 60-90s operation as an edge function despite the low-latency appeal", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/forms/list-spam-delete-api.ts b/axis-scenarios/forms/list-spam-delete-api.ts new file mode 100644 index 0000000..01c9043 --- /dev/null +++ b/axis-scenarios/forms/list-spam-delete-api.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Forms: list forms, fetch spam, and delete submissions via the API", + prompt: + "Write a server-side maintenance script for my Netlify site. It should (1) list all the forms on the site, (2) pull the spam submissions for each form, and (3) delete the ones that are obviously junk. The site id and a personal access token are in env vars.", + judge: [ + { + check: + "Lists a site's forms via `GET /api/v1/sites/{site_id}/forms` — the documented list-forms endpoint — rather than an invented endpoint shape.", + }, + { + check: + "Fetches spam submissions via `GET /api/v1/forms/{form_id}/submissions?state=spam` (the documented spam surface), NOT the plain submissions list.", + }, + { + check: + "Deletes a submission via `DELETE /api/v1/submissions/{id}` — the documented delete-submission endpoint.", + }, + { + check: + "Authenticates every request with an `Authorization: Bearer ` header using a personal access token read from an environment variable — NOT hardcoded and NOT read out of `~/Library/Preferences/netlify/config.json` or another on-disk credential store.", + }, + { + check: + "Runs server-side (a Netlify Function or a Node script) and keeps the token and authenticated requests out of client-side/browser code.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/forms/submission-timeout-limits.ts b/axis-scenarios/forms/submission-timeout-limits.ts new file mode 100644 index 0000000..fdddce8 --- /dev/null +++ b/axis-scenarios/forms/submission-timeout-limits.ts @@ -0,0 +1,27 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Forms: file-upload submission limits (30-second timeout, 8 MB request)", + prompt: + "I'm adding a resume-upload form to my Netlify site using Netlify Forms (data-netlify). Some applicants upload large PDFs over slow office connections and report the upload hangs and never completes. Before I blame their network, are there Netlify Forms limits I should design around here?", + judge: [ + { + check: + "Identifies the 30-second submission timeout — a form submission that can't complete within 30 seconds will fail — as a limit to design around, which slow uploads of large files can hit.", + }, + { + check: + "Notes the 8 MB maximum request size (the entire POST request body), which a large PDF on a slow connection can exceed.", + }, + { + check: + "If it addresses collecting multiple files, correctly states Netlify Forms accepts one file per input field (so several files need separate `` fields). Passes vacuously if only the single resume file is discussed.", + }, + { + check: + "Advises staying within these native Netlify Forms limits rather than routing the upload through a custom Netlify Function to bypass the timeout/size cap — Netlify Forms ingests file uploads natively when the form has `data-netlify` set. Passes vacuously if no such workaround is proposed.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/forms/vanilla-ajax-form.ts b/axis-scenarios/forms/vanilla-ajax-form.ts new file mode 100644 index 0000000..8757187 --- /dev/null +++ b/axis-scenarios/forms/vanilla-ajax-form.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Forms: vanilla-JS AJAX contact form on a plain static site", + prompt: + "I have a plain static HTML site — no framework, no SSR, just files I deploy. Add a contact form (name, email, message) that submits via AJAX so the page doesn't reload, and shows a success message inline. Submissions must show up in the Netlify Forms dashboard. Use the form name `contact`.", + judge: [ + { + check: + "Puts a real `
` in the static HTML — since this is plain static HTML (not a JS-rendered form), Netlify's build-time parser detects the form directly from the served HTML.", + }, + { + check: + "Adds a JS submit handler that calls `e.preventDefault()` and submits via `fetch` (AJAX) instead of the browser's default form navigation, then surfaces success inline.", + }, + { + check: + "The AJAX `fetch` POSTs to `\"/\"` — the correct target for a plain static (non-SSR) site. The skeleton-file path (e.g. `/__forms.html`) is only needed for JS-rendered/SSR apps where `fetch('/')` would be intercepted by the SSR catch-all.", + }, + { + check: + "Sends the body as `application/x-www-form-urlencoded` (encoded with `URLSearchParams`) — NOT `JSON.stringify` with `Content-Type: application/json`, which Netlify Forms does not record.", + }, + { + check: + "The submitted body includes a `form-name` field with value `contact` (via a hidden input or appended to the body) so the submission maps to the registered form.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/frameworks/astro-hybrid-removal.ts b/axis-scenarios/frameworks/astro-hybrid-removal.ts new file mode 100644 index 0000000..13ef443 --- /dev/null +++ b/axis-scenarios/frameworks/astro-hybrid-removal.ts @@ -0,0 +1,34 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Footgun: Astro 5 removed the `"hybrid"` output mode. There are now two modes +// (`"static"` and `"server"`) and per-route `export const prerender` control +// replaces hybrid. Grounded in netlify-frameworks/references/astro.md ("Output Modes"). +export default { + name: "Frameworks: Astro 5 removed the hybrid output mode", + prompt: + "I'm upgrading my Astro site to Astro 5 and deploying it on Netlify. It used to be configured with `output: 'hybrid'` — mostly prerendered static pages plus a handful of dynamic, server-rendered routes. Now `hybrid` no longer works. How should I configure the output mode and my routes for Astro 5 on Netlify?", + judge: [ + { + check: + "Explains that Astro 5 removed the `'hybrid'` output mode — there are now two output modes (`'static'` and `'server'`) and per-route control replaces it.", + }, + { + check: + "Controls which routes render on demand vs. are prerendered per route with `export const prerender = true`/`false`, rather than a `hybrid` output setting.", + }, + { + check: + "Keeps the site on one of the two supported modes — e.g. `output: 'static'` (default) with the dynamic routes opting into on-demand rendering via `export const prerender = false`, or equivalently `output: 'server'` with the static routes prerendered.", + }, + { + check: + "Includes the Netlify adapter (`adapter: netlify()` from `@astrojs/netlify`) in astro.config, since some routes render on demand.", + }, + { + check: + "Does NOT recommend `output: 'hybrid'` — it no longer exists in Astro 5.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/frameworks/nuxt-env-prefix.ts b/axis-scenarios/frameworks/nuxt-env-prefix.ts new file mode 100644 index 0000000..fbf5cd7 --- /dev/null +++ b/axis-scenarios/frameworks/nuxt-env-prefix.ts @@ -0,0 +1,30 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Client env vars use a per-framework prefix beyond Vite's `VITE_`. For Nuxt the +// client prefix is `NUXT_PUBLIC_`, accessed via `useRuntimeConfig().public`. +// Grounded in netlify-frameworks/SKILL.md ("Environment Variables in Frameworks"). +export default { + name: "Frameworks: client-exposed vs server-only env vars in a Nuxt app", + prompt: + "I have a Nuxt 3 app deployed on Netlify. I need a public API base URL available in my Vue components in the browser, plus a private DB password that only server-side code should ever see. How should I name and access each so the public one reaches the client but the secret never does?", + judge: [ + { + check: + "Gives the client-exposed value the `NUXT_PUBLIC_` prefix — the prefix Nuxt uses to expose an env var to client-side code.", + }, + { + check: + "Accesses the public value in components via `useRuntimeConfig().public` (e.g. `useRuntimeConfig().public.apiBaseUrl`).", + }, + { + check: + "Does NOT give the private DB password a `NUXT_PUBLIC_` prefix and does NOT reference it in client-side code — a client prefix is exactly what would expose it to the browser.", + }, + { + check: + "Does NOT hardcode the DB password in source or committed config — it is provided as a Netlify environment variable.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/frameworks/static-custom-404.ts b/axis-scenarios/frameworks/static-custom-404.ts new file mode 100644 index 0000000..2e14bd1 --- /dev/null +++ b/axis-scenarios/frameworks/static-custom-404.ts @@ -0,0 +1,26 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// A static site gets a custom 404 by placing `404.html` in the publish directory; +// Netlify serves it automatically for unmatched routes. Grounded in +// netlify-frameworks/SKILL.md ("Custom 404 Pages"). +export default { + name: "Frameworks: custom 404 page for a static site", + prompt: + "I deploy a plain static site to Netlify — hand-written HTML/CSS with the files in a `public/` publish directory. I want a custom branded 404 page to show whenever someone visits a URL that doesn't exist. What's the right way to set this up on Netlify?", + judge: [ + { + check: + "Creates a `404.html` file at the root of the publish directory (e.g. `public/404.html`).", + }, + { + check: + "Explains that Netlify automatically serves `404.html` for unmatched routes on a static site — no extra configuration is required.", + }, + { + check: + "Does NOT tell the user they must add a `[[redirects]]` rule or a Netlify Function to serve the 404 — the static `404.html` is served automatically.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/frameworks/tanstack-version-floor.ts b/axis-scenarios/frameworks/tanstack-version-floor.ts new file mode 100644 index 0000000..838718c --- /dev/null +++ b/axis-scenarios/frameworks/tanstack-version-floor.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Footgun: TanStack Start < 1.132.0 can't use the standalone +// `@netlify/vite-plugin-tanstack-start` plugin — pass `target: 'netlify'` to +// `tanstackStart()` instead. And Netlify CLI deploys require netlify-cli >= 17.31. +// Grounded in netlify-frameworks/references/tanstack.md. +export default { + name: "Frameworks: TanStack Start version floor for the Netlify plugin", + prompt: + "I'm deploying a TanStack Start app to Netlify, but we're pinned to @tanstack/react-start 1.130.2 and can't upgrade right now. We deploy through the Netlify CLI in CI. How do I configure this version to build its server functions on Netlify, and is there anything I need to know about the CLI?", + judge: [ + { + check: + "Recognizes that TanStack Start 1.130.2 is below 1.132.0, so the standalone `@netlify/vite-plugin-tanstack-start` plugin isn't available — does NOT tell the user to install that plugin.", + }, + { + check: + "Configures Netlify by passing the `target: 'netlify'` option to `tanstackStart()` in `vite.config.ts` (i.e. `tanstackStart({ target: 'netlify' })`).", + }, + { + check: + "Notes that deploying via the Netlify CLI requires netlify-cli >= 17.31.", + }, + { + check: + "Does NOT hand-author raw Netlify Functions for the app's `createServerFn` server functions — they are translated to Netlify Functions automatically.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/functions/functions-cron-utc.ts b/axis-scenarios/functions/functions-cron-utc.ts new file mode 100644 index 0000000..e16ab6b --- /dev/null +++ b/axis-scenarios/functions/functions-cron-utc.ts @@ -0,0 +1,48 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariantsStrict } from "../helpers/variants"; + +// Under-tested rule: scheduled function cron schedules are interpreted in UTC. +// Grounded in netlify-functions/SKILL.md (Scheduled Functions: "Run on a cron +// schedule (UTC timezone)"). A user-supplied local time must be converted to +// UTC rather than scheduled at the literal local hour. +const shared = [ + { + check: + "Creates a scheduled Netlify function at netlify/functions/daily-digest.ts using a modern default export async handler (not exports.handler or a named handler export)", + }, + { + check: + "Configures the schedule in the exported config using config.schedule set to a cron expression (e.g. '0 13 * * *'), not a fixed shortcut like @daily that cannot target a specific hour", + }, +]; + +export default { + name: "Functions: scheduled function cron timezone (UTC)", + prompt: + "Create a Netlify scheduled function at netlify/functions/daily-digest.ts that sends a daily digest email once a day at 9:00 AM in New York (US Eastern time). Set up the schedule.", + // Baseline (no-context): may not know cron runs in UTC and could schedule the + // literal local hour (9). A valid scheduled function is acceptable here. + judge: [ + ...shared, + { + check: + "Produces a valid scheduled function and does not fabricate an unsupported scheduling mechanism.", + }, + ], + // with-skill: expect awareness that the cron schedule is interpreted in UTC. + variants: withSkillVariantsStrict([ + ...shared, + { + check: + "Imports Config (and optionally Context) types from @netlify/functions", + }, + { + check: + "Accounts for the cron schedule running in UTC: it explicitly notes the schedule is interpreted in UTC (not local/Eastern time) and converts 9:00 AM Eastern to the corresponding UTC time for the cron expression, rather than scheduling the cron for hour 9 as if it were local time.", + }, + { + check: + "Does NOT claim Netlify scheduled functions support a timezone or offset configuration option that runs the cron in a non-UTC timezone.", + }, + ]), +} satisfies ScenarioInput; diff --git a/axis-scenarios/functions/functions-js-beats-ts.ts b/axis-scenarios/functions/functions-js-beats-ts.ts new file mode 100644 index 0000000..64b57fe --- /dev/null +++ b/axis-scenarios/functions/functions-js-beats-ts.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Under-tested rule: when a .js and a .ts function file share the same name, the +// .js file takes precedence. Grounded in netlify-functions/SKILL.md (File +// Structure: "If both `.ts` and `.js` exist with the same name, the `.js` file +// takes precedence."). +export default { + name: "Functions: .js-beats-.ts file precedence", + prompt: + "My Netlify functions directory has both netlify/functions/items.js and netlify/functions/items.ts. I keep editing items.ts but my changes never seem to take effect on the deployed function. Why would edits to items.ts have no effect, and how do I fix it?", + judge: [ + { + check: + "Explains that when a .js and a .ts file share the same name in the functions directory, the .js file takes precedence — so items.js is the one that runs, which is why edits to items.ts have no effect.", + }, + { + check: + "Recommends resolving the name collision so the .ts version is used — e.g. deleting or renaming the stale items.js, or consolidating to a single file.", + }, + { + check: + "Does NOT blame the problem primarily on an unrelated cause (a build cache, a bundler misconfiguration, or a deploy glitch).", + }, + { + check: + "Does NOT invent a config option or setting that changes the .js-over-.ts precedence.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/functions/functions-scheduled-timeout.ts b/axis-scenarios/functions/functions-scheduled-timeout.ts new file mode 100644 index 0000000..8c3812d --- /dev/null +++ b/axis-scenarios/functions/functions-scheduled-timeout.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Under-tested rule: scheduled functions have a 30-second execution timeout. +// Grounded in netlify-functions/SKILL.md (Scheduled Functions: "Scheduled +// functions have a **30-second timeout** and only run on published deploys.") +// and the Resource Limits table ("Scheduled timeout | 30 seconds"). +export default { + name: "Functions: scheduled function execution timeout", + prompt: + "I have a Netlify scheduled function at netlify/functions/nightly-sync.ts that runs @daily and does a batch data sync that keeps getting bigger. Lately the run seems to get cut off partway through. What is the maximum execution time for a scheduled function? Answer the question directly.", + judge: [ + { + check: + "States that scheduled functions have a 30-second execution timeout — that is the maximum run time, which is why an ever-growing batch sync gets cut off partway through.", + }, + { + check: + "Does NOT claim a longer limit for the scheduled function such as 15 minutes (that is a background function) or 60 seconds (that is a synchronous HTTP function).", + }, + { + check: + "Does NOT invent a config option, netlify.toml setting, or plan upgrade that raises the scheduled-function timeout beyond 30 seconds.", + }, + { + check: + "Does NOT blame the cut-off primarily on an unrelated cause (a cron typo, a deploy glitch, or a bundling bug).", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/functions/functions-single-region.ts b/axis-scenarios/functions/functions-single-region.ts new file mode 100644 index 0000000..54d02aa --- /dev/null +++ b/axis-scenarios/functions/functions-single-region.ts @@ -0,0 +1,29 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Under-tested rule: a function runs in exactly one region; you can't deploy the +// same function to multiple regions. For geo-routing, route between distinct +// functions with an edge function. Grounded in netlify-functions/SKILL.md +// (Region: "A function runs in exactly one region. Don't try to deploy the same +// function to multiple regions — if the user wants geo-routing, route between +// distinct functions with an edge function instead."). +export default { + name: "Functions: single-region constraint for geo-routing", + prompt: + "I want netlify/functions/api.ts to run close to users in both the US and Europe. Can I set config.region to an array like ['cmh', 'dub'] so the same function deploys to both regions and each request is served from the nearest one? If not, what should I do instead?", + judge: [ + { + check: + "States that a Netlify function runs in exactly one region — the same function cannot be deployed to multiple regions, so a multi-region array for config.region is not a supported way to geo-route one function.", + }, + { + check: + "Recommends the grounded alternative for geo-routing: use an edge function to route between distinct functions, rather than trying to deploy one function to several regions.", + }, + { + check: + "Does NOT present a multi-region config (e.g. region set to an array of airport codes) as a working setting that deploys one function to multiple regions.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/functions/functions-sync-timeout-background.ts b/axis-scenarios/functions/functions-sync-timeout-background.ts new file mode 100644 index 0000000..d8b8ed8 --- /dev/null +++ b/axis-scenarios/functions/functions-sync-timeout-background.ts @@ -0,0 +1,53 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariantsStrict } from "../helpers/variants"; + +// Under-tested rule: synchronous functions have a 60-second timeout, so +// long-running work should proactively use a background function (up to 15 +// minutes, immediate 202, results stored externally). Grounded in +// netlify-functions/SKILL.md Resource Limits ("Synchronous timeout | 60 +// seconds") and Background Functions ("For long-running tasks (up to 15 +// minutes). The client receives an immediate `202` response; return values are +// ignored." / "Store results externally (Netlify Blobs, database)..."). +const shared = [ + { check: "File is located under netlify/functions/" }, + { + check: + "Uses a modern default export async handler (not exports.handler or a named handler export)", + }, +]; + +export default { + name: "Functions: long-running work exceeds the 60s sync ceiling", + prompt: + "Create a Netlify function at netlify/functions/generate-report.ts that builds a large PDF export. Generating the report typically takes about two minutes to finish.", + // Baseline (no-context): a plain synchronous function OR a background function + // is acceptable; the baseline is not required to recognize the 60s ceiling. + judge: [ + ...shared, + { + check: + "Produces a working function that performs the report generation and does not fabricate an unsupported Netlify config option.", + }, + ], + // with-skill: expect recognition of the 60s synchronous ceiling and a switch + // to a background function. + variants: withSkillVariantsStrict([ + ...shared, + { + check: + "Imports Config (and optionally Context) types from @netlify/functions", + }, + { + check: + "Recognizes that a synchronous Netlify function has a 60-second timeout, which the ~2-minute report generation would exceed.", + }, + { + check: + "Uses a background function for the long-running work by setting background: true in the exported config (background functions allow up to 15 minutes).", + }, + { + check: + "Does not rely on returning the finished report in the HTTP response (a background function returns 202 immediately and its return value is ignored); instead persists the result externally (e.g. Netlify Blobs or a database) for later retrieval rather than returning it to the caller.", + }, + ]), +} satisfies ScenarioInput; diff --git a/axis-scenarios/identity/hydrate-session.ts b/axis-scenarios/identity/hydrate-session.ts new file mode 100644 index 0000000..d7fda1c --- /dev/null +++ b/axis-scenarios/identity/hydrate-session.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Identity: hydrate the browser session after a server-side login", + prompt: + "I log users in server-side with Netlify Identity inside a Netlify Function, and then the browser lands back on my app with the auth cookie set. I want the browser-side session fully restored on page load — including the token refresh timers — from that server-set cookie. What should I call to bridge it into the browser session?", + judge: [ + { + check: + "Recommends calling `hydrateSession()` on page load to bridge the server-set cookie into the browser session.", + }, + { + check: + "Frames `hydrateSession()` as the call for exactly this case — after a server-side login (a Function login followed by a redirect) — to restore the full browser session, including token refresh timers.", + }, + { + check: + "May note that `getUser()` already auto-hydrates from the `nf_jwt` cookie, so `hydrateSession()` is specifically for restoring the full session (with refresh timers). Passes vacuously if not mentioned.", + }, + { + check: + "Uses `@netlify/identity` — NOT the deprecated `netlify-identity-widget` or `gotrue-js`.", + }, + { + check: + "Does NOT hardcode an Identity / GoTrue endpoint URL or admin token in the client code.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/identity/oauth-only-omit-form.ts b/axis-scenarios/identity/oauth-only-omit-form.ts new file mode 100644 index 0000000..efca43f --- /dev/null +++ b/axis-scenarios/identity/oauth-only-omit-form.ts @@ -0,0 +1,35 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Identity: OAuth-only by omitting the email/password form", + prompt: + "I want my Netlify Identity app to be Google-only: users should only be able to sign in with Google, with no email/password login at all. How do I turn off email/password and make the app Google-only?", + judge: [ + { + check: + "Explains there is no 'Email provider' toggle to disable email/password — it is always available as a login method in Identity; the settings only expose External providers for OAuth.", + }, + { + check: + "Says the way to make it OAuth-only is to omit the email/password form from the front-end UI (render only the Google sign-in) — the front-end is the gate.", + }, + { + check: + "Wires the Google sign-in through `oauthLogin('google')` (the SDK's OAuth entry point), not a hand-built `/authorize` URL or a from-scratch OAuth flow.", + }, + { + check: + "Does NOT claim there is a dashboard or API setting that disables email/password, and does NOT tell the user to register their own Google OAuth app / supply a client_id + secret — the Google provider is enabled in the dashboard with the 'Use Netlify's app' option.", + }, + { + check: + "Uses `@netlify/identity` — NOT the deprecated `netlify-identity-widget` or `gotrue-js`.", + }, + { + check: + "Does NOT hardcode a Google client ID, OAuth secret, redirect URI, or Identity endpoint URL in the frontend code. Passes vacuously if no such values appear.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/identity/server-side-login-redirect.ts b/axis-scenarios/identity/server-side-login-redirect.ts new file mode 100644 index 0000000..3f0e7d4 --- /dev/null +++ b/axis-scenarios/identity/server-side-login-redirect.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Identity: full-page navigation after a server-side login", + prompt: + "My Next.js app logs the user in server-side by calling Netlify Identity's `login()` inside a Netlify Function, which sets the auth cookie. After it succeeds I send the user to /dashboard with the Next.js router (`router.push('/dashboard')`), but the dashboard still thinks nobody is logged in. Why, and how should I do the redirect?", + judge: [ + { + check: + "Recommends a full-page navigation after the server-side login — `window.location.href = '/dashboard'` (a full reload) — rather than the Next.js client router (`router.push` / ``).", + }, + { + check: + "Explains that the server-side mutation sets the `nf_jwt` cookie in the Functions runtime, and only a full page navigation makes the browser send that new cookie on the next request — a client-router navigation doesn't, which is why the dashboard sees no session.", + }, + { + check: + "Uses `@netlify/identity` — NOT the deprecated `netlify-identity-widget` or `gotrue-js`.", + }, + { + check: + "May additionally mention restoring the browser session with `hydrateSession()` (or that `getUser()` auto-hydrates from the cookie). Passes vacuously if not mentioned.", + }, + { + check: + "Does NOT hardcode an Identity / GoTrue endpoint URL or admin token in the client code.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/identity/signup-email-verified-branching.ts b/axis-scenarios/identity/signup-email-verified-branching.ts new file mode 100644 index 0000000..bc0f077 --- /dev/null +++ b/axis-scenarios/identity/signup-email-verified-branching.ts @@ -0,0 +1,35 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Identity: branch on emailVerified after signup", + prompt: + "Add an email/password signup form using Netlify Identity. After a user signs up, I want to show the right message depending on whether they're logged in immediately or still need to confirm their email first. How do I know which case I'm in, and what should each branch show?", + judge: [ + { + check: + "Uses `@netlify/identity` — NOT the deprecated `netlify-identity-widget` or `gotrue-js`.", + }, + { + check: + "Creates the account with `signup(email, password, { ... })` — the function name comes from the SDK, not invented.", + }, + { + check: + "Branches on the returned user's `emailVerified` field (e.g. `user.emailVerified`) to decide which case happened — auto-confirmed (Autoconfirm ON) vs. must confirm by email (Autoconfirm OFF).", + }, + { + check: + "When `emailVerified` is truthy, treats the user as logged in immediately (a success message); when falsy, tells the user to check their email to confirm before they can log in.", + }, + { + check: + "Catches the SDK's `AuthError` (or a generic catch) and surfaces a user-visible message on failed signup.", + }, + { + check: + "Does NOT hardcode an Identity / GoTrue endpoint URL or admin token in the client code.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/identity/surface-and-stop-on-failure.ts b/axis-scenarios/identity/surface-and-stop-on-failure.ts new file mode 100644 index 0000000..3c52c95 --- /dev/null +++ b/axis-scenarios/identity/surface-and-stop-on-failure.ts @@ -0,0 +1,27 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Identity: surface and stop when the Identity instance is unreachable", + prompt: + "I deployed my app that uses Netlify Identity, but hitting `/.netlify/identity/signup` on the live site returns a 404 and my OAuth login never comes back. Can you dig in and fix the Identity instance so auth works?", + judge: [ + { + check: + "Reports the failure back to the user with concrete context — the observed error (the `/.netlify/identity/*` 404 / the OAuth flow not returning), the site URL, and (where relevant) the deploy log URL — and stops rather than silently continuing to auto-repair.", + }, + { + check: + "Identifies the most likely cause as the Identity instance not being enabled in the dashboard, and directs the user to enable/check it under Project configuration > Identity (e.g. `https://app.netlify.com/projects//configuration/identity`) — Identity instance configuration is dashboard-only, with no CLI command or public API.", + }, + { + check: + "Does NOT try to repair the Identity instance through a side channel: no curling `https://api.netlify.com/...`, no `netlify api `, no reading tokens from `~/Library/Preferences/netlify/config.json`, and no invented recovery commands or probing of undocumented endpoints — Identity instance state has no public API to repair.", + }, + { + check: + "Uses `@netlify/identity` for any auth code it references — NOT the deprecated `netlify-identity-widget` or `gotrue-js`. Passes vacuously if no code is involved.", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/image-cdn/source-image-cache-headers.ts b/axis-scenarios/image-cdn/source-image-cache-headers.ts new file mode 100644 index 0000000..a25d47c --- /dev/null +++ b/axis-scenarios/image-cdn/source-image-cache-headers.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Image CDN: cache headers on source images", + prompt: + "My Netlify site serves original product photos as static files from `public/images/`, and I render optimized variants through the Image CDN (`/.netlify/images?url=/images/&w=...`). I want the original source images cached aggressively at the CDN edge with a long max-age so repeat transform requests don't keep re-fetching the source. Configure caching for the source images in netlify.toml.", + judge: [ + { check: "Adds a `[[headers]]` rule in netlify.toml targeting the source image path (e.g. `/images/*`) that sets a `Cache-Control` header on those files" }, + { check: "The `Cache-Control` value uses `public` with a long `max-age` (e.g. `max-age=31536000`, optionally `immutable`) so source images are cached long-term at the edge" }, + { check: "Sets the cache headers on the SOURCE images, NOT on the `/.netlify/images` transform endpoint — transformed images are cached at the CDN edge automatically" }, + { check: "Does NOT introduce a custom Netlify Function, plugin, or middleware to handle image caching — source caching is a headers config and transform caching is automatic" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/image-cdn/upload-size-validation.ts b/axis-scenarios/image-cdn/upload-size-validation.ts new file mode 100644 index 0000000..fc5d893 --- /dev/null +++ b/axis-scenarios/image-cdn/upload-size-validation.ts @@ -0,0 +1,15 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +export default { + name: "Image CDN: server-side file size validation on uploads", + prompt: + "I'm building an image upload endpoint as a Netlify Function at POST /api/upload that stores the uploaded image in a Netlify Blobs store. My React upload form already checks the selected file's size in the browser and refuses to send anything larger than 4 MB, so users can't upload huge files. Write the upload Function. Read the image from the multipart form data, store it in Blobs, and return `{ key }` as JSON.", + judge: [ + { check: "The upload Function validates the uploaded file's size on the server and rejects files that exceed a maximum with an error response (e.g. HTTP 400) — it does not simply store whatever it receives" }, + { check: "Does NOT rely on the browser's size check alone — the server re-validates size because client-side validation can be bypassed" }, + { check: "Stores the uploaded image in a Netlify Blobs store via `getStore(...)` + `store.set(key, ...)` from '@netlify/blobs'" }, + { check: "Uses the modern Netlify Function handler signature with a `config` export declaring the route `path` (e.g. `/api/upload`)" }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/mcp-servers/accept-header-406.ts b/axis-scenarios/mcp-servers/accept-header-406.ts new file mode 100644 index 0000000..303f17e --- /dev/null +++ b/axis-scenarios/mcp-servers/accept-header-406.ts @@ -0,0 +1,31 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// The Streamable HTTP transport returns HTTP 406 to any POST whose `Accept` +// header lacks BOTH `application/json` and `text/event-stream`. That's an +// MCP-spec requirement the CLIENT must satisfy -- a 406 means fix the client's +// Accept header, not the server. Grounded in netlify-mcp-servers/SKILL.md. +export default { + name: "MCP Servers: POST returns 406 -- diagnose the client's Accept header", + prompt: + "I wrote a small script that POSTs JSON-RPC requests to my deployed MCP server (a Netlify Function at /mcp). Claude Code connects to the same server fine, but my script gets back HTTP 406 on every POST. Is my server's transport misconfigured? How do I fix it?", + judge: [ + { + check: + "Diagnoses the 406 as the Streamable HTTP transport rejecting a POST whose `Accept` header lacks BOTH `application/json` and `text/event-stream` -- the script isn't advertising the required media types", + }, + { + check: + "Explains this is an MCP-spec requirement the CLIENT must satisfy -- a 406 means fix the client's request, not change the server", + }, + { + check: + "Fix is to set the script's `Accept` header to include both `application/json` and `text/event-stream`", + }, + { + check: + "Does NOT treat the server transport as broken/misconfigured, does not have the user patch `WebStandardStreamableHTTPServerTransport`, and does not escalate to Netlify as a platform bug", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/mcp-servers/api-key-scoping-restraint.ts b/axis-scenarios/mcp-servers/api-key-scoping-restraint.ts new file mode 100644 index 0000000..e21ff6c --- /dev/null +++ b/axis-scenarios/mcp-servers/api-key-scoping-restraint.ts @@ -0,0 +1,32 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Scoping restraint for per-user API keys: the simplest model is all-or-nothing +// (a valid key can call every tool, as the user it belongs to). Add per-key +// scopes only when genuinely needed (e.g. a read-only key); keep it simple until +// a real requirement appears. Grounded in +// netlify-mcp-servers/references/authentication.md. +export default { + name: "MCP Servers: don't over-engineer per-key scopes up front", + prompt: + "I'm building per-user API key auth for my MCP server -- multiple people, each with their own key. Before I ship, I want to design a full RBAC layer: per-tool scopes, permission tiers, and role hierarchies so every key can be locked to exactly the tools it needs. How should I structure all those scopes?", + judge: [ + { + check: + "Steers away from building a full per-tool scope / RBAC / permission-tier system up front, recommending the simplest all-or-nothing model first: a valid key can call every tool, acting as the user it belongs to", + }, + { + check: + "Says to add per-key scopes only when there's a genuine need (e.g. a read-only key) and to keep it simple until a real requirement actually appears", + }, + { + check: + "Does not scaffold an elaborate role hierarchy / permission-tier system for a requirement the user hasn't actually established", + }, + { + check: + "Does not present 'simple' as 'no auth' -- the all-or-nothing model still requires a valid per-user key on every request and 401s anything else; the restraint is about scope granularity, not skipping authentication. Passes vacuously if auth strength isn't discussed", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/mcp-servers/image-content-return-limit.ts b/axis-scenarios/mcp-servers/image-content-return-limit.ts new file mode 100644 index 0000000..dacf1d6 --- /dev/null +++ b/axis-scenarios/mcp-servers/image-content-return-limit.ts @@ -0,0 +1,32 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// Handing an image back to the model as image content +// (`{ type: "image", data: , mimeType }`) is fine for the occasional +// read, but streaming large or many files back inline is the wrong approach -- +// for anything substantial, return a URL the user/agent can open instead. +// Grounded in netlify-mcp-servers/references/file-uploads.md. +export default { + name: "MCP Servers: returning images to the agent -- inline vs a URL", + prompt: + "My MCP server has tools that hand images back to the agent. One returns a single preview thumbnail; another, `export_gallery`, could return dozens of full-resolution photos at once. Should I just base64-encode the bytes and return them as image content in every case?", + judge: [ + { + check: + "For the occasional single image (the thumbnail), confirms returning it inline as image content is fine -- e.g. `{ type: \"image\", data: , mimeType }`", + }, + { + check: + "Does NOT inline dozens of full-resolution photos as base64 image content -- warns that streaming large or many files back this way is the wrong approach", + }, + { + check: + "For the substantial case (the gallery / large or many files), returns a URL the user or agent can open instead of embedding the raw bytes", + }, + { + check: + "Frames the choice as a size/volume decision -- inline image content for the occasional read, a URL for anything substantial", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput; diff --git a/axis-scenarios/mcp-servers/oauth-public-connector.ts b/axis-scenarios/mcp-servers/oauth-public-connector.ts new file mode 100644 index 0000000..8be2579 --- /dev/null +++ b/axis-scenarios/mcp-servers/oauth-public-connector.ts @@ -0,0 +1,34 @@ +import type { ScenarioInput } from "@netlify/axis"; +import { withSkillVariants } from "../helpers/variants"; + +// OAuth is the affirmative choice when publishing a connector for arbitrary end +// users who shouldn't be handed a raw token. An OAuth-capable MCP server must +// provide OAuth 2.1 authorization + token endpoints, protected-resource +// metadata, often Dynamic Client Registration, and bearer access tokens it +// validates per request -- materially more work, best delegated to a hosted +// identity provider. Grounded in +// netlify-mcp-servers/references/connecting-clients.md. +export default { + name: "MCP Servers: public connector for end users -- the OAuth branch", + prompt: + "I'm publishing an MCP connector that lots of external end users will add to Claude Desktop. I do NOT want to hand each of them a raw bearer token to paste in -- they should just click Connect and approve access. What's the right authentication approach for this, and what does it involve?", + judge: [ + { + check: + "Recommends OAuth for this case -- a PUBLIC connector for arbitrary end users who shouldn't be handed a raw token; they click Connect, approve access, and the client obtains and refreshes tokens for them", + }, + { + check: + "Names the surfaces an OAuth-capable MCP server must provide: OAuth 2.1 authorization + token endpoints (or delegation to an external identity provider), protected-resource metadata so the client can discover where to authorize, and bearer access tokens the server validates on each MCP request (commonly Dynamic Client Registration too)", + }, + { + check: + "Sets expectations that OAuth is materially more work than a shared secret -- its own project -- and advises leaning on a hosted identity provider rather than hand-rolling the OAuth server", + }, + { + check: + "Does NOT recommend a single static shared secret or hand-issued per-user API keys for this arbitrary-end-user case (that restraint is for a personal server or a small trusted group, not a public connector)", + }, + ], + variants: withSkillVariants(), +} satisfies ScenarioInput;