test(axis): full coverage for under-tested skill rules across all domains (AX-107)

axis-scenarios/agent-runner/error-terminal-state.ts

@@ -0,0 +1,27 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Agent Runner: an `error` status is a terminal failure to inspect, not to wait on",
+  prompt:
+    "I've been polling my Netlify agent task and `netlify agents:show` now reports its status as `error`. What does that mean and what should I do?",
+  judge: [
+    {
+      check:
+        "Explains that `error` is a TERMINAL status meaning the task failed — a task moves `new` → `running` → one of `done`, `error`, or `cancelled`.",
+    },
+    {
+      check:
+        "Advises inspecting the failure on that task — e.g. `netlify agents:show <task-id>` (optionally `--json`) — to see what went wrong.",
+    },
+    {
+      check:
+        "Because `error` is terminal, does NOT tell the user to keep polling/waiting for the same task to finish or to expect it to move on to `done`.",
+    },
+    {
+      check:
+        "Does NOT fabricate task output or claim the task succeeded — an errored task did not produce a successful result.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/agent-runner/link-prerequisite.ts

@@ -0,0 +1,31 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Agent Runner: link or init the project before running agent tasks",
+  prompt:
+    "I want to run a Netlify agent task on this project, but I don't think this directory is linked to a Netlify site yet. How do I get started?",
+  judge: [
+    {
+      check:
+        "Explains the site must be linked to a Netlify project for `netlify agents:*` commands to work — via `netlify link` (to connect an existing site) or `netlify init` (to set up a new project).",
+    },
+    {
+      check:
+        "Notes the alternative: you can skip linking and target any Netlify site directly by passing `--project <name>` (a project ID or name) to `netlify agents:create`.",
+    },
+    {
+      check:
+        "Notes the Netlify CLI must be installed and authenticated.",
+    },
+    {
+      check:
+        "When it comes to creating the task, uses `netlify agents:create` and asks for the user's explicit permission first since it runs on Netlify infrastructure and incurs cost. Passes vacuously if the agent correctly stops at the linking/setup step.",
+    },
+    {
+      check:
+        "Does NOT invent an undocumented linking/targeting flag when `netlify link`, `netlify init`, and `--project <name>` are the documented paths.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/agent-runner/model-selection.ts

@@ -0,0 +1,31 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Agent Runner: pin the model with the --model flag",
+  prompt:
+    "Start a Netlify agent task on my linked site to refactor the checkout module. Use the `claude` agent, and pin it to the Opus model.",
+  judge: [
+    {
+      check:
+        "Selects the model with the documented `-m`/`--model <model>` flag on `netlify agents:create`, passing the requested model — rather than inventing a different flag or silently dropping the model request.",
+    },
+    {
+      check:
+        "Also specifies the agent with `-a`/`--agent claude` (agent type is one of claude, codex, or gemini), keeping the model flag distinct from the agent flag.",
+    },
+    {
+      check:
+        "Asks for the user's explicit permission BEFORE running `netlify agents:create`, since the task runs on Netlify infrastructure and incurs cost. Passes vacuously if the agent correctly stops at the permission step.",
+    },
+    {
+      check:
+        "Does NOT silently run `netlify agents:create` without approval, and does NOT invent an undocumented flag for the model.",
+    },
+    {
+      check:
+        "Treats the task as asynchronous — creation returns once the task is queued, and the outcome must be polled with `netlify agents:show <task-id>`. Passes vacuously if not mentioned.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/agent-runner/permission-request-content.ts

@@ -0,0 +1,31 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Agent Runner: the permission request spells out what, which agent, and why",
+  prompt:
+    "You're helping me build this Netlify site. I'd also like an independent review of the payment integration from a different AI model running remotely. Set that up.",
+  judge: [
+    {
+      check:
+        "Before running anything, asks for the user's explicit permission to run `netlify agents:create`, because the task runs on Netlify infrastructure and incurs cost for the user.",
+    },
+    {
+      check:
+        "The permission request has the documented content: it explains WHAT it wants to run (the command / task), WHICH agent it will use (claude, codex, or gemini), and WHY.",
+    },
+    {
+      check:
+        "Does NOT silently run `netlify agents:create` without approval — proposing the command and stopping to ask is correct; firing it off is not.",
+    },
+    {
+      check:
+        "Plans to hand the remote agent a complete, standalone prompt for the review, since the remote agent starts fresh from the repo with none of this conversation's context.",
+    },
+    {
+      check:
+        "Treats the task as remote and asynchronous — it runs against the pushed branch, creation returns once queued, there are no callbacks, and the outcome must be polled with `netlify agents:show <task-id>`. Passes vacuously if the agent correctly stops at the permission step.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/ai-gateway/limits-context-batch-headers.ts

@@ -0,0 +1,31 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+// The gateway has documented, non-negotiable limits: a 200k-token context
+// window, and no support for batch inference or custom request headers. A
+// skill-aware agent should answer these capability questions from the skill's
+// "Limits" line rather than assuming provider-native features pass through.
+export default {
+  name: "AI Gateway: documented limits (context window, batch, custom headers)",
+  prompt:
+    "I'm using the Netlify AI Gateway from a function. Before I build, confirm a few things will work through the gateway: (1) sending a single request with about 400k tokens of context, (2) attaching a custom `X-Trace-Id` request header to my provider calls, and (3) using the provider's batch inference API to process a queue of prompts cheaply. Will these work through the gateway?",
+  judge: [
+    {
+      check:
+        "States the AI Gateway has a 200k-token context window, so a single ~400k-token request exceeds that limit and won't work as-is.",
+    },
+    {
+      check:
+        "States that batch inference is NOT supported through the gateway.",
+    },
+    {
+      check:
+        "States that custom request headers are NOT supported through the gateway.",
+    },
+    {
+      check:
+        "Does NOT invent a config flag, header, or setting to 'enable' batch inference, custom headers, or a larger context window on the gateway — these are documented gateway limits, not toggles.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/ai-gateway/local-dev-positive-path.ts

@@ -0,0 +1,31 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+// The positive "how do I run gateway calls locally" path (the linked-project
+// scenario tests the failure/diagnosis side). With `netlify dev` (or the vite
+// plugin) in a LINKED directory whose site has already been deployed to
+// production once, the gateway env vars are injected into the local process.
+export default {
+  name: "AI Gateway: run and test gateway calls locally the right way",
+  prompt:
+    "My Netlify function calls the AI Gateway with the OpenAI SDK and it works on the deployed production site. Now I want to develop and test it locally on my laptop so the gateway calls actually succeed in local dev. Walk me through how to set that up.",
+  judge: [
+    {
+      check:
+        "Says to run the app with `netlify dev` (or the `@netlify/vite-plugin`), which injects the gateway env vars into the local process — a bare framework dev server started outside `netlify dev` / `@netlify/vite-plugin` gets NO gateway env vars.",
+    },
+    {
+      check:
+        "Says the working directory must be LINKED to the Netlify site — run `netlify link` (or `netlify init`) — so `netlify dev` can pull the gateway base URL and placeholder key from the linked site's environment.",
+    },
+    {
+      check:
+        "Notes local gateway access still requires the site to have had at least one PRODUCTION deploy already; a brand-new local-only project has no gateway access until it is deployed to production once.",
+    },
+    {
+      check:
+        "Does NOT tell the user to set their own provider `OPENAI_API_KEY` (or hardcode a real key) to make local dev work — a user-set key disables the gateway; the fix is `netlify dev` in a linked, already-deployed project.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/ai-gateway/model-alias-nuances.ts

@@ -0,0 +1,31 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+// Not every model family has an unversioned alias on the gateway. The gpt-5.3
+// family, specifically, is exposed only as `gpt-5.3-chat-latest` and
+// `gpt-5.3-codex` — there is NO bare `gpt-5.3`. A skill-aware agent should catch
+// this when the user asks for the unversioned id and use a real alias instead.
+export default {
+  name: "AI Gateway: handle a model family with no unversioned alias",
+  prompt:
+    "Create a Netlify function at netlify/functions/reason.ts mounted at /api/reason that answers user questions using OpenAI's `gpt-5.3` model through the AI Gateway.",
+  judge: [
+    {
+      check:
+        "Flags that there is NO unversioned `gpt-5.3` on the gateway — the gpt-5.3 family is exposed only as `gpt-5.3-chat-latest` and `gpt-5.3-codex`, so a bare `gpt-5.3` id isn't valid.",
+    },
+    {
+      check:
+        "Pins a real gpt-5.3 alias from the curated list — `gpt-5.3-chat-latest` (the chat model) or `gpt-5.3-codex` — instead of the nonexistent bare `gpt-5.3`.",
+    },
+    {
+      check:
+        "Constructs the OpenAI client bare (`new OpenAI()`, no `apiKey`/`baseURL`) and does NOT read or set `OPENAI_API_KEY`.",
+    },
+    {
+      check:
+        "Uses `Netlify.env.get(...)` (not `process.env`) for any env access and the modern Netlify function signature with a config exporting path: '/api/reason'.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/ai-gateway/universal-gateway-vars.ts

@@ -0,0 +1,35 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+// Besides the per-provider vars, Netlify injects a provider-agnostic pair —
+// `NETLIFY_AI_GATEWAY_BASE_URL` and `NETLIFY_AI_GATEWAY_KEY` — for callers that
+// need an explicit endpoint + key (e.g. a raw `fetch`, not a provider SDK).
+// Both are auto-injected at runtime when AI is enabled.
+export default {
+  name: "AI Gateway: provider-agnostic NETLIFY_AI_GATEWAY_* env vars",
+  prompt:
+    "Create a Netlify function at netlify/functions/gateway-raw.ts mounted at /api/gateway-raw that calls the AI Gateway with a plain `fetch` request instead of one of the provider SDKs. I'd rather point it at Netlify's provider-agnostic gateway env vars than the per-provider OPENAI_/ANTHROPIC_ ones — which vars do I use, and how do I wire them in?",
+  judge: [
+    {
+      check:
+        "Identifies `NETLIFY_AI_GATEWAY_BASE_URL` as the provider-agnostic gateway endpoint / base URL to send the `fetch` request to.",
+    },
+    {
+      check:
+        "Identifies `NETLIFY_AI_GATEWAY_KEY` as the provider-agnostic gateway key used to authenticate the request (e.g. as the API key / Authorization value).",
+    },
+    {
+      check:
+        "Explains both `NETLIFY_AI_GATEWAY_*` vars are auto-injected by Netlify at runtime when AI is enabled — the user does not set them by hand.",
+    },
+    {
+      check:
+        "Reads the vars via `Netlify.env.get(...)` (not `process.env`) and does NOT hardcode the gateway URL or set a real provider API key.",
+    },
+    {
+      check:
+        "Uses the modern Netlify function signature (default export async handler, Web API Request/Response) with a config exporting path: '/api/gateway-raw'.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/ai-gateway/untracked-usage-diagnosis.ts

@@ -0,0 +1,32 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+// Distinct from the "API key missing" error: here the calls SUCCEED but the
+// usage never shows up because a user-set provider key has shadowed Netlify's
+// auto-injection, routing calls straight to the provider and bypassing the
+// gateway. The skill maps this exact symptom ("calls succeed but aren't
+// tracked") to a user-set `*_API_KEY`.
+export default {
+  name: "AI Gateway: calls succeed but usage isn't tracked",
+  prompt:
+    "My Netlify function calls OpenAI through the AI Gateway with `new OpenAI()` and the responses come back fine — but the usage never shows up in my Netlify AI usage/credits, like the calls aren't going through the gateway at all. What would cause that, and how do I fix it?",
+  judge: [
+    {
+      check:
+        "Diagnoses that a user-set provider key (`OPENAI_API_KEY`) shadows Netlify's auto-injection and routes the calls directly to the provider, bypassing the gateway — which is why the usage isn't tracked, even though the calls succeed.",
+    },
+    {
+      check:
+        "Tells the user to check for and remove/unset their own `OPENAI_API_KEY` (and not set any `*_API_KEY`) so the gateway's auto-injected placeholder key and base URL take over again.",
+    },
+    {
+      check:
+        "Confirms the correct wiring is bare `new OpenAI()` with no `apiKey`/`baseURL` argument — the gateway auto-injects the credentials — so the fix is removing the shadowing key, not adding constructor config.",
+    },
+    {
+      check:
+        "Does NOT recommend keeping or rotating the user's own provider key, nor manually pointing a custom `baseURL` at the gateway — a user-set key is the cause, not part of the fix.",
+    },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/blobs/size-and-key-limits.ts

@@ -0,0 +1,16 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Blobs: object, store-name, and key size limits",
+  prompt:
+    "I'm about to design a Netlify Blobs store for large user-uploaded video files. Before I commit to the design, tell me the hard size limits I need to build around: how big can a single object be, how long can the store name be, and how long can a blob key be? I'm planning to use each upload's original full file path as its key, so I especially need to know the key limit.",
+  judge: [
+    { check: "States the maximum object (blob value) size is 5 GB" },
+    { check: "States the store name maximum length is 64 bytes" },
+    { check: "States the blob key maximum length is 600 bytes" },
+    { check: "Applies the 600-byte key limit to the user's plan — flags that a full-file-path key must stay within 600 bytes, so very long paths could exceed the limit" },
+    { check: "Does NOT invent different limit values, and does NOT claim these limits can be raised or worked around" },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/caching/deploy-invalidation-context-scope.ts

@@ -0,0 +1,15 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Caching: deploy-based invalidation is scoped to the deploy context",
+  prompt:
+    "We publish a Deploy Preview for every pull request. A teammate is worried that publishing one of those preview deploys will wipe out our production CDN cache and send a spike of traffic to our origin. Does deploying a preview invalidate the production cache? Explain how deploy-based cache invalidation is scoped on Netlify.",
+  judge: [
+    { check: "Explains that deploy-based cache invalidation is scoped to the deploy context (production vs preview) — it is not global across all contexts" },
+    { check: "Answers the concern directly: publishing a Deploy Preview does NOT invalidate / wipe out the production CDN cache, so it won't cause a production origin traffic spike" },
+    { check: "Correctly notes that a deploy invalidates the cache within its own context (e.g. a production deploy clears the production cache), consistent with static assets being invalidated on every deploy" },
+    { check: "Does NOT claim that any deploy (or a preview deploy specifically) clears the cache across every deploy context at once" },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/caching/purge-entire-site.ts

@@ -0,0 +1,15 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Caching: purge the entire site cache from a deployed function",
+  prompt:
+    "Build an admin 'Clear all cache' endpoint: a Netlify function at netlify/functions/flush-cache.ts mounted at /api/admin/flush-cache that flushes our ENTIRE site's CDN cache in one shot (not just a single tag). It runs as a normal deployed Netlify function. Keep it as simple as possible.",
+  judge: [
+    { check: "Calls `purgeCache()` imported from `@netlify/functions` with NO arguments (no `tags`) — an argument-less `purgeCache()` purges the entire site cache, which matches the 'clear all cache' requirement" },
+    { check: "Does NOT scope the purge to specific cache tags — the requirement is to flush the whole site, not a subset" },
+    { check: "Relies on `purgeCache()` picking up the site ID and credentials automatically because it runs inside a deployed Netlify Function — it does NOT require passing a `token`/`siteID` or hardcoding a Netlify token for this in-function call" },
+    { check: "Uses the modern Netlify function signature with `config.path: '/api/admin/flush-cache'`" },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;

axis-scenarios/caching/static-assets-auto-cache.ts

@@ -0,0 +1,16 @@
+import type { ScenarioInput } from "@netlify/axis";
+import { withSkillVariants } from "../helpers/variants";
+
+export default {
+  name: "Caching: static assets are cached automatically, no config needed",
+  prompt:
+    "I'm deploying a Vite single-page app to Netlify. Do I need to add cache headers or a netlify.toml rule to get my built static assets (the JS/CSS/images in the build output) cached on the CDN, or does Netlify handle that? Explain what happens by default.",
+  judge: [
+    { check: "Explains that Netlify caches static assets automatically with no configuration needed — the agent does NOT tell the user they must hand-write cache headers or a netlify.toml rule just to get ordinary static build output cached" },
+    { check: "States the CDN default: static assets are cached at the CDN for a long time (about 1 year) and automatically invalidated on every deploy, so a new deploy serves the fresh files" },
+    { check: "Notes the browser default: browsers always revalidate (e.g. `max-age=0, must-revalidate`) so users pick up updated files after a deploy" },
+    { check: "Clarifies that it's dynamic responses (Netlify Functions / edge functions / proxied responses) that are NOT cached by default and are where explicit cache headers are needed — not static assets" },
+    { check: "Does NOT claim that a manual netlify.toml `[[headers]]` block for the whole build output is REQUIRED for basic static-asset caching to work" },
+  ],
+  variants: withSkillVariants(),
+} satisfies ScenarioInput;