-
-
-
-
-
-
-
-
-
aiwright QA Agent
-
Give it a story; it plans and runs the pipeline itself — pausing for your OK
- on side-effecting steps and stopping on a real regression. ·
-
Step-by-step studio →
-
-
-
-
User story
-
-
- Start agent
-
-
+
+
+
+
NeuralQA · Agent
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ✦ Autonomous mode
+ Give it a story. The agent runs the pipeline itself.
+ It plans and sequences every step — design, inspect, generate, verify, run, heal — pausing for your OK on side-effecting steps and stopping on a real regression instead of faking green.
+
+
+
-
-
-
-
-
+
+ function onEvent(e) {
+ switch (e.type) {
+ case 'start':
+ line(`
goal: ${esc(e.goal)}`, 'muted'); break;
+ case 'plan':
+ line(`🧠 ${esc(e.text)}`, 'plan'); break;
+ case 'tool':
+ line(`▶
${e.tool} ${e.decision} ${esc(e.reason)} `
+ + (e.input && Object.keys(e.input).length ? `
${esc(JSON.stringify(e.input))} ` : '')); break;
+ case 'awaiting-approval':
+ showGate('approve', `"${e.tool}" — ${e.reason}`, [
+ { label: 'Approve', value: 'yes' },
+ { label: 'Decline', value: 'no', cls: 'ghost' },
+ { label: 'Abort run', value: 'abort', cls: 'bad' }
+ ]); break;
+ case 'declined':
+ line(` ✗ declined "${e.tool}"`, 'muted'); break;
+ case 'result':
+ line(` ${e.ok ? '✓' : '✗'} ${esc(e.summary)}`, e.ok ? 'ok' : 'bad'); break;
+ case 'error':
+ line(` ✗ error (${e.tool}): ${esc(e.message)}`, 'bad'); break;
+ case 'escalation':
+ line(`⚠ ESCALATION: ${esc(e.reason)}`, 'bad');
+ showGate('escalate', e.reason, [
+ { label: 'Halt for triage', value: 'halt', cls: 'bad' },
+ { label: 'Continue anyway', value: 'continue', cls: 'warn' }
+ ]); break;
+ case 'done':
+ hideGate();
+ line(`■ done — ${e.outcome}${e.statePath ? ' · ' + esc(e.statePath) : ''}`, e.outcome === 'completed' ? 'ok' : 'bad');
+ statusEl.textContent = `Finished: ${e.outcome}`;
+ startBtn.disabled = false;
+ break;
+ }
+ }
+
+ startBtn.onclick = async () => {
+ const story = storyEl.value.trim();
+ if (!story) { statusEl.textContent = 'Enter a story first.'; return; }
+ startBtn.disabled = true;
+ statusEl.innerHTML = '
Starting…';
+ logEl.innerHTML = '';
+ runPanel.classList.remove('hidden');
+ try {
+ const r = await fetch('/api/agent/start', {
+ method: 'POST', headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ story })
+ });
+ const data = await r.json();
+ if (!r.ok) throw new Error(data.error || 'failed to start');
+ runId = data.runId;
+ statusEl.innerHTML = '
Running…';
+ const src = new EventSource(`/api/agent/${runId}/events`);
+ src.onmessage = (m) => onEvent(JSON.parse(m.data));
+ src.onerror = () => src.close();
+ } catch (err) {
+ statusEl.textContent = 'Error: ' + err.message;
+ startBtn.disabled = false;
+ }
+ };
+
+
diff --git a/public/favicon.svg b/public/favicon.svg
index 8936f4f..b1e28c8 100644
--- a/public/favicon.svg
+++ b/public/favicon.svg
@@ -1,11 +1,15 @@
-
+
-
-
+
-
+
+
diff --git a/public/scenarios.html b/public/scenarios.html
new file mode 100644
index 0000000..5a61ef9
--- /dev/null
+++ b/public/scenarios.html
@@ -0,0 +1,434 @@
+
+
+
+ NeuralQA · Scenario Studio
+
+
+
+
+
Scenario Studio
+
+
Agent
+
+
+
+
+ ✦ AI-grounded test design
+ From your app to test scenarios
+ Point NeuralQA at your site or API, share your stories, then pick the scenarios and generate a runnable suite — grounded in the real system, not guesses.
+
+
1 Connect
+
→
+
2 Select
+
→
+
3 Generate
+
+
+
+
+
+
+ Site URL
· UI, optional
+
+
+
+
+
+ OpenAPI / Swagger spec
· API, optional
+
+
+
+
+
+ User stories / requirements
· optional
+
+
+
Provide a site URL, an API spec, or stories — at least one.
+
+
+
+
+
+
+
0 selected Pick the scenarios you want, then generate the suite.
+
+
+
+
{
@@ -95,8 +113,10 @@ export async function probeApi(specPath: string, opts: ProbeOptions = {}): Promi
}
const warnings: string[] = [];
- const baseUrl =
- opts.baseUrl ?? spec.servers?.[0]?.url ?? config.apiBaseUrl;
+ const baseUrl = opts.baseUrl ?? baseUrlFromSpec(spec, config.apiBaseUrl);
+ if (spec.swagger && !spec.openapi) {
+ warnings.push(`Swagger 2.0 spec detected (v${spec.swagger}); parsed for endpoints/schemas.`);
+ }
const endpoints: EndpointEntry[] = [];
for (const [p, pathItem] of Object.entries(spec.paths ?? {})) {
@@ -109,11 +129,12 @@ export async function probeApi(specPath: string, opts: ProbeOptions = {}): Promi
operationId: op.operationId,
summary: op.summary,
params: (op.parameters ?? []).map((pr: any) => ({
+ // OpenAPI 3.x: type/example under pr.schema; Swagger 2.0: type/default on the param itself.
name: pr.name,
in: pr.in,
required: !!pr.required,
- type: pr.schema?.type,
- example: pr.schema?.example ?? pr.example
+ type: pr.schema?.type ?? pr.type,
+ example: pr.schema?.example ?? pr.example ?? pr.default
})),
responses: Object.keys(op.responses ?? {}).map((code) => ({
status: code,
diff --git a/src/ai/testDesigner.ts b/src/ai/testDesigner.ts
index d616900..5cd8929 100644
--- a/src/ai/testDesigner.ts
+++ b/src/ai/testDesigner.ts
@@ -1,7 +1,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { getClient, MODEL } from './client';
-import { DESIGNER_SYSTEM } from './prompts';
+import { DESIGNER_SYSTEM, SITE_CONTEXT_INSTRUCTION } from './prompts';
import { redact } from './redact';
import { registerAllSensitive } from '../fixtures/data';
@@ -97,14 +97,31 @@ const DESIGN_SCHEMA = {
additionalProperties: false
} as const;
-export async function designTests(userStory: string, maxScenarios?: number): Promise {
+export async function designTests(
+ userStory: string,
+ maxScenarios?: number,
+ siteContext?: string
+): Promise {
const client = getClient();
// Before going to the LLM: load known secret values + redact the story.
registerAllSensitive();
- const safeStory = redact(userStory);
+ const safeStory = redact(userStory).trim();
+ const hasContext = !!siteContext?.trim();
+ if (!safeStory && !hasContext) {
+ throw new Error('Provide a user story or a system context (site/API) to design tests from.');
+ }
- let content = `Produce a test design for this user story:\n\n${safeStory}`;
+ // With no explicit story, the inspected/probed system IS the requirement — derive the scenarios
+ // from its real capabilities instead of a narrative.
+ let content = safeStory
+ ? `Produce a test design for this user story:\n\n${safeStory}`
+ : `Produce a test design for the system under test. No explicit user story was given — derive the ` +
+ `scenarios from the LIVE CONTEXT below: cover each capability's happy path, key negatives, and ` +
+ `edge/boundary cases, and for APIs also the status codes, contract/response shape, and input validation.`;
+ if (hasContext) {
+ content += `\n\n${SITE_CONTEXT_INSTRUCTION}\n\nLIVE CONTEXT (real UI elements and/or API endpoints):\n\n${redact(siteContext!)}`;
+ }
if (maxScenarios && maxScenarios > 0) {
// Quick-trial mode: keep the design (and therefore generation) small and fast.
content +=
diff --git a/src/web/server.ts b/src/web/server.ts
index 2dabf40..858a884 100644
--- a/src/web/server.ts
+++ b/src/web/server.ts
@@ -1,31 +1,39 @@
import * as path from 'path';
import * as fs from 'fs';
+import * as os from 'os';
import * as crypto from 'crypto';
import express, { Request, Response } from 'express';
import { designTests, renderDesignMarkdown, TestDesign } from '../ai/testDesigner';
import { generateTests, writeArtifacts, correctArtifacts, GeneratedArtifacts } from '../ai/testGenerator';
+import { inspectPage, SelectorMap } from '../ai/pageInspector';
+import { probeApi, EndpointMap } from '../ai/specProbe';
import { verifyTypeScript } from '../ai/verifier';
import { runAgent } from '../agent/orchestrator';
import { AgentIO, AgentEvent } from '../agent/io';
const app = express();
app.use(express.json({ limit: '1mb' }));
-app.use(express.static(path.join(process.cwd(), 'public')));
+// Scenario Studio is the home page ('/'); the older code studio (index.html) is retired.
+app.use(express.static(path.join(process.cwd(), 'public'), { index: 'scenarios.html' }));
const tsScope = (written: string[]) => written.map((w) => w.split(' ')[0]).filter((f) => f.endsWith('.ts'));
-/** Reads the project source files the self-correction step may need to update. */
-function projectSources(): { fileName: string; content: string }[] {
+/** Reads the project source files the self-correction step may need to update (mode-aware). */
+function projectSources(mode: 'ui' | 'api' = 'ui'): { fileName: string; content: string }[] {
+ const dirs = mode === 'api' ? ['src/api', 'src/steps/api'] : ['src/pages', 'src/fixtures'];
const out: { fileName: string; content: string }[] = [];
- for (const dir of ['src/pages', 'src/fixtures']) {
- const abs = path.join(process.cwd(), dir);
- if (!fs.existsSync(abs)) continue;
- for (const f of fs.readdirSync(abs)) {
- if (f.endsWith('.ts') && !f.endsWith('.generated')) {
- out.push({ fileName: f, content: fs.readFileSync(path.join(abs, f), 'utf-8') });
+ const walk = (rel: string) => {
+ const abs = path.join(process.cwd(), rel);
+ if (!fs.existsSync(abs)) return;
+ for (const entry of fs.readdirSync(abs, { withFileTypes: true })) {
+ const childRel = path.join(rel, entry.name);
+ if (entry.isDirectory()) walk(childRel);
+ else if (entry.name.endsWith('.ts') && !entry.name.endsWith('.generated') && !entry.name.endsWith('.bak')) {
+ out.push({ fileName: childRel, content: fs.readFileSync(path.join(process.cwd(), childRel), 'utf-8') });
}
}
- }
+ };
+ for (const d of dirs) walk(d);
return out;
}
@@ -57,6 +65,99 @@ const handler =
}
};
+/** Compact, model-friendly summary of an inspected page — UI grounding for the design. */
+function siteContextFrom(map: SelectorMap): string {
+ const named = map.entries
+ .filter((e) => e.name && !e.unresolved)
+ .slice(0, 40)
+ .map((e) => `- ${e.role ?? e.tag}${e.name ? ` "${e.name}"` : ''}`);
+ return [
+ `UI page title: ${map.title}`,
+ `URL: ${map.url}`,
+ named.length ? `Notable elements on the page:\n${named.join('\n')}` : 'No notable named elements found.'
+ ].join('\n');
+}
+
+/** Compact summary of a probed OpenAPI spec — API grounding for the design. */
+function apiContextFrom(map: EndpointMap): string {
+ const lines = map.endpoints.map(
+ (e) =>
+ `- ${e.method} ${e.path}${e.summary ? ` — ${e.summary}` : ''}` +
+ (e.responses.length ? ` [statuses: ${e.responses.map((r) => r.status).join(', ')}]` : '')
+ );
+ return [
+ `API: ${map.title}${map.version ? ` v${map.version}` : ''}`,
+ `Base URL: ${map.baseUrl}`,
+ lines.length ? `Endpoints:\n${lines.join('\n')}` : 'No endpoints found in the spec.'
+ ].join('\n');
+}
+
+/** Probes an OpenAPI spec given as a URL (fetched) or pasted JSON. Spec-only (no live calls). */
+async function probeSpecInput(input: string): Promise {
+ let json: string;
+ if (/^https?:\/\//i.test(input)) {
+ const r = await fetch(input);
+ if (!r.ok) throw new Error(`Could not fetch the spec (HTTP ${r.status}) from ${input}`);
+ json = await r.text();
+ } else {
+ json = input;
+ }
+ try {
+ JSON.parse(json);
+ } catch {
+ throw new Error('The OpenAPI spec must be valid JSON (paste JSON, or a URL that returns JSON).');
+ }
+ const tmp = path.join(os.tmpdir(), `aiwright-spec-${crypto.randomUUID()}.json`);
+ fs.writeFileSync(tmp, json);
+ try {
+ return await probeApi(tmp, { live: false });
+ } finally {
+ fs.rmSync(tmp, { force: true });
+ }
+}
+
+// PRODUCT flow: { stories, siteUrl?, apiSpec? } -> scenarios. Grounds the test design in the real
+// system — a live UI inspect (siteUrl), an OpenAPI probe (apiSpec = URL or pasted JSON), or both —
+// and returns the scenarios (no code written). "Enter your site and/or API, share your stories,
+// get the scenarios."
+app.post(
+ '/api/scenarios',
+ handler(async (req, res) => {
+ const stories = String(req.body?.stories ?? '').trim();
+ const siteUrl = String(req.body?.siteUrl ?? '').trim();
+ const apiSpec = String(req.body?.apiSpec ?? '').trim();
+ // Stories are optional: a site or an API spec is enough to design from (the system is the
+ // requirement). At least one of the three must be present.
+ if (!stories && !siteUrl && !apiSpec) {
+ res.status(400).json({ error: 'Provide a site URL, an API spec, or your stories.' });
+ return;
+ }
+
+ const contexts: string[] = [];
+ let site: unknown;
+ let api: unknown;
+
+ if (siteUrl) {
+ if (!/^https?:\/\//i.test(siteUrl)) {
+ res.status(400).json({ error: 'Site URL must start with http:// or https://' });
+ return;
+ }
+ const map = await inspectPage(siteUrl);
+ contexts.push(siteContextFrom(map));
+ site = { title: map.title, url: map.url, elements: map.entries.length, warnings: map.warnings };
+ }
+
+ if (apiSpec) {
+ const map = await probeSpecInput(apiSpec);
+ contexts.push(apiContextFrom(map));
+ api = { title: map.title, version: map.version, baseUrl: map.baseUrl, endpoints: map.endpoints.length };
+ }
+
+ const design = await designTests(stories, undefined, contexts.join('\n\n---\n\n'));
+ res.json({ design, site, api });
+ })
+);
+
// Story -> structured test design (the "what to test" layer).
app.post(
'/api/design',
@@ -94,6 +195,60 @@ app.post(
})
);
+// PRODUCT flow step 2: selected scenarios -> a runnable suite written into the project.
+// Re-grounds in the same system (API spec -> endpoint map + API mode, else site -> selector map +
+// UI mode), generates the feature + steps + clients/page objects for ONLY the chosen scenarios,
+// writes them, type-checks, and self-corrects up to 2 rounds so the result compiles.
+app.post(
+ '/api/generate-suite',
+ handler(async (req, res) => {
+ const stories = String(req.body?.stories ?? '').trim();
+ const siteUrl = String(req.body?.siteUrl ?? '').trim();
+ const apiSpec = String(req.body?.apiSpec ?? '').trim();
+ const design = req.body?.design as TestDesign | undefined;
+ const selected = (req.body?.selected as number[] | undefined) ?? [];
+ if (!design) {
+ res.status(400).json({ error: 'Provide the design.' });
+ return;
+ }
+ const scenarios = design.scenarios.filter((_, i) => selected.includes(i));
+ if (scenarios.length === 0) {
+ res.status(400).json({ error: 'Select at least one scenario.' });
+ return;
+ }
+ const approved: TestDesign = { ...design, scenarios };
+
+ // Mode + grounding follow the source: an API spec -> API lane; otherwise the site -> UI lane.
+ const mode: 'ui' | 'api' = apiSpec ? 'api' : 'ui';
+ let mapJson: string | undefined;
+ if (apiSpec) {
+ mapJson = JSON.stringify(await probeSpecInput(apiSpec), null, 2);
+ } else if (siteUrl) {
+ mapJson = JSON.stringify(await inspectPage(siteUrl), null, 2);
+ }
+
+ const storyText = stories || approved.understanding || approved.title;
+ let artifacts = await generateTests(storyText, renderDesignMarkdown(approved), mapJson, undefined, mode);
+ let written = writeArtifacts(artifacts, process.cwd(), { overwrite: true }, mode);
+ let verify = verifyTypeScript(tsScope(written));
+ let rounds = 0;
+ while (!verify.ok && rounds < 2) {
+ rounds++;
+ artifacts = await correctArtifacts(storyText, artifacts, verify.errors, projectSources(mode), mapJson, mode);
+ written = writeArtifacts(artifacts, process.cwd(), { overwrite: true }, mode);
+ verify = verifyTypeScript(tsScope(written));
+ }
+
+ res.json({
+ mode,
+ written,
+ rounds,
+ notes: artifacts.notes,
+ verify: { ok: verify.ok, errors: verify.errors }
+ });
+ })
+);
+
// Write previewed artifacts into the project (never overwrites existing files), then
// type-check them so the UI shows whether the generated code compiles.
app.post(
@@ -257,7 +412,7 @@ const PORT = Number(process.env.PORT ?? 5173);
// Bind to loopback only so the endpoints (which hold the API key and write files)
// are never exposed on the network.
app.listen(PORT, '127.0.0.1', () => {
- console.log(`aiwright web UI running at http://localhost:${PORT}`);
- console.log(` QA agent (live run + approval): http://localhost:${PORT}/agent.html`);
+ console.log(`NeuralQA Scenario Studio running at http://localhost:${PORT}`);
+ console.log(` Agent (live run + approval): http://localhost:${PORT}/agent.html`);
if (TOKEN) console.log('Token auth enabled (AIWRIGHT_TOKEN set).');
});