diff --git a/README.md b/README.md index fdbc20d..eaed49d 100644 --- a/README.md +++ b/README.md @@ -205,9 +205,9 @@ sibling. New page objects come with a fixture-registration snippet in the output ### `run` + `analyze` `run` executes the scenarios in a real browser and retries on failure to tell a **flaky** -scenario (passes on re-run) from a **consistent** one. `analyze` reads the Cucumber report -and classifies each failure as `app-bug | test-bug | flaky | environment` with a root cause -and a concrete fix. +scenario (passes on re-run) from a **consistent** one. `analyze` reads the run's results and +classifies each failure as `app-bug | test-bug | flaky | environment` with a root cause and a +concrete fix. ```bash npm test # all scenarios (parallel) @@ -219,13 +219,21 @@ npm run ai:analyze # → reports/ai-analysis.md ## Running tests ```bash -npm test # all scenarios (parallel) +npm test # all scenarios (UI + API, parallel) → one Allure result set +npm run test:api # only the @api lane (browserless) npm run test:smoke # only @smoke tagged npm run test:ui # Playwright UI mode HEADLESS=false npm test # watch the browser -npm run report # open the Cucumber HTML report +npm run report # generate + open the Allure report ``` +> **Reporting:** [Allure](https://allurereport.org/) is the human-facing report for both lanes +> (UI + API) — run history, per-step detail, and traces/screenshots attached on failure. The test +> scripts clear `reports/allure-results` first, so `npm test` (which runs both projects) gives one +> combined report; `npm run report` renders it (needs Java for the Allure CLI). A Cucumber JSON +> (`reports/*-report.json`) is still emitted, but only as the machine feed the AI pipeline reads +> (`analyze` / `heal-selectors` / `heal-contract`) — not a report you open. + > **CI note:** `npm test` targets the live app (`https://getmobil.com`). Public sites can sit > behind bot challenges that block data-centre IPs (e.g. CI runners), so the browser tests > run locally where a normal browser/IP passes. CI gates on the offline checks (type-check, @@ -240,7 +248,7 @@ lanes share the `features/` tree and are split by tag: `@api` scenarios run in t Playwright project (no browser), everything else in `chromium`. ```bash -npm run test:api # @api scenarios only (browserless) → reports/api-report.{json,html} +npm run test:api # @api scenarios only (browserless) → Allure results + cucumber JSON feed npm run mock:api # run the local mock API standalone (otherwise auto-booted) ``` @@ -368,8 +376,9 @@ giving a number to "how well does it work" instead of a vibe. - [x] **agent** — autonomous orchestrator over the whole pipeline, with guardrails - [x] **self-healing** — compile (`heal`) + runtime selector (`heal-selectors`) + runtime contract (`heal-contract`) - [x] **API lane** — browserless `APIRequestContext` suite with `probe` (OpenAPI → endpoint map) and `heal-contract` +- [x] **`generate` API mode** — endpoint map → @api feature + steps + client/contract files (build API suites end to end) - [x] CI/CD integration (GitHub Actions: type-check + redaction gate) -- [ ] `generate` API mode — endpoint map → API feature + client objects (build API suites end to end, not just heal them) +- [x] **Allure reporting** — combined UI + API report (history, steps, traces/screenshots); cucumber JSON kept as the AI feed - [ ] human-approval web UI over the agent loop (surface run state + confirm/escalate gates) - [ ] Jira integration (pull stories, write results back) - [ ] MCP server (secure tool access layer) diff --git a/docs/api/openapi.json b/docs/api/openapi.json index 1718002..d1e1f56 100644 --- a/docs/api/openapi.json +++ b/docs/api/openapi.json @@ -50,6 +50,15 @@ "404": { "description": "No product with that id", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } + }, + "/api/categories": { + "get": { + "operationId": "listCategories", + "summary": "List product categories with counts", + "responses": { + "200": { "description": "Categories listed", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CategoriesResponse" } } } } + } + } } }, "components": { @@ -74,6 +83,23 @@ "items": { "type": "array", "items": { "$ref": "#/components/schemas/Product" } } } }, + "Category": { + "type": "object", + "required": ["slug", "name", "count"], + "properties": { + "slug": { "type": "string", "example": "telefon" }, + "name": { "type": "string", "example": "Telefon" }, + "count": { "type": "integer", "minimum": 0, "example": 2 } + } + }, + "CategoriesResponse": { + "type": "object", + "required": ["total", "categories"], + "properties": { + "total": { "type": "integer", "minimum": 0 }, + "categories": { "type": "array", "items": { "$ref": "#/components/schemas/Category" } } + } + }, "Error": { "type": "object", "required": ["error"], diff --git a/features/api/categories.feature b/features/api/categories.feature new file mode 100644 index 0000000..8bced30 --- /dev/null +++ b/features/api/categories.feature @@ -0,0 +1,22 @@ +@api +Feature: List product categories + As an API consumer I can list the product categories so that + I can browse products by category. + + Background: + Given the API is up + + Scenario: Happy path – categories list is returned with valid shape + When the categories list is fetched + Then the response status is 200 + And the response body contains a categories array + And every category has slug, name and count fields + And every category count is a positive integer + And the total equals the number of categories returned + + Scenario: Categories endpoint returns no unexpected error fields + When the categories list is fetched + Then the response status is 200 + And the response body contains a categories array + And every category has slug, name and count fields + And the categories array is not empty diff --git a/mock/server.ts b/mock/server.ts index 39e8b80..ada1266 100644 --- a/mock/server.ts +++ b/mock/server.ts @@ -55,6 +55,17 @@ app.get('/api/products/:id', (req, res) => { res.json(product); }); +app.get('/api/categories', (_req, res) => { + const counts = new Map(); + for (const p of CATALOG) counts.set(p.category, (counts.get(p.category) ?? 0) + 1); + const categories = [...counts.entries()].map(([slug, count]) => ({ + slug, + name: slug.charAt(0).toLocaleUpperCase('tr') + slug.slice(1), + count + })); + res.json({ total: categories.length, categories }); +}); + app.listen(PORT, () => { console.log(`Mock search API listening on http://localhost:${PORT}`); }); diff --git a/package-lock.json b/package-lock.json index f3ae29d..ff83543 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,8 @@ "@playwright/test": "^1.60.0", "@types/express": "^5.0.6", "@types/node": "^25.9.3", + "allure-commandline": "^2.43.0", + "allure-playwright": "^3.10.2", "playwright-bdd": "^9.0.0", "ts-node": "^10.9.2", "typescript": "^6.0.3" @@ -409,6 +411,47 @@ "node": ">=0.4.0" } }, + "node_modules/allure-commandline": { + "version": "2.43.0", + "resolved": "https://registry.npmjs.org/allure-commandline/-/allure-commandline-2.43.0.tgz", + "integrity": "sha512-nsqccDOgi80fi9WjvBapZjlFlugodVWudotNtWx6XKiSI7NuN6oTWEkXnkxQBzNg02Yf9lZLtCFlKvjAlcpu6Q==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "allure": "bin/allure" + } + }, + "node_modules/allure-js-commons": { + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/allure-js-commons/-/allure-js-commons-3.10.2.tgz", + "integrity": "sha512-5nAjUF1iNPSQMwKtEsadclSta8C3L705/k/pIzGb9M6P9krgfRaCyaEHt8pkLlCP9NFj5xk3grjtnAhiLeT+Kg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "md5": "^2.3.0" + }, + "peerDependencies": { + "allure-playwright": "3.10.2" + }, + "peerDependenciesMeta": { + "allure-playwright": { + "optional": true + } + } + }, + "node_modules/allure-playwright": { + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/allure-playwright/-/allure-playwright-3.10.2.tgz", + "integrity": "sha512-CGyhxSvx2bIkojhBy0pbQZ12vkLbOE3FBh39WgzPs+katwnuGPUE0whJhryG8aD/dlDALxMehL9G0k8WYBx4tQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "allure-js-commons": "3.10.2" + }, + "peerDependencies": { + "@playwright/test": ">=1.53.0" + } + }, "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -498,6 +541,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/class-transformer": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", @@ -568,6 +621,16 @@ "dev": true, "license": "MIT" }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -951,6 +1014,13 @@ "node": ">= 0.10" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true, + "license": "MIT" + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -1013,6 +1083,18 @@ "node": ">= 0.4" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", diff --git a/package.json b/package.json index 2f4cc1a..910c946 100644 --- a/package.json +++ b/package.json @@ -4,13 +4,13 @@ "description": "AI QA Agent - TypeScript + Playwright + Cucumber BDD framework with Claude-powered test generation and failure analysis", "license": "MIT", "scripts": { - "test": "bddgen && playwright test", - "test:api": "bddgen && REPORT_NAME=api-report playwright test --project=api", + "test": "rm -rf reports/allure-results && bddgen && playwright test", + "test:api": "rm -rf reports/allure-results && bddgen && REPORT_NAME=api-report playwright test --project=api", "mock:api": "ts-node mock/server.ts", "test:smoke": "bddgen --tags @smoke && playwright test", "test:tag": "sh -c 'bddgen --tags \"@negative\" && playwright test'", "test:ui": "bddgen && playwright test --ui", - "report": "open reports/cucumber-report.html", + "report": "allure generate reports/allure-results --clean -o reports/allure-report && allure open reports/allure-report", "web": "ts-node src/web/server.ts", "ai:agent": "ts-node src/cli/index.ts agent", "ai:design": "ts-node src/cli/index.ts design", @@ -29,6 +29,8 @@ "@playwright/test": "^1.60.0", "@types/express": "^5.0.6", "@types/node": "^25.9.3", + "allure-commandline": "^2.43.0", + "allure-playwright": "^3.10.2", "playwright-bdd": "^9.0.0", "ts-node": "^10.9.2", "typescript": "^6.0.3" diff --git a/playwright.config.ts b/playwright.config.ts index 338a0ee..6701747 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -48,8 +48,14 @@ export default defineConfig({ // so `npm test` / ai:analyze keep reading reports/cucumber-report.json as before. reporter: [ ['list'], - cucumberReporter('json', { outputFile: `reports/${REPORT_NAME}.json` }), - cucumberReporter('html', { outputFile: `reports/${REPORT_NAME}.html` }) + // Allure is the human-facing report for BOTH lanes (UI + API): history, per-step detail, + // traces/screenshots attached on failure. Results accumulate in reports/allure-results; + // render with `npm run report`. + ['allure-playwright', { resultsDir: 'reports/allure-results', detail: true, suiteTitle: true }], + // Cucumber JSON is kept as the MACHINE feed for the AI pipeline only (not a report you open): + // failureAnalyzer.extractFailures reads it for `analyze` / `heal-selectors` / `heal-contract`. + // It is a single file overwritten each run (no staleness), unlike the accumulating Allure dir. + cucumberReporter('json', { outputFile: `reports/${REPORT_NAME}.json` }) ], use: { baseURL: process.env.BASE_URL ?? 'https://getmobil.com', diff --git a/src/api/clients/CategoriesApi.ts b/src/api/clients/CategoriesApi.ts new file mode 100644 index 0000000..ecd1ba1 --- /dev/null +++ b/src/api/clients/CategoriesApi.ts @@ -0,0 +1,20 @@ +import { BaseApiClient } from '../BaseApiClient'; +import { validateCategoriesResponse, type CategoriesResponse } from '../contracts/categories'; + +export class CategoriesApi extends BaseApiClient { + /** + * Lists all product categories. + * GET /api/categories + */ + async list(): Promise<{ status: number; body: CategoriesResponse }> { + const res = await this.get('/api/categories'); + const body = (await res.json()) as CategoriesResponse; + if (res.status() === 200) { + const issues = validateCategoriesResponse(body); + if (issues.length) { + throw new Error('Contract violation on /api/categories: ' + issues.join('; ')); + } + } + return { status: res.status(), body }; + } +} diff --git a/src/api/contracts/categories.ts b/src/api/contracts/categories.ts new file mode 100644 index 0000000..a38191d --- /dev/null +++ b/src/api/contracts/categories.ts @@ -0,0 +1,42 @@ +/** + * Contract for GET /api/categories (operationId: listCategories) + */ +export interface Category { + slug: string; + name: string; + count: number; +} + +export interface CategoriesResponse { + categories: Category[]; + /** Optional: some implementations include a top-level total */ + total?: number; +} + +export function validateCategoriesResponse(body: unknown): string[] { + const problems: string[] = []; + if (typeof body !== 'object' || body === null) { + return ['body is not an object']; + } + const b = body as Record; + if (!Array.isArray(b['categories'])) { + problems.push('categories is not an array'); + return problems; + } + (b['categories'] as unknown[]).forEach((item, i) => { + if (typeof item !== 'object' || item === null) { + problems.push(`categories[${i}] is not an object`); + return; + } + const cat = item as Record; + if (typeof cat['slug'] !== 'string') problems.push(`categories[${i}].slug is not a string`); + if (typeof cat['name'] !== 'string') problems.push(`categories[${i}].name is not a string`); + if (typeof cat['count'] !== 'number') problems.push(`categories[${i}].count is not a number`); + else if (!Number.isInteger(cat['count']) || (cat['count'] as number) <= 0) + problems.push(`categories[${i}].count must be a positive integer, got ${cat['count']}`); + }); + if ('total' in b && typeof b['total'] !== 'number') { + problems.push('total is present but not a number'); + } + return problems; +} diff --git a/src/api/fixtures.ts b/src/api/fixtures.ts index 5811130..dc4fde2 100644 --- a/src/api/fixtures.ts +++ b/src/api/fixtures.ts @@ -2,6 +2,7 @@ import { test as base } from 'playwright-bdd'; import type { APIRequestContext } from '@playwright/test'; import { SearchApi } from './clients/SearchApi'; import { ProductApi } from './clients/ProductApi'; +import { CategoriesApi } from './clients/CategoriesApi'; /** * Fixtures for the browserless `api` project (see playwright.config.ts). The `apiClient` is an @@ -27,6 +28,7 @@ type ApiFixtures = { apiClient: APIRequestContext; searchApi: SearchApi; productApi: ProductApi; + categoriesApi: CategoriesApi; apiState: { last?: ApiResponse }; }; @@ -38,5 +40,6 @@ export const test = base.extend({ }, searchApi: async ({ apiClient }, use) => use(new SearchApi(apiClient)), productApi: async ({ apiClient }, use) => use(new ProductApi(apiClient)), + categoriesApi: async ({ apiClient }, use) => use(new CategoriesApi(apiClient)), apiState: async ({}, use) => use({}) }); diff --git a/src/steps/api/categories.api.steps.ts b/src/steps/api/categories.api.steps.ts new file mode 100644 index 0000000..77cdfb9 --- /dev/null +++ b/src/steps/api/categories.api.steps.ts @@ -0,0 +1,47 @@ +import { expect } from '@playwright/test'; +import { When, Then } from './common'; +import type { CategoriesResponse } from '../../api/contracts/categories'; + +When('the categories list is fetched', async ({ categoriesApi, apiState }) => { + apiState.last = await categoriesApi.list(); +}); + +Then('the response body contains a categories array', async ({ apiState }) => { + const body = apiState.last!.body as CategoriesResponse; + expect(body).toHaveProperty('categories'); + expect(Array.isArray(body.categories)).toBe(true); +}); + +Then('every category has slug, name and count fields', async ({ apiState }) => { + const { categories } = apiState.last!.body as CategoriesResponse; + for (const cat of categories) { + expect(cat).toHaveProperty('slug'); + expect(cat).toHaveProperty('name'); + expect(cat).toHaveProperty('count'); + expect(typeof cat.slug).toBe('string'); + expect(typeof cat.name).toBe('string'); + expect(typeof cat.count).toBe('number'); + } +}); + +Then('every category count is a positive integer', async ({ apiState }) => { + const { categories } = apiState.last!.body as CategoriesResponse; + for (const cat of categories) { + expect(Number.isInteger(cat.count)).toBe(true); + expect(cat.count).toBeGreaterThan(0); + } +}); + +Then('the total equals the number of categories returned', async ({ apiState }) => { + const body = apiState.last!.body as CategoriesResponse; + if ('total' in body) { + expect((body as CategoriesResponse & { total: number }).total).toBe(body.categories.length); + } else { + expect(body.categories.length).toBeGreaterThanOrEqual(0); + } +}); + +Then('the categories array is not empty', async ({ apiState }) => { + const { categories } = apiState.last!.body as CategoriesResponse; + expect(categories.length).toBeGreaterThan(0); +});