From 56527262a4648e9faab246ae539628ad928ca33c Mon Sep 17 00:00:00 2001 From: Alexis Moreno Date: Fri, 20 Feb 2026 12:14:10 -0300 Subject: [PATCH 1/4] adding api test generation logic --- qa-config.json.example | 4 +- src/ai/prompts/planning.py | 32 ++++-- src/crawler/crawler.py | 6 + src/executor/assertion_checker.py | 117 +++++++++++++++++++ src/executor/executor.py | 180 +++++++++++++++++++++++++++++- src/models/config.py | 5 +- src/models/site_model.py | 1 + src/models/test_plan.py | 19 +++- src/planner/planner.py | 86 +++++++++++++- src/planner/schema_validator.py | 42 +++++-- src/reporter/html_report.py | 43 +++++++ 11 files changed, 505 insertions(+), 30 deletions(-) diff --git a/qa-config.json.example b/qa-config.json.example index ed3db92..d4a4cf6 100644 --- a/qa-config.json.example +++ b/qa-config.json.example @@ -1,5 +1,6 @@ { "target_url": "https://example.com/", + "backend_url": null, "auth": null, "crawl": { "target_url": "https://example.com/", @@ -20,7 +21,8 @@ "categories": [ "functional", "visual", - "security" + "security", + "api" ], "max_tests_per_run": 30, "max_execution_time_seconds": 1800, diff --git a/src/ai/prompts/planning.py b/src/ai/prompts/planning.py index c4a3259..53fb07c 100644 --- a/src/ai/prompts/planning.py +++ b/src/ai/prompts/planning.py @@ -21,26 +21,26 @@ "test_cases": [ { "test_id": "string (unique ID like tc_001)", - "name": "string (human-readable name)", - "description": "string (what this test verifies)", - "category": "functional | visual | security", + "name": "string (human-readable name; for api category use format '[METHOD] ', e.g. '[GET] Fetch user profile')", + "description": "string (what this test verifies; for api category include the HTTP method, full endpoint URL, and what the test asserts, e.g. 'Sends a GET request to /api/users and verifies the response returns status 200 with a JSON array')", + "category": "functional | visual | security | api", "priority": 1-5, "target_page_id": "string (page_id from Site Model)", "coverage_signature": "string (abstract description for registry matching)", "requires_auth": true, "preconditions": [ { - "action_type": "navigate | click | fill | select | hover | scroll | wait | screenshot | keyboard", - "selector": "string or null", - "value": "string or null", + "action_type": "navigate | click | fill | select | hover | scroll | wait | screenshot | keyboard | api_get | api_post | api_put | api_delete | api_patch", + "selector": "string or null (for API actions: the full endpoint URL goes here)", + "value": "string or null (for api_post/put/patch: JSON-encoded request body)", "description": "string" } ], - "steps": [ (same Action schema as preconditions) ], + "steps": [ "(same Action schema as preconditions)" ], "assertions": [ { - "assertion_type": "element_visible | element_hidden | text_contains | text_equals | text_matches | url_matches | screenshot_diff | element_count | network_request_made | no_console_errors | response_status | ai_evaluate | page_title_contains | page_loaded", - "selector": "string or null", + "assertion_type": "element_visible | element_hidden | text_contains | text_equals | text_matches | url_matches | screenshot_diff | element_count | network_request_made | no_console_errors | response_status | ai_evaluate | page_title_contains | page_loaded | response_body_contains | response_json_path | response_header", + "selector": "string or null (for response_json_path: dot-notation path e.g. 'data.id'; for response_header: header name)", "expected_value": "string or null", "tolerance": "float or null", "description": "string" @@ -58,10 +58,22 @@ 1. **Functional tests:** Test form submissions (valid and invalid data), navigation, CRUD operations, search/filter, pagination, modals, multi-step workflows, and auth flows. 2. **Visual tests:** Use screenshot_diff assertions to compare against baselines. IMPORTANT: Always add a wait step of at least 2000ms before screenshot assertions to allow fonts, images, and animations to fully load. Use element_visible assertions to verify key elements are present. Test responsive behavior across viewports. For screenshot_diff assertions, set tolerance to null (uses default 0.05). 3. **Security tests:** Inject XSS payloads into form fields and verify sanitization. Check HTTPS enforcement, cookie security attributes, open redirect vectors, and error page information leakage. +4. **API tests:** When the site model includes an `api_endpoints` array, generate direct HTTP tests using the API action types (`api_get`, `api_post`, `api_put`, `api_delete`, `api_patch`). These tests make real HTTP calls — they do NOT open a browser page or use visual assertions. + - **CRITICAL: Any test that uses `api_get`, `api_post`, `api_put`, `api_delete`, or `api_patch` action types MUST have `category: "api"`. Never assign these tests `category: "functional"` or any other category. A test that fires an HTTP action but is tagged `functional` will be skipped at runtime.** + - **Name format:** Always use `[METHOD] ` (e.g. `[GET] List products`, `[POST] Create order`, `[DELETE] Remove user`). + - **Description format:** Always describe the HTTP method, the full endpoint URL, and what is being asserted (e.g. `"Sends a POST request to /api/orders with a valid payload and verifies the response returns status 201 with the created order ID"`). + - Put the full endpoint URL in the `selector` field of each action. + - For POST/PUT/PATCH, put the JSON-encoded request body in the `value` field. + - Use `response_status` assertions to verify the HTTP status code (expected_value = status code as a string, e.g. `"200"`). + - Use `response_json_path` to assert values in the JSON body (selector = dot-notation path like `"data.id"`, expected_value = expected substring). + - Use `response_body_contains` to assert a substring appears anywhere in the response body. + - Use `response_header` to assert a response header is present (selector = header name, expected_value = expected substring in the value). + - Focus on endpoints observed during the crawl — do not invent endpoints that are not in the site model. + - Do NOT use `screenshot_diff`, `element_visible`, `page_loaded`, or other browser assertions in API tests. 4. **Prioritization:** Forms and interactive elements get higher priority. Static pages get lower priority. Recently failed areas get highest priority. 5. **Selectors:** Prefer data-testid attributes, then ARIA roles/labels, then stable CSS selectors. Avoid fragile positional selectors. 6. **Test data:** Generate realistic test data for form fills. Use invalid data for negative tests (empty required fields, malformed emails, XSS payloads for security). When a field needs a unique value (e.g., usernames, IDs, vault names), use the dynamic variable `{{$timestamp}}` in the value string (e.g., `"testuser-{{$timestamp}}"`) — it will be replaced with a Unix epoch timestamp at runtime to ensure uniqueness. -7. **Budget:** Respect the max_tests limit. Allocate budget proportionally: ~50% functional, ~30% visual, ~20% security (adjustable by hints). +7. **Budget:** Respect the max_tests limit. ONLY generate tests for the categories listed in the Configuration section — never generate tests for any other category. Allocate the test budget proportionally across those categories (adjustable by hints). Example split when all four are enabled: ~40% functional, ~20% visual, ~20% security, ~20% api. 8. **Assertion robustness:** Prefer behavioral/structural assertions over text matching. This is critical for reliable tests. - After form submissions: assert URL changed (url_matches), form disappeared (element_hidden), or new UI appeared (element_visible). Do NOT assert for specific success/error text you have not observed on the site. - For login flows: assert URL navigated away from the login page, or a logout/profile element appeared, rather than checking for "success" or "welcome" text. diff --git a/src/crawler/crawler.py b/src/crawler/crawler.py index 277b797..c335d9c 100644 --- a/src/crawler/crawler.py +++ b/src/crawler/crawler.py @@ -758,6 +758,8 @@ def _attach_network_listener( self, page: Page, nr_list: list[NetworkRequest] ) -> None: """Attach a network response listener to a page.""" + backend_url = self.config.backend_url + async def on_response(response): try: req = response.request @@ -769,6 +771,9 @@ async def on_response(response): content_type=response.headers.get("content-type", ""), )) if req.resource_type in ("xhr", "fetch"): + is_own = bool( + backend_url and req.url.startswith(backend_url) + ) key = f"{req.method}:{urlparse(req.url).path}" if key not in self._api_endpoints: self._api_endpoints[key] = APIEndpoint( @@ -776,6 +781,7 @@ async def on_response(response): method=req.method, response_content_type=response.headers.get("content-type"), status_codes_seen=[response.status], + is_own_backend=is_own, ) else: ep = self._api_endpoints[key] diff --git a/src/executor/assertion_checker.py b/src/executor/assertion_checker.py index 245466a..b6adffb 100644 --- a/src/executor/assertion_checker.py +++ b/src/executor/assertion_checker.py @@ -6,6 +6,7 @@ import logging import re from pathlib import Path +from typing import Any from playwright.async_api import Page @@ -398,6 +399,122 @@ def _check_response_status(assertion: Assertion, network_log: list[dict] | None) return AssertionResult(False, f"No response with status {expected}") +# --------------------------------------------------------------------------- +# API assertion checker (no browser page — operates on HTTP response data) +# --------------------------------------------------------------------------- + +def check_api_assertion(assertion: Assertion, response_data: dict[str, Any]) -> AssertionResult: + """Evaluate an assertion against HTTP response data from an API call. + + response_data keys: + status (int) — HTTP status code + headers (dict) — response headers (lowercased keys) + body (str) — raw response text + json (Any) — parsed JSON body, or None + url (str) — request URL + """ + try: + match assertion.assertion_type: + case "response_status": + return _check_api_status(assertion, response_data) + case "response_body_contains": + return _check_api_body_contains(assertion, response_data) + case "response_json_path": + return _check_api_json_path(assertion, response_data) + case "response_header": + return _check_api_header(assertion, response_data) + case _: + return AssertionResult(False, f"Unknown API assertion type: {assertion.assertion_type}") + except Exception as e: + logger.debug("API assertion error: %s — %s", assertion.assertion_type, e) + return AssertionResult(False, f"API assertion error: {e}") + + +def _check_api_status(assertion: Assertion, response_data: dict) -> AssertionResult: + if not assertion.expected_value: + return AssertionResult(False, "No expected status code in expected_value") + try: + expected = int(assertion.expected_value) + except ValueError: + return AssertionResult(False, f"expected_value '{assertion.expected_value}' is not a valid status code") + actual = response_data.get("status", 0) + if actual == expected: + return AssertionResult(True, f"Status {actual} matches expected {expected}") + return AssertionResult(False, f"Expected status {expected}, got {actual}") + + +def _check_api_body_contains(assertion: Assertion, response_data: dict) -> AssertionResult: + if not assertion.expected_value: + return AssertionResult(False, "No expected_value for response_body_contains") + body = response_data.get("body", "") + if assertion.expected_value.lower() in body.lower(): + return AssertionResult(True, f"Response body contains '{assertion.expected_value}'") + return AssertionResult(False, f"Response body does not contain '{assertion.expected_value}'") + + +def _check_api_json_path(assertion: Assertion, response_data: dict) -> AssertionResult: + """Traverse a dot-notation path in the JSON response body. + + selector — dot-notation path, e.g. ``data.user.name`` or ``items.0.id`` + expected_value — expected string value at that path (substring match); + omit to just assert the path exists + """ + if not assertion.selector: + return AssertionResult(False, "No JSON path provided in selector field (e.g. 'data.id')") + json_body = response_data.get("json") + if json_body is None: + return AssertionResult(False, "Response body is not valid JSON") + + current: Any = json_body + for part in assertion.selector.split("."): + if isinstance(current, dict): + current = current.get(part) + elif isinstance(current, list): + try: + current = current[int(part)] + except (ValueError, IndexError): + return AssertionResult(False, f"Path '{assertion.selector}': index '{part}' out of range") + else: + return AssertionResult(False, f"Path '{assertion.selector}': cannot traverse into {type(current).__name__}") + if current is None: + return AssertionResult(False, f"Path '{assertion.selector}' is null or missing") + + actual = str(current) + if not assertion.expected_value: + return AssertionResult(True, f"Path '{assertion.selector}' exists (value: '{actual[:80]}')") + if assertion.expected_value.lower() in actual.lower(): + return AssertionResult(True, f"Path '{assertion.selector}' = '{actual[:80]}'") + return AssertionResult( + False, + f"Path '{assertion.selector}': expected '{assertion.expected_value}', got '{actual[:80]}'", + ) + + +def _check_api_header(assertion: Assertion, response_data: dict) -> AssertionResult: + """Assert a response header is present and optionally matches a value. + + selector — header name (case-insensitive) + expected_value — expected substring in the header value (optional) + """ + if not assertion.selector: + return AssertionResult(False, "No header name provided in selector field") + headers: dict = response_data.get("headers", {}) + header_val = next( + (v for k, v in headers.items() if k.lower() == assertion.selector.lower()), + None, + ) + if header_val is None: + return AssertionResult(False, f"Header '{assertion.selector}' not found in response") + if not assertion.expected_value: + return AssertionResult(True, f"Header '{assertion.selector}' present: '{header_val}'") + if assertion.expected_value.lower() in header_val.lower(): + return AssertionResult(True, f"Header '{assertion.selector}': '{header_val}'") + return AssertionResult( + False, + f"Header '{assertion.selector}': expected '{assertion.expected_value}', got '{header_val}'", + ) + + async def _check_ai_evaluate( page: Page, assertion: Assertion, evidence_dir: Path, ai_client: AIClient | None ) -> AssertionResult: diff --git a/src/executor/executor.py b/src/executor/executor.py index 13fc07d..924fefd 100644 --- a/src/executor/executor.py +++ b/src/executor/executor.py @@ -26,7 +26,7 @@ from src.url_utils import page_id_from_url from .action_runner import resolve_dynamic_vars_for_test_case, run_action -from .assertion_checker import check_assertion +from .assertion_checker import check_assertion, check_api_assertion from .evidence_collector import EvidenceCollector from .fallback import FallbackHandler @@ -97,6 +97,18 @@ async def execute(self, plan: TestPlan, baseline_dir: Path | None = None) -> Run async def _run_one(index: int, tc: TestCase) -> TestResult: async with semaphore: + if tc.category not in self.config.categories: + logger.info("Skipping %s — category '%s' not in configured categories %s", + tc.test_id, tc.category, self.config.categories) + return TestResult( + test_id=tc.test_id, test_name=tc.name, + description=tc.description, category=tc.category, + priority=tc.priority, target_page_id=tc.target_page_id, + coverage_signature=tc.coverage_signature, + result="skip", + failure_reason=f"Category '{tc.category}' not in configured categories", + ) + elapsed = time.time() - start_time if elapsed >= self.config.max_execution_time_seconds: logger.warning("Time limit reached, skipping %s", tc.name) @@ -178,6 +190,101 @@ async def _run_one(index: int, tc: TestCase) -> TestResult: ) return run_result + async def _run_api_test(self, context, tc: TestCase) -> TestResult: + """Run an API test using Playwright's request context — no browser page opened. + + All actions must be api_get / api_post / api_put / api_delete / api_patch. + Assertions operate on the HTTP response returned by the last action. + """ + test_start = time.time() + + resolve_dynamic_vars_for_test_case(tc.preconditions + tc.steps) + + api = context.request + step_results: list[StepResult] = [] + assertion_results_list: list[AssertionResultModel] = [] + last_response_data: dict = {} + aborted = False + + for step_idx, action in enumerate(list(tc.preconditions) + list(tc.steps)): + if aborted: + step_results.append(StepResult( + step_index=step_idx, action_type=action.action_type, + selector=action.selector, value=action.value, + description=action.description, status="skip", + error_message="Skipped due to earlier step failure", + )) + continue + try: + resp_data = await _run_api_action(api, action) + last_response_data = resp_data + logger.debug(" API step %d: %s %s -> HTTP %d", + step_idx, action.action_type, action.selector, + resp_data.get("status", 0)) + step_results.append(StepResult( + step_index=step_idx, action_type=action.action_type, + selector=action.selector, value=action.value, + description=action.description, status="pass", + )) + except Exception as e: + logger.warning(" API step %d failed: %s", step_idx, e) + step_results.append(StepResult( + step_index=step_idx, action_type=action.action_type, + selector=action.selector, value=action.value, + description=action.description, status="fail", + error_message=str(e), + )) + aborted = True + + passed_count = 0 + failed_count = 0 + failure_reasons: list[str] = [] + + for a_idx, assertion in enumerate(tc.assertions): + result = check_api_assertion(assertion, last_response_data) + ar = AssertionResultModel( + assertion_type=assertion.assertion_type, + selector=assertion.selector, + expected_value=assertion.expected_value, + description=assertion.description, + passed=result.passed, + message=result.message, + ) + assertion_results_list.append(ar) + if result.passed: + passed_count += 1 + logger.debug(" API assertion %d/%d: PASSED — %s", + a_idx + 1, len(tc.assertions), result.message) + else: + failed_count += 1 + failure_reasons.append(f"{assertion.description or assertion.assertion_type}: {result.message}") + logger.debug(" API assertion %d/%d: FAILED — %s", + a_idx + 1, len(tc.assertions), result.message) + + test_result_status = "pass" if failed_count == 0 and not aborted else "fail" + if aborted and failed_count == 0: + test_result_status = "error" + + return TestResult( + test_id=tc.test_id, + test_name=tc.name, + description=tc.description, + category=tc.category, + priority=tc.priority, + target_page_id=tc.target_page_id, + actual_page_id=tc.target_page_id, + actual_url=last_response_data.get("url", ""), + coverage_signature=tc.coverage_signature, + result=test_result_status, + duration_seconds=round(time.time() - test_start, 2), + failure_reason="; ".join(failure_reasons) if failure_reasons else None, + step_results=step_results, + assertion_results=assertion_results_list, + assertions_passed=passed_count, + assertions_failed=failed_count, + assertions_total=len(tc.assertions), + ) + @staticmethod def _session_invalidated(result: TestResult) -> bool: """Check if a test likely invalidated the auth session (e.g. logout).""" @@ -196,6 +303,9 @@ async def _run_test( self, context, test_case: TestCase, baseline_dir: Path | None, ) -> TestResult: """Run a single test case with full step/assertion detail recording.""" + if test_case.category == "api": + return await self._run_api_test(context, test_case) + tc = test_case test_start = time.time() evidence_dir = self.run_dir / "evidence" / tc.test_id @@ -479,3 +589,71 @@ async def _run_test( ) finally: await page.close() + + +# --------------------------------------------------------------------------- +# Module-level helper for API actions (used by Executor._run_api_test) +# --------------------------------------------------------------------------- + +async def _run_api_action(api, action) -> dict: + """Execute a single API HTTP action and return parsed response data. + + Uses the Playwright APIRequestContext (``context.request``) which + automatically shares cookies and auth state with the browser context. + + action.action_type — one of: api_get, api_post, api_put, api_delete, api_patch + action.selector — full or relative URL + action.value — JSON-encoded request body (for POST / PUT / PATCH) + """ + import json as _json + + url = action.selector or "" + if not url: + raise ValueError( + f"API action '{action.action_type}' requires a URL in the selector field" + ) + + post_data = None + if action.value: + try: + post_data = _json.loads(action.value) + except (_json.JSONDecodeError, TypeError): + post_data = action.value # send as raw string + + # Use `json=` for dict payloads so Playwright sends Content-Type: application/json. + # Raw strings are passed via `data=` as-is. + def _kwargs(body): + if isinstance(body, dict): + return {"json": body} + if body is not None: + return {"data": body} + return {} + + match action.action_type: + case "api_get": + response = await api.get(url) + case "api_post": + response = await api.post(url, **_kwargs(post_data)) + case "api_put": + response = await api.put(url, **_kwargs(post_data)) + case "api_delete": + response = await api.delete(url) + case "api_patch": + response = await api.patch(url, **_kwargs(post_data)) + case _: + raise ValueError(f"Unknown API action type: {action.action_type}") + + body_text = await response.text() + body_json = None + try: + body_json = await response.json() + except Exception: + pass + + return { + "status": response.status, + "headers": dict(response.headers), + "body": body_text, + "json": body_json, + "url": url, + } diff --git a/src/models/config.py b/src/models/config.py index d55d3ab..88fed7d 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -56,6 +56,9 @@ class FrameworkConfig(BaseModel): # Target target_url: str + # Backend URL — when set, captured API endpoints are tagged with is_own_backend + backend_url: Optional[str] = None + # Authentication auth: Optional[AuthConfig] = None @@ -64,7 +67,7 @@ class FrameworkConfig(BaseModel): # Test categories categories: list[str] = Field( - default_factory=lambda: ["functional", "visual", "security"] + default_factory=lambda: ["functional", "visual", "security", "api"] ) # Execution limits diff --git a/src/models/site_model.py b/src/models/site_model.py index 48d2f64..4dbc364 100644 --- a/src/models/site_model.py +++ b/src/models/site_model.py @@ -49,6 +49,7 @@ class APIEndpoint(BaseModel): request_content_type: Optional[str] = None response_content_type: Optional[str] = None status_codes_seen: list[int] = Field(default_factory=list) + is_own_backend: bool = False class AuthFlow(BaseModel): diff --git a/src/models/test_plan.py b/src/models/test_plan.py index e9909e7..c2645fb 100644 --- a/src/models/test_plan.py +++ b/src/models/test_plan.py @@ -8,17 +8,26 @@ class Action(BaseModel): - action_type: str # navigate, click, fill, select, hover, scroll, wait, screenshot, keyboard + # Browser actions: navigate, click, fill, select, hover, scroll, wait, screenshot, keyboard + # API actions (category="api" tests only): api_get, api_post, api_put, api_delete, api_patch + # selector = full endpoint URL; value = JSON-encoded request body (POST/PUT/PATCH) + action_type: str selector: Optional[str] = None value: Optional[str] = None description: str = "" class Assertion(BaseModel): - assertion_type: str # element_visible, element_hidden, text_contains, text_equals, - # text_matches, url_matches, screenshot_diff, element_count, - # network_request_made, no_console_errors, response_status, - # ai_evaluate, page_title_contains, page_loaded + # Browser assertions: element_visible, element_hidden, text_contains, text_equals, + # text_matches, url_matches, screenshot_diff, element_count, + # network_request_made, no_console_errors, response_status, + # ai_evaluate, page_title_contains, page_loaded + # API assertions (category="api" tests only): + # response_status — expected_value = HTTP status code string e.g. "200" + # response_body_contains — expected_value = substring to find in response body + # response_json_path — selector = dot-notation path e.g. "data.id"; expected_value = expected substring + # response_header — selector = header name; expected_value = expected substring in header value + assertion_type: str selector: Optional[str] = None expected_value: Optional[str] = None tolerance: Optional[float] = None diff --git a/src/planner/planner.py b/src/planner/planner.py index 990ec00..23685a8 100644 --- a/src/planner/planner.py +++ b/src/planner/planner.py @@ -56,7 +56,8 @@ def generate_plan( # Build config summary config_summary = ( - f"Categories: {', '.join(self.config.categories)}\n" + f"Categories (ONLY generate tests for these categories — do not generate any others): " + f"{', '.join(self.config.categories)}\n" f"Max tests: {self.config.max_tests_per_run}\n" f"Visual diff tolerance: {self.config.visual_diff_tolerance}\n" f"Viewports: {json.dumps([v.model_dump() for v in self.config.viewports])}\n" @@ -103,21 +104,44 @@ def generate_plan( ] # Inject real credentials in place of placeholder tokens plan = self._inject_credentials(plan) + plan = self._filter_by_categories(plan) logger.info("Generated plan with %d test cases", len(plan.test_cases)) return plan except Exception as e: logger.error("Failed to parse AI plan: %s. Using fallback.", e) - return self._inject_credentials(self._generate_fallback_plan(site_model)) + plan = self._inject_credentials(self._generate_fallback_plan(site_model)) + return self._filter_by_categories(plan) def _summarize_site_model(self, site_model: SiteModel) -> str: """Create a condensed version of the site model for the AI prompt.""" - summary = { + summary: dict = { "base_url": site_model.base_url, "pages": [], "api_endpoints_count": len(site_model.api_endpoints), "has_auth": site_model.auth_flow is not None, } + # Include full API endpoint details when the "api" category is enabled + if "api" in self.config.categories and site_model.api_endpoints: + endpoints = site_model.api_endpoints + # When backend_url is configured, only send own-backend endpoints to the AI + if self.config.backend_url: + endpoints = [ep for ep in endpoints if ep.is_own_backend] + capped = endpoints[:50] + logger.debug("Sending %d API endpoint(s) to AI (total captured: %d):", len(capped), len(endpoints)) + for ep in capped: + logger.debug(" %s %s (status codes: %s)", ep.method, ep.url, ep.status_codes_seen) + summary["api_endpoints"] = [ + { + "url": ep.url, + "method": ep.method, + "response_content_type": ep.response_content_type, + "status_codes_seen": ep.status_codes_seen, + "is_own_backend": ep.is_own_backend, + } + for ep in capped + ] + for page in site_model.pages[:30]: # Limit pages page_summary = { "page_id": page.page_id, @@ -164,11 +188,24 @@ def _parse_plan(self, data: dict, site_model: SiteModel) -> TestPlan: steps = [Action(**a) for a in tc_data.get("steps", [])] assertions = [Assertion(**a) for a in tc_data.get("assertions", [])] + category = tc_data.get("category", "functional") + + # Auto-correct: if any action is an API type, force category to "api" + # regardless of what the AI returned — mistagging causes runtime skips. + _api_action_types = {"api_get", "api_post", "api_put", "api_delete", "api_patch"} + all_actions = preconditions + steps + if any(a.action_type in _api_action_types for a in all_actions) and category != "api": + logger.warning( + "Test case '%s' has api_* actions but category='%s' — correcting to 'api'", + tc_data.get("test_id", "?"), category, + ) + category = "api" + tc = TestCase( test_id=tc_data.get("test_id", f"tc_{uuid.uuid4().hex[:6]}"), name=tc_data.get("name", "Unnamed test"), description=tc_data.get("description", ""), - category=tc_data.get("category", "functional"), + category=category, priority=tc_data.get("priority", 3), target_page_id=tc_data.get("target_page_id", ""), coverage_signature=tc_data.get("coverage_signature", ""), @@ -282,6 +319,34 @@ def _generate_fallback_plan(self, site_model: SiteModel) -> TestPlan: )], )) + # API fallback tests — one GET per captured endpoint + if "api" in self.config.categories: + endpoints = site_model.api_endpoints + if self.config.backend_url: + endpoints = [ep for ep in endpoints if ep.is_own_backend] + for ep in endpoints[:self.config.max_tests_per_run]: + tc_num += 1 + action_type = f"api_{ep.method.lower()}" + test_cases.append(TestCase( + test_id=f"tc_fallback_{tc_num:03d}", + name=f"[{ep.method}] {ep.url}", + description=f"Sends a {ep.method} request to {ep.url} and verifies a successful response", + category="api", + priority=2, + target_page_id="", + coverage_signature=f"api_{ep.method}_{ep.url}", + steps=[Action( + action_type=action_type, + selector=ep.url, + description=f"{ep.method} {ep.url}", + )], + assertions=[Assertion( + assertion_type="response_status", + expected_value=str(ep.status_codes_seen[0]) if ep.status_codes_seen else "200", + description=f"Response status is {ep.status_codes_seen[0] if ep.status_codes_seen else 200}", + )], + )) + return TestPlan( plan_id=f"plan_fallback_{uuid.uuid4().hex[:8]}", generated_at=time.strftime("%Y-%m-%dT%H:%M:%SZ"), @@ -290,6 +355,19 @@ def _generate_fallback_plan(self, site_model: SiteModel) -> TestPlan: estimated_duration_seconds=len(test_cases) * 10, ) + def _filter_by_categories(self, plan: TestPlan) -> TestPlan: + """Remove test cases whose category is not in the configured categories list.""" + allowed = set(self.config.categories) + before = len(plan.test_cases) + plan.test_cases = [tc for tc in plan.test_cases if tc.category in allowed] + removed = before - len(plan.test_cases) + if removed: + logger.info( + "Filtered %d test case(s) with categories not in config %s", + removed, sorted(allowed), + ) + return plan + @staticmethod def _has_auth_placeholders(tc: TestCase) -> bool: """Check if a test case contains any unresolved auth placeholder tokens.""" diff --git a/src/planner/schema_validator.py b/src/planner/schema_validator.py index e7aac02..351ba32 100644 --- a/src/planner/schema_validator.py +++ b/src/planner/schema_validator.py @@ -8,17 +8,23 @@ logger = logging.getLogger(__name__) -VALID_CATEGORIES = {"functional", "visual", "security"} +VALID_CATEGORIES = {"functional", "visual", "security", "api"} VALID_ACTION_TYPES = { "navigate", "click", "fill", "select", "hover", "scroll", "wait", "screenshot", "keyboard", + "api_get", "api_post", "api_put", "api_delete", "api_patch", } -VALID_ASSERTION_TYPES = { +API_ACTION_TYPES = {"api_get", "api_post", "api_put", "api_delete", "api_patch"} +BROWSER_ASSERTION_TYPES = { "element_visible", "element_hidden", "text_contains", "text_equals", "text_matches", "url_matches", "screenshot_diff", "element_count", - "network_request_made", "no_console_errors", "response_status", - "ai_evaluate", "page_title_contains", "page_loaded", + "network_request_made", "no_console_errors", "ai_evaluate", + "page_title_contains", "page_loaded", } +API_ASSERTION_TYPES = { + "response_status", "response_body_contains", "response_json_path", "response_header", +} +VALID_ASSERTION_TYPES = BROWSER_ASSERTION_TYPES | API_ASSERTION_TYPES def validate_test_plan(plan: TestPlan) -> list[str]: @@ -48,17 +54,32 @@ def validate_test_plan(plan: TestPlan) -> list[str]: if not isinstance(tc.requires_auth, bool): errors.append(f"{tc.test_id}: requires_auth must be a boolean, got {type(tc.requires_auth).__name__}") - # At least one step - if not tc.steps: + # Determine whether this is an API test by inspecting actions + all_actions = tc.preconditions + tc.steps + is_api_test = tc.category == "api" or any(a.action_type in API_ACTION_TYPES for a in all_actions) + + # At least one step (API tests may put their action in preconditions) + if not tc.steps and not (is_api_test and tc.preconditions): errors.append(f"{tc.test_id}: no steps defined") + # API tests: name must follow [METHOD] format + if is_api_test and not tc.name.startswith("["): + errors.append( + f"{tc.test_id}: api test name must start with [METHOD] (e.g. '[GET] List users'), got '{tc.name}'" + ) + # Validate actions - for i, action in enumerate(tc.preconditions + tc.steps): + for i, action in enumerate(all_actions): if action.action_type not in VALID_ACTION_TYPES: errors.append( f"{tc.test_id} step {i}: invalid action_type '{action.action_type}'" ) - # Actions that need a selector + # API actions require a URL in selector + if action.action_type in API_ACTION_TYPES and not action.selector: + errors.append( + f"{tc.test_id} step {i}: {action.action_type} requires a URL in the selector field" + ) + # Non-API actions that need a selector if action.action_type in ("click", "fill", "select", "hover") and not action.selector: errors.append( f"{tc.test_id} step {i}: {action.action_type} requires a selector" @@ -73,5 +94,10 @@ def validate_test_plan(plan: TestPlan) -> list[str]: errors.append( f"{tc.test_id} assertion {i}: invalid type '{assertion.assertion_type}'" ) + # API tests must not use browser assertions + if is_api_test and assertion.assertion_type in BROWSER_ASSERTION_TYPES: + errors.append( + f"{tc.test_id} assertion {i}: api test uses browser assertion '{assertion.assertion_type}' — use response_status, response_json_path, response_body_contains, or response_header" + ) return errors diff --git a/src/reporter/html_report.py b/src/reporter/html_report.py index 13e7944..16aaeea 100644 --- a/src/reporter/html_report.py +++ b/src/reporter/html_report.py @@ -200,6 +200,31 @@ def generate_html_report( items += f"
  • {html.escape(r.test_name)} ({r.category}): {r.previous_result} → {r.current_result}{reason}
    • " reg_section = f'

    ⚠ Regressions ({len(regressions)})

      {items}
    ' + # Test type breakdown + category_counts: dict[str, int] = {} + for r in run_result.test_results: + category_counts[r.category] = category_counts.get(r.category, 0) + 1 + + category_order = ["functional", "visual", "security", "api"] + type_stats_html = "" + for cat in category_order: + count = category_counts.get(cat, 0) + if count: + type_stats_html += f'
    {count}
    {cat.capitalize()}
    ' + # Include any unexpected categories too + for cat, count in category_counts.items(): + if cat not in category_order: + type_stats_html += f'
    {count}
    {cat.capitalize()}
    ' + + # Category filter buttons + category_filter_btns = "" + for cat in category_order: + if category_counts.get(cat, 0): + category_filter_btns += f'' + for cat in category_counts: + if cat not in category_order: + category_filter_btns += f'' + # Test cards test_cards = [] for r in run_result.test_results: @@ -236,6 +261,7 @@ def generate_html_report( .badge.functional {{ background: #dbeafe; color: #1e40af; }} .badge.visual {{ background: #e0e7ff; color: #3730a3; }} .badge.security {{ background: #fce7f3; color: #9d174d; }} + .badge.api {{ background: #d1fae5; color: #065f46; }} /* AI / Regression boxes */ .ai-summary {{ background: var(--card); border-radius: 8px; padding: 1.2rem; margin-bottom: 1.5rem; box-shadow: 0 1px 3px rgba(0,0,0,0.08); border-left: 4px solid var(--accent); }} .ai-summary h2 {{ font-size: 1rem; color: var(--accent); margin-bottom: 0.8rem; }} @@ -294,6 +320,9 @@ def generate_html_report( .screenshot-label {{ font-size: 0.75rem; color: var(--muted); margin-top: 0.2rem; }} /* Console */ .console-log {{ background: #1e293b; color: #f1f5f9; padding: 0.8rem; border-radius: 6px; font-size: 0.78rem; overflow-x: auto; max-height: 200px; overflow-y: auto; }} + /* Section heading */ + .section-heading {{ font-size: 0.8rem; color: var(--muted); text-transform: uppercase; letter-spacing: 0.05em; margin: 1rem 0 0.4rem; }} + .type-summary {{ margin-bottom: 1.5rem; }} /* Filter bar */ .filter-bar {{ display: flex; gap: 0.5rem; margin-bottom: 1rem; flex-wrap: wrap; }} .filter-btn {{ padding: 0.3rem 0.8rem; border-radius: 6px; border: 1px solid var(--border); background: var(--card); cursor: pointer; font-size: 0.82rem; }} @@ -313,6 +342,11 @@ def generate_html_report(
    {run_result.errors}
    Errors
    +

    By Test Type

    +
    + {type_stats_html} +
    + {ai_section} {reg_section} @@ -322,6 +356,7 @@ def generate_html_report( + {category_filter_btns} @@ -341,6 +376,14 @@ def generate_html_report( card.style.display = badge && badge.textContent.trim().toLowerCase() === status ? '' : 'none'; }}); }} +function filterByCategory(cat) {{ + document.querySelectorAll('.filter-btn').forEach(b => b.classList.remove('active')); + event.target.classList.add('active'); + document.querySelectorAll('.test-card').forEach(card => {{ + const catBadge = card.querySelector('.test-header .badge.' + cat); + card.style.display = catBadge ? '' : 'none'; + }}); +}} function expandAll() {{ document.querySelectorAll('.test-card').forEach(c => c.classList.add('expanded')); }} From cf3550fd32af23dd2bf40f0acc1f910ad9e963e4 Mon Sep 17 00:00:00 2001 From: Alexis Moreno Date: Fri, 20 Feb 2026 12:16:14 -0300 Subject: [PATCH 2/4] updating readme --- README.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3380e4e..540da5d 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ python -m src.cli run - **Zero test scripts** - AI generates tests by understanding your site - **Self-healing** - When selectors break, AI analyzes screenshots and fixes them -- **Comprehensive coverage** - Functional, visual, and security testing in one pass +- **Comprehensive coverage** - Functional, visual, security, and API testing in one pass +- **API testing** - Real HTTP calls via Playwright's request context, not a headless browser - **Natural language hints** - Guide priorities without writing test specs - **Coverage memory** - Tracks what's been tested, focuses on gaps @@ -53,7 +54,7 @@ Create `qa-config.json`: ```json { "target_url": "https://yoursite.com", - "categories": ["functional", "visual", "security"], + "categories": ["functional", "visual", "security", "api"], "hints": [ "The checkout flow is our most critical path" ] @@ -110,6 +111,7 @@ open qa-reports/report_*.html - **Functional tests** - Forms, navigation, workflows, CRUD - **Visual regression** - Screenshot baselines, responsive design - **Security checks** - XSS, HTTPS, cookies, headers +- **API tests** - Direct HTTP calls against observed endpoints, with JSON path and status assertions - **Evidence collection** - Screenshots, logs, network activity **→ [See all features in detail](./OVERVIEW.md#key-features)** @@ -154,6 +156,23 @@ open qa-reports/report_*.html Hints guide AI priorities without writing test specifications. The AI interprets them and adjusts test generation accordingly. +### With API Testing + +```json +{ + "target_url": "https://yoursite.com", + "backend_url": "https://api.yoursite.com", + "categories": ["functional", "api"] +} +``` + +The crawler captures every XHR/fetch request made by the browser during crawling. When `"api"` is in `categories`, the AI generates direct HTTP tests for those observed endpoints — no browser page is opened, requests go through Playwright's `APIRequestContext` and share the authenticated session automatically. + +- **`backend_url`** (optional) — when set, only endpoints whose URL starts with this value are sent to the AI. Use this to focus on your own backend and exclude third-party calls (analytics, CDN, etc.). +- **Test names** follow the format `[METHOD] description` — e.g. `[GET] List products`, `[POST] Create order`. +- **Supported assertions:** `response_status`, `response_json_path`, `response_body_contains`, `response_header`. +- **No browser assertions** (`element_visible`, `screenshot_diff`, etc.) are allowed in API tests. + **→ [Complete configuration reference](./REQUIREMENTS.md#configuration)** ## CLI Commands @@ -293,6 +312,7 @@ open qa-reports/report_*.html - XSS vulnerability in product review form → Flagged - Visual regression: Logo alignment shifted → Screenshot diff - Checkout flow: 100% passing +- `[POST] /api/orders` returns 500 on valid payload → Flagged **→ [See full example walkthrough](./OVERVIEW.md#real-world-example)** From e933312c39945235d6d5a41f1842b13df07df72f Mon Sep 17 00:00:00 2001 From: Alexis Moreno Date: Tue, 24 Feb 2026 16:23:54 -0300 Subject: [PATCH 3/4] adding unit tests --- tests/test_api_assertion_checker.py | 286 +++++++++++++++++ tests/test_api_executor.py | 427 +++++++++++++++++++++++++ tests/test_models_config.py | 2 +- tests/test_planner_schema_validator.py | 127 ++++++++ 4 files changed, 841 insertions(+), 1 deletion(-) create mode 100644 tests/test_api_assertion_checker.py create mode 100644 tests/test_api_executor.py diff --git a/tests/test_api_assertion_checker.py b/tests/test_api_assertion_checker.py new file mode 100644 index 0000000..6bfaa00 --- /dev/null +++ b/tests/test_api_assertion_checker.py @@ -0,0 +1,286 @@ +"""Tests for API assertion checker — check_api_assertion and helpers.""" + +import pytest + +from src.executor.assertion_checker import check_api_assertion +from src.models.test_plan import Assertion + + +def _response(status=200, body="", json=None, headers=None): + """Build a minimal response_data dict.""" + return { + "status": status, + "headers": headers or {}, + "body": body, + "json": json, + "url": "http://localhost:8000/api/test", + } + + +# --------------------------------------------------------------------------- +# response_status +# --------------------------------------------------------------------------- + +class TestCheckApiStatus: + + def test_matching_status_passes(self): + assertion = Assertion(assertion_type="response_status", expected_value="200") + result = check_api_assertion(assertion, _response(status=200)) + assert result.passed is True + assert "200" in result.message + + def test_mismatched_status_fails(self): + assertion = Assertion(assertion_type="response_status", expected_value="200") + result = check_api_assertion(assertion, _response(status=404)) + assert result.passed is False + assert "404" in result.message + + def test_status_4xx_expected_passes(self): + assertion = Assertion(assertion_type="response_status", expected_value="404") + result = check_api_assertion(assertion, _response(status=404)) + assert result.passed is True + + def test_missing_expected_value_fails(self): + assertion = Assertion(assertion_type="response_status", expected_value=None) + result = check_api_assertion(assertion, _response(status=200)) + assert result.passed is False + assert "expected" in result.message.lower() + + def test_non_numeric_expected_value_fails(self): + assertion = Assertion(assertion_type="response_status", expected_value="ok") + result = check_api_assertion(assertion, _response(status=200)) + assert result.passed is False + assert "not a valid status code" in result.message.lower() + + +# --------------------------------------------------------------------------- +# response_body_contains +# --------------------------------------------------------------------------- + +class TestCheckApiBodyContains: + + def test_substring_present_passes(self): + assertion = Assertion(assertion_type="response_body_contains", expected_value="success") + result = check_api_assertion(assertion, _response(body='{"status": "success"}')) + assert result.passed is True + + def test_substring_absent_fails(self): + assertion = Assertion(assertion_type="response_body_contains", expected_value="error") + result = check_api_assertion(assertion, _response(body='{"status": "ok"}')) + assert result.passed is False + + def test_case_insensitive_match(self): + assertion = Assertion(assertion_type="response_body_contains", expected_value="SUCCESS") + result = check_api_assertion(assertion, _response(body='{"status": "success"}')) + assert result.passed is True + + def test_missing_expected_value_fails(self): + assertion = Assertion(assertion_type="response_body_contains", expected_value=None) + result = check_api_assertion(assertion, _response(body="anything")) + assert result.passed is False + + +# --------------------------------------------------------------------------- +# response_json_path +# --------------------------------------------------------------------------- + +class TestCheckApiJsonPath: + + def test_simple_key_passes(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="id", + expected_value="42", + ) + result = check_api_assertion(assertion, _response(json={"id": 42})) + assert result.passed is True + + def test_nested_path_passes(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="data.user.name", + expected_value="alice", + ) + result = check_api_assertion( + assertion, + _response(json={"data": {"user": {"name": "alice"}}}), + ) + assert result.passed is True + + def test_array_index_passes(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="items.0.id", + expected_value="1", + ) + result = check_api_assertion( + assertion, + _response(json={"items": [{"id": 1}, {"id": 2}]}), + ) + assert result.passed is True + + def test_path_exists_no_expected_value(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="data.id", + expected_value=None, + ) + result = check_api_assertion(assertion, _response(json={"data": {"id": 99}})) + assert result.passed is True + assert "exists" in result.message.lower() + + def test_wrong_value_fails(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="status", + expected_value="pending", + ) + result = check_api_assertion(assertion, _response(json={"status": "complete"})) + assert result.passed is False + assert "complete" in result.message + + def test_missing_key_fails(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="data.missing", + expected_value="x", + ) + result = check_api_assertion(assertion, _response(json={"data": {}})) + assert result.passed is False + + def test_non_json_response_fails(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="id", + expected_value="1", + ) + result = check_api_assertion(assertion, _response(json=None, body="not json")) + assert result.passed is False + assert "not valid json" in result.message.lower() + + def test_no_selector_fails(self): + assertion = Assertion( + assertion_type="response_json_path", + selector=None, + expected_value="1", + ) + result = check_api_assertion(assertion, _response(json={"id": 1})) + assert result.passed is False + + def test_array_index_out_of_range_fails(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="items.5.id", + expected_value="1", + ) + result = check_api_assertion(assertion, _response(json={"items": [{"id": 1}]})) + assert result.passed is False + assert "out of range" in result.message.lower() + + def test_substring_match_in_value(self): + assertion = Assertion( + assertion_type="response_json_path", + selector="message", + expected_value="created", + ) + result = check_api_assertion( + assertion, + _response(json={"message": "resource created successfully"}), + ) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# response_header +# --------------------------------------------------------------------------- + +class TestCheckApiHeader: + + def test_header_present_with_matching_value(self): + assertion = Assertion( + assertion_type="response_header", + selector="content-type", + expected_value="application/json", + ) + result = check_api_assertion( + assertion, + _response(headers={"content-type": "application/json; charset=utf-8"}), + ) + assert result.passed is True + + def test_header_absent_fails(self): + assertion = Assertion( + assertion_type="response_header", + selector="x-custom-header", + expected_value="value", + ) + result = check_api_assertion(assertion, _response(headers={})) + assert result.passed is False + assert "not found" in result.message.lower() + + def test_header_present_wrong_value_fails(self): + assertion = Assertion( + assertion_type="response_header", + selector="content-type", + expected_value="text/html", + ) + result = check_api_assertion( + assertion, + _response(headers={"content-type": "application/json"}), + ) + assert result.passed is False + + def test_case_insensitive_header_name(self): + assertion = Assertion( + assertion_type="response_header", + selector="Content-Type", + expected_value="json", + ) + result = check_api_assertion( + assertion, + _response(headers={"content-type": "application/json"}), + ) + assert result.passed is True + + def test_no_expected_value_just_presence_check(self): + assertion = Assertion( + assertion_type="response_header", + selector="x-request-id", + expected_value=None, + ) + result = check_api_assertion( + assertion, + _response(headers={"x-request-id": "abc123"}), + ) + assert result.passed is True + assert "present" in result.message.lower() + + def test_no_selector_fails(self): + assertion = Assertion( + assertion_type="response_header", + selector=None, + expected_value="json", + ) + result = check_api_assertion(assertion, _response(headers={"content-type": "json"})) + assert result.passed is False + + +# --------------------------------------------------------------------------- +# check_api_assertion dispatch +# --------------------------------------------------------------------------- + +class TestCheckApiAssertionDispatch: + + def test_unknown_assertion_type_fails(self): + assertion = Assertion(assertion_type="element_visible", selector=".foo") + result = check_api_assertion(assertion, _response()) + assert result.passed is False + assert "unknown" in result.message.lower() + + def test_exception_is_caught_and_returns_fail(self): + """An unexpected exception inside a helper must not propagate.""" + assertion = Assertion(assertion_type="response_json_path", selector="a.b") + # Pass a response_data that will cause an unexpected error path + result = check_api_assertion(assertion, _response(json={"a": {"b": "v"}}, status=200)) + # Should complete without raising — result can be pass or fail + assert isinstance(result.passed, bool) diff --git a/tests/test_api_executor.py b/tests/test_api_executor.py new file mode 100644 index 0000000..0057932 --- /dev/null +++ b/tests/test_api_executor.py @@ -0,0 +1,427 @@ +"""Tests for API test execution — _run_api_action and Executor._run_api_test.""" + +import pytest +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +from src.executor.executor import Executor, _run_api_action +from src.models.config import CrawlConfig, FrameworkConfig, ViewportConfig +from src.models.test_plan import Action, Assertion, TestCase, TestPlan + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_config(): + return FrameworkConfig( + target_url="http://localhost:8000", + categories=["api"], + crawl=CrawlConfig( + target_url="http://localhost:8000", + viewport=ViewportConfig(width=1280, height=720, name="desktop"), + ), + max_tests_per_run=50, + max_execution_time_seconds=1800, + selector_timeout_seconds=5, + ai_model="claude-opus-4-6", + ai_max_fallback_calls_per_test=3, + ai_max_planning_tokens=32000, + visual_diff_tolerance=0.05, + report_output_dir="./test-reports", + ) + + +def _make_api_test_case( + test_id="tc_api_001", + steps=None, + assertions=None, + preconditions=None, +) -> TestCase: + return TestCase( + test_id=test_id, + name="[GET] List items", + category="api", + priority=2, + target_page_id="", + requires_auth=False, + coverage_signature=f"sig_{test_id}", + steps=steps or [Action( + action_type="api_get", + selector="http://localhost:8000/api/items", + description="GET /api/items", + )], + assertions=assertions or [Assertion( + assertion_type="response_status", + expected_value="200", + description="Status is 200", + )], + preconditions=preconditions or [], + timeout_seconds=30, + ) + + +def _make_mock_api_response(status=200, body='{"items":[]}', headers=None, json_data=None): + """Return an AsyncMock mimicking Playwright's APIResponse.""" + resp = AsyncMock() + resp.status = status + resp.headers = headers or {"content-type": "application/json"} + resp.text = AsyncMock(return_value=body) + resp.json = AsyncMock(return_value=json_data if json_data is not None else {"items": []}) + return resp + + +# --------------------------------------------------------------------------- +# _run_api_action +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +class TestRunApiAction: + + async def test_get_calls_api_get(self): + api = AsyncMock() + api.get = AsyncMock(return_value=_make_mock_api_response()) + action = Action(action_type="api_get", selector="http://localhost/api/users") + + result = await _run_api_action(api, action) + + api.get.assert_awaited_once_with("http://localhost/api/users") + assert result["status"] == 200 + assert result["url"] == "http://localhost/api/users" + + async def test_post_with_json_body(self): + api = AsyncMock() + api.post = AsyncMock(return_value=_make_mock_api_response(status=201)) + action = Action( + action_type="api_post", + selector="http://localhost/api/users", + value='{"name": "alice"}', + ) + + result = await _run_api_action(api, action) + + api.post.assert_awaited_once_with( + "http://localhost/api/users", + json={"name": "alice"}, + ) + assert result["status"] == 201 + + async def test_post_with_raw_string_body(self): + """Unparseable JSON value is sent as raw data string.""" + api = AsyncMock() + api.post = AsyncMock(return_value=_make_mock_api_response(status=200)) + action = Action( + action_type="api_post", + selector="http://localhost/api/raw", + value="not-json", + ) + + result = await _run_api_action(api, action) + + api.post.assert_awaited_once_with( + "http://localhost/api/raw", + data="not-json", + ) + + async def test_put_with_json_body(self): + api = AsyncMock() + api.put = AsyncMock(return_value=_make_mock_api_response()) + action = Action( + action_type="api_put", + selector="http://localhost/api/users/1", + value='{"name": "bob"}', + ) + + await _run_api_action(api, action) + + api.put.assert_awaited_once_with( + "http://localhost/api/users/1", + json={"name": "bob"}, + ) + + async def test_delete_no_body(self): + api = AsyncMock() + api.delete = AsyncMock(return_value=_make_mock_api_response(status=204, body="")) + action = Action( + action_type="api_delete", + selector="http://localhost/api/users/1", + ) + + result = await _run_api_action(api, action) + + api.delete.assert_awaited_once_with("http://localhost/api/users/1") + assert result["status"] == 204 + + async def test_patch_with_json_body(self): + api = AsyncMock() + api.patch = AsyncMock(return_value=_make_mock_api_response()) + action = Action( + action_type="api_patch", + selector="http://localhost/api/users/1", + value='{"active": true}', + ) + + await _run_api_action(api, action) + + api.patch.assert_awaited_once_with( + "http://localhost/api/users/1", + json={"active": True}, + ) + + async def test_missing_url_raises(self): + api = AsyncMock() + action = Action(action_type="api_get", selector=None) + + with pytest.raises(ValueError, match="requires a URL"): + await _run_api_action(api, action) + + async def test_unknown_action_type_raises(self): + api = AsyncMock() + action = Action(action_type="api_head", selector="http://localhost/api") + + with pytest.raises(ValueError, match="Unknown API action type"): + await _run_api_action(api, action) + + async def test_returns_parsed_json(self): + api = AsyncMock() + json_data = {"id": 1, "name": "alice"} + api.get = AsyncMock(return_value=_make_mock_api_response( + body='{"id":1,"name":"alice"}', json_data=json_data + )) + action = Action(action_type="api_get", selector="http://localhost/api/users/1") + + result = await _run_api_action(api, action) + + assert result["json"] == json_data + assert result["body"] == '{"id":1,"name":"alice"}' + + async def test_non_json_response_json_is_none(self): + """If response.json() raises, result['json'] is None.""" + resp = AsyncMock() + resp.status = 200 + resp.headers = {"content-type": "text/plain"} + resp.text = AsyncMock(return_value="plain text") + resp.json = AsyncMock(side_effect=Exception("not json")) + + api = AsyncMock() + api.get = AsyncMock(return_value=resp) + action = Action(action_type="api_get", selector="http://localhost/api/plain") + + result = await _run_api_action(api, action) + + assert result["json"] is None + assert result["body"] == "plain text" + + async def test_no_body_post_sends_no_kwargs(self): + """POST with no value sends no body kwargs.""" + api = AsyncMock() + api.post = AsyncMock(return_value=_make_mock_api_response(status=201)) + action = Action(action_type="api_post", selector="http://localhost/api/ping") + + await _run_api_action(api, action) + + api.post.assert_awaited_once_with("http://localhost/api/ping") + + +# --------------------------------------------------------------------------- +# Executor._run_api_test +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +class TestRunApiTest: + + async def test_passing_test_returns_pass(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_resp = _make_mock_api_response(status=200) + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=mock_resp) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case( + assertions=[Assertion( + assertion_type="response_status", + expected_value="200", + )] + ) + + result = await executor._run_api_test(context, tc) + + assert result.result == "pass" + assert result.assertions_passed == 1 + assert result.assertions_failed == 0 + + async def test_failing_assertion_returns_fail(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_resp = _make_mock_api_response(status=404) + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=mock_resp) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case( + assertions=[Assertion( + assertion_type="response_status", + expected_value="200", + )] + ) + + result = await executor._run_api_test(context, tc) + + assert result.result == "fail" + assert result.assertions_failed == 1 + assert result.failure_reason is not None + + async def test_step_error_aborts_remaining_steps(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_api = AsyncMock() + mock_api.get = AsyncMock(side_effect=Exception("Connection refused")) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case( + steps=[ + Action(action_type="api_get", selector="http://localhost/api/one"), + Action(action_type="api_get", selector="http://localhost/api/two"), + ], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + + result = await executor._run_api_test(context, tc) + + # When a step aborts and assertions run against empty response (fail), + # result is "fail". "error" only when aborted with zero assertion failures. + assert result.result in ("fail", "error") + skipped = [s for s in result.step_results if s.status == "skip"] + assert len(skipped) == 1 # second step was skipped + + async def test_no_page_is_opened(self, tmp_path): + """API tests must not open a browser page.""" + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=_make_mock_api_response()) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case() + await executor._run_api_test(context, tc) + + context.new_page.assert_not_called() + + async def test_uses_context_request(self, tmp_path): + """Executor uses context.request, not a separate API context.""" + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=_make_mock_api_response()) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case() + await executor._run_api_test(context, tc) + + mock_api.get.assert_awaited_once() + + async def test_precondition_actions_are_executed(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=_make_mock_api_response()) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case( + preconditions=[Action( + action_type="api_get", + selector="http://localhost/api/setup", + )], + steps=[Action( + action_type="api_get", + selector="http://localhost/api/items", + )], + ) + + result = await executor._run_api_test(context, tc) + + assert mock_api.get.await_count == 2 + assert result.result == "pass" + + async def test_multiple_assertions_all_pass(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + json_data = {"items": [{"id": 1}], "total": 1} + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=_make_mock_api_response( + status=200, + body='{"items":[{"id":1}],"total":1}', + json_data=json_data, + )) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case( + assertions=[ + Assertion(assertion_type="response_status", expected_value="200"), + Assertion(assertion_type="response_json_path", selector="total", expected_value="1"), + Assertion(assertion_type="response_body_contains", expected_value="items"), + ], + ) + + result = await executor._run_api_test(context, tc) + + assert result.result == "pass" + assert result.assertions_passed == 3 + assert result.assertions_failed == 0 + + async def test_result_contains_response_url(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=_make_mock_api_response()) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case( + steps=[Action( + action_type="api_get", + selector="http://localhost:8000/api/items", + )] + ) + + result = await executor._run_api_test(context, tc) + assert result.actual_url == "http://localhost:8000/api/items" + + async def test_category_is_preserved_in_result(self, tmp_path): + config = _make_config() + executor = Executor(config, ai_client=None, runs_dir=tmp_path) + + mock_api = AsyncMock() + mock_api.get = AsyncMock(return_value=_make_mock_api_response()) + + context = AsyncMock() + context.request = mock_api + + tc = _make_api_test_case() + result = await executor._run_api_test(context, tc) + + assert result.category == "api" diff --git a/tests/test_models_config.py b/tests/test_models_config.py index 848f2e4..e19f10a 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -149,7 +149,7 @@ def test_minimal_config(self): def test_default_values(self): """Test FrameworkConfig has correct default values.""" config = FrameworkConfig(target_url="https://example.com") - assert config.categories == ["functional", "visual", "security"] + assert config.categories == ["functional", "visual", "security", "api"] assert config.max_tests_per_run == 20 assert config.max_execution_time_seconds == 1800 assert config.max_parallel_contexts == 3 diff --git a/tests/test_planner_schema_validator.py b/tests/test_planner_schema_validator.py index 459b50a..21efac0 100644 --- a/tests/test_planner_schema_validator.py +++ b/tests/test_planner_schema_validator.py @@ -416,3 +416,130 @@ def test_multiple_errors_returned(self): assert any("category" in e.lower() for e in errors) assert any("priority" in e.lower() for e in errors) assert any("action_type" in e.lower() for e in errors) + + +class TestValidateApiTests: + """Tests for API-specific validation rules.""" + + def _make_api_plan(self, test_cases): + return TestPlanModel( + plan_id="plan-api", + generated_at="2025-01-01T00:00:00Z", + target_url="https://example.com", + test_cases=test_cases, + ) + + def test_valid_api_test_passes(self): + tc = TestCaseModel( + test_id="tc_api_001", + name="[GET] List users", + category="api", + steps=[Action( + action_type="api_get", + selector="http://localhost/api/users", + )], + assertions=[Assertion( + assertion_type="response_status", + expected_value="200", + )], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert errors == [] + + def test_all_api_action_types_are_valid(self): + for method in ("api_get", "api_post", "api_put", "api_delete", "api_patch"): + tc = TestCaseModel( + test_id=f"tc_{method}", + name=f"[{method.split('_')[1].upper()}] test", + category="api", + steps=[Action(action_type=method, selector="http://localhost/api/x")], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert not any("invalid action_type" in e.lower() for e in errors), \ + f"{method} should be a valid action type" + + def test_all_api_assertion_types_are_valid(self): + for atype in ("response_status", "response_body_contains", "response_json_path", "response_header"): + tc = TestCaseModel( + test_id=f"tc_{atype}", + name="[GET] test", + category="api", + steps=[Action(action_type="api_get", selector="http://localhost/api/x")], + assertions=[Assertion(assertion_type=atype, selector="key", expected_value="val")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert not any("invalid" in e.lower() and "assertion" in e.lower() for e in errors), \ + f"{atype} should be a valid assertion type" + + def test_api_action_without_url_fails(self): + tc = TestCaseModel( + test_id="tc_no_url", + name="[GET] missing url", + category="api", + steps=[Action(action_type="api_get", selector=None)], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert any("requires a url" in e.lower() for e in errors) + + def test_api_test_name_without_bracket_format_fails(self): + tc = TestCaseModel( + test_id="tc_bad_name", + name="Get all users", # missing [METHOD] prefix + category="api", + steps=[Action(action_type="api_get", selector="http://localhost/api/users")], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert any("[method]" in e.lower() for e in errors) + + def test_api_test_with_correct_name_format_passes(self): + for name in ["[GET] List items", "[POST] Create user", "[DELETE] Remove item", "[PATCH] Update"]: + tc = TestCaseModel( + test_id=f"tc_{name[:5]}", + name=name, + category="api", + steps=[Action(action_type="api_get", selector="http://localhost/api/x")], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert not any("[method]" in e.lower() for e in errors), \ + f"Name '{name}' should pass format check" + + def test_api_test_with_browser_assertion_fails(self): + tc = TestCaseModel( + test_id="tc_bad_assert", + name="[GET] bad assertion", + category="api", + steps=[Action(action_type="api_get", selector="http://localhost/api/x")], + assertions=[Assertion(assertion_type="element_visible", selector=".foo")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert any("browser assertion" in e.lower() for e in errors) + + def test_api_test_actions_in_preconditions_only_no_step_error(self): + """API test with actions only in preconditions should not raise 'no steps defined'.""" + tc = TestCaseModel( + test_id="tc_precond_only", + name="[GET] preconditions only", + category="api", + preconditions=[Action(action_type="api_get", selector="http://localhost/api/x")], + steps=[], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + assert not any("no steps" in e.lower() for e in errors) + + def test_api_category_detected_via_action_type(self): + """A test with api_* actions but category='functional' is still treated as API for validation.""" + tc = TestCaseModel( + test_id="tc_mislabeled", + name="Get users", # wrong format — should be flagged + category="functional", + steps=[Action(action_type="api_get", selector="http://localhost/api/users")], + assertions=[Assertion(assertion_type="response_status", expected_value="200")], + ) + errors = validate_test_plan(self._make_api_plan([tc])) + # Should flag the name format issue because it detected api actions + assert any("[method]" in e.lower() for e in errors) From 0c381dea6a1020f8173b746d23160d716ac3c499 Mon Sep 17 00:00:00 2001 From: Alexis Moreno Date: Wed, 25 Feb 2026 12:09:01 -0300 Subject: [PATCH 4/4] fixed strings and readme --- README.md | 2 +- src/ai/prompts/planning.py | 2 +- src/planner/planner.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0ab3b42..998757f 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ open qa-reports/report_*.html - **Functional tests** - Forms, navigation, workflows, CRUD - **Visual regression** - Screenshot baselines, responsive design - **Security checks** - XSS, HTTPS, cookies, headers -- **API tests** - Direct HTTP calls against observed endpoints, with JSON path and status assertions +- **API tests** - Direct HTTP calls against observed endpoints of the configured backend, with JSON path and status assertions - **Evidence collection** - Screenshots, logs, network activity **→ [See all features in detail](./OVERVIEW.md#key-features)** diff --git a/src/ai/prompts/planning.py b/src/ai/prompts/planning.py index 53fb07c..6c73c4f 100644 --- a/src/ai/prompts/planning.py +++ b/src/ai/prompts/planning.py @@ -59,7 +59,7 @@ 2. **Visual tests:** Use screenshot_diff assertions to compare against baselines. IMPORTANT: Always add a wait step of at least 2000ms before screenshot assertions to allow fonts, images, and animations to fully load. Use element_visible assertions to verify key elements are present. Test responsive behavior across viewports. For screenshot_diff assertions, set tolerance to null (uses default 0.05). 3. **Security tests:** Inject XSS payloads into form fields and verify sanitization. Check HTTPS enforcement, cookie security attributes, open redirect vectors, and error page information leakage. 4. **API tests:** When the site model includes an `api_endpoints` array, generate direct HTTP tests using the API action types (`api_get`, `api_post`, `api_put`, `api_delete`, `api_patch`). These tests make real HTTP calls — they do NOT open a browser page or use visual assertions. - - **CRITICAL: Any test that uses `api_get`, `api_post`, `api_put`, `api_delete`, or `api_patch` action types MUST have `category: "api"`. Never assign these tests `category: "functional"` or any other category. A test that fires an HTTP action but is tagged `functional` will be skipped at runtime.** + - **CRITICAL: Any test that uses `api_get`, `api_post`, `api_put`, `api_delete`, or `api_patch` action types MUST have `category: "api"`** - **Name format:** Always use `[METHOD] ` (e.g. `[GET] List products`, `[POST] Create order`, `[DELETE] Remove user`). - **Description format:** Always describe the HTTP method, the full endpoint URL, and what is being asserted (e.g. `"Sends a POST request to /api/orders with a valid payload and verifies the response returns status 201 with the created order ID"`). - Put the full endpoint URL in the `selector` field of each action. diff --git a/src/planner/planner.py b/src/planner/planner.py index 23685a8..9ef7827 100644 --- a/src/planner/planner.py +++ b/src/planner/planner.py @@ -56,8 +56,7 @@ def generate_plan( # Build config summary config_summary = ( - f"Categories (ONLY generate tests for these categories — do not generate any others): " - f"{', '.join(self.config.categories)}\n" + f"Categories: {self.config.categories}\n" f"Max tests: {self.config.max_tests_per_run}\n" f"Visual diff tolerance: {self.config.visual_diff_tolerance}\n" f"Viewports: {json.dumps([v.model_dump() for v in self.config.viewports])}\n" @@ -191,7 +190,6 @@ def _parse_plan(self, data: dict, site_model: SiteModel) -> TestPlan: category = tc_data.get("category", "functional") # Auto-correct: if any action is an API type, force category to "api" - # regardless of what the AI returned — mistagging causes runtime skips. _api_action_types = {"api_get", "api_post", "api_put", "api_delete", "api_patch"} all_actions = preconditions + steps if any(a.action_type in _api_action_types for a in all_actions) and category != "api": @@ -319,7 +317,7 @@ def _generate_fallback_plan(self, site_model: SiteModel) -> TestPlan: )], )) - # API fallback tests — one GET per captured endpoint + # API fallback tests — one call per captured endpoint if "api" in self.config.categories: endpoints = site_model.api_endpoints if self.config.backend_url: