Adding API as a test category

README.md

@@ -29,7 +29,8 @@ python -m src.cli run
 
 - **Zero test scripts** - AI generates tests by understanding your site
 - **Self-healing** - When selectors break, AI analyzes screenshots and fixes them
-- **Comprehensive coverage** - Functional, visual, and security testing in one pass
+- **Comprehensive coverage** - Functional, visual, security, and API testing in one pass
+- **API testing** - Real HTTP calls via Playwright's request context, not a headless browser
 - **Natural language hints** - Guide priorities without writing test specs
 - **Coverage memory** - Tracks what's been tested, focuses on gaps
 
@@ -53,7 +54,7 @@ Create `qa-config.json`:
 ```json
 {
   "target_url": "https://yoursite.com",
-  "categories": ["functional", "visual", "security"],
+  "categories": ["functional", "visual", "security", "api"],
   "hints": [
     "The checkout flow is our most critical path"
   ]
@@ -118,6 +119,7 @@ open qa-reports/report_*.html
 - **Functional tests** - Forms, navigation, workflows, CRUD
 - **Visual regression** - Screenshot baselines, responsive design
 - **Security checks** - XSS, HTTPS, cookies, headers
+- **API tests** - Direct HTTP calls against observed endpoints of the configured backend, with JSON path and status assertions
 - **Evidence collection** - Screenshots, logs, network activity
 
 **→ [See all features in detail](./OVERVIEW.md#key-features)**
@@ -162,6 +164,23 @@ open qa-reports/report_*.html
 
 Hints guide AI priorities without writing test specifications. The AI interprets them and adjusts test generation accordingly.
 
+### With API Testing
+
+```json
+{
+  "target_url": "https://yoursite.com",
+  "backend_url": "https://api.yoursite.com",
+  "categories": ["functional", "api"]
+}
+```
+
+The crawler captures every XHR/fetch request made by the browser during crawling. When `"api"` is in `categories`, the AI generates direct HTTP tests for those observed endpoints — no browser page is opened, requests go through Playwright's `APIRequestContext` and share the authenticated session automatically.
+
+- **`backend_url`** (optional) — when set, only endpoints whose URL starts with this value are sent to the AI. Use this to focus on your own backend and exclude third-party calls (analytics, CDN, etc.).
+- **Test names** follow the format `[METHOD] description` — e.g. `[GET] List products`, `[POST] Create order`.
+- **Supported assertions:** `response_status`, `response_json_path`, `response_body_contains`, `response_header`.
+- **No browser assertions** (`element_visible`, `screenshot_diff`, etc.) are allowed in API tests.
+
 **→ [Complete configuration reference](./REQUIREMENTS.md#configuration)**
 
 ## CLI Commands
@@ -303,6 +322,7 @@ open qa-reports/report_*.html
 - XSS vulnerability in product review form → Flagged
 - Visual regression: Logo alignment shifted → Screenshot diff
 - Checkout flow: 100% passing
+- `[POST] /api/orders` returns 500 on valid payload → Flagged
 
 **→ [See full example walkthrough](./OVERVIEW.md#real-world-example)**
 

qa-config.json.example

@@ -1,5 +1,6 @@
 {
   "target_url": "https://example.com/",
+  "backend_url": null,
   "auth": null,
   "crawl": {
     "target_url": "https://example.com/",
@@ -20,7 +21,8 @@
   "categories": [
     "functional",
     "visual",
-    "security"
+    "security",
+    "api"
   ],
   "max_tests_per_run": 30,
   "max_execution_time_seconds": 1800,

src/ai/prompts/planning.py

@@ -21,26 +21,26 @@
   "test_cases": [
     {
       "test_id": "string (unique ID like tc_001)",
-      "name": "string (human-readable name)",
-      "description": "string (what this test verifies)",
-      "category": "functional | visual | security",
+      "name": "string (human-readable name; for api category use format '[METHOD] <short description>', e.g. '[GET] Fetch user profile')",
+      "description": "string (what this test verifies; for api category include the HTTP method, full endpoint URL, and what the test asserts, e.g. 'Sends a GET request to /api/users and verifies the response returns status 200 with a JSON array')",
+      "category": "functional | visual | security | api",
       "priority": 1-5,
       "target_page_id": "string (page_id from Site Model)",
       "coverage_signature": "string (abstract description for registry matching)",
       "requires_auth": true,
       "preconditions": [
         {
-          "action_type": "navigate | click | fill | select | hover | scroll | wait | screenshot | keyboard",
-          "selector": "string or null",
-          "value": "string or null",
+          "action_type": "navigate | click | fill | select | hover | scroll | wait | screenshot | keyboard | api_get | api_post | api_put | api_delete | api_patch",
+          "selector": "string or null  (for API actions: the full endpoint URL goes here)",
+          "value": "string or null     (for api_post/put/patch: JSON-encoded request body)",
           "description": "string"
         }
       ],
-      "steps": [ (same Action schema as preconditions) ],
+      "steps": [ "(same Action schema as preconditions)" ],
       "assertions": [
         {
-          "assertion_type": "element_visible | element_hidden | text_contains | text_equals | text_matches | url_matches | screenshot_diff | element_count | network_request_made | no_console_errors | response_status | ai_evaluate | page_title_contains | page_loaded",
-          "selector": "string or null",
+          "assertion_type": "element_visible | element_hidden | text_contains | text_equals | text_matches | url_matches | screenshot_diff | element_count | network_request_made | no_console_errors | response_status | ai_evaluate | page_title_contains | page_loaded | response_body_contains | response_json_path | response_header",
+          "selector": "string or null  (for response_json_path: dot-notation path e.g. 'data.id'; for response_header: header name)",
           "expected_value": "string or null",
           "tolerance": "float or null",
           "description": "string"
@@ -58,10 +58,22 @@
 1. **Functional tests:** Test form submissions (valid and invalid data), navigation, CRUD operations, search/filter, pagination, modals, multi-step workflows, and auth flows.
 2. **Visual tests:** Use screenshot_diff assertions to compare against baselines. IMPORTANT: Always add a wait step of at least 2000ms before screenshot assertions to allow fonts, images, and animations to fully load. Use element_visible assertions to verify key elements are present. Test responsive behavior across viewports. For screenshot_diff assertions, set tolerance to null (uses default 0.05).
 3. **Security tests:** Inject XSS payloads into form fields and verify sanitization. Check HTTPS enforcement, cookie security attributes, open redirect vectors, and error page information leakage.
+4. **API tests:** When the site model includes an `api_endpoints` array, generate direct HTTP tests using the API action types (`api_get`, `api_post`, `api_put`, `api_delete`, `api_patch`). These tests make real HTTP calls — they do NOT open a browser page or use visual assertions.
+   - **CRITICAL: Any test that uses `api_get`, `api_post`, `api_put`, `api_delete`, or `api_patch` action types MUST have `category: "api"`**
+   - **Name format:** Always use `[METHOD] <short description>` (e.g. `[GET] List products`, `[POST] Create order`, `[DELETE] Remove user`).
+   - **Description format:** Always describe the HTTP method, the full endpoint URL, and what is being asserted (e.g. `"Sends a POST request to /api/orders with a valid payload and verifies the response returns status 201 with the created order ID"`).
+   - Put the full endpoint URL in the `selector` field of each action.
+   - For POST/PUT/PATCH, put the JSON-encoded request body in the `value` field.
+   - Use `response_status` assertions to verify the HTTP status code (expected_value = status code as a string, e.g. `"200"`).
+   - Use `response_json_path` to assert values in the JSON body (selector = dot-notation path like `"data.id"`, expected_value = expected substring).
+   - Use `response_body_contains` to assert a substring appears anywhere in the response body.
+   - Use `response_header` to assert a response header is present (selector = header name, expected_value = expected substring in the value).
+   - Focus on endpoints observed during the crawl — do not invent endpoints that are not in the site model.
+   - Do NOT use `screenshot_diff`, `element_visible`, `page_loaded`, or other browser assertions in API tests.
 4. **Prioritization:** Forms and interactive elements get higher priority. Static pages get lower priority. Recently failed areas get highest priority.
 5. **Selectors:** Prefer data-testid attributes, then ARIA roles/labels, then stable CSS selectors. Avoid fragile positional selectors.
 6. **Test data:** Generate realistic test data for form fills. Use invalid data for negative tests (empty required fields, malformed emails, XSS payloads for security). When a field needs a unique value (e.g., usernames, IDs, vault names), use the dynamic variable `{{$timestamp}}` in the value string (e.g., `"testuser-{{$timestamp}}"`) — it will be replaced with a Unix epoch timestamp at runtime to ensure uniqueness.
-7. **Budget:** Respect the max_tests limit. Allocate budget proportionally: ~50% functional, ~30% visual, ~20% security (adjustable by hints).
+7. **Budget:** Respect the max_tests limit. ONLY generate tests for the categories listed in the Configuration section — never generate tests for any other category. Allocate the test budget proportionally across those categories (adjustable by hints). Example split when all four are enabled: ~40% functional, ~20% visual, ~20% security, ~20% api.
 8. **Assertion robustness:** Prefer behavioral/structural assertions over text matching. This is critical for reliable tests.
    - After form submissions: assert URL changed (url_matches), form disappeared (element_hidden), or new UI appeared (element_visible). Do NOT assert for specific success/error text you have not observed on the site.
    - For login flows: assert URL navigated away from the login page, or a logout/profile element appeared, rather than checking for "success" or "welcome" text.

src/crawler/crawler.py

@@ -758,6 +758,8 @@ def _attach_network_listener(
         self, page: Page, nr_list: list[NetworkRequest]
     ) -> None:
         """Attach a network response listener to a page."""
+        backend_url = self.config.backend_url
+
         async def on_response(response):
             try:
                 req = response.request
@@ -769,13 +771,17 @@ async def on_response(response):
                     content_type=response.headers.get("content-type", ""),
                 ))
                 if req.resource_type in ("xhr", "fetch"):
+                    is_own = bool(
+                        backend_url and req.url.startswith(backend_url)
+                    )
                     key = f"{req.method}:{urlparse(req.url).path}"
                     if key not in self._api_endpoints:
                         self._api_endpoints[key] = APIEndpoint(
                             url=req.url,
                             method=req.method,
                             response_content_type=response.headers.get("content-type"),
                             status_codes_seen=[response.status],
+                            is_own_backend=is_own,
                         )
                     else:
                         ep = self._api_endpoints[key]

src/executor/assertion_checker.py

@@ -6,6 +6,7 @@
 import logging
 import re
 from pathlib import Path
+from typing import Any
 
 from playwright.async_api import Page
 
@@ -398,6 +399,122 @@ def _check_response_status(assertion: Assertion, network_log: list[dict] | None)
     return AssertionResult(False, f"No response with status {expected}")
 
 
+# ---------------------------------------------------------------------------
+# API assertion checker (no browser page — operates on HTTP response data)
+# ---------------------------------------------------------------------------
+
+def check_api_assertion(assertion: Assertion, response_data: dict[str, Any]) -> AssertionResult:
+    """Evaluate an assertion against HTTP response data from an API call.
+
+    response_data keys:
+        status  (int)   — HTTP status code
+        headers (dict)  — response headers (lowercased keys)
+        body    (str)   — raw response text
+        json    (Any)   — parsed JSON body, or None
+        url     (str)   — request URL
+    """
+    try:
+        match assertion.assertion_type:
+            case "response_status":
+                return _check_api_status(assertion, response_data)
+            case "response_body_contains":
+                return _check_api_body_contains(assertion, response_data)
+            case "response_json_path":
+                return _check_api_json_path(assertion, response_data)
+            case "response_header":
+                return _check_api_header(assertion, response_data)
+            case _:
+                return AssertionResult(False, f"Unknown API assertion type: {assertion.assertion_type}")
+    except Exception as e:
+        logger.debug("API assertion error: %s — %s", assertion.assertion_type, e)
+        return AssertionResult(False, f"API assertion error: {e}")
+
+
+def _check_api_status(assertion: Assertion, response_data: dict) -> AssertionResult:
+    if not assertion.expected_value:
+        return AssertionResult(False, "No expected status code in expected_value")
+    try:
+        expected = int(assertion.expected_value)
+    except ValueError:
+        return AssertionResult(False, f"expected_value '{assertion.expected_value}' is not a valid status code")
+    actual = response_data.get("status", 0)
+    if actual == expected:
+        return AssertionResult(True, f"Status {actual} matches expected {expected}")
+    return AssertionResult(False, f"Expected status {expected}, got {actual}")
+
+
+def _check_api_body_contains(assertion: Assertion, response_data: dict) -> AssertionResult:
+    if not assertion.expected_value:
+        return AssertionResult(False, "No expected_value for response_body_contains")
+    body = response_data.get("body", "")
+    if assertion.expected_value.lower() in body.lower():
+        return AssertionResult(True, f"Response body contains '{assertion.expected_value}'")
+    return AssertionResult(False, f"Response body does not contain '{assertion.expected_value}'")
+
+
+def _check_api_json_path(assertion: Assertion, response_data: dict) -> AssertionResult:
+    """Traverse a dot-notation path in the JSON response body.
+
+    selector      — dot-notation path, e.g. ``data.user.name`` or ``items.0.id``
+    expected_value — expected string value at that path (substring match);
+                     omit to just assert the path exists
+    """
+    if not assertion.selector:
+        return AssertionResult(False, "No JSON path provided in selector field (e.g. 'data.id')")
+    json_body = response_data.get("json")
+    if json_body is None:
+        return AssertionResult(False, "Response body is not valid JSON")
+
+    current: Any = json_body
+    for part in assertion.selector.split("."):
+        if isinstance(current, dict):
+            current = current.get(part)
+        elif isinstance(current, list):
+            try:
+                current = current[int(part)]
+            except (ValueError, IndexError):
+                return AssertionResult(False, f"Path '{assertion.selector}': index '{part}' out of range")
+        else:
+            return AssertionResult(False, f"Path '{assertion.selector}': cannot traverse into {type(current).__name__}")
+        if current is None:
+            return AssertionResult(False, f"Path '{assertion.selector}' is null or missing")
+
+    actual = str(current)
+    if not assertion.expected_value:
+        return AssertionResult(True, f"Path '{assertion.selector}' exists (value: '{actual[:80]}')")
+    if assertion.expected_value.lower() in actual.lower():
+        return AssertionResult(True, f"Path '{assertion.selector}' = '{actual[:80]}'")
+    return AssertionResult(
+        False,
+        f"Path '{assertion.selector}': expected '{assertion.expected_value}', got '{actual[:80]}'",
+    )
+
+
+def _check_api_header(assertion: Assertion, response_data: dict) -> AssertionResult:
+    """Assert a response header is present and optionally matches a value.
+
+    selector      — header name (case-insensitive)
+    expected_value — expected substring in the header value (optional)
+    """
+    if not assertion.selector:
+        return AssertionResult(False, "No header name provided in selector field")
+    headers: dict = response_data.get("headers", {})
+    header_val = next(
+        (v for k, v in headers.items() if k.lower() == assertion.selector.lower()),
+        None,
+    )
+    if header_val is None:
+        return AssertionResult(False, f"Header '{assertion.selector}' not found in response")
+    if not assertion.expected_value:
+        return AssertionResult(True, f"Header '{assertion.selector}' present: '{header_val}'")
+    if assertion.expected_value.lower() in header_val.lower():
+        return AssertionResult(True, f"Header '{assertion.selector}': '{header_val}'")
+    return AssertionResult(
+        False,
+        f"Header '{assertion.selector}': expected '{assertion.expected_value}', got '{header_val}'",
+    )
+
+
 async def _check_ai_evaluate(
     page: Page, assertion: Assertion, evidence_dir: Path, ai_client: AIClient | None
 ) -> AssertionResult: