From 9a131a006afca91a2478b3abf679e647ed8cb96b Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Tue, 14 Jul 2026 10:55:27 +0530 Subject: [PATCH 1/6] uts: align layout with ably-java and add integration/proxy infra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the UTS target to mirror ably-java's uts module (tier-then-module): Harness/ -> infra/unit/, Tests//unit/ -> unit//. Add the integration tier's infrastructure, mirroring ably-java's infra/integration: - SandboxApp: provisions/deletes throwaway sandbox apps from ably-common's test-app-setup.json - proxy/ProxyManager (macOS-only): syncs (downloads, SHA-256-verifies, caches at ~/.cache/uts-proxy — shared with ably-java) and launches the pinned uts-proxy release; UTS_PROXY_LOCAL_PATH override; atexit reaper - proxy/ProxySession: typed client for the proxy control REST API (rules, imperative actions, typed event log) + connectThroughProxy client wiring - infra/Utils.swift: shared async pollUntil/awaitState/awaitChannelState and the URLSession plumbing the integration infra is built on Bring the unit infra to helper parity with ably-java: .connectedMessage, respondWithSuccess(_:), respondWithDelay, PendingHTTPConnection.queryParams, pendingScheduledCount, dummy-key seeding in makeRealtime/makeRest, and a configurable MockTimeProvider clock origin. Add env-gated (UTS_INTEGRATION_SMOKE) acceptance tests for both integration tiers: IntegrationSmokeTest (direct sandbox, run per protocol variant: JSON + msgpack) and ProxyInfraSmokeTests (full proxy chain, JSON-only with token auth, as the proxy requires). Rewrite Test/UTS/README.md as a comprehensive human-readable guide mirroring ably-java's uts/README.md (tier walkthroughs, request-flow diagrams, per-file reference, java-to-cocoa helper map) and sync the uts-to-swift skill; update spec links for the helpers/proxy.md -> uts/docs/proxy.md move. --- .claude/skills/uts-to-swift/SKILL.md | 54 +- Test/UTS/README.md | 925 +++++++++++++++++- Test/UTS/infra/Utils.swift | 119 +++ Test/UTS/infra/integration/SandboxApp.swift | 100 ++ .../integration/proxy/ProxyManager.swift | 270 +++++ .../integration/proxy/ProxySession.swift | 296 ++++++ .../{Harness => infra/unit}/Captured.swift | 0 .../unit}/CapturingLog.swift | 0 .../unit}/MockHTTPClient.swift | 28 +- .../unit}/MockTimeProvider.swift | 17 +- .../unit}/MockWebSocket.swift | 16 +- .../unit}/NoOpReachability.swift | 0 .../unit}/ProtocolMessage.swift | 7 + .../{Harness => infra/unit}/UTSTestCase.swift | 10 +- .../proxy/ProxyInfraSmokeTests.swift | 70 ++ .../standard/IntegrationSmokeTest.swift | 66 ++ .../realtime}/ConnectionRecoveryTests.swift | 2 +- .../rest/unit => unit/rest}/TimeTests.swift | 0 18 files changed, 1914 insertions(+), 66 deletions(-) create mode 100644 Test/UTS/infra/Utils.swift create mode 100644 Test/UTS/infra/integration/SandboxApp.swift create mode 100644 Test/UTS/infra/integration/proxy/ProxyManager.swift create mode 100644 Test/UTS/infra/integration/proxy/ProxySession.swift rename Test/UTS/{Harness => infra/unit}/Captured.swift (100%) rename Test/UTS/{Harness => infra/unit}/CapturingLog.swift (100%) rename Test/UTS/{Harness => infra/unit}/MockHTTPClient.swift (81%) rename Test/UTS/{Harness => infra/unit}/MockTimeProvider.swift (91%) rename Test/UTS/{Harness => infra/unit}/MockWebSocket.swift (96%) rename Test/UTS/{Harness => infra/unit}/NoOpReachability.swift (100%) rename Test/UTS/{Harness => infra/unit}/ProtocolMessage.swift (89%) rename Test/UTS/{Harness => infra/unit}/UTSTestCase.swift (93%) create mode 100644 Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift create mode 100644 Test/UTS/integration/standard/IntegrationSmokeTest.swift rename Test/UTS/{Tests/realtime/unit/connection => unit/realtime}/ConnectionRecoveryTests.swift (99%) rename Test/UTS/{Tests/rest/unit => unit/rest}/TimeTests.swift (100%) diff --git a/.claude/skills/uts-to-swift/SKILL.md b/.claude/skills/uts-to-swift/SKILL.md index 92a119978..2819e3e04 100644 --- a/.claude/skills/uts-to-swift/SKILL.md +++ b/.claude/skills/uts-to-swift/SKILL.md @@ -67,12 +67,15 @@ Read the file at `$ARGUMENTS`. Identify all the test cases — each has a title, Map the spec path to a test path: -Mirror the spec's `uts/` directory layout under `Test/UTS/Tests/`: keep the same `rest`/`realtime` top-level folder and the `unit`/`integration` tier below it. This stops files that share a name in different parts of the spec from colliding (e.g. `uts/realtime/integration/auth.md` and `uts/rest/integration/auth.md`). +Tests are organised **by tier, then by module** under `Test/UTS/` (kept in sync with ably-java's `uts` module — see `Test/UTS/README.md` §4): `unit//` for mocked-transport tests, `integration/standard//` and `integration/proxy//` for real-backend tests. The per-module folder keeps same-named specs from different parts of the spec from colliding (e.g. `uts/realtime/integration/auth.md` and `uts/rest/integration/auth.md`). | Spec file | Test file | |---|---| -| `.../uts/rest/unit/.md` | `Test/UTS/Tests/rest/unit/Tests.swift` | -| `.../uts/realtime/unit//.md` | `Test/UTS/Tests/realtime/unit//Tests.swift` | +| `.../uts/rest/unit/.md` | `Test/UTS/unit/rest/Tests.swift` | +| `.../uts/realtime/unit//.md` | `Test/UTS/unit/realtime/Tests.swift` | +| `.../uts/rest/integration/.md` | `Test/UTS/integration/standard/rest/Tests.swift` | +| `.../uts/realtime/integration/.md` | `Test/UTS/integration/standard/realtime/Tests.swift` | +| `.../uts/realtime/integration/proxy/.md` | `Test/UTS/integration/proxy/realtime/Tests.swift` | Class name: take the file name, strip a trailing `_test`, convert `snake_case` → `PascalCase`, append `Tests`. Example: `connection_recovery_test.md` → `ConnectionRecoveryTests`. Each test is a **Swift Testing** suite — `@Suite(.serialized) final class Tests: UTSTestCase`. @@ -80,9 +83,16 @@ If a suitable suite already exists, add the new test methods to it rather than c --- -## Step 3 — Read the harness +## Step 3 — Read the UTS infra -Read ALL files in `Test/UTS/Harness/` before generating any code (you need the exact method names/signatures). +Read ALL files in `Test/UTS/infra/unit/` before generating any code (you need the exact method names/signatures). For an **integration** spec, additionally read `Test/UTS/infra/Utils.swift` (async `pollUntil` / `awaitState` / `awaitChannelState`) and ALL files in `Test/UTS/infra/integration/` (`SandboxApp`, and for proxy specs `ProxyManager`/`ProxySession`), plus the reference smoke tests under `Test/UTS/integration/` and the guide's §11 in `Test/UTS/README.md` — especially the per-tier walkthroughs (§11.4–11.5) and the request-flow diagram (§11.6). + +Integration-tier rules that differ from the unit tier: + +- Clients are built with a **real** transport (plain `ARTClientOptions` + `ARTRealtime`/`ARTRest`, no mocks): direct-sandbox tests point `realtimeHost`/`restHost` at `SandboxApp.sandboxHost` (TLS stays on, plain key auth works — `IntegrationSmokeTest.swift` is the shape); proxy tests call `options.connectThroughProxy(session)` and MUST authenticate via an `authCallback` that signs a `TokenRequest` locally (basic auth is TLS-only, RSA1 — `ProxyInfraSmokeTests.swift` is the shape). +- Proxy suites call `try await ProxyManager.shared.ensureProxy()` in setup and always `await session.close()` in teardown; proxy files are wrapped in `#if os(macOS)`. +- Wait on real network/proxy state with `await awaitState(client, .connected)` / `await awaitChannelState(…)` / `await pollUntil("…") { … }` — never a fixed sleep. +- **Protocol variants**: when a direct-sandbox spec declares the `PROTOCOL` dimension (json/msgpack), parameterise the test — `@Test(arguments: [false, true])` over a `useBinaryProtocol: Bool` parameter, applied via `options.useBinaryProtocol` (see `IntegrationSmokeTest.swift`). Proxy tests are **always JSON** (the proxy can't inspect binary frames) — `connectThroughProxy` already forces it, so no parameterisation there. --- @@ -90,9 +100,22 @@ Read ALL files in `Test/UTS/Harness/` before generating any code (you need the e Apply the translation rules below, then write the file. +### Accessing SDK internals (`import Ably.Private`) + +The SDK is Objective-C, so Swift access levels (`internal`/`package`/`private`) don't apply to it — visibility is controlled by **headers + the module map**: + +- `import Ably` → the public API (headers in `Source/include/Ably/`). +- `import Ably.Private` → the internal API: the private headers listed in the `explicit module Private` block of `Source/include/module.modulemap` (files under `Source/PrivateHeaders/Ably/`, e.g. `ARTClientOptions+TestConfiguration.h` for `testOptions`, `ART*+Private.h` for class internals). This is how the UTS infra reaches the injection seams, and how a test reaches internal fields the spec asserts on. See `Test/UTS/README.md` §4–§5. + +When a spec needs an internal class/method/field, work down this list: + +1. **Check it's already exposed**: `grep -r "" Source/PrivateHeaders/Ably/` — if it's declared in a listed private header, just `import Ably.Private` and use it. +2. **Declared only in a `.m` file** (class extension, ivar, private method)? It is invisible to Swift, period. To expose it, declare it in a header under `Source/PrivateHeaders/Ably/` and register that header in **both** module maps (`Source/include/module.modulemap` for SPM and `Source/Ably.modulemap` for Xcode) — the repo's CLAUDE.md convention. Only do this for small, test-motivated exposure; mirror how existing `+Private.h` headers are written. +3. **Truly private state with no reasonable seam** (or exposing it would distort the SDK)? Don't hack around it — keep the spec's line as a comment, note why no assertion is emitted (see "Comments and assertion fidelity"), and record it in `deviations.md` under **Mock Infrastructure Limitations**. + ### Client construction -Set `ClientOptions` fields (`key`, `autoConnect`, `recover`, `disconnectedRetryTimeout`, etc.) in the `makeRealtime`/`makeRest` configuration block. Set `options.key` to "app.key:secret". +Set `ClientOptions` fields (`key`, `autoConnect`, `recover`, `disconnectedRetryTimeout`, etc.) in the `makeRealtime`/`makeRest` configuration block. Both factories already seed the dummy key `appId.keyId:keySecret` (matching ably-java's `ClientOptionsBuilder`), so only set `options.key` when the spec pseudocode sets one — then use the spec's value. | Pseudocode | Swift | |---|---| @@ -126,7 +149,7 @@ AWAIT_STATE client.connection.state == ConnectionState.connected ws_connection = mock_ws.events.find(e => e.type == CONNECTION_SUCCESS).connection ``` -Swift (the harness exposes `activeConnection` as the cocoa equivalent of the spec's `events.find(CONNECTION_SUCCESS).connection`): +Swift (the UTS infra exposes `activeConnection` as the cocoa equivalent of the spec's `events.find(CONNECTION_SUCCESS).connection`): ```swift let wsProvider = MockWebSocketProvider(onConnectionAttempt: { connection in @@ -215,11 +238,11 @@ let rest = makeRest { $0.key = "app.key:secret" } ### Variable declarations -Take spec variable names (adopted to camel case), for new ones, if needed don't use "noname" names (like "fields"), make the name concrete. +Take spec variable names (adapted to camelCase), for new ones, if needed don't use "noname" names (like "fields"), make the name concrete. ### Capturing connection attempts / requests -The target is built in the Swift 6 language mode. Since mock handler closures are`@Sendable` and run on the SDK's queues, a plain `var array` captured into them is a compile error (a data race). Use the harness's thread-safe (lock-guarded) `Captured` instead: +The target is built in the Swift 6 language mode. Since mock handler closures are`@Sendable` and run on the SDK's queues, a plain `var array` captured into them is a compile error (a data race). Use the UTS infra's thread-safe (lock-guarded) `Captured` instead: Spec pseudocode: @@ -254,11 +277,11 @@ let wsProvider = MockWebSocketProvider(onConnectionAttempt: { connection in #expect(capturedConnectionAttempts[0].queryParams["recover"] == "...") ``` -`Captured` exposes `append`, `all`, `count`, `first`, and `subscript(Int)`. Use `capturedConnectionAttempts.count` to change outcome for different connection attemts. +`Captured` exposes `append`, `all`, `count`, `first`, and `subscript(Int)`. Use `capturedConnectionAttempts.count` to change outcome for different connection attempts. ### Inspecting outgoing frames -The spec inspects client-sent frames through the mock's event timeline (`mock_ws.events.filter(e => e.type == "ws_frame" AND e.direction == "client_to_server")`); the cocoa harness exposes the decoded equivalent as `ws.sentMessages`. Capture after the channel/connection state confirms the send happened: +The spec inspects client-sent frames through the mock's event timeline (`mock_ws.events.filter(e => e.type == "ws_frame" AND e.direction == "client_to_server")`); the cocoa UTS infra exposes the decoded equivalent as `ws.sentMessages`. Capture after the channel/connection state confirms the send happened: Spec pseudocode: @@ -292,11 +315,13 @@ let attachFrames = ws.sentMessages.filter { $0.action == .attach && $0.channel = | Pseudocode | Swift | |---|---| | `conn.respond_with_success()` | `connection.respondWithSuccess()` | +| `conn.respond_with_success(message)` | `connection.respondWithSuccess(message)` — accept + deliver in one call (e.g. `.connectedMessage`) | | `conn.respond_with_refused()` | `connection.respondWithRefused()` | | `mock_ws.send_to_client(msg)` | `connection.sendToClient(msg)` | | `mock_ws.send_to_client_and_close(msg)` | `connection.sendToClientAndClose(msg)` | | `mock_ws.simulate_disconnect()` | `connection.simulateDisconnect()` | | `req.respond_with(200, {...})` | `request.respondWith(status: 200, body: [...])` | +| `req.respond_with_delay(delay, status, body)` | `request.respondWithDelay(delay, status: …, body: …)` | | `req.respond_with_timeout()` | `request.respondWithTimeout()` | | `conn.respond_with_refused/timeout/dns_error()` (HTTP) | `connection.respondWithRefused()` / `respondWithTimeout()` / `respondWithDNSError()` | @@ -311,6 +336,7 @@ Server-to-client messages are described with the `Sendable` `ProtocolMessage` fa | `ProtocolMessage(action: ERROR, error: ErrorInfo(code, statusCode, message))` | `.error(code:, statusCode:, message:)` | | `ProtocolMessage(action: ACK, msgSerial, count)` | `.ack(msgSerial:, count:)` | | `ProtocolMessage(action: CLOSED)` | `.closed()` | +| `CONNECTED_MESSAGE` (ready-made default) | `.connectedMessage` | | `connectionStateTtl: 2000` (wire ms) | seconds here: `connectionStateTtl: 2` | | `ConnectionState.connected` / `ChannelState.attached` | `.connected` / `.attached` (`ARTRealtimeConnectionState` / `ARTRealtimeChannelState`) | @@ -418,7 +444,7 @@ private func awaitTime(_ rest: ARTRest, sourceLocation: SourceLocation = #_sourc } ``` -Put these `async` helpers in the test class extention at the bottom of the test file. Inside the harness, report non-assertion failures (timeouts, unexpected errors) with `Issue.record("...", sourceLocation:)`. +Put these `async` helpers in the test class extension at the bottom of the test file. Inside these helpers, report non-assertion failures (timeouts, unexpected errors) with `Issue.record("...", sourceLocation:)`. ### Test naming and annotation @@ -467,7 +493,7 @@ final class Tests: UTSTestCase { } ``` -Helper methods return types should be ready for use in the test code without additional casting (if you need "value as? String" in the test, move this convertion to the helper instead). Don't return optionals from these methods - assert not `nil` within the method itself, unless test expects optional. Don't create additional helper types for parsing dictionaries - just use a dictionary with the most suited type for the test, accessing its fields by subscript. For dictionaries containing values of different types use `[String: Any]`. Don't wrap dictionary constant initialization into helper method. Put helper methods for the suite into an extension at the bottom of the file. Keep the spec's original tests order in the generated test file. Keep test segmentation by adding comments like "// Setup", "// Test Steps", "// Assertions". +Helper methods return types should be ready for use in the test code without additional casting (if you need "value as? String" in the test, move this conversion to the helper instead). Don't return optionals from these methods - assert not `nil` within the method itself, unless test expects optional. Don't create additional helper types for parsing dictionaries - just use a dictionary with the most suited type for the test, accessing its fields by subscript. For dictionaries containing values of different types use `[String: Any]`. Don't wrap dictionary constant initialization into helper method. Put helper methods for the suite into an extension at the bottom of the file. Keep the spec's original tests order in the generated test file. Keep test segmentation by adding comments like "// Setup", "// Test Steps", "// Assertions". --- @@ -527,7 +553,7 @@ Reproduce with `RUN_DEVIATIONS=1 swift test --filter /`. **Never use the accommodate-both pattern** Every test must assert either spec behaviour or the SDK's actual behaviour — never both at once. -Note: a *harness-driving* difference (e.g. using fake timers where the spec uses real ones, or a queue-ordering workaround) is **not** an SDK deviation — explain it in a code comment, not `deviations.md`. `deviations.md` is only for SDK non-compliance and mock-capability gaps. +Note: an *infra-driving* difference (e.g. using fake timers where the spec uses real ones, or a queue-ordering workaround) is **not** an SDK deviation — explain it in a code comment, not `deviations.md`. `deviations.md` is only for SDK non-compliance and mock-capability gaps. ### Deviations file diff --git a/Test/UTS/README.md b/Test/UTS/README.md index 1cb209bb4..6edf9a858 100644 --- a/Test/UTS/README.md +++ b/Test/UTS/README.md @@ -1,45 +1,904 @@ -# UTS — Universal Test Suite (ably-cocoa) +# UTS in ably-cocoa — A Human-Readable Guide -Tests derived from the language-neutral specs in [`ably/specification`](https://github.com/ably/specification) under [`uts/`](https://github.com/ably/specification/blob/main/uts). -This is a **standalone Swift Testing target** (`UTS`), separate from `AblyTests`, and deliberately does **not** depend on Nimble or XCTest. It is built in the **Swift 6 language mode** (strict concurrency checking) so the compiler catches data races in the harness and tests. +> A practical, end-to-end explanation of the **Universal Test Specification (UTS)** and how it is +> realised in the `ably-cocoa` repository. Written for a developer who has never touched UTS before +> and needs to understand *what it is*, *why it exists*, and *exactly how the Swift code under +> `Test/UTS/` makes the unit tests work* — and what is still to be built. +> +> This guide mirrors the structure of the equivalent document in `ably-java` (`uts/README.md`); the +> directory layout here is kept **in sync with the Kotlin `uts` module** so a developer can move +> between the two SDKs without re-learning the map. -## Layout +--- +## Table of Contents + +1. [Introduction: What is UTS?](#1-introduction-what-is-uts) +2. [The Three Test Tiers](#2-the-three-test-tiers) +3. [The UTS Documents (the source of truth)](#3-the-uts-documents-the-source-of-truth) +4. [The Cocoa Setup: the `UTS` target](#4-the-cocoa-setup-the-uts-target) +5. [How a Test Reaches the SDK: the hook points](#5-how-a-test-reaches-the-sdk-the-hook-points) +6. [Unit-Test Infrastructure (mocked transports)](#6-unit-test-infrastructure-mocked-transports) +7. [Walkthrough: the Realtime Unit Test (`ConnectionRecoveryTests`)](#7-walkthrough-the-realtime-unit-test-connectionrecoverytests) +8. [Walkthrough: the REST Unit Test (`TimeTests`)](#8-walkthrough-the-rest-unit-test-timetests) +9. [Deviations: when the SDK disagrees with the spec](#9-deviations-when-the-sdk-disagrees-with-the-spec) +10. [How to Run the Tests](#10-how-to-run-the-tests) +11. [Integration & Proxy Infrastructure](#11-integration--proxy-infrastructure) +12. [Quick Reference / Cheat-Sheet](#12-quick-reference--cheat-sheet) +13. [Appendix: Per-File API Reference](#13-appendix-per-file-api-reference) + +--- + +## 1. Introduction: What is UTS? + +**UTS (Universal Test Specification)** is Ably's language-neutral catalogue of tests for its client +SDKs. Ably ships many SDKs (JavaScript, Dart, Kotlin/Java, Swift, Go, …), and every one of them must +obey the *same* behavioural contract — the **Ably features spec** +(`specification/specifications/features.md`, whose requirements are tagged `RSC16`, `RTN16g`, +`RTL4f`, etc.). Without a shared test definition, each SDK would re-invent its own tests, drift +apart, and leave gaps. + +UTS fixes this by separating **what to test** from **how to test it in a given language**: + +```text + ┌──────────────────────────────┐ + │ Ably features spec │ ← the ultimate authority (RSC*, RTN*, RTL* …) + │ (features.md) │ + └──────────────┬───────────────┘ + │ distilled into portable test specs + ▼ + ┌──────────────────────────────┐ + │ UTS test specs (.md) │ ← language-neutral pseudocode, one file per feature + │ │ e.g. realtime/unit/connection/connection_recovery_test.md + └──────────────┬───────────────┘ + │ translated ("derived") per SDK + ▼ + ┌──────────────────────────────┐ + │ Derived tests │ ← concrete, runnable tests in the SDK's language + │ (this repo: Swift in │ e.g. ConnectionRecoveryTests.swift + │ Test/UTS/) │ + └──────────────────────────────┘ ``` + +Four concepts you will see constantly: + +| Term | Meaning | +|------|---------| +| **Spec point** | A tagged requirement in the features spec, e.g. `RTN16g`, `RSC16`. Test names embed these. | +| **UTS spec** | A markdown file of portable pseudocode describing the setup, steps, and assertions for one feature. The *source of truth for what to test.* | +| **Derived test** | A faithful translation of a UTS spec into a real test in a specific SDK/language. This is what lives in `Test/UTS/`. | +| **Deviation** | A documented case where the SDK's actual behaviour diverges from the spec. Recorded in `deviations.md`. | + +The golden rule (from +[`writing-derived-tests.md`](https://github.com/ably/specification/blob/main/uts/docs/writing-derived-tests.md)): +**translate the UTS spec faithfully** — same structure, same assertions, same naming — don't +optimise or skip steps. Every derived test carries a `// UTS: ` comment immediately above the +test method, linking it back to its spec (e.g. `// UTS: realtime/unit/RTN16g/recovery-key-structure-0`). + +--- + +## 2. The Three Test Tiers + +UTS divides tests into three tiers by *what infrastructure they need* and *what confidence they give*: + +| Tier | Transport | Backend | Purpose | Status in ably-cocoa | +|------|-----------|---------|---------|----------------------| +| **Unit** | **Mocked** (`MockWebSocket`, `MockHTTPClient`) | none | Client-side logic: state machines, request formation, response parsing, timer behaviour. Fast & deterministic. | ✅ Implemented — `unit/realtime/`, `unit/rest/` | +| **Direct sandbox integration** | Real network | Real Ably sandbox | Happy-path interop: connect, publish, subscribe. No fault injection. | 🚧 Infra ready + smoke-tested (§11) — spec-derived tests TODO | +| **Proxy integration** | Real network **through a programmable proxy** | Real Ably sandbox | Fault behaviour: dropped connections, injected errors, timeouts, re-auth. | 🚧 Infra ready + smoke-tested (§11) — spec-derived tests TODO | + +Each tier folder is organised **by module** (`realtime`, `rest`, …), so a feature's tests sit +together by SDK area — e.g. `unit/realtime/ConnectionRecoveryTests.swift`. + +Key principles (from +[`integration-testing.md`](https://github.com/ably/specification/blob/main/uts/docs/integration-testing.md)): + +- **Integration tests do not replace unit tests.** A spec point covered by a proxy test should + *also* have a unit test. The unit test proves the client logic; the proxy test proves the client + and the real server agree. +- **Proxy tests prefer "late fault injection".** Let the real handshake complete against the real + server, *then* inject the fault as the final interaction. +- **Proxy tests always use JSON** (`useBinaryProtocol = false`) — the proxy can only inspect text + WebSocket frames, not binary msgpack. (The cocoa unit infra *also* forces JSON, so its mock can + decode the frames the SDK sends.) **Direct-sandbox tests**, by contrast, run once per protocol + variant — JSON *and* msgpack — via `@Test(arguments: [false, true])` over `useBinaryProtocol`. + +--- + +## 3. The UTS Documents (the source of truth) + +These documents live in the **specification repo** at +[`uts/docs/`](https://github.com/ably/specification/blob/main/uts/docs/). They are the +policy/authoring guides; the Swift code in this directory is the *implementation* of what they +describe: + +- [`writing-test-specs.md`](https://github.com/ably/specification/blob/main/uts/docs/writing-test-specs.md) + — how to author a portable UTS spec: test types, **test IDs** + (`//-`), the mock pseudocode interfaces + (`MockWebSocket`, `MockHttpClient`, `PendingConnection`, `PendingRequest`), WebSocket closing + semantics (`send_to_client_and_close` for DISCONNECTED / connection-level ERROR vs + `send_to_client` for channel-level ERROR), and anti-flake conventions (no fixed waits — use + `AWAIT_STATE`, polling, and fake timers). +- [`writing-derived-tests.md`](https://github.com/ably/specification/blob/main/uts/docs/writing-derived-tests.md) + — how to translate a spec into a real SDK test, and the **decision tree** when a translated test + fails: spec wrong → fix test + record a UTS spec error; translation wrong → fix test; + SDK non-compliant → gate the spec-correct assertion behind `RUN_DEVIATIONS` and record a + **deviation** (§9). +- [`integration-testing.md`](https://github.com/ably/specification/blob/main/uts/docs/integration-testing.md) + — the policy for the two integration tiers (directory layout, sandbox provisioning, proxy session + lifecycle, late fault injection). Implemented by the §11 infrastructure. +- [`completion-status.md`](https://github.com/ably/specification/blob/main/uts/docs/completion-status.md) + — the coverage matrix mapping every features-spec group to its UTS specs; the tracker for what's + done and what's missing. + +> There is also a fifth, *referenced* spec: +> [`docs/proxy.md`](https://github.com/ably/specification/blob/main/uts/docs/proxy.md) +> — it defines the proxy's control API, rule format, action types, and the **protocol message +> action-number table** (CONNECTED=4, ATTACH=10, AUTH=17, …). `ProxySession` (§11.3) is the Swift +> client for exactly that API. + +The specs currently derived here: + +- `RTN16` (connection recovery) → unit spec + [`connection_recovery_test.md`](https://github.com/ably/specification/blob/main/uts/realtime/unit/connection/connection_recovery_test.md) + → **`unit/realtime/ConnectionRecoveryTests.swift`**. +- `RSC16` (server time) → unit spec + [`time.md`](https://github.com/ably/specification/blob/main/uts/rest/unit/time.md) + → **`unit/rest/TimeTests.swift`**. + +--- + +## 4. The Cocoa Setup: the `UTS` target + +`Test/UTS/` is a **standalone SPM test target** (`UTS` in `Package.swift`), separate from the main +`AblyTests` suite. Its distinguishing choices: + +- **Swift Testing, not XCTest/Nimble.** Suites are `@Suite(.serialized) final class … : UTSTestCase` + and tests are `@Test func …`. Swift Testing creates a **fresh suite instance per `@Test`**, so + each test gets its own clients/mocks and `deinit` is the per-test teardown. +- **Swift 6 language mode** (strict concurrency checking), applied via the + `-swift-version 6` compiler flag in `Package.swift` — the compiler itself catches data races in + the infra and tests. This is why the infra has types like `Captured` (§6.6) instead of plain + captured arrays. +- **`import Ably.Private`** — the SDK is Objective-C; its internal API is exposed to tests through + the `explicit module Private` block in `Source/include/module.modulemap`. This is how the infra + reaches the injection seams in §5. +- **SPM-only.** The target is not part of `Ably.xcodeproj`; SPM discovers the sources automatically, + so adding a test file requires no project-file changes. + +### Directory layout + +The layout is deliberately **in sync with `ably-java`'s Kotlin `uts` module** +(`uts/src/test/kotlin/io/ably/lib/uts/`): **infrastructure** under `infra/` (no `@Test`s), tests +organised **by tier, then by module**. + +```text Test/UTS/ - Harness/ - UTSTestCase.swift # base case: installMock, makeRealtime/makeRest, awaitConnectionState/awaitChannelState, advanceTime - MockWebSocket.swift # MockWebSocketProvider (the spec's mock_ws) + MockWebSocket (fake socket) + factories - MockHTTPClient.swift # MockHTTPClient (fake ARTHTTPExecutor) + PendingHTTPConnection / PendingHTTPRequest - MockTimeProvider.swift # deterministic ARTTimeProvider (enable_fake_timers / ADVANCE_TIME) - Captured.swift # thread-safe collector for the spec's local captured_* arrays (Swift 6 safe) - CapturingLog.swift # ARTLog that records log lines for assertions - NoOpReachability.swift # disables OS network monitoring in unit tests - ProtocolMessage.swift # Sendable server-message factories (.connected/.attached/.error/.ack/.closed) - Tests/ - ConnectionRecoveryTests.swift # RTN16 - TimeTests.swift # RSC16 - deviations.md # SDK deviations found during evaluation -``` - -## How the harness maps the UTS primitives - -| UTS pseudocode | cocoa mapping | +├── README.md # this guide +├── deviations.md # the catalogue of SDK-vs-spec divergences +│ +├── infra/ # ── TEST INFRASTRUCTURE (no @Test methods) ── +│ ├── Utils.swift # shared helpers: async pollUntil / awaitState / +│ │ # awaitChannelState + HTTP support (≈ Kotlin's infra/Utils.kt) +│ │ +│ ├── unit/ # UNIT infra (mocked transports) +│ │ ├── UTSTestCase.swift # base case: installMock, makeRealtime/makeRest, +│ │ │ # awaitConnectionState/awaitChannelState/poll, advanceTime +│ │ │ # (≈ Kotlin's infra/unit/ClientFactories.kt) +│ │ ├── MockWebSocket.swift # MockWebSocketProvider + MockWebSocket + the two factories +│ │ ├── MockHTTPClient.swift # fake ARTHTTPExecutor + PendingHTTPConnection/Request +│ │ ├── MockTimeProvider.swift # virtual clock + virtual timers (deterministic time) +│ │ ├── ProtocolMessage.swift # Sendable server-message factories (.connected/.attached/…) +│ │ ├── Captured.swift # thread-safe captured_* collector (Swift 6 safe) +│ │ ├── CapturingLog.swift # ARTLog recording log lines for assertions +│ │ └── NoOpReachability.swift # disables OS network monitoring in unit tests +│ │ +│ └── integration/ # INTEGRATION infra (real backend) — see §11 +│ ├── SandboxApp.swift # provisions/deletes a sandbox app +│ └── proxy/ # (macOS-only — spawns a local process) +│ ├── ProxyManager.swift # syncs (downloads/caches) + launches the uts-proxy binary +│ └── ProxySession.swift # proxy session: rules, actions, log + connectThroughProxy +│ +├── unit/ # ── UNIT TESTS (mock transport) ── · per module +│ ├── realtime/ +│ │ └── ConnectionRecoveryTests.swift# RTN16* (mocked WS, fake timers) +│ └── rest/ +│ └── TimeTests.swift # RSC16 (mocked HTTP) +│ +└── integration/ # ── INTEGRATION TESTS (real backend) ── · per module + ├── standard/ # direct sandbox: happy-path, no fault injection + │ └── IntegrationSmokeTest.swift # env-gated acceptance test: SandboxApp + real TLS client (JSON + msgpack) + └── proxy/ # sandbox through the fault-injecting uts-proxy + └── ProxyInfraSmokeTests.swift # env-gated acceptance test: ProxyManager + ProxySession +``` + +The mental model (same as ably-java): **`infra/unit/` powers the unit tests, `infra/integration/` +powers both integration kinds (`standard` + `proxy`), and `infra/Utils.swift` serves all of +them.** One deliberate difference from Kotlin: cocoa's `UTSTestCase` (in `infra/unit/`, like +Kotlin's `infra/unit/ClientFactories.kt`) also carries the unit tier's synchronous `AWAIT_STATE` +helpers, while the shared `infra/Utils.swift` provides the async `pollUntil` / `awaitState` / +`awaitChannelState` the integration tier uses for real-network waits (Kotlin keeps all of these +together in `infra/Utils.kt`). + +--- + +## 5. How a Test Reaches the SDK: the hook points + +A test can only mock transports because the SDK exposes **pluggable seams** on +`ARTClientOptions.testOptions` (an `ARTTestClientOptions`, reachable via `import Ably.Private` — +the cocoa analogue of ably-java's `DebugOptions`): + +| Seam | Type | Mock installed there | +|------|------|----------------------| +| `testOptions.transportFactory` | `RealtimeTransportFactory` | `MockWebSocketTransportFactory` (→ `MockWebSocket`) | +| `testOptions.httpExecutor` | `ARTHTTPExecutor` | `MockHTTPClient` | +| `testOptions.timeProvider` | `ARTTimeProvider` | `MockTimeProvider` | +| `testOptions.reachabilityClass` | `ARTReachability` class | `NoOpReachability` | +| `options.logHandler` | `ARTLog` | `CapturingLog` (when a test asserts on log output) | + +So the recipe is: + +- Want to fake the **WebSocket**? Set `transportFactory` — note this still builds a **real** + `ARTWebSocketTransport`, so URL and query-param construction (`recover`, `resume`, `format`, …) + is exercised by production code; only the socket underneath is faked, through the + `ARTWebSocketFactory` seam. +- Want to fake **HTTP**? Set `httpExecutor` — every REST request the SDK makes flows through it. +- Want to control **time** (timeouts, retries, TTL expiry) deterministically? Set `timeProvider` — + every SDK timer is created through `scheduleAfter:queue:block:` on it. +- Want to stop the OS **network monitor** from interfering? Set `reachabilityClass`. + +Tests never touch `testOptions` directly — `UTSTestCase.makeRealtime()`/`makeRest()` wire all of +this (§6.1). + +**Reaching further internals.** `import Ably.Private` exposes exactly the private headers listed in +the `explicit module Private` block of `Source/include/module.modulemap` — anything declared only +in a `.m` file (class extensions, ivars, private methods) is invisible to Swift. The SDK being +Objective-C also means Swift access levels (`internal`/`package`) play no part. If a spec needs an +internal symbol that isn't exposed: check `Source/PrivateHeaders/Ably/` first; if absent, declare +it in a private header there and register the header in **both** module maps +(`Source/include/module.modulemap` and `Source/Ably.modulemap`, per the repo's CLAUDE.md); if the +state has no reasonable seam, record the gap in `deviations.md` under *Mock Infrastructure +Limitations* rather than hacking around it. (The `uts-to-swift` skill's Step 4 carries the same +decision list for test translation.) + +--- + +## 6. Unit-Test Infrastructure (mocked transports) + +### 6.1 The base case — `infra/unit/UTSTestCase.swift` + +Every UTS suite subclasses `UTSTestCase`. It provides the cocoa mappings of the UTS infra +primitives: + +| UTS pseudocode | `UTSTestCase` API | +|---|---| +| `install_mock(mock_ws)` / `install_mock(mock_http)` | `installMock(_:)` — registers the mock; the **next** `makeRealtime()` / `makeRest()` picks it up. The mock is never passed to the client constructor (a mistake per `writing-test-specs.md`). | +| client construction | `makeRealtime { options in … }` / `makeRest { options in … }` — seeds a dummy key (`appId.keyId:keySecret`, like ably-java's `ClientOptionsBuilder`; the configure block can override it), forces JSON (`useBinaryProtocol = false`), gives each client its own internal/callback dispatch queues, and wires every installed mock into `testOptions`. `makeRealtime` *requires* a `MockWebSocketProvider`; it also wires an installed `MockHTTPClient` into the client's REST layer (auth callbacks, fallback hosts, …). | +| `AWAIT_STATE conn.state == s` | `awaitConnectionState(client, .connected)` — proceeds immediately if the condition already holds, otherwise polls until it does or the timeout (default 2 s) expires, then records a failure. | +| `AWAIT_STATE channel.state == s` | `awaitChannelState(channel, .attached)` | +| generic awaiting (e.g. "a frame has been sent") | `poll("description") { condition }` | +| `enable_fake_timers()` | `enableFakeTimers()` — clients built *after* this call use the `MockTimeProvider`; by default they run on the real clock (mirroring the spec, where only timeout/retry tests opt into fake timers). | +| `ADVANCE_TIME(ms)` | `advanceTime(byMilliseconds:)` | +| `CLOSE_CLIENT` | `closeClient(_:)` | + +Teardown is `deinit` (Swift Testing releases the suite instance after each `@Test`): it closes all +clients created by `makeRealtime` and calls `timeProvider?.cancelAllScheduled()` — the +**timer-leak safety net** the derived-tests guide asks for. + +### 6.2 `MockWebSocket.swift` — the fake realtime transport + +Three cooperating types: + +- **`MockWebSocketProvider`** — the UTS `mock_ws`, the object the *test* holds. It outlives any + single socket (the SDK creates a new `MockWebSocket` per connection attempt), carries the + `onConnectionAttempt` handler, and exposes `activeConnection` (the UTS `active_connection` — the + most recent attempt). Tests that need the full history capture sockets into a `Captured` inside + `onConnectionAttempt`. +- **`MockWebSocket`** — a single simulated connection. It conforms to `ARTWebSocket` so the real + `ARTWebSocketTransport` can drive it, and exposes the UTS **server-side API** to tests: + + | Method | What it does | Use for | + |--------|--------------|---------| + | `respondWithSuccess()` | accepts the connection (`readyState = .open`, fires `webSocketDidOpen`) | the transport-level accept; follow with `sendToClient(.connected(…))` | + | `respondWithSuccess(_:)` | accept + deliver a message in one call | the common accept-then-CONNECTED pair, e.g. `respondWithSuccess(.connectedMessage)` | + | `sendToClient(_:)` | delivers a protocol message, connection stays open | CONNECTED, ATTACHED, channel-level ERROR, ACK, normal messages | + | `sendToClientAndClose(_:)` | delivers a message then closes (code 1000) | DISCONNECTED, connection-level (fatal) ERROR | + | `respondWithRefused()` | closes with code 1003, no message | retryable connection refusal (`realtimeTransportRefused:`) | + | `simulateDisconnect()` | closes with code 1001, no message | unexpected network drop → DISCONNECTED/resume | + + For **inspection** it exposes `request` / `url` / `queryParams` (the real production-built URL — + this is how `recover=`/`resume=` assertions work) and `sentMessages` (every protocol message the + SDK sent towards the "server", decoded, in order — the client→server `ws_frame` log). +- **`MockWebSocketFactory` / `MockWebSocketTransportFactory`** — the adapters. The transport factory + is what `makeRealtime` installs; it builds a real `ARTWebSocketTransport` backed by a + `MockWebSocketFactory`, which creates `MockWebSocket`s, registers them with the provider, and + fires `onConnectionAttempt` on the client's work queue (timed so the transport has already wired + `delegate` and `delegateDispatchQueue`). + +All simulated server activity is delivered on the transport's `delegateDispatchQueue`, matching the +real `ARTSRWebSocket` contract. + +> The cocoa UTS infra currently implements the **handler style** (`onConnectionAttempt` closure) from +> the spec. The await style (`await_connection_attempt()`), which ably-java also implements, hasn't +> been needed yet — add it when a spec requires different answers for first connection vs +> reconnection attempts that a handler can't express. + +### 6.3 `MockHTTPClient.swift` — the fake REST transport + +A fake `ARTHTTPExecutor` mirroring the spec's `mock_http.md`. The cocoa HTTP seam is +**request-level** (`execute(_:completion:)`), so each request is a standalone two-phase attempt: + +1. **Connection phase** — `onConnectionAttempt` receives a `PendingHTTPConnection` (`host`, `port`, + `tls`, `queryParams`, derived from the request URL) and returns the connection verdict: + `respondWithSuccess()` (nil error — proceed), `respondWithRefused()`, `respondWithTimeout()`, or + `respondWithDNSError()` (the corresponding `NSURLErrorDomain` errors). +2. **Request phase** — unless the connection failed, `onRequest` receives a `PendingHTTPRequest` + exposing `url`, `method`, `headers`, `body`, `queryParams`, and the response API: + `respondWith(status:body:headers:)` (body may be `Data`, `String`, or a JSON-serialisable value; + defaults to a JSON content type), `respondWithDelay(_:status:body:)` (same, delivered after a + delay — for slow-server specs), or `respondWithTimeout()`. + +This lets REST unit tests assert on outgoing request shape (path, headers, query) and feed canned +responses back — all without a socket. + +### 6.4 `MockTimeProvider.swift` — deterministic time + +A deterministic `ARTTimeProvider`: both the wall clock and the continuous clock start at a fixed +origin (the wall-clock origin is overridable via `init(initialWallClockMilliseconds:)`, java's +`FakeClock(initialTimeMs)`) and **only move when the test calls `advanceTime(byMilliseconds:)`** — +nothing fires on its own. Every block the SDK schedules through `scheduleAfter:queue:block:` is +recorded. + +`advanceTime` does more than bump a counter — it runs the SDK to quiescence: + +1. **Drains** the SDK's queues first, so already-queued work finishes *registering its timer* + before the clock moves past it. +2. Advances both clocks. +3. **Fires every now-due block** (in fire-time order) and drains the cascade it triggers, repeating + while the cascade schedules further blocks that are themselves already due. + +After it returns, the SDK has fully reacted to the elapsed time. Chained retries scheduled +*relative to the new time* remain in the future and need a subsequent `advanceTime` — matching real +elapsed time. `pendingScheduledCount` exposes how many blocks are currently scheduled — useful for +asserting retry state (the counterpart of ably-java's `FakeClock.pendingTaskCount`; cocoa timers +have no names, so it is a single overall count). `cancelAllScheduled()` is the teardown safety +net (§6.1). + +### 6.5 `ProtocolMessage.swift` — server-message factories + +A `Sendable` value type describing a server→client message, built lazily into a real +`ARTProtocolMessage` at delivery time (on the delegate queue). Factories: +`.connected(connectionId:connectionKey:maxIdleInterval:connectionStateTtl:)` (defaults: TTL 120 s, +max-idle 15 s), `.attached(channel:channelSerial:)`, `.error(code:statusCode:message:)`, +`.ack(msgSerial:count:)`, `.closed()`. Extend this enum as new specs need more actions. + +`.connectedMessage` is a ready-to-use default CONNECTED message (ably-java's `CONNECTED_MESSAGE`: +connectionId `test-connection-id`, key `test-connection-key`) so most tests don't hand-build one — +and being a value type, each use is inherently a fresh instance. + +### 6.6 `Captured.swift` — the spec's `captured_*` arrays, Swift 6-safe + +Mock handlers run on SDK queues while the test reads results on the test thread; a plain captured +`var array` is a data race the Swift 6 compiler **rejects**. `Captured` is a small +lock-guarded, `Sendable` collector (`append`, `all`, `count`, `first`, subscript) that lets a test +keep a *local* collector — the spec's pattern — while staying race-free. + +### 6.7 `CapturingLog.swift` — asserting on log output + +An `ARTLog` that records every message the SDK logs (regardless of `logLevel`, and keeping the +console quiet). Install via `options.logHandler`; assert with +`contains(level: .error, message: "substring")`. Used by specs like RTN16f1 ("an error is logged"). + +### 6.8 `NoOpReachability.swift` + +A reachability implementation that never reports network changes, installed via +`testOptions.reachabilityClass` so the SDK doesn't start OS-level network monitoring during a unit +test (which must not touch the real network). + +### 6.9 Where each ably-java helper lives in cocoa + +For a developer coming from the Kotlin `uts` module — every java helper's capability exists here, +though some map onto a different shape: + +| ably-java helper | cocoa equivalent | |---|---| -| `install_mock(mock_ws)` | `installMock(_:)` — registers the `MockWebSocketProvider`; the next `makeRealtime()` wires it into `ARTClientOptions.testOptions.transportFactory` (per-client DI). The mock is never passed to the constructor. | -| `mock_ws = MockWebSocket(onConnectionAttempt:)` | `MockWebSocketProvider(onConnectionAttempt:)` — builds a real `ARTWebSocketTransport` over a fake `MockWebSocket` per connection | -| `mock_ws.active_connection.respond_with_success()` / `send_to_client(...)` | `MockWebSocket.respondWithSuccess()` / `sendToClient(_:)` | -| `AWAIT_STATE conn.state == s` | `awaitConnectionState(client, s)` — poll until it holds or timeout (state getters sync onto the internal queue) | -| `AWAIT_STATE channel.state == s` | `awaitChannelState(channel, s)` | -| `enable_fake_timers()` / `ADVANCE_TIME(ms)` | shared `MockTimeProvider` / `advanceTime(byMilliseconds:)` | -| `// UTS: ` | comment immediately above each test method | +| `infra/Utils.kt` (`awaitState`, `awaitChannelState`, `pollUntil`) | `infra/Utils.swift` (`awaitState`, `awaitChannelState`, `pollUntil` — async, for the integration tier) + the synchronous `UTSTestCase` awaits for the unit tier | +| `infra/Utils.kt` `withRealTimeout` | Not needed — Swift Testing has no virtual-time scheduler for `withTimeout` to be fooled by | +| `unit/ClientFactories.kt` (seeds the dummy key) | `UTSTestCase.makeRealtime` / `makeRest` (seed the same dummy key) | +| `unit/MockWebSocket.kt` + `MockWebSocketEngineFactory.kt` | `MockWebSocket.swift` (provider + socket + the two factories) | +| `unit/MockHttpClient.kt` + `MockHttpEngine.kt` + `PendingConnection/Request` (+ `Default*`) | `MockHTTPClient.swift` (`MockHTTPClient`, `PendingHTTPConnection`, `PendingHTTPRequest`) — cocoa's HTTP seam is one protocol, so no engine/adapter split | +| `unit/MockEvent.kt` (typed event timeline) | `MockWebSocket.sentMessages` (client→server frames) + `Captured` collectors in `onConnectionAttempt` — same assertions, no separate event type | +| await-style mocking (`awaitConnectionAttempt()`, …) and the per-frame handlers (`onMessageFromClient`, `onText/BinaryDataFrame`) | Handler style + a counter in `onConnectionAttempt`, and `sentMessages` polling (see §6.2's note; add these when a spec needs what they can't express) | +| `CONNECTED_MESSAGE` | `ProtocolMessage.connectedMessage` | +| `parseQueryString` | `parseQueryParams(of:)` in `infra/Utils.swift` | +| `unit/FakeClock.kt` (`advance`, `pendingTaskCount`, `initialTimeMs`) | `MockTimeProvider` (`advanceTime(byMilliseconds:)`, `pendingScheduledCount`, `init(initialWallClockMilliseconds:)`) | +| `unit/Utils.kt` (`ConnectionDetails { }` reflective builder) | `ProtocolMessage.connected(…)` builds `ARTConnectionDetails` directly — `import Ably.Private` exposes the initialiser, no reflection needed | +| `integration/SandboxApp.kt`, `integration/proxy/ProxyManager.kt` / `ProxySession.kt` | Same names, same shape — `infra/integration/` (§11) | + +### 6.10 How the pieces connect (request flow, no network) + +A unit test installs the mock transport into `testOptions`; the SDK believes it is talking to a +real socket, while every byte is intercepted in-process and surfaced to the test: + +```text + ┌──────────────────────── TEST (Swift Testing suite : UTSTestCase) ────────────────────────┐ + │ │ + │ installMock(wsProvider); let client = makeRealtime { $0.autoConnect = false } │ + │ │ client.connect() ▲ awaitConnectionState(client, .connected)│ + │ ▼ │ │ + │ ┌─────────────────────────────┐ testOptions.transportFactory │ + │ │ ARTRealtime + the REAL │ ────────▶ MockWebSocketTransportFactory │ + │ │ ARTWebSocketTransport │ │ creates one per connection attempt │ + │ └──────────┬──────────────────┘ ▼ │ + │ │ send(frame) ─────────────▶ ┌────────────────────┐ │ + │ didReceiveMessage ◀────────────────────│ MockWebSocket │◀── wsProvider │ + │ ▲ │ • sentMessages │ .activeConnection │ + │ │ │ • queryParams │ (the test's handle) │ + │ │ └────────────────────┘ │ + │ TEST drives the "server" side: TEST inspects: │ + │ connection.respondWithSuccess(.connectedMessage) ws.queryParams (recover/resume…) │ + │ connection.sendToClient(.attached(…)) ws.sentMessages (ATTACH, ACK…) │ + │ connection.sendToClientAndClose(…) / .simulateDisconnect() │ + │ │ + │ MockTimeProvider (testOptions.timeProvider): │ + │ enableFakeTimers() + advanceTime(byMilliseconds:) — fires due timers, settles cascades │ + └──────────────────────────────────────────────────────────────────────────────────────────┘ + + No TCP, no DNS, no real time. Everything is in-process and deterministic. +``` + +(The HTTP path is identical in shape: `MockHTTPClient` → `testOptions.httpExecutor` → +`PendingHTTPConnection` then `PendingHTTPRequest`, with `respondWith(status:body:)`. +The integration-tier counterpart of this picture — real network through the proxy — is §11.6.) + +--- + +## 7. Walkthrough: the Realtime Unit Test (`ConnectionRecoveryTests`) + +**File:** `unit/realtime/ConnectionRecoveryTests.swift` +**Tier:** Unit (mocked WebSocket, no network). +**Spec area:** RTN16 — connection recovery via the `recover` option and `createRecoveryKey()`. +**Spec:** [`connection_recovery_test.md`](https://github.com/ably/specification/blob/main/uts/realtime/unit/connection/connection_recovery_test.md) -URL/query-param construction (`recover`, `resume`, `format`, …) is exercised by **real** SDK code; -only the socket is faked, through the `ARTWebSocketFactory` seam. +Six tests, each carrying a `// UTS: realtime/unit/RTN16…/…` tag: -## Running +- **`RTN16g` (+`RTN16g1`) — recovery-key structure (incl. Unicode).** Connects (the + `onConnectionAttempt` handler responds with success + CONNECTED carrying a known key), attaches + two channels — one ASCII, one Unicode (`channel-éàü-世界`) — feeding each an ATTACHED with a + `channelSerial` via `sendToClient`. Asserts `createRecoveryKey()` contains the connection key, + `msgSerial == 0`, and both channel serials — including an encode→decode round-trip proving the + Unicode name survives. +- **`RTN16g2` — `createRecoveryKey()` returns nil in inactive states.** Walks the connection + through INITIALIZED → CONNECTED → CLOSING/CLOSED → FAILED → SUSPENDED asserting nil in every + inactive state. The SUSPENDED leg uses `enableFakeTimers()` + `advanceTime` to expire the + connection-state TTL deterministically — no real sleeping. +- **`RTN16k` — `recover` adds the `recover` query param.** Builds the client with + `options.recover = ` and asserts via `MockWebSocket.queryParams` (real production URL + building) that the first attempt carries `recover=` and a post-disconnect reconnect switches to + `resume=` — recover is a one-shot bootstrap. +- **`RTN16f` — `recover` initialises `msgSerial` from the recovery key**, verified through an ACK + round-trip (`.ack(msgSerial:count:)`). +- **`RTN16f1` — malformed `recover` key degrades gracefully.** A garbage key must be logged + (asserted via `CapturingLog`) and ignored: the client connects normally, with no + `recover`/`resume` params. +- **`RTN16j` (+`RTN16i`) — `recover` instantiates channels with their serials**, not auto-attached; + a manual `attach()` sends an ATTACH frame carrying the recovered serial (verified via + `sentMessages`). + +**What this file teaches about the infra:** the handler-style mock, `sendToClient` vs a real close, +fake-timer-driven SUSPENDED, `queryParams`/`sentMessages` as the two inspection surfaces, and +`CapturingLog` assertions. + +--- + +## 8. Walkthrough: the REST Unit Test (`TimeTests`) + +**File:** `unit/rest/TimeTests.swift` +**Tier:** Unit (mocked HTTP, no network). +**Spec area:** RSC16 — `ARTRest.time()`. + +Five tests (`// UTS: rest/unit/RSC16/…`): the returned `Date` matches the server's millisecond +timestamp; the request is a `GET /time` (asserted via `PendingHTTPRequest.url`/`method`); no +authentication is required; it works without TLS; and an error response (`respondWith(status: 500, +…)`) propagates as an error. Each test builds a `MockHTTPClient` with the two handlers and a +`makeRest` client — the minimal REST-unit-test shape to copy for new REST specs. + +--- + +## 9. Deviations: when the SDK disagrees with the spec + +`deviations.md` (next to this file) is the single catalogue of every place ably-cocoa behaves +differently from the features spec, discovered during translation. The mechanism (from +`writing-derived-tests.md`): the test keeps the **spec-correct** assertion but gates it behind the +`RUN_DEVIATIONS` environment variable, so normal runs stay green while each deviation stays +individually reproducible: + +```swift +@Test(.enabled(if: ProcessInfo.processInfo.environment["RUN_DEVIATIONS"] != nil)) +``` ```bash +RUN_DEVIATIONS=1 swift test --filter UTS./ +``` + +No deviations are recorded yet. When one is found, follow the decision tree (§3, +`writing-derived-tests.md`) and record: the spec point, what the spec requires, what the SDK does, +the root cause (file/function, where known), the workaround in the test, and the affected tests. +Deviations are **valuable output**, not failures — each one is a precise, reproducible bug report, +and the gated test becomes the acceptance test for the fix. + +--- + +## 10. How to Run the Tests + +```bash +# The whole UTS target (fast — fully mocked, no network): swift test --filter UTS -swift test --filter UTS.ConnectionRecoveryTests/ + +# One suite / one test: +swift test --filter UTS.ConnectionRecoveryTests +swift test --filter UTS.ConnectionRecoveryTests/test_RTN16k_recover_option_adds_recover_query_param_to_WebSocket_URL + +# Turn on the spec-correct (currently failing) deviation assertions: +RUN_DEVIATIONS=1 swift test --filter UTS./ + +# The integration-infra acceptance tests (need network; skipped without the env var). +# IntegrationSmokeTest hits the Ably sandbox directly (once per protocol variant: JSON + msgpack); +# ProxyInfraSmokeTests (JSON-only, as the proxy requires) additionally syncs +# the uts-proxy binary from GitHub releases on first run and is macOS-only: +UTS_INTEGRATION_SMOKE=1 swift test --filter "UTS.IntegrationSmokeTest|UTS.ProxyInfraSmokeTests" + +# Run the proxy infra against a locally built uts-proxy instead of a GitHub release: +UTS_PROXY_LOCAL_PATH=/path/to/uts-proxy UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.ProxyInfraSmokeTests ``` + +**Where CI runs them:** there is currently **no UTS-specific CI job** — the UTS target runs as part +of the full test suite (the `ably-cocoa` scheme driven by the fastlane lanes in +`.github/workflows/integration-test.yaml`). Since `swift test --filter UTS` is sub-second and +network-free, a dedicated fast PR-gate step (e.g. in `check-spm.yaml`, mirroring ably-java's +`runUtsUnitTests` gate in its `check.yml`) is a cheap improvement worth making — especially once +the integration tier exists and the unit/integration split needs separate CI cadences. + +--- + +## 11. Integration & Proxy Infrastructure + +The `infra/integration/` layer mirrors ably-java's `infra/integration/` (see its `uts/README.md` +§7 for the reference design) and is verified end-to-end by two env-gated acceptance tests (§10): +`integration/standard/IntegrationSmokeTest.swift` (SandboxApp + a real TLS client, no proxy; run once per protocol variant — JSON + msgpack) and +`integration/proxy/ProxyInfraSmokeTests.swift` (the full proxy chain). Three components +(§11.1–11.3), then a walkthrough of each tier's reference test (§11.4–11.5) and the request-flow +picture (§11.6): + +### 11.1 `SandboxApp.swift` — a throwaway app on the real sandbox + +Provisions a real backend app **directly** (not through the proxy, so provisioning is independent +of any fault rules under test): fetches the canonical `test-app-setup.json` from ably-common, +`POST`s its `post_apps` body to `https://sandbox.realtime.ably-nonprod.net/apps` (GETs are retried +with backoff; the POST is never retried, to avoid duplicate apps), and exposes `appId`, +`defaultKey` (full-capability `appId.keyId:keySecret`), and the full `keys` list. `delete()` +removes the app in teardown (best-effort — sandbox apps also auto-expire). Owns the single +upstream host constant `SandboxApp.sandboxHost` — the `nonprod:sandbox` endpoint used uniformly +across the integration specs; both `ProxySession` targets and direct-sandbox clients point at it. + +`SandboxApp` is the shared backbone of *both* integration kinds: **proxy** tests pair it with a +`ProxySession`, while **direct sandbox** tests use it alone — connecting straight to +`SandboxApp.sandboxHost` over TLS, where plain key (basic) auth works; +`IntegrationSmokeTest.swift` is the reference shape. + +### 11.2 `proxy/ProxyManager.swift` — syncs and launches the proxy binary + +The proxy is a small Go program — [`ably/uts-proxy`](https://github.com/ably/uts-proxy) — that +forwards traffic to the sandbox and injects faults on command. `ProxyManager` (an actor singleton) +owns the *process*: + +- `try await ProxyManager.shared.ensureProxy()` (call in suite setup) is **idempotent**: if a proxy + is already healthy on the control port (**10100**), it's a no-op. Otherwise it **downloads** the + pinned release (`v0.3.0`) archive for the current arch from GitHub releases, **verifies its + SHA-256 checksum**, extracts the binary (via the system `tar` — one deliberate simplification + over ably-java's hand-rolled JDK-only tar reader), caches it under + `~/.cache/uts-proxy//` — the **same cache ably-java uses**, so the two SDKs share one + download — and launches it with `--port 10100`. +- The download is serialised across concurrently launched test processes by an exclusive `flock`, + and within the process by the actor. +- A spawned `Process` does **not** die with its parent, so an `atexit` reaper kills it when the + test process exits; `stopProxy()` stops it explicitly. +- Override knob: set `UTS_PROXY_LOCAL_PATH` to a **locally built** proxy binary or `.tar.gz` to + skip the download + checksum (for testing against an unreleased proxy). This mirrors ably-java's + `-Duts.proxy.localPath` / `UTS_PROXY_LOCAL_PATH`; there is no separate "sync" build task in + either SDK — the sync *is* `ensureProxy()`. +- **macOS-only**: spawning a local process needs `Foundation.Process`, which doesn't exist on + iOS/tvOS — the `proxy/` infra and proxy suites are wrapped in `#if os(macOS)` and compile to + nothing on the simulator platforms CI also builds. + +### 11.3 `proxy/ProxySession.swift` — one test's window into the proxy + +The proxy exposes a **control REST API** on port 10100; `ProxySession` is the typed Swift client +for it, per +[`docs/proxy.md`](https://github.com/ably/specification/blob/main/uts/docs/proxy.md). +One session per test: + +- `ProxySession.create(rules:)` → `POST /sessions` with a `target` (the sandbox hosts) and an + initial **rule list**; the proxy assigns a `sessionId` and a fresh **listening port**. +- `addRules(_:position:)` → add rules mid-test; `triggerAction(_:)` → fire an **imperative** action + right now (late fault injection, e.g. `["type": "inject_to_client", "message": ["action": 17]]`). +- `getLog()` → the session's ordered, typed `[ProxyEvent]` — each carries `type` (`ws_connect`, + `ws_frame`, `http_request`, …), `direction`, `queryParams`, and the parsed protocol `message` + (introspect via `message?["action"] as? Int`). Proxy-log assertions are a proxy test's primary + verification. +- `close()` → `DELETE /sessions/{id}` — always call it in teardown. + +**Rules** = `match` + `action` (+ optional `times`), built with `wsConnectRule` / +`wsFrameToClientRule` / `wsFrameToServerRule` / `httpRequestRule`. Rules evaluate in order, first +match wins, unmatched traffic passes through, `times: N` auto-removes after N firings. + +**Wiring a client through the proxy** — `options.connectThroughProxy(session)` sets +`realtimeHost`/`restHost` = localhost, `port` = the session's port, `tls = false`, and +`useBinaryProtocol = false` (the proxy only understands text frames). ⚠️ Because the proxy serves +plain ws, **basic (key) auth is rejected** (RSA1: basic auth is TLS-only) — authenticate through +the proxy with an `authCallback` that signs a `TokenRequest` locally using the sandbox key (see +`ProxyInfraSmokeTests` for the shape; ably-java's `AuthReauthTest` does the same). + +**The shared async helpers** both integration walkthroughs below lean on (from +`infra/Utils.swift` — java's `infra/Utils.kt`, see §6.9). All are wall-clock and poll-based +(never a fixed sleep); on timeout they record a Swift Testing `Issue` (and return `false`) +rather than throwing: + +| Helper | Shape | Purpose | +|--------|-------|---------| +| `awaitState` | `await awaitState(client, .connected, timeout: 15)` | suspend until `connection.state == target` (or already there) | +| `awaitChannelState` | `await awaitChannelState(channel, .attached, timeout: 15)` | same, for a channel's state | +| `pollUntil` | `await pollUntil("what you wait for", timeout: 15, interval: 0.1) { condition }` | suspend until an arbitrary predicate holds — e.g. `pollUntil("re-auth") { count.count > original }` | + +### 11.4 Walkthrough: a direct-sandbox test (`IntegrationSmokeTest`) + +**File:** `integration/standard/IntegrationSmokeTest.swift` +**Tier:** Direct-sandbox integration (real network, real Ably sandbox, **no** proxy, **no** fault +injection). + +This is the reference for the **middle tier** — the shape every happy-path interop spec +(connect/publish/subscribe/presence/history) follows. Step by step: + +1. **Suite setup** — provision a real app: `let app = try await SandboxApp.create()`. (A + spec-derived suite would do this once per suite; `delete()` in teardown.) +2. **Protocol variants** — the test takes `useBinaryProtocol: Bool` via + `@Test(arguments: [false, true])`, the cocoa realisation of the spec's `PROTOCOL` dimension + (§2): each case runs the whole scenario once over JSON and once over msgpack. +3. **A real client, wired straight to the sandbox** — plain `ARTClientOptions(key: app.defaultKey)` + with `realtimeHost`/`restHost` = `SandboxApp.sandboxHost`. TLS stays on, so basic key auth is + fine here (unlike through the proxy), and explicit hosts auto-disable fallback hosts (REC2c2). + No mocks anywhere — this drives the SDK's real `ARTWebSocketTransport`. +4. **Connect and wait, never sleep** — `client.connect()` then + `await awaitState(client, .connected)`: real network, so the async wall-clock waits from + `infra/Utils.swift` (§6.9), not the unit tier's fake timers. +5. **A fresh channel per run** — `"smoke-\(UUID().uuidString)"`, so variants and retries never + collide on server-side channel state. +6. **The round-trip** — subscribe first (capturing into a `Captured` — the callback + arrives on the SDK's queue, §6.6), publish, then + `await pollUntil("published message is echoed back…") { received.count == 1 }` — the real + backend is eventually consistent, so poll on observable state rather than assuming timing. +7. **Teardown** — `client.close()` → `await awaitState(client, .closed)` → `await app.delete()`. + +**What this teaches about the infra:** `SandboxApp`-only provisioning, direct-sandbox client +wiring, protocol-variant parameterisation, `Captured` for cross-queue capture, and `pollUntil` +over real network state. + +### 11.5 Walkthrough: a proxy test (`ProxyInfraSmokeTests`) + +**File:** `integration/proxy/ProxyInfraSmokeTests.swift` (macOS-only, §11.2) +**Tier:** Proxy integration (real sandbox, traffic routed through the local `uts-proxy`). + +Step by step: + +1. **Suite setup** — `try await ProxyManager.shared.ensureProxy()` (syncs + launches the proxy if + it isn't already healthy, §11.2), then `SandboxApp.create()` — the app is provisioned + **directly** against the sandbox, not through the proxy, so provisioning is independent of any + fault rules. +2. **A session with no rules** — `try await ProxySession.create()`. Starting rule-less is the + **late-fault-injection** principle (§2): the connect handshake runs against the real server + unmodified; a spec test injects its fault *afterwards*, as the final interaction. +3. **Token auth, not the key** — the proxy serves plain ws (`tls = false`), and basic key auth is + TLS-only (RSA1), so the client authenticates via an `authCallback` that signs a `TokenRequest` + locally using the sandbox key (through a separate TLS `ARTRest` "token signer"). +4. **Wire the client through the proxy** — `options.connectThroughProxy(session)` (§11.3), then + `client.connect()` and `await awaitState(client, .connected)` — the SDK believes it is talking + to Ably; every byte actually flows through the proxy. +5. **The proxy log is the primary verification** — `try await session.getLog()` and filter the + typed events: the smoke asserts a `ws_connect` event and a server→client `ws_frame` whose + `message?["action"] as? Int == 4` (CONNECTED). Spec tests assert on exactly this log — e.g. + "the client sent an AUTH frame (17) carrying non-nil `auth` details". +6. **Teardown** — close the client and await CLOSED, then **always** `await session.close()` + (leaked sessions hold proxy listeners), then `app.delete()`. + +**What a full spec-derived proxy test adds** (ably-java's `AuthReauthTest`, RTN22/RTC8a, is the +reference): snapshot `connection.id` and a callback counter after connecting; inject the fault +imperatively — `try await session.triggerAction(["type": "inject_to_client", "message": +["action": 17]])` (a server-initiated AUTH); `await pollUntil("re-auth round-trip") { … }` on the +counter; then assert the callback re-fired, the connection stayed CONNECTED with an **unchanged** +`connection.id`, and the log contains the client→server AUTH frame. Declarative faults use +`addRules` / the rule builders instead (§11.3). + +### 11.6 How the pieces connect (request flow) + +```text + ┌───────────────────── TEST (Swift Testing) ─────────────────────┐ + │ setup: try await ProxyManager.shared.ensureProxy() │ syncs binary, launches proxy, + │ let app = try await SandboxApp.create() ────────────────────── POST /apps ──────────┐ + │ let session = try await ProxySession.create(rules: []) ──┐ │ control REST :10100 │ (direct, TLS) + │ options.connectThroughProxy(session) │ │ │ + └──────────────┬────────────────────────────────────────────┼─────┘ │ + │ client.connect() │ │ + │ (host=localhost, port=session.proxyPort, │ │ + │ tls=false, JSON) │ ▼ + ▼ │ + ┌──────────────────┐ ws/http (plain) ┌──────────┴───────────┐ ws/http (TLS) ┌───────────────────────────┐ + │ ARTRealtime │ ◀──────────────────▶ │ uts-proxy │ ◀───────────────▶ │ Ably sandbox │ + │ (REAL transport) │ data plane │ • forwards traffic │ │ sandbox.realtime. │ + └──────────────────┘ │ • applies rules │ │ ably-nonprod.net │ + ▲ │ • records event log │ └───────────────────────────┘ + │ TEST controls the proxy: └──────────┬───────────┘ + │ session.triggerAction(["type": "inject_to_client", "message": ["action": 17]]) + │ session.addRules([…]) │ control plane (REST :10100) + │ TEST verifies via: │ + │ session.getLog() — filter type / direction / message?["action"] + │ await awaitState(…) / await pollUntil("…") { … } + └── everything before the injected fault is REAL client ↔ server traffic +``` + +**Why two channels to the proxy?** The **data plane** (the SDK's ws/http traffic on +`session.proxyPort`) is separate from the **control plane** (the test's REST calls on the control +port 10100 to create sessions, add rules, trigger actions, read the log). The SDK never sees the +control plane; the test never speaks the data plane directly. A direct-sandbox test (§11.4) is +this same picture with the proxy column removed — the client talks TLS straight to the sandbox, +and the test's only side channel is `SandboxApp` provisioning. + +### 11.7 What remains TODO + +- **`integration/standard//`** — spec-derived direct-sandbox happy-path tests (SandboxApp + only, real transport, protocol-variant parameterisation via Swift Testing + `@Test(arguments: [false, true])` over `useBinaryProtocol` — `IntegrationSmokeTest` demonstrates + the shape). +- **`integration/proxy//`** — spec-derived fault-injection tests (SandboxApp + + ProxySession). +- **CI wiring** — a dedicated job/lane for the integration tier (macOS, network access), separate + from the fast unit gate. + +Design constraints to carry over: late fault injection, JSON-only through the proxy, poll — never +sleep — on real-network state (`pollUntil`), and every proxy test also keeping a unit-tier +counterpart. + +--- + +## 12. Quick Reference / Cheat-Sheet + +**The seams that make unit tests possible** (`ARTClientOptions.testOptions`, via +`import Ably.Private`): `transportFactory` (WS) · `httpExecutor` (HTTP) · `timeProvider` (time) · +`reachabilityClass` (network monitor) — plus `options.logHandler` (log assertions). + +**Build a realtime unit-test client:** +```swift +let wsProvider = MockWebSocketProvider(onConnectionAttempt: { connection in + connection.respondWithSuccess() + connection.sendToClient(.connected(connectionId: "connection-1", connectionKey: "key-1")) +}) +installMock(wsProvider) +let client = makeRealtime { options in + options.key = "appId.keyId:keySecret" + options.autoConnect = false +} +client.connect() +awaitConnectionState(client, .connected) +``` + +**Build a REST unit-test client:** +```swift +let mockHTTP = MockHTTPClient( + onConnectionAttempt: { connection in connection.respondWithSuccess() }, + onRequest: { request in request.respondWith(status: 200, body: [1_704_067_200_000]) } +) +installMock(mockHTTP) +let rest = makeRest { options in options.key = "appId.keyId:keySecret" } +``` + +**Build a proxy-test client (macOS only; token auth — basic auth is TLS-only):** +```swift +try await ProxyManager.shared.ensureProxy() // suite setup +let app = try await SandboxApp.create() // suite setup +let session = try await ProxySession.create(rules: []) +let options = ARTClientOptions() +options.authCallback = { params, callback in + tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in callback(tokenRequest, error) } +} +options.connectThroughProxy(session) +// … scenario: session.triggerAction(…), await pollUntil("…") { … }, session.getLog() … +await session.close() // always, in teardown +``` + +**Server→client (mock WS):** `sendToClient` (stays open — ATTACHED, channel ERROR, ACK) · +`sendToClientAndClose` (DISCONNECTED / fatal ERROR) · `respondWithRefused` (1003 refusal) · +`simulateDisconnect` (1001 drop). + +**Inspect what the SDK did:** `mockWebSocket.queryParams` / `.sentMessages` (WS) · +`PendingHTTPRequest.url/method/headers/body/queryParams` (HTTP) · `CapturingLog.contains(…)` (logs). + +**Wait (never sleep):** unit tier — `awaitConnectionState` · `awaitChannelState` · +`poll("…") { … }` · `enableFakeTimers()` + `advanceTime(byMilliseconds:)`; integration tier +(async, wall-clock) — `await awaitState(client, .connected)` · `await awaitChannelState(…)` · +`await pollUntil("…") { … }`. + +**Protocol action numbers** (used in proxy rules & log assertions): CONNECTED=4, +DISCONNECTED=6, ERROR=9, ATTACH=10, ATTACHED=11, DETACH=12, DETACHED=13, **AUTH=17**. + +**Test ID format:** `//-` → +`// UTS: realtime/unit/RTN16g/recovery-key-structure-0` (comment immediately above each test). + +**The decision tree when a translated test fails:** spec wrong → fix test + record UTS spec error; +translation wrong → fix test; SDK non-compliant → gate the spec-correct assertion behind +`RUN_DEVIATIONS` and record in `deviations.md`. + +--- + +## 13. Appendix: Per-File API Reference + +### Infrastructure — `infra/` (shared) + +| File | Key public surface | Role | +|------|--------------------|------| +| `infra/Utils.swift` | `pollUntil`, `awaitState`, `awaitChannelState` (async, wall-clock); `parseQueryParams`; `httpRequest`, `jsonRequest`, `makeURLSession`, `HTTPError` | Shared helpers: the integration tier's wall-clock waits (never sleep), the query-param parsing both mocks use, and the `URLSession` plumbing the integration infra is built on. | + +### Infrastructure — `infra/unit/` + +| File | Key public surface | Role | +|------|--------------------|------| +| `infra/unit/UTSTestCase.swift` | `installMock(_:)` ×2, `makeRealtime { }`, `makeRest { }`, `awaitConnectionState`, `awaitChannelState`, `poll`, `enableFakeTimers`, `advanceTime(byMilliseconds:)`, `closeClient`, `defaultAwaitTimeout` (2 s) | Base class for every UTS suite; seeds the dummy key; wires all seams; `deinit` closes clients + cancels leaked timers. | +| `infra/unit/MockWebSocket.swift` | `MockWebSocketProvider` (`onConnectionAttempt`, `activeConnection`); `MockWebSocket` (`respondWithSuccess`/`respondWithSuccess(_:)`, `sendToClient`, `sendToClientAndClose`, `respondWithRefused`, `simulateDisconnect`, `request`/`url`/`queryParams`, `sentMessages`); `MockWebSocketFactory`; `MockWebSocketTransportFactory` | Fake realtime transport (handler style). Real `ARTWebSocketTransport` on top → production URL building exercised. Close codes: 1000 closed / 1001 disconnected / 1003 refused. | +| `infra/unit/MockHTTPClient.swift` | `MockHTTPClient(onConnectionAttempt:onRequest:)`; `PendingHTTPConnection` (`host`/`port`/`tls`/`queryParams`, `respondWithSuccess/Refused/Timeout/DNSError`); `PendingHTTPRequest` (`url`, `method`, `headers`, `body`, `queryParams`, `respondWith(status:body:headers:)`, `respondWithDelay(_:status:body:)`, `respondWithTimeout`) | Fake REST transport (`ARTHTTPExecutor`); two-phase connect→request per call. | +| `infra/unit/MockTimeProvider.swift` | `init(initialWallClockMilliseconds:)`, `advanceTime(byMilliseconds:)`, `pendingScheduledCount`, `cancelAllScheduled()`; implements `wallClockNow`, `continuousClockNow`, `schedule(after:queue:block:)` | Virtual clocks + recorded timers; `advanceTime` drains SDK queues, fires due blocks, and settles cascades. | +| `infra/unit/ProtocolMessage.swift` | `.connected(…)`, `.attached(…)`, `.error(…)`, `.ack(…)`, `.closed()`; `.connectedMessage` (default CONNECTED); `makeProtocolMessage()` | `Sendable` server→client message descriptions, materialised at delivery time. | +| `infra/unit/Captured.swift` | `append`, `all`, `count`, `first`, subscript | Thread-safe local collector for the spec's `captured_*` pattern (Swift 6 race-free). | +| `infra/unit/CapturingLog.swift` | `entries`, `contains(level:message:)` | `ARTLog` recording everything regardless of `logLevel`; install via `options.logHandler`. | +| `infra/unit/NoOpReachability.swift` | (`ARTReachability` conformance) | Never reports network changes; keeps OS monitoring out of unit tests. | + +### Infrastructure — `infra/integration/` + +| File | Key public surface | Role | +|------|--------------------|------| +| `infra/integration/SandboxApp.swift` | `SandboxApp.create()`, `delete()`, `appId`, `defaultKey`, `keys`; `SandboxApp.sandboxHost` | Provisions/tears down a throwaway sandbox app from ably-common's `test-app-setup.json`; owns the upstream sandbox host constant. | +| `infra/integration/proxy/ProxyManager.swift` | `ProxyManager.shared.ensureProxy(timeout:)`, `stopProxy()`, `ProxyManager.controlPort` (10100); `UTS_PROXY_LOCAL_PATH` override | Syncs (downloads, checksum-verifies, caches at `~/.cache/uts-proxy//`) and launches the pinned `uts-proxy` release; `atexit` reaper. **macOS-only** (`#if os(macOS)`). | +| `infra/integration/proxy/ProxySession.swift` | `ProxySession.create(rules:port:timeoutMs:realtimeHost:restHost:)`, `addRules`, `triggerAction`, `getLog() -> [ProxyEvent]`, `close`, `sessionId`, `proxyPort`, `proxyHost`; `ProxyEvent`; `ProxyRule` + `wsConnectRule`/`wsFrameToClientRule`/`wsFrameToServerRule`/`httpRequestRule`; `ARTClientOptions.connectThroughProxy(_:)` | Typed client for the proxy control REST API + client wiring. **macOS-only**. | + +### Tests and docs + +| File | Contents | Notes | +|------|----------|-------| +| `unit/realtime/ConnectionRecoveryTests.swift` | 6 `@Test`s: RTN16g/g1, RTN16g2, RTN16k, RTN16f, RTN16f1, RTN16j/i | Mocked WS + fake timers; see §7. | +| `unit/rest/TimeTests.swift` | 5 `@Test`s: RSC16 ×5 | Mocked HTTP; see §8. | +| `integration/standard/IntegrationSmokeTest.swift` | 1 `@Test` × {JSON, msgpack} (`arguments: [false, true]`), gated behind `UTS_INTEGRATION_SMOKE` | Acceptance test for the direct-sandbox tier (not spec-derived): sandbox app → real TLS client → publish/subscribe round-trip, once per protocol variant. | +| `integration/proxy/ProxyInfraSmokeTests.swift` | 1 `@Test`, gated behind `UTS_INTEGRATION_SMOKE` | End-to-end acceptance test for the proxy infra (not spec-derived): binary sync → proxy launch → sandbox app → real client through the proxy → log assertions. macOS-only. | +| `deviations.md` | none recorded yet | Catalogue of SDK-vs-spec divergences + the `RUN_DEVIATIONS` pattern. | + +> **Coverage note:** the infrastructure is built out beyond what the current suites exercise +> (full HTTP fault verdicts, `.error`/`.closed` message factories, `CapturingLog`, all four proxy +> rule builders), anticipating the broader UTS coverage catalogued in +> [`completion-status.md`](https://github.com/ably/specification/blob/main/uts/docs/completion-status.md). +> The spec-derived integration tests are the big missing piece — see §11.7. + +--- + +### Source map (where each fact in this doc comes from) + +| Topic | File | +|-------|------| +| Authoring portable specs, test IDs, mock pseudocode | [`uts/docs/writing-test-specs.md`](https://github.com/ably/specification/blob/main/uts/docs/writing-test-specs.md) | +| Translating specs, deviation patterns, decision tree | [`uts/docs/writing-derived-tests.md`](https://github.com/ably/specification/blob/main/uts/docs/writing-derived-tests.md) | +| Integration/proxy policy, late fault injection, tiers | [`uts/docs/integration-testing.md`](https://github.com/ably/specification/blob/main/uts/docs/integration-testing.md) | +| Coverage matrix | [`uts/docs/completion-status.md`](https://github.com/ably/specification/blob/main/uts/docs/completion-status.md) | +| Proxy control API, rule format, action numbers | [`uts/docs/proxy.md`](https://github.com/ably/specification/blob/main/uts/docs/proxy.md) | +| SDK seams | `Source/PrivateHeaders/Ably/ARTClientOptions+TestConfiguration.h` (`ARTTestClientOptions`), `Source/include/module.modulemap` (`Ably.Private`) | +| Target wiring | `Package.swift` (the `UTS` test target) | +| Unit mocks | `Test/UTS/infra/unit/*` | +| Shared helpers | `Test/UTS/infra/Utils.swift` | +| Integration helpers | `Test/UTS/infra/integration/*` (+ `proxy/*`) | +| The reference tests | `unit/realtime/ConnectionRecoveryTests.swift`, `unit/rest/TimeTests.swift`, `integration/standard/IntegrationSmokeTest.swift`, `integration/proxy/ProxyInfraSmokeTests.swift` | +| Deviations | `Test/UTS/deviations.md` | +| The ably-java counterpart this guide mirrors | `uts/README.md` in the `ably-java` repository | diff --git a/Test/UTS/infra/Utils.swift b/Test/UTS/infra/Utils.swift new file mode 100644 index 000000000..cdde5e308 --- /dev/null +++ b/Test/UTS/infra/Utils.swift @@ -0,0 +1,119 @@ +import Foundation +import Testing +import Ably +import Ably.Private + +/// Shared helpers used by every tier (the cocoa counterpart of ably-java's `infra/Utils.kt`). +/// +/// The unit tier's state waits live on `UTSTestCase` (`awaitConnectionState` / `awaitChannelState` / +/// `poll`) — they poll synchronously, which is fine when a frozen `MockTimeProvider` settles the SDK +/// in microseconds. The integration tier waits on *real* network and proxy state, where blocking the +/// test thread for seconds is wasteful, so it uses the async helpers below (the UTS specs' +/// "poll — never sleep" rule). + +/// Suspends until `condition` returns `true`, polling every `interval` seconds, or records a test +/// failure once `timeout` (wall-clock seconds) elapses. Returns whether the condition was met. +/// +/// The integration analogue of `UTSTestCase.poll` — e.g. +/// `await pollUntil("auth callback re-invoked") { authCallbackCount.count > original }`. +@discardableResult +func pollUntil(_ description: String, + timeout: TimeInterval = 15, + interval: TimeInterval = 0.1, + sourceLocation: SourceLocation = #_sourceLocation, + _ condition: () async -> Bool) async -> Bool { + let deadline = Date().addingTimeInterval(timeout) + while !(await condition()) { + if Date() >= deadline { + Issue.record("Timed out after \(timeout)s polling for: \(description)", sourceLocation: sourceLocation) + return false + } + try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + } + return true +} + +/// Suspends until `client.connection.state == expected` (the integration tier's `AWAIT_STATE`, +/// against a real backend — the async counterpart of `UTSTestCase.awaitConnectionState`). +@discardableResult +func awaitState(_ client: ARTRealtime, + _ expected: ARTRealtimeConnectionState, + timeout: TimeInterval = 15, + sourceLocation: SourceLocation = #_sourceLocation) async -> Bool { + await pollUntil("connection.state == \(ARTRealtimeConnectionStateToStr(expected))", + timeout: timeout, sourceLocation: sourceLocation) { + client.connection.state == expected + } +} + +/// Suspends until `channel.state == expected` (the integration tier's `AWAIT_STATE` for channels). +@discardableResult +func awaitChannelState(_ channel: ARTRealtimeChannel, + _ expected: ARTRealtimeChannelState, + timeout: TimeInterval = 15, + sourceLocation: SourceLocation = #_sourceLocation) async -> Bool { + await pollUntil("channel '\(channel.name)'.state == \(ARTRealtimeChannelStateToStr(expected))", + timeout: timeout, sourceLocation: sourceLocation) { + channel.state == expected + } +} + +/// Query parameters parsed from a URL (UTS `url.query_params`) — the counterpart of ably-java's +/// `parseQueryString`. Shared by the WS and HTTP mocks so the parsing lives in one place. +func parseQueryParams(of url: URL?) -> [String: String] { + guard let url, + let components = URLComponents(url: url, resolvingAgainstBaseURL: true), + let items = components.queryItems else { return [:] } + var result: [String: String] = [:] + for item in items where item.value != nil { + result[item.name] = item.value + } + return result +} + +// MARK: - HTTP support for integration infra + +/// Error thrown by `httpRequest` when the response is not an HTTP response, or by callers on an +/// unexpected status code. +struct HTTPError: Error, CustomStringConvertible { + let description: String + init(_ description: String) { self.description = description } +} + +/// Minimal async wrapper over `URLSession.dataTask` used by the integration infrastructure +/// (`SandboxApp`, `ProxyManager`, `ProxySession`). A continuation-based helper rather than +/// `URLSession.data(for:)` so the target's deployment floor stays where the rest of the suite is. +/// Returns the body and status code; callers decide which statuses are acceptable. +func httpRequest(_ request: URLRequest, session: URLSession) async throws -> (data: Data, status: Int) { + try await withCheckedThrowingContinuation { continuation in + session.dataTask(with: request) { data, response, error in + if let error { + continuation.resume(throwing: error) + return + } + guard let http = response as? HTTPURLResponse else { + continuation.resume(throwing: HTTPError("Non-HTTP response for \(request.url?.absoluteString ?? "?")")) + return + } + continuation.resume(returning: (data ?? Data(), http.statusCode)) + }.resume() + } +} + +/// Builds a JSON request. `body` (when present) is serialised with `JSONSerialization`. +func jsonRequest(_ method: String, _ url: URL, body: Any? = nil) throws -> URLRequest { + var request = URLRequest(url: url) + request.httpMethod = method + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + if let body { + request.httpBody = try JSONSerialization.data(withJSONObject: body) + } + return request +} + +/// A `URLSession` with finite timeouts so a stalled endpoint fails fast instead of hanging a suite. +func makeURLSession(requestTimeout: TimeInterval) -> URLSession { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = requestTimeout + return URLSession(configuration: configuration) +} diff --git a/Test/UTS/infra/integration/SandboxApp.swift b/Test/UTS/infra/integration/SandboxApp.swift new file mode 100644 index 000000000..db3487426 --- /dev/null +++ b/Test/UTS/infra/integration/SandboxApp.swift @@ -0,0 +1,100 @@ +import Foundation + +/// A test app provisioned in the Ably sandbox (`sandbox.realtime.ably-nonprod.net`). +/// +/// Integration test suites provision one app in suite setup (`create()`) and tear it down in suite +/// teardown (`delete()`). The app is created against the sandbox directly (not through the proxy) +/// so provisioning is independent of any fault rules under test. +/// +/// ```swift +/// let app = try await SandboxApp.create() +/// defer { /* in teardown: */ } +/// let key = app.defaultKey // "appId.keyId:keySecret" +/// // … tests … +/// await app.delete() +/// ``` +/// +/// Mirrors ably-java's `infra/integration/SandboxApp.kt`. +final class SandboxApp: Sendable { + + /// The Ably **nonprod sandbox** host — the `nonprod:sandbox` endpoint (used uniformly across the + /// realtime/objects/rest integration specs), resolved to a hostname. Realtime and REST share + /// this single host, so point both transports at it: set `realtimeHost` and/or `restHost` from + /// here. + static let sandboxHost = "sandbox.realtime.ably-nonprod.net" + + private static let sandboxBaseURL = URL(string: "https://\(sandboxHost)")! + + /// The canonical app spec shared across all Ably SDK test suites. + private static let appSetupURL = URL(string: "https://raw.githubusercontent.com/ably/ably-common/refs/heads/main/test-resources/test-app-setup.json")! + + private static let session = makeURLSession(requestTimeout: 30) + + /// The provisioned app's id. + let appId: String + /// A full-capability API key string in `appId.keyId:keySecret` form. + let defaultKey: String + /// API keys with different capabilities, in `appId.keyId:keySecret` form. The first is the + /// default key. See `test-app-setup.json` in ably-common for what each key can do. + let keys: [String] + + private init(appId: String, defaultKey: String, keys: [String]) { + self.appId = appId + self.defaultKey = defaultKey + self.keys = keys + } + + /// Provisions a fresh sandbox app and returns its id and keys. + static func create() async throws -> SandboxApp { + let appSpec = try await loadAppCreationJSON() + let request = try jsonRequest("POST", sandboxBaseURL.appendingPathComponent("apps"), body: appSpec) + let (data, status) = try await httpRequest(request, session: session) + guard (200..<300).contains(status) else { + throw HTTPError("Sandbox POST /apps returned \(status): \(String(decoding: data, as: UTF8.self))") + } + guard let body = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let appId = body["appId"] as? String, + let keyObjects = body["keys"] as? [[String: Any]] else { + throw HTTPError("Sandbox POST /apps returned an unexpected body") + } + let keys = keyObjects.compactMap { $0["keyStr"] as? String } + guard let defaultKey = keys.first else { + throw HTTPError("Sandbox POST /apps returned no keys") + } + return SandboxApp(appId: appId, defaultKey: defaultKey, keys: keys) + } + + /// Deletes the provisioned app. Errors are ignored so teardown never masks a test failure + /// (sandbox apps also auto-expire server-side). + func delete() async { + var request = URLRequest(url: Self.sandboxBaseURL.appendingPathComponent("apps/\(appId)")) + request.httpMethod = "DELETE" + let basic = Data(defaultKey.utf8).base64EncodedString() + request.setValue("Basic \(basic)", forHTTPHeaderField: "Authorization") + _ = try? await httpRequest(request, session: Self.session) + } + + /// Fetches the `post_apps` body from the shared `test-app-setup.json` in ably-common. + /// Retried (idempotent GET only — retrying POST /apps could provision duplicate apps). + private static func loadAppCreationJSON() async throws -> Any { + var lastError: Error = HTTPError("unreachable") + for attempt in 0..<5 { + do { + let (data, status) = try await httpRequest(URLRequest(url: appSetupURL), session: session) + guard status == 200 else { + throw HTTPError("GET test-app-setup.json returned \(status)") + } + guard let setup = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let postApps = setup["post_apps"] else { + throw HTTPError("test-app-setup.json has no post_apps key") + } + return postApps + } catch { + lastError = error + // Exponential backoff: 0.5s, 1s, 2s, 4s. + try? await Task.sleep(nanoseconds: UInt64(500_000_000 * (1 << attempt))) + } + } + throw lastError + } +} diff --git a/Test/UTS/infra/integration/proxy/ProxyManager.swift b/Test/UTS/infra/integration/proxy/ProxyManager.swift new file mode 100644 index 000000000..5c1f81014 --- /dev/null +++ b/Test/UTS/infra/integration/proxy/ProxyManager.swift @@ -0,0 +1,270 @@ +// The proxy runs as a locally spawned child process, which requires `Foundation.Process` — +// unavailable on iOS/tvOS. Proxy integration tests are therefore macOS-only (on the simulator +// platforms this file compiles to nothing and the proxy suites are excluded by the same condition). +#if os(macOS) + +import Foundation +import CryptoKit + +/// Manages the lifecycle of the `uts-proxy` binary used for proxy integration tests. +/// +/// Downloads the binary from GitHub releases on first use, caching it at +/// `~/.cache/uts-proxy//uts-proxy` (the same cache location ably-java uses, so the two +/// SDKs share a download). The download is serialised across OS processes by an exclusive `flock` +/// on `uts-proxy.lock`, and within this process by the actor's isolation. Note: only the *download* +/// is cross-process locked — process startup relies on the shared health check on `controlPort`. +/// +/// The spawned process does **not** die with its parent, so it is reaped by an `atexit` hook; +/// `stopProxy()` stops it explicitly. +/// +/// To run against a locally built proxy instead of downloading a release, set the +/// `UTS_PROXY_LOCAL_PATH` environment variable to a local `uts-proxy` binary or a `.tar.gz` +/// distributive. When set, the download and checksum verification are skipped. +/// +/// Call `try await ProxyManager.shared.ensureProxy()` in suite setup for every proxy integration +/// test suite — if the proxy is already healthy (e.g. started by a previous suite in the same run), +/// it is a no-op. +/// +/// Mirrors ably-java's `infra/integration/proxy/ProxyManager.kt`. One deliberate difference: the +/// `.tar.gz` is extracted by spawning `/usr/bin/tar` (always present on macOS) instead of the +/// hand-rolled JDK-only tar reader ably-java needs. +actor ProxyManager { + + static let shared = ProxyManager() + + /// The proxy's control REST API port (shared by all sessions; data-plane ports are per-session). + static let controlPort = 10100 + + private static let proxyVersion = "v0.3.0" + private static let versionBare = "0.3.0" + private static let githubBase = "https://github.com/ably/uts-proxy/releases/download/\(proxyVersion)" + + /// SHA-256 checksums of the release archives (from the release's checksums.txt). + private static let checksums: [String: String] = [ + "uts-proxy_\(versionBare)_darwin_amd64.tar.gz": + "1355526543c3022f87efb7f564f55200b78edc68d84c7dba2e49f63429e3b788", + "uts-proxy_\(versionBare)_darwin_arm64.tar.gz": + "a948f99b7daf9b3bffff742f6405637d40a79947389309eed5f87e59026de9a5", + "uts-proxy_\(versionBare)_linux_amd64.tar.gz": + "de741ba21f3630fea4f59714d00585638d565005599ecd84179931eba248f280", + "uts-proxy_\(versionBare)_linux_arm64.tar.gz": + "15b5ca87c40c2c4ff350c94af1911cea0ad6be5a2d890ba41029bc4b8bc52c61", + ] + + private static var archiveName: String { + #if arch(arm64) + let arch = "arm64" + #elseif arch(x86_64) + let arch = "amd64" + #else + #error("Unsupported architecture for uts-proxy") + #endif + return "uts-proxy_\(versionBare)_darwin_\(arch).tar.gz" + } + + private static var cacheDir: URL { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".cache/uts-proxy/\(proxyVersion)", isDirectory: true) + } + + private static var binaryURL: URL { cacheDir.appendingPathComponent("uts-proxy") } + + /// Optional path to a locally built `uts-proxy` binary or `.tar.gz` distributive, from the + /// `UTS_PROXY_LOCAL_PATH` environment variable. When present, the release download + checksum + /// check are bypassed. + private static var localDistributive: String? { + ProcessInfo.processInfo.environment["UTS_PROXY_LOCAL_PATH"].flatMap { $0.isEmpty ? nil : $0 } + } + + // Short per-attempt timeout: /health is local, so a slow answer means "not up yet". + private static let healthSession = makeURLSession(requestTimeout: 2) + // Generous timeout for the release download. + private static let downloadSession = makeURLSession(requestTimeout: 60) + + private var proxyProcess: Process? + + private init() {} + + /// Ensures the `uts-proxy` process is running on `controlPort`. + /// + /// If the proxy is already healthy (e.g. started by a previous test suite in the same run, or + /// launched manually), this is a no-op. Otherwise it downloads + verifies the binary and starts + /// the process. + /// + /// - Parameter timeout: Maximum wall-clock seconds to wait for the process to become healthy. + func ensureProxy(timeout: TimeInterval = 15) async throws { + if await Self.isHealthy() { return } + try await Self.ensureBinary() + + let process = Process() + process.executableURL = Self.binaryURL + process.arguments = ["--port", "\(Self.controlPort)"] + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + proxyProcess = process + ProxyProcessReaper.shared.register(process) + + try await waitForHealth(timeout: timeout) + } + + /// Stops the shared proxy process if this test run started one. + /// + /// The process is normally left running for the lifetime of the test run (it is reused across + /// suites) and reaped by the `atexit` hook. This method is exposed for explicit teardown. + func stopProxy() { + proxyProcess?.terminate() + proxyProcess = nil + } + + // MARK: - Internal + + static func isHealthy() async -> Bool { + let url = URL(string: "http://localhost:\(controlPort)/health")! + let result = try? await httpRequest(URLRequest(url: url), session: healthSession) + return result?.status == 200 + } + + private func waitForHealth(timeout: TimeInterval) async throws { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if await Self.isHealthy() { return } + try? await Task.sleep(nanoseconds: 200_000_000) + } + proxyProcess?.terminate() + proxyProcess = nil + throw HTTPError("uts-proxy did not become healthy within \(timeout)s") + } + + /// Ensures the binary is present in the cache, downloading and extracting if needed. + private static func ensureBinary() async throws { + let fileManager = FileManager.default + try fileManager.createDirectory(at: cacheDir, withIntermediateDirectories: true) + + if let local = localDistributive { + try installLocalDistributive(local) + return + } + + // flock serialises the download across concurrently launched test processes. + let lockURL = cacheDir.appendingPathComponent("uts-proxy.lock") + let lockFd = open(lockURL.path, O_CREAT | O_WRONLY, 0o644) + guard lockFd >= 0 else { throw HTTPError("Cannot open \(lockURL.path) for locking") } + defer { close(lockFd) } + flock(lockFd, LOCK_EX) + defer { flock(lockFd, LOCK_UN) } + + // The archive (not the extracted binary) is checksum-verified at download time, and the + // cache dir is keyed on the version, so a present+executable binary is a hit. + if fileManager.isExecutableFile(atPath: binaryURL.path) { + return + } + + let archiveBytes = try await downloadArchive() + try verifyChecksum(archiveBytes) + try extractBinary(fromArchiveBytes: archiveBytes) + } + + /// Installs a locally provided distributive into the cache, skipping download + checksum. + /// The path may be a raw `uts-proxy` binary or a `.tar.gz` archive containing one. + private static func installLocalDistributive(_ path: String) throws { + guard FileManager.default.fileExists(atPath: path) else { + throw HTTPError("Local uts-proxy distributive not found at \(path)") + } + FileHandle.standardError.write(Data("Using local uts-proxy distributive: \(path)\n".utf8)) + if path.hasSuffix(".tar.gz") { + try extractBinary(fromArchiveBytes: Data(contentsOf: URL(fileURLWithPath: path))) + } else { + try? FileManager.default.removeItem(at: binaryURL) + try FileManager.default.copyItem(at: URL(fileURLWithPath: path), to: binaryURL) + try makeExecutable(binaryURL) + } + } + + private static func downloadArchive() async throws -> Data { + FileHandle.standardError.write(Data("Downloading uts-proxy \(proxyVersion) (\(archiveName))…\n".utf8)) + // GitHub release downloads 302-redirect to the asset CDN; URLSession follows by default. + let url = URL(string: "\(githubBase)/\(archiveName)")! + let (data, status) = try await httpRequest(URLRequest(url: url), session: downloadSession) + guard status == 200 else { + throw HTTPError("Failed to download uts-proxy from \(url): HTTP \(status)") + } + return data + } + + private static func verifyChecksum(_ bytes: Data) throws { + guard let expected = checksums[archiveName] else { + throw HTTPError("No checksum for \(archiveName) — unsupported platform/arch") + } + let actual = SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + guard actual == expected else { + throw HTTPError("Checksum mismatch for \(archiveName): expected \(expected), got \(actual)") + } + } + + /// Extracts the `uts-proxy` binary from the (already verified) `.tar.gz` bytes into the cache, + /// using the system `tar`. + private static func extractBinary(fromArchiveBytes bytes: Data) throws { + let fileManager = FileManager.default + let stagingDir = fileManager.temporaryDirectory + .appendingPathComponent("uts-proxy-extract-\(UUID().uuidString)", isDirectory: true) + try fileManager.createDirectory(at: stagingDir, withIntermediateDirectories: true) + defer { try? fileManager.removeItem(at: stagingDir) } + + let archiveURL = stagingDir.appendingPathComponent(archiveName) + try bytes.write(to: archiveURL) + + let tar = Process() + tar.executableURL = URL(fileURLWithPath: "/usr/bin/tar") + tar.arguments = ["-xzf", archiveURL.path, "-C", stagingDir.path] + try tar.run() + tar.waitUntilExit() + guard tar.terminationStatus == 0 else { + throw HTTPError("tar failed to extract \(archiveName) (exit \(tar.terminationStatus))") + } + + let extracted = stagingDir.appendingPathComponent("uts-proxy") + guard fileManager.fileExists(atPath: extracted.path) else { + throw HTTPError("uts-proxy binary not found in archive '\(archiveName)'") + } + try? fileManager.removeItem(at: binaryURL) + try fileManager.moveItem(at: extracted, to: binaryURL) + try makeExecutable(binaryURL) + } + + private static func makeExecutable(_ url: URL) throws { + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: url.path) + } +} + +/// Kills any proxy process this test run spawned when the test process exits — a `Process` child +/// does not die with its parent. Registered via `atexit` (the closure must not capture anything, +/// so it reaches the reaper through the `shared` global). +private final class ProxyProcessReaper: @unchecked Sendable { + static let shared = ProxyProcessReaper() + + private let lock = NSLock() + private var processes: [Process] = [] + + private init() { + atexit { ProxyProcessReaper.shared.killAll() } + } + + func register(_ process: Process) { + lock.lock() + processes.append(process) + lock.unlock() + } + + private func killAll() { + lock.lock() + let toKill = processes + processes.removeAll() + lock.unlock() + for process in toKill where process.isRunning { + process.terminate() + } + } +} + +#endif diff --git a/Test/UTS/infra/integration/proxy/ProxySession.swift b/Test/UTS/infra/integration/proxy/ProxySession.swift new file mode 100644 index 000000000..29c620d58 --- /dev/null +++ b/Test/UTS/infra/integration/proxy/ProxySession.swift @@ -0,0 +1,296 @@ +// Proxy sessions talk to a locally spawned uts-proxy (see ProxyManager), which is macOS-only. +#if os(macOS) + +import Foundation +import Ably + +// MARK: - Rule type + factory helpers + +/// A proxy rule: a dictionary with at minimum `"match"` and `"action"` keys. +/// +/// Use the factory helpers (`wsConnectRule`, `wsFrameToClientRule`, `wsFrameToServerRule`, +/// `httpRequestRule`) to construct rules without hard-coding dictionary literals everywhere. +/// +/// Rules are evaluated in order; the first matching rule wins. Unmatched traffic passes through. +/// When `"times"` is set the rule auto-removes after that many firings. +typealias ProxyRule = [String: any Sendable] + +/// Builds a rule that matches WebSocket connection attempts. +/// +/// - Parameters: +/// - action: The action to take (e.g. `["type": "refuse_connection"]`). +/// - count: 1-based occurrence index; `2` matches only the 2nd connection attempt. +/// - queryContains: Match only if the WS URL query params contain these key/value pairs. +/// Use `"*"` as a wildcard value (matches any non-null value). +/// - times: Auto-remove the rule after this many firings. +func wsConnectRule(action: ProxyRule, + count: Int? = nil, + queryContains: [String: String]? = nil, + times: Int? = nil) -> ProxyRule { + var match: ProxyRule = ["type": "ws_connect"] + if let count { match["count"] = count } + if let queryContains { match["queryContains"] = queryContains } + var rule: ProxyRule = ["match": match, "action": action] + if let times { rule["times"] = times } + return rule +} + +/// Builds a rule that matches WebSocket frames travelling **server → client**. +/// +/// - Parameters: +/// - action: The action to take (e.g. `["type": "suppress"]`). +/// - messageAction: The Ably protocol message action number to match (see the action table in +/// the spec repo's `uts/docs/proxy.md`; e.g. `4` = CONNECTED, `11` = ATTACHED). +/// - channel: If set, additionally match only frames for this channel name. +/// - times: Auto-remove the rule after this many firings. +func wsFrameToClientRule(action: ProxyRule, + messageAction: Int? = nil, + channel: String? = nil, + times: Int? = nil) -> ProxyRule { + var match: ProxyRule = ["type": "ws_frame_to_client"] + // The proxy's MatchConfig.Action is a Go string (action name or numeric string). + if let messageAction { match["action"] = String(messageAction) } + if let channel { match["channel"] = channel } + var rule: ProxyRule = ["match": match, "action": action] + if let times { rule["times"] = times } + return rule +} + +/// Builds a rule that matches WebSocket frames travelling **client → server**. +/// +/// - Parameters: +/// - action: The action to take. +/// - messageAction: The Ably protocol message action number to match +/// (e.g. `10` = ATTACH, `17` = AUTH). +/// - channel: If set, additionally match only frames for this channel name. +/// - times: Auto-remove the rule after this many firings. +func wsFrameToServerRule(action: ProxyRule, + messageAction: Int? = nil, + channel: String? = nil, + times: Int? = nil) -> ProxyRule { + var match: ProxyRule = ["type": "ws_frame_to_server"] + // The proxy's MatchConfig.Action is a Go string (action name or numeric string). + if let messageAction { match["action"] = String(messageAction) } + if let channel { match["channel"] = channel } + var rule: ProxyRule = ["match": match, "action": action] + if let times { rule["times"] = times } + return rule +} + +/// Builds a rule that matches HTTP requests passing through the proxy. +/// +/// - Parameters: +/// - action: The action to take (e.g. `["type": "http_respond", "status": 401]`). +/// - pathContains: Match only requests whose path contains this substring. +/// - method: Match only requests with this HTTP method (e.g. `"GET"`, `"POST"`). +/// - times: Auto-remove the rule after this many firings. +func httpRequestRule(action: ProxyRule, + pathContains: String? = nil, + method: String? = nil, + times: Int? = nil) -> ProxyRule { + var match: ProxyRule = ["type": "http_request"] + if let pathContains { match["pathContains"] = pathContains } + if let method { match["method"] = method } + var rule: ProxyRule = ["match": match, "action": action] + if let times { rule["times"] = times } + return rule +} + +// MARK: - Event + +/// A single event recorded in a `ProxySession`'s log, returned by `ProxySession.getLog()`. +/// +/// Mirrors the proxy's `Event` struct. Fields that are absent from a given event are `nil` +/// (Go's `omitempty` tags), so every property is optional. A thin typed wrapper over the raw JSON +/// object so the arbitrary protocol `message` stays introspectable +/// (`event.message?["action"] as? Int`). +struct ProxyEvent: @unchecked Sendable { + /// The raw event JSON. + let raw: [String: Any] + + /// RFC3339 timestamp, e.g. `2026-06-22T21:43:56.747996Z`. + var timestamp: String? { raw["timestamp"] as? String } + /// `ws_connect`, `ws_frame`, `ws_disconnect`, `http_request`, `http_response`, or `action`. + var type: String? { raw["type"] as? String } + /// `client_to_server` or `server_to_client`. + var direction: String? { raw["direction"] as? String } + var url: String? { raw["url"] as? String } + var queryParams: [String: String]? { raw["queryParams"] as? [String: String] } + /// The protocol message carried by a `ws_frame` event. Introspect via + /// `message?["action"] as? Int` etc. + var message: [String: Any]? { raw["message"] as? [String: Any] } + var method: String? { raw["method"] as? String } + var path: String? { raw["path"] as? String } + var status: Int? { raw["status"] as? Int } + /// `client`, `server`, or `proxy`. + var initiator: String? { raw["initiator"] as? String } + var closeCode: Int? { raw["closeCode"] as? Int } + var ruleMatched: String? { raw["ruleMatched"] as? String } + var headers: [String: String]? { raw["headers"] as? [String: String] } +} + +// MARK: - ProxySession + +/// A single proxy session wrapping the `uts-proxy` control REST API. +/// +/// Each test should create one session, run its scenario, and always `close()` it in teardown: +/// +/// ```swift +/// let session = try await ProxySession.create(rules: [ +/// wsConnectRule(action: ["type": "refuse_connection"], count: 2), +/// ]) +/// let options = ARTClientOptions(key: app.defaultKey) +/// options.connectThroughProxy(session) +/// let client = ARTRealtime(options: options) +/// // … test scenario … +/// await session.close() // always — `defer` can't await, so close at the end of every path +/// ``` +/// +/// > Note: `getLog()` returns typed `ProxyEvent`s; the raw protocol message is exposed as +/// > `ProxyEvent.message` — introspect it via `message?["action"]`. +/// +/// Mirrors ably-java's `infra/integration/proxy/ProxySession.kt`. +final class ProxySession: Sendable { + + /// Opaque session identifier assigned by the proxy. + let sessionId: String + /// The port on `localhost` that the proxy is listening on for this session. + let proxyPort: Int + /// Always `"localhost"`. Exposed for use by `connectThroughProxy`. + let proxyHost = "localhost" + + // Finite timeouts so a stalled local proxy/control endpoint fails fast instead of hanging teardown. + private static let session = makeURLSession(requestTimeout: 15) + + private init(sessionId: String, proxyPort: Int) { + self.sessionId = sessionId + self.proxyPort = proxyPort + } + + /// Creates a new proxy session pointing at the Ably sandbox. + /// + /// - Parameters: + /// - rules: Initial rule set applied to all traffic through this session. + /// - port: Specific port to listen on; `0` (default) lets the proxy choose. + /// - timeoutMs: Session idle-timeout in ms; `nil` uses the proxy default (30 000 ms). + /// - realtimeHost: Upstream Ably realtime host (defaults to sandbox). + /// - restHost: Upstream Ably REST host (defaults to sandbox). + static func create(rules: [ProxyRule] = [], + port: Int = 0, + timeoutMs: Int? = nil, + realtimeHost: String = SandboxApp.sandboxHost, + restHost: String = SandboxApp.sandboxHost) async throws -> ProxySession { + var body: [String: Any] = [ + "target": ["realtimeHost": realtimeHost, "restHost": restHost], + "rules": rules, + ] + if port != 0 { body["port"] = port } + if let timeoutMs { body["timeoutMs"] = timeoutMs } + + let data = try await controlPost("/sessions", body: body) + guard let response = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let sessionId = response["sessionId"] as? String, + let proxy = response["proxy"] as? [String: Any], + let proxyPort = proxy["port"] as? Int else { + throw HTTPError("Proxy POST /sessions returned an unexpected body") + } + return ProxySession(sessionId: sessionId, proxyPort: proxyPort) + } + + /// Appends or prepends `rules` to this session's active rule list. + /// + /// - Parameters: + /// - rules: Rules to add. + /// - position: `"append"` (default) or `"prepend"`. + func addRules(_ rules: [ProxyRule], position: String = "append") async throws { + _ = try await Self.controlPost("/sessions/\(sessionId)/rules", + body: ["rules": rules, "position": position]) + } + + /// Triggers an imperative action on the current active WebSocket connection. + /// + /// Common actions: + /// ```swift + /// try await session.triggerAction(["type": "disconnect"]) + /// try await session.triggerAction(["type": "close", "closeCode": 1000]) + /// try await session.triggerAction(["type": "inject_to_client", "message": ["action": 6]]) + /// ``` + func triggerAction(_ action: ProxyRule) async throws { + _ = try await Self.controlPost("/sessions/\(sessionId)/actions", body: action) + } + + /// Returns the ordered event log recorded by the proxy for this session as typed `ProxyEvent`s. + /// + /// Common `type` values: `ws_connect`, `ws_frame`, `ws_disconnect`, `http_request`, + /// `http_response`, `action`. The raw protocol message is available via `ProxyEvent.message`. + func getLog() async throws -> [ProxyEvent] { + let data = try await Self.controlGet("/sessions/\(sessionId)/log") + guard let body = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw HTTPError("Proxy GET /log returned an unexpected body") + } + let events = body["events"] as? [[String: Any]] ?? [] + return events.map(ProxyEvent.init(raw:)) + } + + /// Closes this session and stops its proxy listener. + /// Should always be called in teardown after a test completes. Cleanup errors are ignored. + func close() async { + await Self.controlDelete("/sessions/\(sessionId)") + } + + // MARK: - Control-plane HTTP helpers + + private static func controlURL(_ path: String) -> URL { + URL(string: "http://localhost:\(ProxyManager.controlPort)\(path)")! + } + + private static func controlPost(_ path: String, body: Any) async throws -> Data { + let request = try jsonRequest("POST", controlURL(path), body: body) + let (data, status) = try await httpRequest(request, session: session) + guard (200..<300).contains(status) else { + throw HTTPError("Proxy control API returned \(status) for POST \(path): \(String(decoding: data, as: UTF8.self))") + } + return data + } + + private static func controlGet(_ path: String) async throws -> Data { + let (data, status) = try await httpRequest(URLRequest(url: controlURL(path)), session: session) + guard (200..<300).contains(status) else { + throw HTTPError("Proxy control API returned \(status) for GET \(path): \(String(decoding: data, as: UTF8.self))") + } + return data + } + + private static func controlDelete(_ path: String) async { + var request = URLRequest(url: controlURL(path)) + request.httpMethod = "DELETE" + guard let (_, status) = try? await httpRequest(request, session: session) else { return } + if !(200..<300).contains(status) { + // Teardown should never throw, but a failed delete leaks a session — make it visible. + FileHandle.standardError.write(Data("Proxy control API returned \(status) for DELETE \(path)\n".utf8)) + } + } +} + +// MARK: - Client wiring + +extension ARTClientOptions { + /// Routes a client through the given proxy `session`. + /// + /// Sets `realtimeHost` and `restHost` to the proxy host, `port` to the session's assigned + /// port, `tls = false` (the proxy serves plain HTTP/WS; TLS is only used upstream to the + /// sandbox), and `useBinaryProtocol = false` (the proxy can only inspect text frames — the + /// UTS proxy specs require JSON). + /// + /// Setting explicit hosts disables fallback hosts automatically (REC2c2), so no + /// `fallbackHosts` juggling is needed. + func connectThroughProxy(_ session: ProxySession) { + realtimeHost = session.proxyHost + restHost = session.proxyHost + port = session.proxyPort + tls = false + useBinaryProtocol = false + } +} + +#endif diff --git a/Test/UTS/Harness/Captured.swift b/Test/UTS/infra/unit/Captured.swift similarity index 100% rename from Test/UTS/Harness/Captured.swift rename to Test/UTS/infra/unit/Captured.swift diff --git a/Test/UTS/Harness/CapturingLog.swift b/Test/UTS/infra/unit/CapturingLog.swift similarity index 100% rename from Test/UTS/Harness/CapturingLog.swift rename to Test/UTS/infra/unit/CapturingLog.swift diff --git a/Test/UTS/Harness/MockHTTPClient.swift b/Test/UTS/infra/unit/MockHTTPClient.swift similarity index 81% rename from Test/UTS/Harness/MockHTTPClient.swift rename to Test/UTS/infra/unit/MockHTTPClient.swift index 2ee8b81cf..cc3ebb84d 100644 --- a/Test/UTS/Harness/MockHTTPClient.swift +++ b/Test/UTS/infra/unit/MockHTTPClient.swift @@ -50,6 +50,8 @@ struct PendingHTTPConnection { let host: String let port: Int let tls: Bool + /// Query parameters parsed from the request URL (UTS `url.query_params`). + let queryParams: [String: String] init(request: URLRequest) { guard let url = request.url, let host = url.host else { @@ -60,6 +62,7 @@ struct PendingHTTPConnection { self.tls = (url.scheme?.lowercased() == "https") self.host = host self.port = url.port ?? (tls ? 443 : 80) + self.queryParams = parseQueryParams(of: url) } /// Connection succeeds; requests proceed (UTS `respond_with_success`). @@ -99,11 +102,7 @@ struct PendingHTTPRequest { /// Query parameters parsed from the request URL (UTS `url.query_params`). var queryParams: [String: String] { - guard let components = URLComponents(url: url, resolvingAgainstBaseURL: true), - let items = components.queryItems else { return [:] } - var result: [String: String] = [:] - for item in items where item.value != nil { result[item.name] = item.value } - return result + parseQueryParams(of: request.url) } /// Sends an HTTP response (UTS `respond_with`). `body` may be `Data`, `String`, or a @@ -117,6 +116,18 @@ struct PendingHTTPRequest { completion?(response, Self.data(from: body), nil) } + /// Sends an HTTP response after `delay` seconds (UTS `respond_with_delay` — for slow-server + /// specs). The response is built up front; only the delivery is deferred. + func respondWithDelay(_ delay: TimeInterval, status: Int, body: Any) { + let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"]) + let data = Self.data(from: body) + let completion = DelayedCompletionBox(completion: self.completion) + DispatchQueue.global().asyncAfter(deadline: .now() + delay) { + completion.completion?(response, data, nil) + } + } + /// Simulates a request timeout after the connection was established (UTS `respond_with_timeout`). func respondWithTimeout() { completion?(nil, nil, NSError(domain: NSURLErrorDomain, code: URLError.timedOut.rawValue, userInfo: nil)) @@ -131,6 +142,13 @@ struct PendingHTTPRequest { } } +/// Carries a completion handler across the `asyncAfter` hop for `respondWithDelay`. The handler is +/// only ever invoked once, from that single deferred block, so the unchecked-Sendable wrapper is safe. +private final class DelayedCompletionBox: @unchecked Sendable { + let completion: ((HTTPURLResponse?, Data?, Error?) -> Void)? + init(completion: ((HTTPURLResponse?, Data?, Error?) -> Void)?) { self.completion = completion } +} + /// No-op cancellable returned by `MockHTTPClient.execute` (the response is delivered synchronously, so /// there's nothing to cancel). private final class NoopCancellable: NSObject, ARTCancellable { diff --git a/Test/UTS/Harness/MockTimeProvider.swift b/Test/UTS/infra/unit/MockTimeProvider.swift similarity index 91% rename from Test/UTS/Harness/MockTimeProvider.swift rename to Test/UTS/infra/unit/MockTimeProvider.swift index bd74dc996..c694eac05 100644 --- a/Test/UTS/Harness/MockTimeProvider.swift +++ b/Test/UTS/infra/unit/MockTimeProvider.swift @@ -18,8 +18,14 @@ final class MockTimeProvider: NSObject, TimeProvider, @unchecked Sendable { private let lock = NSLock() - /// Wall-clock origin: a fixed, arbitrary point in time (ms since the Unix epoch). - private var wallClockMilliseconds: Double = 1_600_000_000_000 + /// Wall-clock origin (ms since the Unix epoch): a fixed, arbitrary point by default, + /// overridable per test for wall-clock-sensitive specs (java's `FakeClock(initialTimeMs)`). + private var wallClockMilliseconds: Double + + init(initialWallClockMilliseconds: Double = 1_600_000_000_000) { + wallClockMilliseconds = initialWallClockMilliseconds + super.init() + } /// Continuous-clock origin. private var continuousNanoseconds: Nanoseconds = 1_000_000_000 @@ -177,6 +183,13 @@ final class MockTimeProvider: NSObject, TimeProvider, @unchecked Sendable { } } + /// Number of blocks currently scheduled (not yet fired, not cancelled) — useful for asserting + /// retry state (the counterpart of ably-java's `FakeClock.pendingTaskCount`; cocoa timers have + /// no names, so this is a single overall count). + var pendingScheduledCount: Int { + withLock { pendingCount } + } + /// Cancels every scheduled block. Used in teardown as the timer-leak safety net. func cancelAllScheduled() { withLock { scheduledBlocks.removeAll() } diff --git a/Test/UTS/Harness/MockWebSocket.swift b/Test/UTS/infra/unit/MockWebSocket.swift similarity index 96% rename from Test/UTS/Harness/MockWebSocket.swift rename to Test/UTS/infra/unit/MockWebSocket.swift index ebe851411..9c8a190b7 100644 --- a/Test/UTS/Harness/MockWebSocket.swift +++ b/Test/UTS/infra/unit/MockWebSocket.swift @@ -63,14 +63,7 @@ final class MockWebSocket: NSObject, ARTWebSocket, @unchecked Sendable { var url: URL { request.url ?? URL(string: "wss://invalid")! } /// Query parameters parsed from the connection URL (UTS `url.query_params`). var queryParams: [String: String] { - guard let url = request.url, - let components = URLComponents(url: url, resolvingAgainstBaseURL: true), - let items = components.queryItems else { return [:] } - var result: [String: String] = [:] - for item in items where item.value != nil { - result[item.name] = item.value - } - return result + parseQueryParams(of: request.url) } /// Protocol messages the SDK has sent towards the server, decoded, in order @@ -124,6 +117,13 @@ final class MockWebSocket: NSObject, ARTWebSocket, @unchecked Sendable { } } + /// Accepts the connection and immediately delivers `message` — usually `.connectedMessage` + /// (UTS `respond_with_success(message)`, the convenience ably-java also offers). + func respondWithSuccess(_ message: ProtocolMessage) { + respondWithSuccess() + sendToClient(message) + } + /// Delivers a protocol message to the client, leaving the connection open /// (UTS `send_to_client`). func sendToClient(_ message: ProtocolMessage) { diff --git a/Test/UTS/Harness/NoOpReachability.swift b/Test/UTS/infra/unit/NoOpReachability.swift similarity index 100% rename from Test/UTS/Harness/NoOpReachability.swift rename to Test/UTS/infra/unit/NoOpReachability.swift diff --git a/Test/UTS/Harness/ProtocolMessage.swift b/Test/UTS/infra/unit/ProtocolMessage.swift similarity index 89% rename from Test/UTS/Harness/ProtocolMessage.swift rename to Test/UTS/infra/unit/ProtocolMessage.swift index df61608fc..ad8e56ece 100644 --- a/Test/UTS/Harness/ProtocolMessage.swift +++ b/Test/UTS/infra/unit/ProtocolMessage.swift @@ -16,6 +16,13 @@ struct ProtocolMessage: Sendable { private let kind: Kind + /// A ready-to-use default `CONNECTED` message (ably-java's `CONNECTED_MESSAGE`) so most tests + /// don't hand-build one: connectionId `test-connection-id`, key `test-connection-key`, + /// TTL 120 s, max-idle 15 s. A value type, so it is always a fresh instance. + static var connectedMessage: ProtocolMessage { + .connected(connectionId: "test-connection-id", connectionKey: "test-connection-key") + } + /// A `CONNECTED` message carrying connection details (UTS `ProtocolMessage(action: CONNECTED, ...)`). static func connected(connectionId: String, connectionKey: String, diff --git a/Test/UTS/Harness/UTSTestCase.swift b/Test/UTS/infra/unit/UTSTestCase.swift similarity index 93% rename from Test/UTS/Harness/UTSTestCase.swift rename to Test/UTS/infra/unit/UTSTestCase.swift index 5d413e634..e1b0ca09f 100644 --- a/Test/UTS/Harness/UTSTestCase.swift +++ b/Test/UTS/infra/unit/UTSTestCase.swift @@ -9,7 +9,7 @@ import Ably.Private /// Swift Testing creates a fresh instance per `@Test`, so each test gets its own clients/mocks and /// `deinit` tears them down. /// -/// Provides the cocoa mappings of the UTS harness primitives: +/// Provides the cocoa mappings of the UTS infra primitives: /// - `installMock(_:)` — the spec's `install_mock`: registers the `MockWebSocketProvider` / `MockHTTPClient` /// so the next client built by `makeRealtime()` / `makeRest()` picks it up. The mock is never /// passed to the client constructor (a mistake per `uts/docs/writing-test-specs.md`). @@ -64,7 +64,9 @@ class UTSTestCase { fatalError("No MockWebSocketProvider installed") } - let options = ARTClientOptions() + // Seed a dummy key (ably-java's ClientOptionsBuilder does the same) so specs that don't + // exercise auth need no explicit key; the configure block can override it. + let options = ARTClientOptions(key: "appId.keyId:keySecret") options.useBinaryProtocol = false let suffix = UUID().uuidString @@ -97,7 +99,9 @@ class UTSTestCase { fatalError("No MockHTTPClient installed") } - let options = ARTClientOptions() + // Seed a dummy key (ably-java's ClientOptionsBuilder does the same) so specs that don't + // exercise auth need no explicit key; the configure block can override it. + let options = ARTClientOptions(key: "appId.keyId:keySecret") options.useBinaryProtocol = false let suffix = UUID().uuidString diff --git a/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift b/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift new file mode 100644 index 000000000..3fd73d9f9 --- /dev/null +++ b/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift @@ -0,0 +1,70 @@ +// Proxy integration tests spawn a local uts-proxy process — macOS-only (see ProxyManager). +#if os(macOS) + +import Testing +import Foundation +import Ably + +/// Acceptance test for the integration infrastructure itself (`SandboxApp`, `ProxyManager`, +/// `ProxySession`) — not derived from a UTS spec. It proves the full chain works end-to-end: +/// binary sync → proxy launch → sandbox provisioning → a real client connecting through the proxy +/// → typed event-log assertions → teardown. +/// +/// Needs outbound network (GitHub releases on first run, then the Ably sandbox), so it is gated +/// behind an env var and skipped by default: +/// +/// ```bash +/// UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.ProxyInfraSmokeTests +/// ``` +@Suite(.serialized) +final class ProxyInfraSmokeTests { + + @Test(.enabled(if: ProcessInfo.processInfo.environment["UTS_INTEGRATION_SMOKE"] != nil)) + func proxy_and_sandbox_infra_work_end_to_end() async throws { + // Suite setup (what a real proxy test does in setup) + try await ProxyManager.shared.ensureProxy() + let app = try await SandboxApp.create() + #expect(app.defaultKey.hasPrefix(app.appId + ".")) + + // Session with no rules — traffic passes through to the real sandbox. + let session = try await ProxySession.create() + #expect(session.proxyPort > 0) + + // A REAL client (no mocks) wired through the proxy. The proxy serves plain ws (tls=false), + // and basic (key) auth is TLS-only (RSA1) — so, as in ably-java's proxy tests, authenticate + // via an authCallback that signs a TokenRequest locally with the sandbox key. + let signerOptions = ARTClientOptions(key: app.defaultKey) + signerOptions.restHost = SandboxApp.sandboxHost + let tokenSigner = ARTRest(options: signerOptions) + + let options = ARTClientOptions() + options.authCallback = { params, callback in + tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } + } + options.connectThroughProxy(session) + options.autoConnect = false + let client = ARTRealtime(options: options) + + client.connect() + await awaitState(client, .connected) + #expect(client.connection.state == .connected) + + // The proxy recorded the handshake: a ws_connect and a server→client CONNECTED frame (4). + let log = try await session.getLog() + #expect(log.contains { $0.type == "ws_connect" }) + #expect(log.contains { event in + event.type == "ws_frame" && event.direction == "server_to_client" + && event.message?["action"] as? Int == 4 + }) + + // Teardown (always runs in this linear happy path; failures above skip straight to Issue). + client.close() + await awaitState(client, .closed, timeout: 10) + await session.close() + await app.delete() + } +} + +#endif diff --git a/Test/UTS/integration/standard/IntegrationSmokeTest.swift b/Test/UTS/integration/standard/IntegrationSmokeTest.swift new file mode 100644 index 000000000..21c69e899 --- /dev/null +++ b/Test/UTS/integration/standard/IntegrationSmokeTest.swift @@ -0,0 +1,66 @@ +import Testing +import Foundation +import Ably + +/// Acceptance test for the direct-sandbox integration infrastructure (`SandboxApp` alone — no +/// proxy, no fault rules) — not derived from a UTS spec. It proves the middle tier's shape works +/// end-to-end: sandbox provisioning → a real client connecting straight to the sandbox over TLS +/// (basic key auth is fine here, unlike through the proxy) → a publish/subscribe round-trip → +/// teardown. +/// +/// Runs once per protocol variant (the UTS integration specs' `PROTOCOL` dimension, ably-java's +/// `@ParameterizedTest` over `useBinaryProtocol`): `false` = JSON, `true` = msgpack. Only the +/// proxy tier is JSON-only (the proxy can't inspect binary frames) — direct-sandbox tests must +/// exercise both. +/// +/// Needs outbound network (the Ably sandbox), so it is gated behind an env var and skipped by +/// default: +/// +/// ```bash +/// UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.IntegrationSmokeTest +/// ``` +@Suite(.serialized) +final class IntegrationSmokeTest { + + @Test(.enabled(if: ProcessInfo.processInfo.environment["UTS_INTEGRATION_SMOKE"] != nil), + arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack + func sandbox_infra_works_end_to_end(useBinaryProtocol: Bool) async throws { + // Suite setup (what a real direct-sandbox test does in setup) + let app = try await SandboxApp.create() + #expect(app.defaultKey.hasPrefix(app.appId + ".")) + + // A REAL client (no mocks) wired straight to the sandbox — TLS stays on, so the plain + // sandbox key works (RSA1). Explicit hosts auto-disable fallback hosts (REC2c2). + let options = ARTClientOptions(key: app.defaultKey) + options.realtimeHost = SandboxApp.sandboxHost + options.restHost = SandboxApp.sandboxHost + options.useBinaryProtocol = useBinaryProtocol + options.autoConnect = false + let client = ARTRealtime(options: options) + + client.connect() + await awaitState(client, .connected) + #expect(client.connection.state == .connected) + + // Publish/subscribe round-trip on a fresh channel. + let channel = client.channels.get("smoke-\(UUID().uuidString)") + channel.attach() + await awaitChannelState(channel, .attached, timeout: 10) + + let received = Captured() + channel.subscribe { message in + received.append(message) + } + channel.publish("event", data: "payload") + await pollUntil("published message is echoed back to the subscriber", timeout: 10) { + received.count == 1 + } + #expect(received.first?.name == "event") + #expect(received.first?.data as? String == "payload") + + // Teardown + client.close() + await awaitState(client, .closed, timeout: 10) + await app.delete() + } +} diff --git a/Test/UTS/Tests/realtime/unit/connection/ConnectionRecoveryTests.swift b/Test/UTS/unit/realtime/ConnectionRecoveryTests.swift similarity index 99% rename from Test/UTS/Tests/realtime/unit/connection/ConnectionRecoveryTests.swift rename to Test/UTS/unit/realtime/ConnectionRecoveryTests.swift index ffaf38e3c..4076ca941 100644 --- a/Test/UTS/Tests/realtime/unit/connection/ConnectionRecoveryTests.swift +++ b/Test/UTS/unit/realtime/ConnectionRecoveryTests.swift @@ -118,7 +118,7 @@ final class ConnectionRecoveryTests: UTSTestCase { #expect(client.connection.createRecoveryKey() == nil) // The mock server doesn't auto-respond to the client's CLOSE frame, so deliver the server's - // CLOSED to drive the connection from CLOSING to CLOSED (harness-driving detail: + // CLOSED to drive the connection from CLOSING to CLOSED (infra-driving detail: wsConnection.sendToClient(.closed()) awaitConnectionState(client, .closed) diff --git a/Test/UTS/Tests/rest/unit/TimeTests.swift b/Test/UTS/unit/rest/TimeTests.swift similarity index 100% rename from Test/UTS/Tests/rest/unit/TimeTests.swift rename to Test/UTS/unit/rest/TimeTests.swift From 10ae406b40f4049753259031a5d3bfc3d279856e Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Tue, 14 Jul 2026 11:37:50 +0530 Subject: [PATCH 2/6] uts: address review feedback, add integration base test cases, split doc roles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes: - pollUntil / ProxyManager.waitForHealth / SandboxApp's retry backoff now handle task cancellation instead of hot-looping or burning retries; the backoff sleeps only between attempts and its comment matches the delays - UTS_PROXY_LOCAL_PATH installs land in a separate uts-proxy-local file, so a local build can never masquerade as the checksum-verified release cache - ProxySession's doc example shows token auth (the key-based example would throw at runtime: basic auth is TLS-only, RSA1); controlDelete now logs transport-level failures, not just non-2xx statuses - ProxyManager documents the one-test-process-at-a-time advisory (ably-java's single-fork equivalent) rather than a per-process-port redesign - dropped the unverifiable "sandbox apps auto-expire" claim DRY setup/teardown: Swift Testing has no setUp/tearDown hooks and deinit cannot await, so integration suites now subclass scoped-resource base cases: IntegrationTestCase (withSandboxApp / withRealtimeClient) and ProxyTestCase (withProxySession, proxyClientOptions), built on a shared runThenCleanUp(_:body:cleanup:) engine that always tears down and rethrows the body's error after cleanup. Both smoke tests are rewritten on top, every wait is guarded so a timeout can't cascade into secondary failures, and each carries a TODO to remove it once spec-derived tests cover its tier. Docs: split responsibilities — Test/UTS/README.md now only documents the existing setup (authoring templates and procedures moved to the uts-to-swift skill, which gains integration/proxy file templates); README gains the base test cases section, updated walkthroughs, and the java lifecycle mapping. --- .claude/skills/uts-to-swift/SKILL.md | 75 +++++++++- Test/UTS/README.md | 131 ++++++++---------- Test/UTS/infra/Utils.swift | 11 +- .../integration/IntegrationTestCase.swift | 58 ++++++++ Test/UTS/infra/integration/SandboxApp.swift | 14 +- .../integration/proxy/ProxyManager.swift | 60 +++++--- .../integration/proxy/ProxySession.swift | 19 ++- .../integration/proxy/ProxyTestCase.swift | 62 +++++++++ .../proxy/ProxyInfraSmokeTests.swift | 63 ++++----- .../standard/IntegrationSmokeTest.swift | 60 ++++---- 10 files changed, 386 insertions(+), 167 deletions(-) create mode 100644 Test/UTS/infra/integration/IntegrationTestCase.swift create mode 100644 Test/UTS/infra/integration/proxy/ProxyTestCase.swift diff --git a/.claude/skills/uts-to-swift/SKILL.md b/.claude/skills/uts-to-swift/SKILL.md index 2819e3e04..790784736 100644 --- a/.claude/skills/uts-to-swift/SKILL.md +++ b/.claude/skills/uts-to-swift/SKILL.md @@ -89,8 +89,9 @@ Read ALL files in `Test/UTS/infra/unit/` before generating any code (you need th Integration-tier rules that differ from the unit tier: -- Clients are built with a **real** transport (plain `ARTClientOptions` + `ARTRealtime`/`ARTRest`, no mocks): direct-sandbox tests point `realtimeHost`/`restHost` at `SandboxApp.sandboxHost` (TLS stays on, plain key auth works — `IntegrationSmokeTest.swift` is the shape); proxy tests call `options.connectThroughProxy(session)` and MUST authenticate via an `authCallback` that signs a `TokenRequest` locally (basic auth is TLS-only, RSA1 — `ProxyInfraSmokeTests.swift` is the shape). -- Proxy suites call `try await ProxyManager.shared.ensureProxy()` in setup and always `await session.close()` in teardown; proxy files are wrapped in `#if os(macOS)`. +- Suites subclass the tier's base case, which owns setup/teardown via scoped methods: direct-sandbox suites extend `IntegrationTestCase` and wrap the scenario in `withSandboxApp { app in … withRealtimeClient(options) { client in … } }`; proxy suites extend `ProxyTestCase` and use `withProxySession(rules:) { app, session in … }` — teardown (client close, session close, app delete) always runs, so never hand-roll it. +- Clients are built with a **real** transport (plain `ARTClientOptions` + `ARTRealtime`/`ARTRest`, no mocks): direct-sandbox tests point `realtimeHost`/`restHost` at `SandboxApp.sandboxHost` (TLS stays on, plain key auth works — `IntegrationSmokeTest.swift` is the shape); proxy tests get their options from `proxyClientOptions(for:through:)`, which wires the proxy and token auth (basic auth is TLS-only, RSA1 — `ProxyInfraSmokeTests.swift` is the shape). +- Proxy files are wrapped in `#if os(macOS)`. - Wait on real network/proxy state with `await awaitState(client, .connected)` / `await awaitChannelState(…)` / `await pollUntil("…") { … }` — never a fixed sleep. - **Protocol variants**: when a direct-sandbox spec declares the `PROTOCOL` dimension (json/msgpack), parameterise the test — `@Test(arguments: [false, true])` over a `useBinaryProtocol: Bool` parameter, applied via `options.useBinaryProtocol` (see `IntegrationSmokeTest.swift`). Proxy tests are **always JSON** (the proxy can't inspect binary frames) — `connectThroughProxy` already forces it, so no parameterisation there. @@ -495,6 +496,76 @@ final class Tests: UTSTestCase { Helper methods return types should be ready for use in the test code without additional casting (if you need "value as? String" in the test, move this conversion to the helper instead). Don't return optionals from these methods - assert not `nil` within the method itself, unless test expects optional. Don't create additional helper types for parsing dictionaries - just use a dictionary with the most suited type for the test, accessing its fields by subscript. For dictionaries containing values of different types use `[String: Any]`. Don't wrap dictionary constant initialization into helper method. Put helper methods for the suite into an extension at the bottom of the file. Keep the spec's original tests order in the generated test file. Keep test segmentation by adding comments like "// Setup", "// Test Steps", "// Assertions". +### File template — integration tiers + +Direct-sandbox spec (`integration/standard//`) — subclass `IntegrationTestCase` (setup/teardown are automatic); parameterise over the spec's `PROTOCOL` dimension when it declares one: + +```swift +import Testing +import Foundation +import Ably + +/// () +/// Derived from +@Suite(.serialized) +final class Tests: IntegrationTestCase { + + // UTS: + @Test(arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack + func test__(useBinaryProtocol: Bool) async throws { + try await withSandboxApp { app in + let options = ARTClientOptions(key: app.defaultKey) // TLS stays on → plain key auth is fine + options.realtimeHost = SandboxApp.sandboxHost + options.restHost = SandboxApp.sandboxHost + options.useBinaryProtocol = useBinaryProtocol + options.autoConnect = false + try await withRealtimeClient(options) { client in + client.connect() + guard await awaitState(client, .connected) else { return } + // … spec steps — guard every wait; the scopes above handle all teardown … + } + } + } +} +``` + +Proxy spec (`integration/proxy//`) — subclass `ProxyTestCase`; wrap the whole file in `#if os(macOS)`: + +```swift +// Proxy tests spawn a local uts-proxy process — macOS-only. +#if os(macOS) + +import Testing +import Foundation +import Ably + +/// () +/// Derived from +@Suite(.serialized) +final class Tests: ProxyTestCase { + + // UTS: + @Test + func test__() async throws { + try await withProxySession(rules: []) { app, session in // rule-less start = late fault injection + let options = proxyClientOptions(for: app, through: session) // token auth; JSON forced + options.autoConnect = false + try await withRealtimeClient(options) { client in + client.connect() + guard await awaitState(client, .connected) else { return } + // Late imperative fault injection, then poll — never sleep — and verify via the log: + try await session.triggerAction(["type": "inject_to_client", "message": ["action": 17]]) + await pollUntil("client reacted to the injected frame") { /* observable state */ true } + let log = try await session.getLog() + #expect(log.contains { $0.type == "ws_frame" && $0.direction == "client_to_server" }) + } + } // client closed, session closed, app deleted — even on failure + } +} + +#endif +``` + --- ## Step 5 — Compile diff --git a/Test/UTS/README.md b/Test/UTS/README.md index 6edf9a858..7a35463b6 100644 --- a/Test/UTS/README.md +++ b/Test/UTS/README.md @@ -194,8 +194,12 @@ Test/UTS/ │ │ └── NoOpReachability.swift # disables OS network monitoring in unit tests │ │ │ └── integration/ # INTEGRATION infra (real backend) — see §11 +│ ├── IntegrationTestCase.swift # base case for direct-sandbox suites: withSandboxApp / +│ │ # withRealtimeClient scoped setup + always-run teardown │ ├── SandboxApp.swift # provisions/deletes a sandbox app │ └── proxy/ # (macOS-only — spawns a local process) +│ ├── ProxyTestCase.swift # base case for proxy suites: withProxySession + +│ │ # token-auth proxyClientOptions │ ├── ProxyManager.swift # syncs (downloads/caches) + launches the uts-proxy binary │ └── ProxySession.swift # proxy session: rules, actions, log + connectThroughProxy │ @@ -253,13 +257,10 @@ this (§6.1). **Reaching further internals.** `import Ably.Private` exposes exactly the private headers listed in the `explicit module Private` block of `Source/include/module.modulemap` — anything declared only in a `.m` file (class extensions, ivars, private methods) is invisible to Swift. The SDK being -Objective-C also means Swift access levels (`internal`/`package`) play no part. If a spec needs an -internal symbol that isn't exposed: check `Source/PrivateHeaders/Ably/` first; if absent, declare -it in a private header there and register the header in **both** module maps -(`Source/include/module.modulemap` and `Source/Ably.modulemap`, per the repo's CLAUDE.md); if the -state has no reasonable seam, record the gap in `deviations.md` under *Mock Infrastructure -Limitations* rather than hacking around it. (The `uts-to-swift` skill's Step 4 carries the same -decision list for test translation.) +Objective-C also means Swift access levels (`internal`/`package`) play no part. When a new test +needs an internal symbol that isn't exposed yet, the `uts-to-swift` skill (Step 4, "Accessing SDK +internals") carries the decision list: check the private headers, extend them (registering in both +module maps), or record the gap in `deviations.md`. --- @@ -405,6 +406,7 @@ though some map onto a different shape: | `infra/Utils.kt` (`awaitState`, `awaitChannelState`, `pollUntil`) | `infra/Utils.swift` (`awaitState`, `awaitChannelState`, `pollUntil` — async, for the integration tier) + the synchronous `UTSTestCase` awaits for the unit tier | | `infra/Utils.kt` `withRealTimeout` | Not needed — Swift Testing has no virtual-time scheduler for `withTimeout` to be fooled by | | `unit/ClientFactories.kt` (seeds the dummy key) | `UTSTestCase.makeRealtime` / `makeRest` (seed the same dummy key) | +| JUnit suite lifecycle (`@BeforeAll` / `@AfterAll` in integration suites) | Swift Testing has no setUp/tearDown and `deinit` can't `await` — integration suites subclass `IntegrationTestCase` / `ProxyTestCase`, whose scoped `with…` methods own setup + always-run teardown (§11) | | `unit/MockWebSocket.kt` + `MockWebSocketEngineFactory.kt` | `MockWebSocket.swift` (provider + socket + the two factories) | | `unit/MockHttpClient.kt` + `MockHttpEngine.kt` + `PendingConnection/Request` (+ `Default*`) | `MockHTTPClient.swift` (`MockHTTPClient`, `PendingHTTPConnection`, `PendingHTTPRequest`) — cocoa's HTTP seam is one protocol, so no engine/adapter split | | `unit/MockEvent.kt` (typed event timeline) | `MockWebSocket.sentMessages` (client→server frames) + `Captured` collectors in `onConnectionAttempt` — same assertions, no separate event type | @@ -567,8 +569,20 @@ The `infra/integration/` layer mirrors ably-java's `infra/integration/` (see its §7 for the reference design) and is verified end-to-end by two env-gated acceptance tests (§10): `integration/standard/IntegrationSmokeTest.swift` (SandboxApp + a real TLS client, no proxy; run once per protocol variant — JSON + msgpack) and `integration/proxy/ProxyInfraSmokeTests.swift` (the full proxy chain). Three components -(§11.1–11.3), then a walkthrough of each tier's reference test (§11.4–11.5) and the request-flow -picture (§11.6): +(§11.1–11.3), the base test cases that own setup/teardown (below), then a walkthrough of each +tier's reference test (§11.4–11.5) and the request-flow picture (§11.6). + +**The base test cases.** Swift Testing has no `setUp()`/`tearDown()` hooks and `deinit` cannot +`await`, so integration suites subclass a base case whose **scoped-resource methods** own the +lifecycle — provision, run your body, and *always* tear down (rethrowing any error after cleanup, +so failures stay attributed and nothing is orphaned): + +| Base class | Subclass it for | Scoped methods | +|---|---|---| +| `IntegrationTestCase` | direct-sandbox suites | `withSandboxApp { app in … }` (create → body → `delete()`), `withRealtimeClient(options) { client in … }` (build → body → `close()` + await CLOSED) | +| `ProxyTestCase` (: `IntegrationTestCase`, macOS-only) | proxy suites | `withProxySession(rules:) { app, session in … }` (`ensureProxy` + app + session → body → `session.close()` + app delete), plus `proxyClientOptions(for:through:)` (token-auth options wired through the proxy) | + +Components: ### 11.1 `SandboxApp.swift` — a throwaway app on the real sandbox @@ -577,7 +591,7 @@ of any fault rules under test): fetches the canonical `test-app-setup.json` from `POST`s its `post_apps` body to `https://sandbox.realtime.ably-nonprod.net/apps` (GETs are retried with backoff; the POST is never retried, to avoid duplicate apps), and exposes `appId`, `defaultKey` (full-capability `appId.keyId:keySecret`), and the full `keys` list. `delete()` -removes the app in teardown (best-effort — sandbox apps also auto-expire). Owns the single +removes the app in teardown (best-effort — cleanup must never mask a test failure). Owns the single upstream host constant `SandboxApp.sandboxHost` — the `nonprod:sandbox` endpoint used uniformly across the integration specs; both `ProxySession` targets and direct-sandbox clients point at it. @@ -600,7 +614,9 @@ owns the *process*: `~/.cache/uts-proxy//` — the **same cache ably-java uses**, so the two SDKs share one download — and launches it with `--port 10100`. - The download is serialised across concurrently launched test processes by an exclusive `flock`, - and within the process by the actor. + and within the process by the actor. Process *startup* relies only on the shared health check, + so run proxy suites from **one test process at a time** (ably-java's single-fork advisory) — + concurrent runners could race to bind the control port or reap each other's proxy. - A spawned `Process` does **not** die with its parent, so an `atexit` reaper kills it when the test process exits; `stopProxy()` stops it explicitly. - Override knob: set `UTS_PROXY_LOCAL_PATH` to a **locally built** proxy binary or `.tar.gz` to @@ -659,8 +675,10 @@ injection). This is the reference for the **middle tier** — the shape every happy-path interop spec (connect/publish/subscribe/presence/history) follows. Step by step: -1. **Suite setup** — provision a real app: `let app = try await SandboxApp.create()`. (A - spec-derived suite would do this once per suite; `delete()` in teardown.) +1. **Setup/teardown via the base class** — the suite subclasses `IntegrationTestCase` and wraps + the scenario in `withSandboxApp { app in … withRealtimeClient(options) { client in … } }`: + the app is provisioned up front, and app deletion + client close always run afterwards, even + when the scenario throws or a wait fails. 2. **Protocol variants** — the test takes `useBinaryProtocol: Bool` via `@Test(arguments: [false, true])`, the cocoa realisation of the spec's `PROTOCOL` dimension (§2): each case runs the whole scenario once over JSON and once over msgpack. @@ -677,11 +695,13 @@ This is the reference for the **middle tier** — the shape every happy-path int arrives on the SDK's queue, §6.6), publish, then `await pollUntil("published message is echoed back…") { received.count == 1 }` — the real backend is eventually consistent, so poll on observable state rather than assuming timing. -7. **Teardown** — `client.close()` → `await awaitState(client, .closed)` → `await app.delete()`. +7. **Guarded waits** — the scenario lives in a helper where every wait is `guard`-ed: a timeout + has already recorded its `Issue`, so the scenario stops instead of cascading into secondary + failures, and the base class's teardown still runs. -**What this teaches about the infra:** `SandboxApp`-only provisioning, direct-sandbox client -wiring, protocol-variant parameterisation, `Captured` for cross-queue capture, and `pollUntil` -over real network state. +**What this teaches about the infra:** `IntegrationTestCase`'s scoped setup/teardown, +`SandboxApp`-only provisioning, direct-sandbox client wiring, protocol-variant parameterisation, +`Captured` for cross-queue capture, and `pollUntil` over real network state. ### 11.5 Walkthrough: a proxy test (`ProxyInfraSmokeTests`) @@ -690,25 +710,29 @@ over real network state. Step by step: -1. **Suite setup** — `try await ProxyManager.shared.ensureProxy()` (syncs + launches the proxy if - it isn't already healthy, §11.2), then `SandboxApp.create()` — the app is provisioned - **directly** against the sandbox, not through the proxy, so provisioning is independent of any - fault rules. -2. **A session with no rules** — `try await ProxySession.create()`. Starting rule-less is the - **late-fault-injection** principle (§2): the connect handshake runs against the real server - unmodified; a spec test injects its fault *afterwards*, as the final interaction. -3. **Token auth, not the key** — the proxy serves plain ws (`tls = false`), and basic key auth is - TLS-only (RSA1), so the client authenticates via an `authCallback` that signs a `TokenRequest` - locally using the sandbox key (through a separate TLS `ARTRest` "token signer"). -4. **Wire the client through the proxy** — `options.connectThroughProxy(session)` (§11.3), then +1. **Setup/teardown via the base class** — the suite subclasses `ProxyTestCase` and wraps the + scenario in `withProxySession(rules: []) { app, session in … }`: the proxy is ensured running + (§11.2), the app is provisioned **directly** against the sandbox (not through the proxy, so + provisioning is independent of any fault rules), and session close + app deletion always run + afterwards. +2. **A session with no rules** — starting rule-less is the **late-fault-injection** principle + (§2): the connect handshake runs against the real server unmodified; a spec test injects its + fault *afterwards*, as the final interaction. +3. **Token auth, not the key** — `proxyClientOptions(for: app, through: session)`: the proxy + serves plain ws (`tls = false`), and basic key auth is TLS-only (RSA1), so the client + authenticates via an `authCallback` that signs a `TokenRequest` locally using the sandbox key + (through a separate TLS `ARTRest` "token signer"). The options come back already wired through + the proxy (§11.3). +4. **Run the client in a scope** — `withRealtimeClient(options) { client in … }`, then `client.connect()` and `await awaitState(client, .connected)` — the SDK believes it is talking to Ably; every byte actually flows through the proxy. 5. **The proxy log is the primary verification** — `try await session.getLog()` and filter the typed events: the smoke asserts a `ws_connect` event and a server→client `ws_frame` whose `message?["action"] as? Int == 4` (CONNECTED). Spec tests assert on exactly this log — e.g. "the client sent an AUTH frame (17) carrying non-nil `auth` details". -6. **Teardown** — close the client and await CLOSED, then **always** `await session.close()` - (leaked sessions hold proxy listeners), then `app.delete()`. +6. **Teardown is automatic** — the scopes unwind in order: client closed and awaited CLOSED, + then `session.close()` (leaked sessions hold proxy listeners), then `app.delete()` — even when + the scenario failed or threw. **What a full spec-derived proxy test adds** (ably-java's `AuthReauthTest`, RTN22/RTC8a, is the reference): snapshot `connection.id` and a callback counter after connecting; inject the fault @@ -775,44 +799,10 @@ counterpart. `import Ably.Private`): `transportFactory` (WS) · `httpExecutor` (HTTP) · `timeProvider` (time) · `reachabilityClass` (network monitor) — plus `options.logHandler` (log assertions). -**Build a realtime unit-test client:** -```swift -let wsProvider = MockWebSocketProvider(onConnectionAttempt: { connection in - connection.respondWithSuccess() - connection.sendToClient(.connected(connectionId: "connection-1", connectionKey: "key-1")) -}) -installMock(wsProvider) -let client = makeRealtime { options in - options.key = "appId.keyId:keySecret" - options.autoConnect = false -} -client.connect() -awaitConnectionState(client, .connected) -``` - -**Build a REST unit-test client:** -```swift -let mockHTTP = MockHTTPClient( - onConnectionAttempt: { connection in connection.respondWithSuccess() }, - onRequest: { request in request.respondWith(status: 200, body: [1_704_067_200_000]) } -) -installMock(mockHTTP) -let rest = makeRest { options in options.key = "appId.keyId:keySecret" } -``` - -**Build a proxy-test client (macOS only; token auth — basic auth is TLS-only):** -```swift -try await ProxyManager.shared.ensureProxy() // suite setup -let app = try await SandboxApp.create() // suite setup -let session = try await ProxySession.create(rules: []) -let options = ARTClientOptions() -options.authCallback = { params, callback in - tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in callback(tokenRequest, error) } -} -options.connectThroughProxy(session) -// … scenario: session.triggerAction(…), await pollUntil("…") { … }, session.getLog() … -await session.close() // always, in teardown -``` +**Writing a new test?** This guide documents the *existing* setup — the actionable authoring +material (file templates for every tier, pseudocode→Swift translation tables, deviation patterns) +lives in the `uts-to-swift` skill (`.claude/skills/uts-to-swift/SKILL.md`), which carries out UTS +spec translation and evaluation. The reference tests to crib from are listed in §13. **Server→client (mock WS):** `sendToClient` (stays open — ATTACHED, channel ERROR, ACK) · `sendToClientAndClose` (DISCONNECTED / fatal ERROR) · `respondWithRefused` (1003 refusal) · @@ -832,9 +822,6 @@ DISCONNECTED=6, ERROR=9, ATTACH=10, ATTACHED=11, DETACH=12, DETACHED=13, **AUTH= **Test ID format:** `//-` → `// UTS: realtime/unit/RTN16g/recovery-key-structure-0` (comment immediately above each test). -**The decision tree when a translated test fails:** spec wrong → fix test + record UTS spec error; -translation wrong → fix test; SDK non-compliant → gate the spec-correct assertion behind -`RUN_DEVIATIONS` and record in `deviations.md`. --- @@ -863,6 +850,8 @@ translation wrong → fix test; SDK non-compliant → gate the spec-correct asse | File | Key public surface | Role | |------|--------------------|------| +| `infra/integration/IntegrationTestCase.swift` | `withSandboxApp { }`, `withRealtimeClient(_:) { }`; `runThenCleanUp(_:body:cleanup:)` (the scoped-resource engine subclasses build new scopes on) | Base case for direct-sandbox suites: scoped setup + always-run async teardown (rethrows the body's error after cleanup). | +| `infra/integration/proxy/ProxyTestCase.swift` | `withProxySession(rules:) { }`, `proxyClientOptions(for:through:)` | Base case for proxy suites (extends `IntegrationTestCase`): ensureProxy + app + session lifecycle; token-auth options wired through the proxy. **macOS-only**. | | `infra/integration/SandboxApp.swift` | `SandboxApp.create()`, `delete()`, `appId`, `defaultKey`, `keys`; `SandboxApp.sandboxHost` | Provisions/tears down a throwaway sandbox app from ably-common's `test-app-setup.json`; owns the upstream sandbox host constant. | | `infra/integration/proxy/ProxyManager.swift` | `ProxyManager.shared.ensureProxy(timeout:)`, `stopProxy()`, `ProxyManager.controlPort` (10100); `UTS_PROXY_LOCAL_PATH` override | Syncs (downloads, checksum-verifies, caches at `~/.cache/uts-proxy//`) and launches the pinned `uts-proxy` release; `atexit` reaper. **macOS-only** (`#if os(macOS)`). | | `infra/integration/proxy/ProxySession.swift` | `ProxySession.create(rules:port:timeoutMs:realtimeHost:restHost:)`, `addRules`, `triggerAction`, `getLog() -> [ProxyEvent]`, `close`, `sessionId`, `proxyPort`, `proxyHost`; `ProxyEvent`; `ProxyRule` + `wsConnectRule`/`wsFrameToClientRule`/`wsFrameToServerRule`/`httpRequestRule`; `ARTClientOptions.connectThroughProxy(_:)` | Typed client for the proxy control REST API + client wiring. **macOS-only**. | diff --git a/Test/UTS/infra/Utils.swift b/Test/UTS/infra/Utils.swift index cdde5e308..37ece5cba 100644 --- a/Test/UTS/infra/Utils.swift +++ b/Test/UTS/infra/Utils.swift @@ -9,7 +9,8 @@ import Ably.Private /// `poll`) — they poll synchronously, which is fine when a frozen `MockTimeProvider` settles the SDK /// in microseconds. The integration tier waits on *real* network and proxy state, where blocking the /// test thread for seconds is wasteful, so it uses the async helpers below (the UTS specs' -/// "poll — never sleep" rule). +/// anti-flake rule: no fixed blind waits — poll on observable state, with a short interval +/// between checks). /// Suspends until `condition` returns `true`, polling every `interval` seconds, or records a test /// failure once `timeout` (wall-clock seconds) elapses. Returns whether the condition was met. @@ -28,7 +29,13 @@ func pollUntil(_ description: String, Issue.record("Timed out after \(timeout)s polling for: \(description)", sourceLocation: sourceLocation) return false } - try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + do { + try await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + } catch { + // Task cancelled — bail out instead of spinning a hot loop until the deadline. + Issue.record("Cancelled while polling for: \(description)", sourceLocation: sourceLocation) + return false + } } return true } diff --git a/Test/UTS/infra/integration/IntegrationTestCase.swift b/Test/UTS/infra/integration/IntegrationTestCase.swift new file mode 100644 index 000000000..5338ee110 --- /dev/null +++ b/Test/UTS/infra/integration/IntegrationTestCase.swift @@ -0,0 +1,58 @@ +import Foundation +import Testing +import Ably + +/// Base class for **direct-sandbox** integration suites (the counterpart of `UTSTestCase` for the +/// unit tier): `@Suite(.serialized) final class FooTests: IntegrationTestCase`. +/// +/// Swift Testing has no `setUp()`/`tearDown()` hooks and `deinit` cannot `await`, so setup and +/// async teardown are provided as **scoped-resource methods** instead: each `with…` method +/// provisions its resource, runs your body, and *always* tears the resource down — even when the +/// body throws or a wait inside it failed. A thrown error is rethrown after cleanup, so the test +/// still fails with the original error and nothing is orphaned. +/// +/// ```swift +/// try await withSandboxApp { app in +/// let options = ARTClientOptions(key: app.defaultKey) +/// … +/// try await withRealtimeClient(options) { client in +/// // scenario — client is closed and app deleted afterwards, no matter what +/// } +/// } +/// ``` +class IntegrationTestCase { + + /// Provisions a fresh sandbox app, runs `body`, then always deletes the app. + func withSandboxApp(_ body: (SandboxApp) async throws -> Void) async throws { + let app = try await SandboxApp.create() + try await runThenCleanUp(app, body: body) { app in + await app.delete() + } + } + + /// Builds a real (unmocked) `ARTRealtime` from `options`, runs `body`, then always closes the + /// client and waits for CLOSED. + func withRealtimeClient(_ options: ARTClientOptions, _ body: (ARTRealtime) async throws -> Void) async throws { + let client = ARTRealtime(options: options) + try await runThenCleanUp(client, body: body) { client in + client.close() + await awaitState(client, .closed, timeout: 10) + } + } + + /// The scoped-resource engine every `with…` method is built on: run `body` over `resource`, + /// then always run `cleanup` — rethrowing the body's error afterwards so the test fails with + /// the original error and nothing is orphaned. Subclasses use this for their own scopes. + func runThenCleanUp(_ resource: Resource, + body: (Resource) async throws -> Void, + cleanup: (Resource) async -> Void) async throws { + var thrown: Error? + do { + try await body(resource) + } catch { + thrown = error + } + await cleanup(resource) + if let thrown { throw thrown } + } +} diff --git a/Test/UTS/infra/integration/SandboxApp.swift b/Test/UTS/infra/integration/SandboxApp.swift index db3487426..bc91884cc 100644 --- a/Test/UTS/infra/integration/SandboxApp.swift +++ b/Test/UTS/infra/integration/SandboxApp.swift @@ -8,10 +8,9 @@ import Foundation /// /// ```swift /// let app = try await SandboxApp.create() -/// defer { /* in teardown: */ } /// let key = app.defaultKey // "appId.keyId:keySecret" /// // … tests … -/// await app.delete() +/// await app.delete() // always, in teardown — even when the scenario failed /// ``` /// /// Mirrors ably-java's `infra/integration/SandboxApp.kt`. @@ -64,8 +63,8 @@ final class SandboxApp: Sendable { return SandboxApp(appId: appId, defaultKey: defaultKey, keys: keys) } - /// Deletes the provisioned app. Errors are ignored so teardown never masks a test failure - /// (sandbox apps also auto-expire server-side). + /// Deletes the provisioned app. Errors are ignored — best-effort cleanup must never mask a + /// test failure. func delete() async { var request = URLRequest(url: Self.sandboxBaseURL.appendingPathComponent("apps/\(appId)")) request.httpMethod = "DELETE" @@ -91,8 +90,11 @@ final class SandboxApp: Sendable { return postApps } catch { lastError = error - // Exponential backoff: 0.5s, 1s, 2s, 4s. - try? await Task.sleep(nanoseconds: UInt64(500_000_000 * (1 << attempt))) + if attempt < 4 { + // Exponential backoff between attempts: 0.5s, 1s, 2s, 4s. A cancelled task + // propagates out of the throwing sleep instead of burning the remaining retries. + try await Task.sleep(nanoseconds: UInt64(500_000_000 * (1 << attempt))) + } } } throw lastError diff --git a/Test/UTS/infra/integration/proxy/ProxyManager.swift b/Test/UTS/infra/integration/proxy/ProxyManager.swift index 5c1f81014..a43d11a8a 100644 --- a/Test/UTS/infra/integration/proxy/ProxyManager.swift +++ b/Test/UTS/infra/integration/proxy/ProxyManager.swift @@ -12,7 +12,10 @@ import CryptoKit /// `~/.cache/uts-proxy//uts-proxy` (the same cache location ably-java uses, so the two /// SDKs share a download). The download is serialised across OS processes by an exclusive `flock` /// on `uts-proxy.lock`, and within this process by the actor's isolation. Note: only the *download* -/// is cross-process locked — process startup relies on the shared health check on `controlPort`. +/// is cross-process locked — process startup relies on the shared health check on `controlPort`, +/// so run proxy suites from **one test process at a time** (the same advisory ably-java's +/// ProxyManager makes for Gradle workers); concurrent runners could race to bind the control port +/// or reap a proxy the other is still using. /// /// The spawned process does **not** die with its parent, so it is reaped by an `atexit` hook; /// `stopProxy()` stops it explicitly. @@ -69,9 +72,14 @@ actor ProxyManager { private static var binaryURL: URL { cacheDir.appendingPathComponent("uts-proxy") } + /// Where `UTS_PROXY_LOCAL_PATH` installs land — deliberately a *separate* file from the + /// checksum-verified release at `binaryURL`, so a local build can never be mistaken for a + /// verified cache hit by a later run without the override. + private static var localBinaryURL: URL { cacheDir.appendingPathComponent("uts-proxy-local") } + /// Optional path to a locally built `uts-proxy` binary or `.tar.gz` distributive, from the /// `UTS_PROXY_LOCAL_PATH` environment variable. When present, the release download + checksum - /// check are bypassed. + /// check are bypassed (the artifact is installed to `localBinaryURL`, re-copied on every run). private static var localDistributive: String? { ProcessInfo.processInfo.environment["UTS_PROXY_LOCAL_PATH"].flatMap { $0.isEmpty ? nil : $0 } } @@ -94,10 +102,10 @@ actor ProxyManager { /// - Parameter timeout: Maximum wall-clock seconds to wait for the process to become healthy. func ensureProxy(timeout: TimeInterval = 15) async throws { if await Self.isHealthy() { return } - try await Self.ensureBinary() + let binary = try await Self.ensureBinary() let process = Process() - process.executableURL = Self.binaryURL + process.executableURL = binary process.arguments = ["--port", "\(Self.controlPort)"] process.standardOutput = FileHandle.nullDevice process.standardError = FileHandle.nullDevice @@ -127,9 +135,16 @@ actor ProxyManager { private func waitForHealth(timeout: TimeInterval) async throws { let deadline = Date().addingTimeInterval(timeout) - while Date() < deadline { - if await Self.isHealthy() { return } - try? await Task.sleep(nanoseconds: 200_000_000) + do { + while Date() < deadline { + if await Self.isHealthy() { return } + try await Task.sleep(nanoseconds: 200_000_000) + } + } catch { + // Task cancelled — reap the process we just spawned and propagate. + proxyProcess?.terminate() + proxyProcess = nil + throw error } proxyProcess?.terminate() proxyProcess = nil @@ -137,13 +152,15 @@ actor ProxyManager { } /// Ensures the binary is present in the cache, downloading and extracting if needed. - private static func ensureBinary() async throws { + /// Returns the binary to launch: the verified release, or the separate local install when + /// `UTS_PROXY_LOCAL_PATH` is set. + private static func ensureBinary() async throws -> URL { let fileManager = FileManager.default try fileManager.createDirectory(at: cacheDir, withIntermediateDirectories: true) if let local = localDistributive { try installLocalDistributive(local) - return + return localBinaryURL } // flock serialises the download across concurrently launched test processes. @@ -157,12 +174,13 @@ actor ProxyManager { // The archive (not the extracted binary) is checksum-verified at download time, and the // cache dir is keyed on the version, so a present+executable binary is a hit. if fileManager.isExecutableFile(atPath: binaryURL.path) { - return + return binaryURL } let archiveBytes = try await downloadArchive() try verifyChecksum(archiveBytes) - try extractBinary(fromArchiveBytes: archiveBytes) + try extractBinary(fromArchiveBytes: archiveBytes, to: binaryURL) + return binaryURL } /// Installs a locally provided distributive into the cache, skipping download + checksum. @@ -173,11 +191,11 @@ actor ProxyManager { } FileHandle.standardError.write(Data("Using local uts-proxy distributive: \(path)\n".utf8)) if path.hasSuffix(".tar.gz") { - try extractBinary(fromArchiveBytes: Data(contentsOf: URL(fileURLWithPath: path))) + try extractBinary(fromArchiveBytes: Data(contentsOf: URL(fileURLWithPath: path)), to: localBinaryURL) } else { - try? FileManager.default.removeItem(at: binaryURL) - try FileManager.default.copyItem(at: URL(fileURLWithPath: path), to: binaryURL) - try makeExecutable(binaryURL) + try? FileManager.default.removeItem(at: localBinaryURL) + try FileManager.default.copyItem(at: URL(fileURLWithPath: path), to: localBinaryURL) + try makeExecutable(localBinaryURL) } } @@ -202,9 +220,9 @@ actor ProxyManager { } } - /// Extracts the `uts-proxy` binary from the (already verified) `.tar.gz` bytes into the cache, - /// using the system `tar`. - private static func extractBinary(fromArchiveBytes bytes: Data) throws { + /// Extracts the `uts-proxy` binary from the `.tar.gz` bytes to `destination`, using the + /// system `tar`. + private static func extractBinary(fromArchiveBytes bytes: Data, to destination: URL) throws { let fileManager = FileManager.default let stagingDir = fileManager.temporaryDirectory .appendingPathComponent("uts-proxy-extract-\(UUID().uuidString)", isDirectory: true) @@ -227,9 +245,9 @@ actor ProxyManager { guard fileManager.fileExists(atPath: extracted.path) else { throw HTTPError("uts-proxy binary not found in archive '\(archiveName)'") } - try? fileManager.removeItem(at: binaryURL) - try fileManager.moveItem(at: extracted, to: binaryURL) - try makeExecutable(binaryURL) + try? fileManager.removeItem(at: destination) + try fileManager.moveItem(at: extracted, to: destination) + try makeExecutable(destination) } private static func makeExecutable(_ url: URL) throws { diff --git a/Test/UTS/infra/integration/proxy/ProxySession.swift b/Test/UTS/infra/integration/proxy/ProxySession.swift index 29c620d58..e567a4515 100644 --- a/Test/UTS/infra/integration/proxy/ProxySession.swift +++ b/Test/UTS/infra/integration/proxy/ProxySession.swift @@ -139,7 +139,12 @@ struct ProxyEvent: @unchecked Sendable { /// let session = try await ProxySession.create(rules: [ /// wsConnectRule(action: ["type": "refuse_connection"], count: 2), /// ]) -/// let options = ARTClientOptions(key: app.defaultKey) +/// // The proxy serves plain ws (`tls = false`) and basic (key) auth is TLS-only (RSA1), +/// // so authenticate with a TokenRequest signed locally by a TLS "token signer" client: +/// let options = ARTClientOptions() +/// options.authCallback = { params, callback in +/// tokenSigner.auth.createTokenRequest(params, options: nil) { callback($0, $1) } +/// } /// options.connectThroughProxy(session) /// let client = ARTRealtime(options: options) /// // … test scenario … @@ -264,10 +269,14 @@ final class ProxySession: Sendable { private static func controlDelete(_ path: String) async { var request = URLRequest(url: controlURL(path)) request.httpMethod = "DELETE" - guard let (_, status) = try? await httpRequest(request, session: session) else { return } - if !(200..<300).contains(status) { - // Teardown should never throw, but a failed delete leaks a session — make it visible. - FileHandle.standardError.write(Data("Proxy control API returned \(status) for DELETE \(path)\n".utf8)) + // Teardown should never throw, but a failed delete leaks a session — make it visible. + do { + let (_, status) = try await httpRequest(request, session: session) + if !(200..<300).contains(status) { + FileHandle.standardError.write(Data("Proxy control API returned \(status) for DELETE \(path)\n".utf8)) + } + } catch { + FileHandle.standardError.write(Data("Proxy DELETE \(path) failed: \(error)\n".utf8)) } } } diff --git a/Test/UTS/infra/integration/proxy/ProxyTestCase.swift b/Test/UTS/infra/integration/proxy/ProxyTestCase.swift new file mode 100644 index 000000000..9414e8647 --- /dev/null +++ b/Test/UTS/infra/integration/proxy/ProxyTestCase.swift @@ -0,0 +1,62 @@ +// Proxy tests spawn a local uts-proxy process — macOS-only (see ProxyManager). +#if os(macOS) + +import Foundation +import Testing +import Ably + +/// Base class for **proxy** integration suites: +/// `@Suite(.serialized) final class FooTests: ProxyTestCase`. +/// +/// Extends `IntegrationTestCase` with the proxy tier's setup/teardown: `withProxySession` ensures +/// the proxy is running, provisions the sandbox app, creates the session, and always closes the +/// session and deletes the app afterwards — the shape every proxy test needs. +/// +/// ```swift +/// try await withProxySession(rules: []) { app, session in +/// let options = proxyClientOptions(for: app, through: session) +/// options.autoConnect = false +/// try await withRealtimeClient(options) { client in +/// // scenario — late fault injection via session.triggerAction(…), +/// // verification via session.getLog() +/// } +/// } +/// ``` +class ProxyTestCase: IntegrationTestCase { + + /// Ensures the proxy is running, provisions a sandbox app and a `ProxySession` with `rules`, + /// runs `body`, then always closes the session and deletes the app. + func withProxySession(rules: [ProxyRule] = [], + _ body: (SandboxApp, ProxySession) async throws -> Void) async throws { + try await ProxyManager.shared.ensureProxy() + try await withSandboxApp { app in + let session = try await ProxySession.create(rules: rules) + try await runThenCleanUp(session, body: { session in + try await body(app, session) + }) { session in + await session.close() + } + } + } + + /// Client options wired through the proxy with **token auth**: the proxy serves plain ws + /// (`tls = false`) and basic (key) auth is TLS-only (RSA1), so the client authenticates via an + /// `authCallback` that signs a `TokenRequest` locally using the sandbox key (through a + /// separate TLS "token signer" client, as in ably-java's proxy tests). + func proxyClientOptions(for app: SandboxApp, through session: ProxySession) -> ARTClientOptions { + let signerOptions = ARTClientOptions(key: app.defaultKey) + signerOptions.restHost = SandboxApp.sandboxHost + let tokenSigner = ARTRest(options: signerOptions) + + let options = ARTClientOptions() + options.authCallback = { params, callback in + tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } + } + options.connectThroughProxy(session) + return options + } +} + +#endif diff --git a/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift b/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift index 3fd73d9f9..d0b5eebf0 100644 --- a/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift +++ b/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift @@ -6,9 +6,9 @@ import Foundation import Ably /// Acceptance test for the integration infrastructure itself (`SandboxApp`, `ProxyManager`, -/// `ProxySession`) — not derived from a UTS spec. It proves the full chain works end-to-end: -/// binary sync → proxy launch → sandbox provisioning → a real client connecting through the proxy -/// → typed event-log assertions → teardown. +/// `ProxySession`, `ProxyTestCase`) — not derived from a UTS spec. It proves the full chain works +/// end-to-end: binary sync → proxy launch → sandbox provisioning → a real client connecting +/// through the proxy → typed event-log assertions → teardown (handled by the base class). /// /// Needs outbound network (GitHub releases on first run, then the Ably sandbox), so it is gated /// behind an env var and skipped by default: @@ -16,54 +16,51 @@ import Ably /// ```bash /// UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.ProxyInfraSmokeTests /// ``` +// TODO: Remove this infra acceptance (smoke) test once spec-derived integration/proxy +// tests exist and cover this ground (see Test/UTS/README.md §11.7). @Suite(.serialized) -final class ProxyInfraSmokeTests { +final class ProxyInfraSmokeTests: ProxyTestCase { @Test(.enabled(if: ProcessInfo.processInfo.environment["UTS_INTEGRATION_SMOKE"] != nil)) func proxy_and_sandbox_infra_work_end_to_end() async throws { - // Suite setup (what a real proxy test does in setup) - try await ProxyManager.shared.ensureProxy() - let app = try await SandboxApp.create() - #expect(app.defaultKey.hasPrefix(app.appId + ".")) + // Session with no rules — traffic passes through to the real sandbox. The base class + // ensures the proxy is running and always closes the session + deletes the app. + try await withProxySession(rules: []) { app, session in + #expect(app.defaultKey.hasPrefix(app.appId + ".")) + #expect(session.proxyPort > 0) - // Session with no rules — traffic passes through to the real sandbox. - let session = try await ProxySession.create() - #expect(session.proxyPort > 0) + // A REAL client (no mocks) wired through the proxy with token auth (see + // proxyClientOptions — basic key auth is TLS-only, RSA1). + let options = proxyClientOptions(for: app, through: session) + options.autoConnect = false - // A REAL client (no mocks) wired through the proxy. The proxy serves plain ws (tls=false), - // and basic (key) auth is TLS-only (RSA1) — so, as in ably-java's proxy tests, authenticate - // via an authCallback that signs a TokenRequest locally with the sandbox key. - let signerOptions = ARTClientOptions(key: app.defaultKey) - signerOptions.restHost = SandboxApp.sandboxHost - let tokenSigner = ARTRest(options: signerOptions) - - let options = ARTClientOptions() - options.authCallback = { params, callback in - tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in - callback(tokenRequest, error) + try await withRealtimeClient(options) { client in + await runScenario(client, session) } } - options.connectThroughProxy(session) - options.autoConnect = false - let client = ARTRealtime(options: options) + } + /// The happy-path scenario. Each wait is guarded (a timeout already records an `Issue`), and + /// the control-plane `getLog()` failure is recorded rather than thrown — the base class's + /// teardown runs regardless. + private func runScenario(_ client: ARTRealtime, _ session: ProxySession) async { client.connect() - await awaitState(client, .connected) + guard await awaitState(client, .connected) else { return } #expect(client.connection.state == .connected) // The proxy recorded the handshake: a ws_connect and a server→client CONNECTED frame (4). - let log = try await session.getLog() + let log: [ProxyEvent] + do { + log = try await session.getLog() + } catch { + Issue.record("Proxy getLog() failed: \(error)") + return + } #expect(log.contains { $0.type == "ws_connect" }) #expect(log.contains { event in event.type == "ws_frame" && event.direction == "server_to_client" && event.message?["action"] as? Int == 4 }) - - // Teardown (always runs in this linear happy path; failures above skip straight to Issue). - client.close() - await awaitState(client, .closed, timeout: 10) - await session.close() - await app.delete() } } diff --git a/Test/UTS/integration/standard/IntegrationSmokeTest.swift b/Test/UTS/integration/standard/IntegrationSmokeTest.swift index 21c69e899..72242654a 100644 --- a/Test/UTS/integration/standard/IntegrationSmokeTest.swift +++ b/Test/UTS/integration/standard/IntegrationSmokeTest.swift @@ -2,11 +2,11 @@ import Testing import Foundation import Ably -/// Acceptance test for the direct-sandbox integration infrastructure (`SandboxApp` alone — no -/// proxy, no fault rules) — not derived from a UTS spec. It proves the middle tier's shape works -/// end-to-end: sandbox provisioning → a real client connecting straight to the sandbox over TLS -/// (basic key auth is fine here, unlike through the proxy) → a publish/subscribe round-trip → -/// teardown. +/// Acceptance test for the direct-sandbox integration infrastructure (`SandboxApp` + +/// `IntegrationTestCase` — no proxy, no fault rules) — not derived from a UTS spec. It proves the +/// middle tier's shape works end-to-end: sandbox provisioning → a real client connecting straight +/// to the sandbox over TLS (basic key auth is fine here, unlike through the proxy) → a +/// publish/subscribe round-trip → teardown (handled by the base class). /// /// Runs once per protocol variant (the UTS integration specs' `PROTOCOL` dimension, ably-java's /// `@ParameterizedTest` over `useBinaryProtocol`): `false` = JSON, `true` = msgpack. Only the @@ -19,48 +19,54 @@ import Ably /// ```bash /// UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.IntegrationSmokeTest /// ``` +// TODO: Remove this infra acceptance (smoke) test once spec-derived integration/standard +// tests exist and cover this ground (see Test/UTS/README.md §11.7). @Suite(.serialized) -final class IntegrationSmokeTest { +final class IntegrationSmokeTest: IntegrationTestCase { @Test(.enabled(if: ProcessInfo.processInfo.environment["UTS_INTEGRATION_SMOKE"] != nil), arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack func sandbox_infra_works_end_to_end(useBinaryProtocol: Bool) async throws { - // Suite setup (what a real direct-sandbox test does in setup) - let app = try await SandboxApp.create() - #expect(app.defaultKey.hasPrefix(app.appId + ".")) + try await withSandboxApp { app in + #expect(app.defaultKey.hasPrefix(app.appId + ".")) - // A REAL client (no mocks) wired straight to the sandbox — TLS stays on, so the plain - // sandbox key works (RSA1). Explicit hosts auto-disable fallback hosts (REC2c2). - let options = ARTClientOptions(key: app.defaultKey) - options.realtimeHost = SandboxApp.sandboxHost - options.restHost = SandboxApp.sandboxHost - options.useBinaryProtocol = useBinaryProtocol - options.autoConnect = false - let client = ARTRealtime(options: options) + // A REAL client (no mocks) wired straight to the sandbox — TLS stays on, so the plain + // sandbox key works (RSA1). Explicit hosts auto-disable fallback hosts (REC2c2). + let options = ARTClientOptions(key: app.defaultKey) + options.realtimeHost = SandboxApp.sandboxHost + options.restHost = SandboxApp.sandboxHost + options.useBinaryProtocol = useBinaryProtocol + options.autoConnect = false + try await withRealtimeClient(options) { client in + await runScenario(client) + } + } + } + + /// The happy-path scenario. Each wait is guarded: on timeout an `Issue` is already recorded + /// by the helper, so just stop instead of driving a client in the wrong state — the base + /// class's teardown runs regardless. + private func runScenario(_ client: ARTRealtime) async { client.connect() - await awaitState(client, .connected) + guard await awaitState(client, .connected) else { return } #expect(client.connection.state == .connected) - // Publish/subscribe round-trip on a fresh channel. + // Publish/subscribe round-trip on a fresh channel (fresh name per variant/retry, so runs + // never collide on server-side channel state). let channel = client.channels.get("smoke-\(UUID().uuidString)") channel.attach() - await awaitChannelState(channel, .attached, timeout: 10) + guard await awaitChannelState(channel, .attached, timeout: 10) else { return } let received = Captured() channel.subscribe { message in received.append(message) } channel.publish("event", data: "payload") - await pollUntil("published message is echoed back to the subscriber", timeout: 10) { + guard await pollUntil("published message is echoed back to the subscriber", timeout: 10, { received.count == 1 - } + }) else { return } #expect(received.first?.name == "event") #expect(received.first?.data as? String == "payload") - - // Teardown - client.close() - await awaitState(client, .closed, timeout: 10) - await app.delete() } } From 539081b7b8eae118fd28c83316611044f4380f69 Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Wed, 15 Jul 2026 00:38:12 +0530 Subject: [PATCH 3/6] uts: rewrite uts-to-swift skill to module-directory flow, mirroring uts-to-kotlin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuild the skill to match ably-java's uts-to-kotlin step for step: it now takes a whole UTS module directory (not a single spec file) and runs in two phases — scripted selection (resolve module -> confirm mapping -> pick tier -> pick specs -> translate-only vs translate-and-evaluate), then per-spec translation with a deterministic faithfulness audit. New alongside SKILL.md (v2.0.0): - scripts/resolve_uts.py — validates a module dir, reads uts-package-mapping.json, and emits per-tier target directories plus spec files with derived Swift class names (minimal-diff port of kotlin's; Tests suffix, no package concept). - scripts/audit_translation.py — Step 7 review aid: Test-ID coverage (missing/orphan // UTS: tags) and a per-test spec-line ledger with an assertion-shortfall tripwire. Improves on the kotlin original by skipping comment-only lines when counting assertions, so a commented-out #expect can't mask a dropped one. - uts-package-mapping.json — spec module -> Test/UTS tier-directory mapping (realtime, rest, objects). - references/objects-mapping.md — intentionally-empty placeholder for the future ably-js -> ably-cocoa LiveObjects type map. SKILL.md keeps all cocoa-specific translation content (Ably.Private access ladder, Captured/Sendable rules, mock tables, per-tier file templates) at kotlin's structure, adds the three-outcome evaluation model incl. the spec-error fail-fast pattern, and a dedicated integration/proxy section built on the Test/UTS scoped-resource base cases. Verified: resolver and audit exercised against the full spec corpus (146 files, zero extractor failures) and both existing UTS suite/spec pairs; swift test --filter UTS passes (13/13). --- .claude/skills/uts-to-swift/SKILL.md | 881 ++++++++++++++---- .../references/objects-mapping.md | 7 + .../uts-to-swift/scripts/audit_translation.py | 251 +++++ .../uts-to-swift/scripts/resolve_uts.py | 178 ++++ .../uts-to-swift/uts-package-mapping.json | 22 + 5 files changed, 1174 insertions(+), 165 deletions(-) create mode 100644 .claude/skills/uts-to-swift/references/objects-mapping.md create mode 100755 .claude/skills/uts-to-swift/scripts/audit_translation.py create mode 100755 .claude/skills/uts-to-swift/scripts/resolve_uts.py create mode 100644 .claude/skills/uts-to-swift/uts-package-mapping.json diff --git a/.claude/skills/uts-to-swift/SKILL.md b/.claude/skills/uts-to-swift/SKILL.md index 790784736..8b4e999aa 100644 --- a/.claude/skills/uts-to-swift/SKILL.md +++ b/.claude/skills/uts-to-swift/SKILL.md @@ -1,133 +1,240 @@ --- name: uts-to-swift -description: Translate a UTS (Universal Test Suite) pseudocode test spec into Swift tests in the UTS target. Run as /uts-to-swift +description: "Translate the UTS pseudocode test specs in a whole module directory into runnable Swift tests in the ably-cocoa UTS test target. Takes a UTS module directory (e.g. .../specification/uts/objects), validates its structure, resolves the target test directories, lets you pick a tier (unit/integration/proxy) and which specs, then derives a Swift test per spec. Usage: /uts-to-swift " license: proprietary -allowed-tools: Bash, Read, Edit, Write +allowed-tools: Bash, Read, Edit, Write, WebFetch metadata: team: engineering - version: "1.0.0" + version: "2.0.0" tags: testing, uts, swift, cocoa, ably-cocoa, test-translation marketplace: false --- -# UTS to Swift +Translate the UTS pseudocode test specs under the **module directory** `$ARGUMENTS` into runnable Swift +tests in the ably-cocoa `UTS` test target (`Test/UTS`). -Translate the UTS pseudocode test spec at `$ARGUMENTS` into a runnable Swift test in the `UTS` test target (`Test/UTS`). +`$ARGUMENTS` is a UTS *module* directory — a directory sitting directly under `.../specification/uts/`, +e.g. `/Users/sachinsh/ably-specification/specification/uts/objects`. Its name (`objects`, `realtime`, +`rest`, …) is the **source module**. A module directory holds many spec files, organised into tiers +(`unit/`, `integration/`, and `integration/proxy/`). -Reference: [Writing Derived Tests](https://raw.githubusercontent.com/ably/specification/refs/heads/main/uts/docs/writing-derived-tests.md) +The work happens in two phases: + +- **Phase 1 — Selection (Steps A–E below):** a bundled resolver script validates the directory and works + out the target `Test/UTS` directories, the spec files, and their class names; you then pick a tier, pick + which specs, and choose whether to also evaluate. +- **Phase 2 — Per-spec translation (Steps 1–7):** for each selected spec file, derive a Swift test. + +## Required reading — fetch first + +Always fetch [writing-derived-tests.md](https://raw.githubusercontent.com/ably/specification/refs/heads/main/uts/docs/writing-derived-tests.md) first (once per run) — don't rely on memory or the inlined summaries; the manual is updated over time. --- -## Step 0 — Validate arguments +# Phase 1 — Selection -**If `$ARGUMENTS` is empty or blank**, stop immediately and tell the user: +Path validation, the directory mapping, spec discovery, and class-name derivation are all mechanical, so a +bundled script does them — that keeps selection byte-for-byte deterministic instead of relying on the model +to re-eyeball regexes, join paths, and hand-convert `snake_case` → `PascalCase` each run. -``` -Please re-run the command with the path to a UTS pseudocode spec file. +> **If `$ARGUMENTS` is empty or blank**, stop and show: `Usage: /uts-to-swift ` +> — with the example `/uts-to-swift /Users/sachinsh/ably-specification/specification/uts/objects`. -Usage: /uts-to-swift or /uts-to-swift https://github.com/ably/specification/blob/main/uts/realtime/unit/connection/connection_recovery_test.md +## Step A — Resolve the module -Example: - /uts-to-swift /path/to/spec/file.md - /uts-to-swift https://github.com/ably/specification/blob/main/uts/realtime/unit/connection/connection_recovery_test.md +Run the resolver on the directory passed in (substitute the real path; the script path is relative to the +ably-cocoa repo root): +```bash +python3 .claude/skills/uts-to-swift/scripts/resolve_uts.py "" ``` -Do not proceed to Step 1. +It prints one JSON object. **If `ok` is `false`, relay `message` to the user and stop** — error codes: +`NOT_A_UTS_MODULE_PATH` (not a `.../uts/` directory), `DIR_NOT_FOUND`, `NO_TIER_DIRS` (no `unit/` +or `integration/`). On success it gives `sourceModule`, `mapped`, `testRoot`, `translationNotes`, and a +`tiers` object with one entry per tier (`unit` / `integration` / `proxy`), each carrying `present`, +`sourceDir`, `targetDir`, and `specs` (a list of `{file, className}`). Everything downstream reads from +this output — treat it as the single source of truth and don't recompute paths or names by hand. -**If `$ARGUMENTS` is provided but does not end in `.md`**, stop and tell the user: +`translationNotes` is the path to a per-module ably-js → ably-cocoa type/interface map when the module +declares one (its `notes` field in `uts-package-mapping.json`, e.g. `objects` → +`references/objects-mapping.md`), else `null`. When it's non-null, it is **required reading before +Phase 2** — see Step 1. (The `objects` notes file is currently a **placeholder** — if it still only +carries the "intentionally empty" stub, tell the user the module's mapping hasn't been authored yet and +treat the module as not-yet-translatable rather than guessing a mapping.) -``` -Error: "" does not look like a spec file path (expected a .md file). +## Step B — Confirm or create the target mapping -``` +The target dirs come from `uts-package-mapping.json` (alongside this skill); spec and ably-cocoa module +names don't always match (ably-java, for example, maps `objects` → `liveobjects`), which is why it's +explicit. -Do not proceed to Step 1. +- **If `mapped` is `true`**: show the resolved `targetDir` for each present tier and ask the user to confirm. + If they say the mapping is wrong, ask for the correct ably-cocoa module base name and re-run with `--create` + (below) to overwrite the entry, then re-resolve. +- **If `mapped` is `false`**: there's no mapping for `sourceModule` yet. Ask for the target ably-cocoa module + base name (default to `sourceModule`; suggest a rename only when the SDK uses different terminology), then + create it deterministically and re-resolve: -**If `$ARGUMENTS` ends in `.md` but the file does not exist** (check with `test -f "$ARGUMENTS"`), stop and tell the user: + ```bash + python3 .claude/skills/uts-to-swift/scripts/resolve_uts.py "" --create + ``` -``` -Error: file not found: "" + This adds `unit/`, `integration/standard/`, and `integration/proxy/` under + `packages` and re-prints the resolved output. (`` must be a simple module base name — letters, + digits, underscore; the script returns `BAD_TARGET_NAME` otherwise, so just ask again.) -``` +## Step C — Choose the tier + +Offer the tiers whose `present` is `true`. The chosen tier fixes the `targetDir` and `specs` (from Step A) +**and** the translation flow Phase 2 uses — don't re-detect any of it per spec: -Do not proceed to Step 1. +| Tier | Translation flow | +|---|---| +| **unit** | mocked transport — Steps 3–4 below | +| **integration** (direct sandbox) | real sandbox, no faults — **Direct-sandbox integration tests** section | +| **proxy** | real sandbox + fault injection — **Integration tests** section (proxy subsections) | -Only continue to Step 1 once the file is confirmed to exist. +## Step D — Choose which specs to translate ---- +The chosen tier's `specs` list (from Step A) is the candidate set — each entry already has its source `file` +and derived `className`. Present it and ask whether to translate **all** of them or a **selected subset**. +Then continue to Step E. + +## Step E — Translate only, or also evaluate? -## Step 1 — Read the spec +`writing-derived-tests.md` splits the work into **Translation** (always) and **Evaluation** (only +meaningful once the SDK implementation for this module exists). Ask the user which they want, and carry the +answer into Phase 2: -Read the file at `$ARGUMENTS`. Identify all the test cases — each has a title, a structured `Test ID` like `realtime/unit/RTN16g/recovery-key-structure-0` and a requirement. +- **Translate only** — generate each test and make it **compile** (Steps 1–5 and the Step 7 review). Don't + run the tests. Use this when the SDK feature isn't implemented yet, so there's nothing to run against. +- **Translate and evaluate** — all of the above **plus** running the tests and **fixing until they pass** + (Step 6): work the decision tree, and where the SDK genuinely diverges, gate/adapt the assertion and + record a deviation. Use this when the implementation exists. + +If you can't tell whether the implementation exists, ask the user rather than guessing. --- -## Step 2 — Determine output path and class name +# Phase 2 — Per-spec translation -Map the spec path to a test path: +Run this for **each** spec file selected in Step D. **Step 6 only applies in "translate and evaluate" mode +(Step E)** — in "translate only" mode, stop after compiling (Step 5) and reviewing (Step 7), and skip +Step 6 entirely. -Tests are organised **by tier, then by module** under `Test/UTS/` (kept in sync with ably-java's `uts` module — see `Test/UTS/README.md` §4): `unit//` for mocked-transport tests, `integration/standard//` and `integration/proxy//` for real-backend tests. The per-module folder keeps same-named specs from different parts of the spec from colliding (e.g. `uts/realtime/integration/auth.md` and `uts/rest/integration/auth.md`). +When translating several specs, do Steps 1–4 (generate the file) for every spec first, then run Step 5 +(compile) once for the whole module, then per file run Step 6 (only if evaluating) and the Step 7 review — +compiling once is faster than per-file and surfaces cross-file issues together. For a single spec, just go +through the steps in order. -| Spec file | Test file | -|---|---| -| `.../uts/rest/unit/.md` | `Test/UTS/unit/rest/Tests.swift` | -| `.../uts/realtime/unit//.md` | `Test/UTS/unit/realtime/Tests.swift` | -| `.../uts/rest/integration/.md` | `Test/UTS/integration/standard/rest/Tests.swift` | -| `.../uts/realtime/integration/.md` | `Test/UTS/integration/standard/realtime/Tests.swift` | -| `.../uts/realtime/integration/proxy/.md` | `Test/UTS/integration/proxy/realtime/Tests.swift` | +## Step 1 — Read the spec (and any module translation notes) -Class name: take the file name, strip a trailing `_test`, convert `snake_case` → `PascalCase`, append `Tests`. Example: `connection_recovery_test.md` → `ConnectionRecoveryTests`. Each test is a **Swift Testing** suite — `@Suite(.serialized) final class Tests: UTSTestCase`. +**If Step A reported a non-null `translationNotes`, read that file first (once per run).** UTS specs are +written in a language-agnostic pseudocode that mirrors the *ably-js* API; for modules whose ably-cocoa +types diverge, the notes map each spec symbol to its ably-cocoa equivalent. Skipping them yields tests +that read like ably-js and won't compile. (If the notes file is still the "intentionally empty" +placeholder, stop and tell the user — see Step A.) -If a suitable suite already exists, add the new test methods to it rather than creating a duplicate. +Then read the current spec file (the one being translated from the Step D selection). Identify: +- All test cases — each has a structured ID like `realtime/unit/RSA4c2/callback-error-connecting-disconnected-0` and a description +- The protocol used (WebSocket for Realtime, HTTP for REST) +- Any timer usage (`enable_fake_timers`, `ADVANCE_TIME`) +- Any **protocol-variant dimension** — a `PROTOCOL` (`json` / `msgpack`) matrix the spec header says to run "once per variant". In swift this becomes a `useBinaryProtocol` parameterised test — `@Test(arguments: [false, true])` (see the **Direct-sandbox integration tests** section), not a plain `@Test`. --- -## Step 3 — Read the UTS infra +## Step 2 — Output path -Read ALL files in `Test/UTS/infra/unit/` before generating any code (you need the exact method names/signatures). For an **integration** spec, additionally read `Test/UTS/infra/Utils.swift` (async `pollUntil` / `awaitState` / `awaitChannelState`) and ALL files in `Test/UTS/infra/integration/` (`SandboxApp`, and for proxy specs `ProxyManager`/`ProxySession`), plus the reference smoke tests under `Test/UTS/integration/` and the guide's §11 in `Test/UTS/README.md` — especially the per-tier walkthroughs (§11.4–11.5) and the request-flow diagram (§11.6). +Don't derive anything here — the resolver (Step A) already produced it. For the chosen tier use its +`targetDir`, and for the spec being translated use its `className` from that tier's `specs` list. Write the +test to `/.swift`. -Integration-tier rules that differ from the unit tier: +The spec's own `` grouping (e.g. `connection/`, `channels/`) is **not** reflected in the output — every +test sits directly in `targetDir` (the resolver flattens it). The chosen tier also fixes the translation +flow: **unit** → the rules in Steps 3–4 below; **integration** (direct sandbox) → the **Direct-sandbox +integration tests** section; **proxy** → the proxy subsections of the **Integration tests** section. -- Suites subclass the tier's base case, which owns setup/teardown via scoped methods: direct-sandbox suites extend `IntegrationTestCase` and wrap the scenario in `withSandboxApp { app in … withRealtimeClient(options) { client in … } }`; proxy suites extend `ProxyTestCase` and use `withProxySession(rules:) { app, session in … }` — teardown (client close, session close, app delete) always runs, so never hand-roll it. -- Clients are built with a **real** transport (plain `ARTClientOptions` + `ARTRealtime`/`ARTRest`, no mocks): direct-sandbox tests point `realtimeHost`/`restHost` at `SandboxApp.sandboxHost` (TLS stays on, plain key auth works — `IntegrationSmokeTest.swift` is the shape); proxy tests get their options from `proxyClientOptions(for:through:)`, which wires the proxy and token auth (basic auth is TLS-only, RSA1 — `ProxyInfraSmokeTests.swift` is the shape). -- Proxy files are wrapped in `#if os(macOS)`. -- Wait on real network/proxy state with `await awaitState(client, .connected)` / `await awaitChannelState(…)` / `await pollUntil("…") { … }` — never a fixed sleep. -- **Protocol variants**: when a direct-sandbox spec declares the `PROTOCOL` dimension (json/msgpack), parameterise the test — `@Test(arguments: [false, true])` over a `useBinaryProtocol: Bool` parameter, applied via `options.useBinaryProtocol` (see `IntegrationSmokeTest.swift`). Proxy tests are **always JSON** (the proxy can't inspect binary frames) — `connectThroughProxy` already forces it, so no parameterisation there. +Each test file is a **Swift Testing** suite whose base class matches the tier: +`@Suite(.serialized) final class : UTSTestCase` (unit), `… : IntegrationTestCase` +(direct sandbox), or `… : ProxyTestCase` (proxy). If a suitable suite already exists, add the new test +methods to it rather than creating a duplicate. --- +## Step 3 — Read infrastructure files + +> **Orientation — read `Test/UTS/README.md` first.** It's the human-readable guide to this target: the +> tier model (unit / direct-sandbox / proxy, §2), the run commands (§10), the integration & proxy +> infrastructure (§11), and a per-file API reference for every infra helper (§13). Skim it for the *why* +> and the *what's available*; the per-file list below is the *what to open for exact signatures* before +> writing code. + +Infrastructure is split by tier under `Test/UTS/infra/`: + +- `infra/Utils.swift` — shared async helpers used by the integration tiers (`awaitState`, + `awaitChannelState`, `pollUntil`, `parseQueryParams`). +- `infra/unit/` — unit-test base class and mocks (`UTSTestCase.swift` with the `makeRealtime`/`makeRest` + factories and the synchronous `awaitConnectionState`/`awaitChannelState`/`poll` waits, + `MockWebSocket.swift`, `MockHTTPClient.swift`, `MockTimeProvider.swift`, `ProtocolMessage.swift`, + `Captured.swift`, `CapturingLog.swift`, `NoOpReachability.swift`). +- `infra/integration/` + `infra/integration/proxy/` — direct-sandbox + proxy helpers (`SandboxApp.swift`, + `IntegrationTestCase.swift`, `ProxyManager.swift`, `ProxySession.swift`, `ProxyTestCase.swift`) — see + the **Integration tests** section. + +For a **unit** test, read ALL files under `infra/unit/` plus `infra/Utils.swift` before generating any code +(you need exact method signatures). For an **integration** or **proxy** spec, follow the reading list in +the **Integration tests** section instead. + ## Step 4 — Generate the Swift test file Apply the translation rules below, then write the file. ### Accessing SDK internals (`import Ably.Private`) -The SDK is Objective-C, so Swift access levels (`internal`/`package`/`private`) don't apply to it — visibility is controlled by **headers + the module map**: +The SDK is Objective-C, so Swift access levels (`internal`/`package`/`private`) don't apply to it — +visibility is controlled by **headers + the module map**: - `import Ably` → the public API (headers in `Source/include/Ably/`). -- `import Ably.Private` → the internal API: the private headers listed in the `explicit module Private` block of `Source/include/module.modulemap` (files under `Source/PrivateHeaders/Ably/`, e.g. `ARTClientOptions+TestConfiguration.h` for `testOptions`, `ART*+Private.h` for class internals). This is how the UTS infra reaches the injection seams, and how a test reaches internal fields the spec asserts on. See `Test/UTS/README.md` §4–§5. +- `import Ably.Private` → the internal API: the private headers listed in the `explicit module Private` + block of `Source/include/module.modulemap` (files under `Source/PrivateHeaders/Ably/`, e.g. + `ARTClientOptions+TestConfiguration.h` for `testOptions`, `ART*+Private.h` for class internals). This is + how the UTS infra reaches the injection seams, and how a test reaches internal fields the spec asserts + on. See `Test/UTS/README.md` §4–§5. When a spec needs an internal class/method/field, work down this list: -1. **Check it's already exposed**: `grep -r "" Source/PrivateHeaders/Ably/` — if it's declared in a listed private header, just `import Ably.Private` and use it. -2. **Declared only in a `.m` file** (class extension, ivar, private method)? It is invisible to Swift, period. To expose it, declare it in a header under `Source/PrivateHeaders/Ably/` and register that header in **both** module maps (`Source/include/module.modulemap` for SPM and `Source/Ably.modulemap` for Xcode) — the repo's CLAUDE.md convention. Only do this for small, test-motivated exposure; mirror how existing `+Private.h` headers are written. -3. **Truly private state with no reasonable seam** (or exposing it would distort the SDK)? Don't hack around it — keep the spec's line as a comment, note why no assertion is emitted (see "Comments and assertion fidelity"), and record it in `deviations.md` under **Mock Infrastructure Limitations**. +1. **Check it's already exposed**: `grep -r "" Source/PrivateHeaders/Ably/` — if it's declared in a + listed private header, just `import Ably.Private` and use it. +2. **Declared only in a `.m` file** (class extension, ivar, private method)? It is invisible to Swift, + period. To expose it, declare it in a header under `Source/PrivateHeaders/Ably/` and register that + header in **both** module maps (`Source/include/module.modulemap` for SPM and `Source/Ably.modulemap` + for Xcode) — the repo's CLAUDE.md convention. Only do this for small, test-motivated exposure; mirror + how existing `+Private.h` headers are written. +3. **Truly private state with no reasonable seam** (or exposing it would distort the SDK)? Don't hack + around it — keep the spec's line as a comment, note why no assertion is emitted (see "Comments and + assertion fidelity"), and record it in `deviations.md` under **Mock Infrastructure Limitations**. ### Client construction -Set `ClientOptions` fields (`key`, `autoConnect`, `recover`, `disconnectedRetryTimeout`, etc.) in the `makeRealtime`/`makeRest` configuration block. Both factories already seed the dummy key `appId.keyId:keySecret` (matching ably-java's `ClientOptionsBuilder`), so only set `options.key` when the spec pseudocode sets one — then use the spec's value. +Set `ClientOptions` fields (`key`, `autoConnect`, `recover`, `disconnectedRetryTimeout`, etc.) in the +`makeRealtime`/`makeRest` configuration block. Both factories already seed the dummy key +`appId.keyId:keySecret` (matching ably-java's `ClientOptionsBuilder`), so only set `options.key` when the +spec pseudocode sets one — then use the spec's value. | Pseudocode | Swift | |---|---| | `install_mock(mock_http)` / `install_mock(mock_ws)` | `installMock(mockHTTPClient)` / `installMock(wsProvider)` — **before** `makeRealtime`/`makeRest` (the mock is injected at construction) | | `Rest(options: ClientOptions(key: "..."))` | `let rest = makeRest { $0.key = "..." }` | | `Realtime(options: ClientOptions(key: "...", autoConnect: false))` | `let client = makeRealtime { $0.key = "..."; $0.autoConnect = false }` | +| `Realtime(options: ClientOptions(autoConnect: false))` | `let client = makeRealtime { $0.autoConnect = false }` — no key set; the seeded dummy key applies | | `enable_fake_timers()` | `enableFakeTimers()` — **before** `makeRealtime`/`makeRest` (the clock is injected at construction) | ### Mock setup — WebSocket -Use the **handler pattern**: configure the simulated server in `onConnectionAttempt`. As in the spec, opening the socket and delivering the `CONNECTED` message are two separate calls — `respond_with_success()` then `send_to_client(...)`. +Use the **handler pattern**: configure the simulated server in `onConnectionAttempt`. As in the spec, +opening the socket and delivering the `CONNECTED` message are two separate calls — +`respond_with_success()` then `send_to_client(...)`. Spec pseudocode: @@ -150,7 +257,8 @@ AWAIT_STATE client.connection.state == ConnectionState.connected ws_connection = mock_ws.events.find(e => e.type == CONNECTION_SUCCESS).connection ``` -Swift (the UTS infra exposes `activeConnection` as the cocoa equivalent of the spec's `events.find(CONNECTION_SUCCESS).connection`): +Swift (the UTS infra exposes `activeConnection` as the cocoa equivalent of the spec's +`events.find(CONNECTION_SUCCESS).connection`): ```swift let wsProvider = MockWebSocketProvider(onConnectionAttempt: { connection in @@ -167,7 +275,8 @@ awaitConnectionState(client, .connected) let ws = try #require(wsProvider.activeConnection) ``` -When attempts need different behaviour (e.g. first succeeds, reconnects refused), branch on a counter inside the handler: +When attempts need different behaviour (e.g. first succeeds, reconnects refused), branch on a counter +inside the handler: Spec pseudocode: @@ -239,11 +348,14 @@ let rest = makeRest { $0.key = "app.key:secret" } ### Variable declarations -Take spec variable names (adapted to camelCase), for new ones, if needed don't use "noname" names (like "fields"), make the name concrete. +Take spec variable names (adapted to camelCase), for new ones, if needed don't use "noname" names (like +"fields"), make the name concrete. ### Capturing connection attempts / requests -The target is built in the Swift 6 language mode. Since mock handler closures are`@Sendable` and run on the SDK's queues, a plain `var array` captured into them is a compile error (a data race). Use the UTS infra's thread-safe (lock-guarded) `Captured` instead: +The target is built in the Swift 6 language mode. Since mock handler closures are `@Sendable` and run on +the SDK's queues, a plain `var array` captured into them is a compile error (a data race). Use the UTS +infra's thread-safe (lock-guarded) `Captured` instead: Spec pseudocode: @@ -278,11 +390,15 @@ let wsProvider = MockWebSocketProvider(onConnectionAttempt: { connection in #expect(capturedConnectionAttempts[0].queryParams["recover"] == "...") ``` -`Captured` exposes `append`, `all`, `count`, `first`, and `subscript(Int)`. Use `capturedConnectionAttempts.count` to change outcome for different connection attempts. +`Captured` exposes `append`, `all`, `count`, `first`, and `subscript(Int)`. Use +`capturedConnectionAttempts.count` to change outcome for different connection attempts. ### Inspecting outgoing frames -The spec inspects client-sent frames through the mock's event timeline (`mock_ws.events.filter(e => e.type == "ws_frame" AND e.direction == "client_to_server")`); the cocoa UTS infra exposes the decoded equivalent as `ws.sentMessages`. Capture after the channel/connection state confirms the send happened: +The spec inspects client-sent frames through the mock's event timeline +(`mock_ws.events.filter(e => e.type == "ws_frame" AND e.direction == "client_to_server")`); the cocoa UTS +infra exposes the decoded equivalent as `ws.sentMessages`. Capture after the channel/connection state +confirms the send happened: Spec pseudocode: @@ -311,7 +427,9 @@ let attachFrames = ws.sentMessages.filter { $0.action == .attach && $0.channel = ### Mock method reference -`connection` is the object the relevant handler hands you — for WebSocket it's the `MockWebSocket` passed to `onConnectionAttempt` (the spec's `mock_ws`/`conn`); for HTTP it's the `PendingHTTPConnection` passed to the `MockHTTPClient` `onConnectionAttempt`. `request` is the `PendingHTTPRequest` passed to `onRequest`. +`connection` is the object the relevant handler hands you — for WebSocket it's the `MockWebSocket` passed +to `onConnectionAttempt` (the spec's `mock_ws`/`conn`); for HTTP it's the `PendingHTTPConnection` passed to +the `MockHTTPClient` `onConnectionAttempt`. `request` is the `PendingHTTPRequest` passed to `onRequest`. | Pseudocode | Swift | |---|---| @@ -326,9 +444,19 @@ let attachFrames = ws.sentMessages.filter { $0.action == .attach && $0.channel = | `req.respond_with_timeout()` | `request.respondWithTimeout()` | | `conn.respond_with_refused/timeout/dns_error()` (HTTP) | `connection.respondWithRefused()` / `respondWithTimeout()` / `respondWithDNSError()` | +**Known WS-mock gap:** the WS-level `conn.respond_with_timeout()` / `conn.respond_with_dns_error()` some +specs use have **no** `MockWebSocket` counterpart yet (only success/refused/disconnect exist; the +timeout/refused/DNS-error trio exists HTTP-side on `PendingHTTPConnection`). When a spec calls them, either +extend `MockWebSocket` (small — mirror `respondWithRefused()`, plumbing the matching transport error) or +record the case in `deviations.md` under **Mock Infrastructure Limitations** — never silently substitute +`respondWithRefused()`. + ### Protocol messages and types -Server-to-client messages are described with the `Sendable` `ProtocolMessage` factories — `sendToClient`/`sendToClientAndClose` take a `ProtocolMessage` and build the real `ARTProtocolMessage` at delivery, so no non-Sendable value crosses the queue hop. Use these factories; add a new one (and the matching `makeProtocolMessage()` case) if you need another action: +Server-to-client messages are described with the `Sendable` `ProtocolMessage` factories — +`sendToClient`/`sendToClientAndClose` take a `ProtocolMessage` and build the real `ARTProtocolMessage` at +delivery, so no non-Sendable value crosses the queue hop. Use these factories; add a new one (and the +matching `makeProtocolMessage()` case) if you need another action: | Pseudocode | Swift | |---|---| @@ -343,7 +471,10 @@ Server-to-client messages are described with the `Sendable` `ProtocolMessage` fa ### Awaiting state -`AWAIT_STATE x.state == ConnectionState.X` → `awaitConnectionState(client, .x)` (default 2s timeout, or `timeout:`). Channels: `awaitChannelState(channel, .attached)`. For other conditions (e.g. "a frame was sent") use `poll("description") { }`. Don't use poll unless you can't find alternative or spec says to do so. +`AWAIT_STATE x.state == ConnectionState.X` → `awaitConnectionState(client, .x)` (default 2s timeout, or +`timeout:`). Channels: `awaitChannelState(channel, .attached)`. For other conditions (e.g. "a frame was +sent") use `poll("description") { }`. Don't use poll unless you can't find alternative or spec says +to do so. ### Timer control @@ -401,9 +532,13 @@ let client = makeRealtime { $0.key = "..."; $0.logHandler = log } ### Comments and assertion fidelity -Carry over **every** comment from the spec pseudocode verbatim, as a `//` comment at the matching step — these explain *why* each step exists and must not be dropped or paraphrased. +Carry over **every** comment from the spec pseudocode verbatim, as a `//` comment at the matching step — +these explain *why* each step exists and must not be dropped or paraphrased. -Translate every spec `ASSERT`/`AWAIT` into a Swift assertion at the same place. If an assertion genuinely has no Swift equivalent (e.g. it checks a language-specific construct, or the SDK exposes no observable hook for it), **do not silently omit it** — keep the spec's comment/line and add a `//` note explaining why no assertion is emitted. Never delete the spec line. +Translate every spec `ASSERT`/`AWAIT` into a Swift assertion at the same place. If an assertion genuinely +has no Swift equivalent (e.g. it checks a language-specific construct, or the SDK exposes no observable +hook for it), **do not silently omit it** — keep the spec's comment/line and add a `//` note explaining why +no assertion is emitted. Never delete the spec line. ```swift // The recovery key should round-trip the connection id ← spec comment, copied verbatim @@ -424,14 +559,19 @@ Use Swift Testing macros (`import Testing`, **not** `XCTest`): | `ASSERT x == y` | `#expect(x == y)` | | `ASSERT x IS NOT null` | `let x = try #require(optional)` (or `#expect(x != nil)`) | | `ASSERT x IS null` | `#expect(x == nil)` | +| `ASSERT x IS Auth` (type check) | `#expect(x is ARTAuth)` — or `let auth = try #require(x as? ARTAuth)` when later lines use the value | +| `ASSERT x matches pattern "..."` | `#expect(x.range(of: "...", options: .regularExpression) != nil)` | +| `ASSERT list CONTAINS_IN_ORDER [a, b, c]` | `#expect(list.filter { [a, b, c].contains($0) } == [a, b, c])` — or walk an index as the spec does | | `ASSERT "k" IN map` / `NOT IN` | `#expect(map["k"] != nil)` / `#expect(map["k"] == nil)` | | `ASSERT list.length == N` | `#expect(list.count == N)` | | `ASSERT x == y (with message)` | `#expect(x == y, "message")` | | `AWAIT op FAILS WITH error` | capture the error (see async below) and `#expect(error.code == ...)` / `#expect(error.statusCode == ...)` | -`#expect(...)` records a failure and continues; `try #require(...)` unwraps/asserts and stops the test on failure (use it when later lines depend on the value, like `XCTUnwrap`). +`#expect(...)` records a failure and continues; `try #require(...)` unwraps/asserts and stops the test on +failure (use it when later lines depend on the value, like `XCTUnwrap`). -REST calls will callback (e.g. `rest.time { ... }`) — make the test `async throws` and create a helper method that bridges the completion handler with a continuation: +REST calls will callback (e.g. `rest.time { ... }`) — make the test `async throws` and create a helper +method that bridges the completion handler with a continuation: ```swift private func awaitTime(_ rest: ARTRest, sourceLocation: SourceLocation = #_sourceLocation) async -> Date { @@ -445,12 +585,18 @@ private func awaitTime(_ rest: ARTRest, sourceLocation: SourceLocation = #_sourc } ``` -Put these `async` helpers in the test class extension at the bottom of the test file. Inside these helpers, report non-assertion failures (timeouts, unexpected errors) with `Issue.record("...", sourceLocation:)`. +Put these `async` helpers in the test class extension at the bottom of the test file. Inside these helpers, +report non-assertion failures (timeouts, unexpected errors) with `Issue.record("...", sourceLocation:)`. ### Test naming and annotation -- `// UTS: ` comment immediately above each test, then the `@Test` attribute. -- Method name: `test__` — keep the spec point intact, join words with underscores. Take the description from the spec test title. Keep camelCase for symbol names. Mark the function `throws` (and `async` if it awaits). +- `// UTS: ` comment immediately above each test, then the `@Test` attribute. Copy the spec's + **full** structured id verbatim (e.g. `realtime/unit/RTN16g/recovery-key-structure-0`; for the objects + module it would start `objects/unit/…`). Don't hand-build the prefix — use what the spec file declares. + (A hand-built prefix silently breaks the Step 7 audit's ID matching.) +- Method name: `test__` — keep the spec point intact, join words with + underscores. Take the description from the spec test title. Keep camelCase for symbol names. Mark the + function `throws` (and `async` if it awaits). ```swift // UTS: realtime/unit/RTN16g/recovery-key-structure-0 @@ -460,7 +606,11 @@ func test_RTN16g_createRecoveryKey_returns_a_recovery_key() throws { } ``` -### File template +### File template (unit tier) + +This scaffold is for the **unit** tier — it wires the mocked transport (`MockWebSocketProvider`, +`installMock`, `makeRealtime`). For the **integration** (direct sandbox) and **proxy** tiers, start from +the file templates in the **Integration tests** section instead, not from this one. ```swift import Testing @@ -471,7 +621,7 @@ import Ably.Private /// () /// Derived from @Suite(.serialized) -final class Tests: UTSTestCase { +final class : UTSTestCase { // UTS: @Test @@ -494,11 +644,319 @@ final class Tests: UTSTestCase { } ``` -Helper methods return types should be ready for use in the test code without additional casting (if you need "value as? String" in the test, move this conversion to the helper instead). Don't return optionals from these methods - assert not `nil` within the method itself, unless test expects optional. Don't create additional helper types for parsing dictionaries - just use a dictionary with the most suited type for the test, accessing its fields by subscript. For dictionaries containing values of different types use `[String: Any]`. Don't wrap dictionary constant initialization into helper method. Put helper methods for the suite into an extension at the bottom of the file. Keep the spec's original tests order in the generated test file. Keep test segmentation by adding comments like "// Setup", "// Test Steps", "// Assertions". +Helper methods return types should be ready for use in the test code without additional casting (if you +need "value as? String" in the test, move this conversion to the helper instead). Don't return optionals +from these methods - assert not `nil` within the method itself, unless test expects optional. Don't create +additional helper types for parsing dictionaries - just use a dictionary with the most suited type for the +test, accessing its fields by subscript. For dictionaries containing values of different types use +`[String: Any]`. Don't wrap dictionary constant initialization into helper method. Put helper methods for +the suite into an extension at the bottom of the file. Keep the spec's original tests order in the +generated test file. Keep test segmentation by adding comments like "// Setup", "// Test Steps", +"// Assertions". + +--- + +## Step 5 — Compile + +```bash +swift build --build-tests +``` + +Fix any compilation errors and recompile until clean. Common issues: +- Missing `import Ably.Private` when the test touches internals. +- Mock method names differing from what you read in Step 3's files — use the exact signatures. +- Swift 6 `@Sendable` capture errors — a plain `var` captured in a mock handler is a compile error; use + `Captured` (see "Capturing connection attempts / requests"). + +--- + +## Step 6 — Run tests *(evaluate mode only)* + +Skip this whole step in "translate only" mode. In "translate and evaluate" mode, run the test and resolve +every failure via the decision tree below. Each test must end in exactly one of these states — never an +**unexplained** red: +- the spec-correct assertion **passes**; or +- a documented **SDK deviation** — env-gated or adapted, stays green (SDK ≠ spec; fix belongs in the SDK); or +- a documented **UTS spec error** — **fails fast** (the spec is wrong; fix belongs in the spec). This is the + one acceptable red. + +Run the suite with the resolver's `className` as the filter: + +```bash +# unit tier (fast, fully mocked, no network) +swift test --filter "UTS." +# a single test (Swift Testing matches the function name) +swift test --filter "UTS./" +``` + +(`swift test --filter UTS` still runs the whole UTS target. The integration and proxy tiers additionally +need network access and the suite's env gate — see `Test/UTS/README.md` §10 for the current gating.) + +Handle test failures using this decision tree (the **Required reading** doc you fetched up front has the full detail): + +``` +Test fails + | + +-- Does UTS spec match features spec? (fetch the features spec — see Required reading) + | NO → UTS SPEC ERROR — fix the spec at source, not the test. Record in deviations.md + | (UTS Spec Errors) + emit a fail-fast test (below) so it's fixed early. + | YES + | +-- Does test accurately translate the UTS spec? + | NO → fix the test (no deviation entry needed) + | YES → SDK deviation — adapt test, record in deviations file +``` + +### Test patterns for a diagnosed failure + +Two patterns are for an **SDK deviation** (both write the spec-correct assertions); the third, +**spec-error fail-fast**, is for a **UTS spec error** and is not a deviation. + +**Env-gated skip (preferred)** — test contains spec-correct assertions but is disabled by default via the +Swift Testing `.enabled(if:)` trait, so it only runs when `RUN_DEVIATIONS` is set: + +```swift +// UTS: realtime/unit/RSA4c2/callback-error-connecting-disconnected-0 +// DEVIATION: see deviations.md +@Test(.enabled(if: ProcessInfo.processInfo.environment["RUN_DEVIATIONS"] != nil)) +func test_RSA4c2_callback_error_connecting_disconnected() throws { + // ... spec-correct setup and assertions ... +} +``` + +Reproduce with `RUN_DEVIATIONS=1 swift test --filter "UTS./"`. + +**Adapted assertion** — when you still want to assert on the SDK's actual behaviour to prevent regressions: + +```swift +// DEVIATION: spec requires error code 40106, SDK returns 40160 — see deviations.md +#expect(error.code == 40160) +``` + +**Spec-error fail-fast** *(UTS spec error — NOT an SDK deviation)* — when the decision tree's first branch +is **NO** (the UTS spec contradicts the features spec, or is internally inconsistent), the spec is the +fixable source of truth, so the test must **fail loudly** pointing at the deviation entry, rather than be +quietly adapted to green. This is the opposite of the SDK-deviation patterns above (which stay green / +assert actual behaviour) — failing fast is the forcing function that gets the spec fixed early. + +```swift +// UTS: objects/unit/RTLC7c2/local-source-no-sitetimeserials-0 +@Test +func test_RTLC7c2_LOCAL_source_does_not_write_siteTimeserials() throws { + // SPEC ERROR RTLC7c2: replayed ACK serial "t:1:0" contradicts the harness ("ack-0:0"). + // Fix the UTS spec first — see deviations.md (UTS Spec Errors). + Issue.record("UTS spec error RTLC7c2 — fix the spec first; see deviations.md") +} +``` + +**Never use the accommodate-both pattern** (accept either spec or SDK behaviour). Every test must assert +either spec behaviour or the SDK's actual behaviour — never both at once. + +Note: an *infra-driving* difference (e.g. using fake timers where the spec uses real ones, or a +queue-ordering workaround) is **not** an SDK deviation — explain it in a code comment, not `deviations.md`. +`deviations.md` is only for SDK non-compliance and mock-capability gaps. + +### Deviations file + +Append to `Test/UTS/deviations.md`, using the manual's **Recording deviations** entry format and sections +(the `writing-derived-tests.md` you fetched up front — not a home-grown format). The ably-cocoa-specific +mapping: a **UTS Spec Error** (test fails fast — fix in the spec) goes under the manual's *UTS Spec Errors* +section (the file doesn't have that section yet — add the heading on first use, alongside the existing +three); an **SDK deviation** (env-gated/adapted — fix in the SDK) goes under *Failing Tests* / *Adapted +Tests*; a mock-capability gap goes under *Mock Infrastructure Limitations*. + +--- + +## Step 7 — Review generated output against the spec + +The translation isn't done when it compiles — it's done when **every line of the source spec is faithfully +represented** in the Swift: each test case, each setup step, each operation, and each assertion. Missing a +single `ASSERT` produces a test that compiles, passes, and silently checks less than the spec demands. This +review runs in **both** modes (it's static — it doesn't need the tests to have run). + +### Deterministic faithfulness audit — run the script first + +Eyeballing two files for "did I translate every line?" is exactly the kind of mechanical comparison the +model does inconsistently, so a bundled script extracts the ledger for you — same inputs, same report every +time. Run it for each translated spec — don't hand-build paths; use the resolver's output (Step A): the +**source spec file** is that spec's `specs[].file` (already an absolute path) and the **generated Swift +file** is the chosen tier's `targetDir` + `className` (Step 2): + +```bash +python3 .claude/skills/uts-to-swift/scripts/audit_translation.py \ + "" \ + "/.swift" +``` + +It prints one JSON report and exits non-zero (2) when there are missing or orphan Test IDs. It does **no** +semantic judgement — it only extracts, deterministically, what you must then reconcile by hand: + +- **`idCoverage`** — the spec's `**Test ID**` set vs the Swift file's `// UTS:` tag set. + - `missingInSwift` **must be empty.** Each entry is a spec test case with no `@Test` method — implement it. + - `orphanInSwift` **must be empty** (or every entry explained) — a `// UTS:` tag that no longer matches + any spec Test ID means a stale or hand-edited tag. +- **`perTest[]`** — for each spec test case, every non-comment code line inside the spec's `pseudo` blocks, + **grouped by section** (`Setup` / `Test Steps` / `Assertions` / …) in `sections[]` and each line tagged + `assert` (an `ASSERT*` outcome), `await` (an `AWAIT*` / `EXPECT` wait), or `step` (setup, mock/client + construction, an operation). `specAsserts` / `specAwaits` are flat convenience views of the first two; + `specCodeLineTotal` is the size of the spec test. The matching Swift method's assertion/await/poll calls + and their count come alongside. + +The audit is a review aid, not a gate — it never crashes, it degrades. It always prints one JSON object +(an `error` field instead of a report if a file is unreadable). If `idCoverage.specCount` is `0` for a file +you know is a real spec, the extractor couldn't find any `**Test ID**` markers (an unusual spec format) — +treat that as "couldn't verify deterministically" and fall back to a manual side-by-side read for that +file. The known case is a **fixture-driven** spec (e.g. `rest/unit/encoding/msgpack_interop.md` — one +parameterised test looping over external fixture data instead of enumerated test cases): there the audit +will also report your `// UTS:` tag as an *orphan* — expected, not a defect; explain it in the review and +verify that file by hand. + +### Coverage check — every test case is present + +From the audit's `idCoverage`: +- [ ] `missingInSwift` is empty (every spec Test ID has an `@Test` with that ID in its `// UTS:` comment) +- [ ] `orphanInSwift` is empty, or each orphan is a deliberate, explained rename +- [ ] Each method name contains the spec ID and a meaningful description + +### Line-by-line completeness — every spec line is translated + +Walk the audit's `perTest[].sections` and reconcile **each** extracted spec line against the Swift method. +This is the guarantee that no setup step, operation, or assertion was dropped — the whole point of this step: +- [ ] Every `assert` line maps to a concrete Swift assertion (`#expect`, `try #require`, …) — not a + comment, not a weaker check +- [ ] Every `await` line (state waits **and** awaited operations like `AWAIT channel.attach()`) is + performed via `awaitConnectionState` / `awaitChannelState` / `poll` (unit tier) or `awaitState` / + `awaitChannelState` / `pollUntil` (integration tiers) or the corresponding SDK call +- [ ] Every `step` line — client/mock construction, `ClientOptions`, installed mocks, channel ops — is + reflected in the test setup or body. Multi-line spec constructs split across several `step` lines; + reconcile them as a group. +- [ ] The pseudocode's inline `#` comments are carried over as `//` comments at the matching steps verbatim +- [ ] A spec line with no Swift equivalent is annotated per "Comments and assertion fidelity" (spec line + kept + `//` note), never silently dropped +- [ ] `assertionShortfall > 0` for a test (`summary.testsWithShortfall`) is a **tripwire** — fewer Swift + assertions than spec `ASSERT`s strongly suggests a dropped assertion; open that test and account for + each one. (A negative shortfall is fine — the SDK mapping often needs *more* Swift lines per spec + `ASSERT`; an annotated omission — e.g. a type check satisfied by a helper's return type — is a valid + account.) + +### Setup fidelity — preconditions match the spec + +For each test case, verify: +- [ ] Client options (`autoConnect`, `recover`, timeouts, etc.) match the spec's `ClientOptions` +- [ ] Mock responses (success / refused / timeout / DNS error / custom messages) match the spec's prescribed network behaviour +- [ ] Timer setup (`enableFakeTimers`, `advanceTime(byMilliseconds:)`) matches every `enable_fake_timers` / `ADVANCE_TIME` in the spec +- [ ] Channel operations (attach, detach, publish) are performed in the order the spec requires + +### Deviation honesty *(evaluate mode)* + +Deviations are discovered by running, so this check applies in evaluate mode. For any place where the +generated test diverges from the spec pseudocode (adapted assertion, env-gated skip, or omitted step): +- [ ] A `// DEVIATION:` comment explains why +- [ ] The deviation is recorded in `Test/UTS/deviations.md` + +If you find gaps during this review, fix them, then **re-run the audit script** until `missingInSwift` / +`orphanInSwift` are empty and every `perTest` entry reconciles, and re-run Step 5 (compile) — and, in +evaluate mode, Step 6 — before finishing. + +--- + +## Integration tests (direct sandbox + proxy) + +Some specs are **integration tests** — they run against the **real Ably sandbox** instead of a mocked transport. Two tiers share one foundation and differ only in *transport*: + +- **Direct sandbox** — the client connects straight to the sandbox. Happy-path interop (connect, publish, subscribe, presence); no faults. +- **Proxy** — the client is routed through the [`ably/uts-proxy`](https://github.com/ably/uts-proxy), a programmable HTTP/WebSocket proxy that forwards traffic transparently but can inject faults (dropped connections, modified/injected/delayed frames, error responses) via rules. + +**Shared foundation (both tiers, covered once):** `SandboxApp` provisioning and teardown via the tier's +base test case, real (unmocked) clients, and the async `awaitState` / `awaitChannelState` / `pollUntil` +waits — never a fixed sleep, since real network is involved (use generous 10–30s timeouts; the helpers +default to 15s). Swift Testing has no `setUp()`/`tearDown()` and `deinit` can't `await`, so the base cases +provide **scoped-resource methods** instead — each `with…` method provisions its resource, runs your body, +and *always* tears it down, rethrowing the body's error afterwards. **Never hand-roll teardown**; wrap the +scenario in the scopes. Only the *wiring* differs per tier; that's what the two tier subsections below +cover. + +**Required reading for the proxy tier:** before translating any proxy spec, read +`uts/docs/proxy.md` (once per run) — it defines the proxy session/control API, the rule and action +vocabulary, and the protocol action-number table the subsections below rely on. Fetch +[the raw URL](https://raw.githubusercontent.com/ably/specification/refs/heads/main/uts/docs/proxy.md); +if that 404s (the `docs/` layout may not be merged upstream yet), read it from the spec checkout the +module directory came from instead: `/../docs/proxy.md`. + +Recognise a **proxy** spec by a reference to `create_proxy_session()`, proxy `rules`, `trigger_action`, +`get_log`, or a pointer to `uts/docs/proxy.md`. A spec with none of those is **direct sandbox**. + +### Which integration tier? + +| Test type | When the spec uses it | +|---|---| +| **Unit test** (mock HTTP/WS — the rest of this skill) | Client-side logic, state machines, request formation, error parsing. Fast, deterministic. | +| **Direct sandbox integration** | Happy-path behaviour (connect, publish, subscribe, presence). No fault injection. | +| **Proxy integration test** | Fault behaviour against the real backend: connection failures, resume, heartbeat starvation, token renewal under network errors, channel error injection. | + +### Direct-sandbox integration tests + +A **direct-sandbox** spec (no `create_proxy_session`, no rules — just happy-path interop against +`nonprod:sandbox`) uses the same `SandboxApp` provisioning as a proxy test but **drops all proxy wiring**: +no `ProxyManager`, no `ProxySession`, no `connectThroughProxy`. The client connects straight to the +sandbox host. `IntegrationSmokeTest.swift` (under `Test/UTS/integration/standard/`) is the reference +example — read it before translating a direct-sandbox spec. (It's an infra acceptance test slated for +removal once spec-derived integration suites exist — the reference then migrates to the first of those; +see `Test/UTS/README.md` §11.7.) + +Suites subclass `IntegrationTestCase` and wrap the scenario in the scoped-resource methods: +`withSandboxApp { app in … }` provisions a throwaway sandbox app and always deletes it; +`withRealtimeClient(options) { client in … }` builds a real `ARTRealtime` and always closes it (waiting +for CLOSED). + +**Client wiring** — point both transports at the sandbox host; TLS stays on, so the plain sandbox key +works (RSA1). Explicit hosts auto-disable fallback hosts (REC2c2), so no `fallbackHosts`: + +```swift +let options = ARTClientOptions(key: app.defaultKey) +options.realtimeHost = SandboxApp.sandboxHost // sandbox.realtime.ably-nonprod.net +options.restHost = SandboxApp.sandboxHost +options.useBinaryProtocol = useBinaryProtocol +options.autoConnect = false +``` + +**Class docstring** — use a direct-sandbox variant (no proxy/fault-injection wording): + +```swift +/// Direct-sandbox integration test against the Ably Sandbox (`sandbox.realtime.ably-nonprod.net`, +/// via SandboxApp.sandboxHost) — no proxy, no fault injection. Provisions a throwaway SandboxApp +/// and connects real clients straight to the sandbox. +``` + +**Protocol variants (`json` / `msgpack`)** — when the spec header declares a `PROTOCOL` dimension and says +each test runs once per variant, translate it to a parameterised test over `useBinaryProtocol` (Swift +Testing has this built in — no extra imports), not a plain `@Test`. The `// UTS:` tag and method name stay +singular — the parameter expresses the variant: + +```swift +// UTS: realtime/integration/RTL10d/history-cross-client-0 +@Test(arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack +func test_RTL10d_history_contains_messages_published_by_another_client(useBinaryProtocol: Bool) async throws { + // … +} +``` + +A spec with **no** protocol dimension stays a plain `@Test`. Only the direct-sandbox tier is +parameterised — the proxy tier is always JSON (see "Connecting through the proxy"). + +**Awaiting real server outcomes** — integration specs assert on real backend state, so never sleep; await +or poll: + +| Pseudocode | Swift | +|---|---| +| `AWAIT channel.attach()` | `channel.attach()` then `guard await awaitChannelState(channel, .attached, timeout: 10) else { return }` | +| `AWAIT channel.publish(name, data)` (await the ack) | bridge the callback overload (`publish(_:data:callback:)`) with a continuation, resuming on the callback and recording an `Issue` on error — same helper pattern as Step 4's `awaitTime` | +| `poll_until(() => AWAIT channel.history().items.length == N, …)` | `await pollUntil("history has N items") { await historyCount(channel) == N }` — with a continuation-bridged `history` helper | -### File template — integration tiers +Use generous timeouts (10–30s) — real network is involved. Everything else is the shared foundation +described at the top of this section; a direct-sandbox test just skips the proxy-only subsections +(`ProxySession`, rule factories, the event log). -Direct-sandbox spec (`integration/standard//`) — subclass `IntegrationTestCase` (setup/teardown are automatic); parameterise over the spec's `PROTOCOL` dimension when it declares one: +**File template — direct sandbox** (`integration/standard//`): ```swift import Testing @@ -508,7 +966,7 @@ import Ably /// () /// Derived from @Suite(.serialized) -final class Tests: IntegrationTestCase { +final class : IntegrationTestCase { // UTS: @Test(arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack @@ -529,127 +987,220 @@ final class Tests: IntegrationTestCase { } ``` -Proxy spec (`integration/proxy//`) — subclass `ProxyTestCase`; wrap the whole file in `#if os(macOS)`: +### Infrastructure + +The helpers live under `Test/UTS/infra/integration/`. **Read the ones your tier uses before translating an +integration spec** — they hold the exact method signatures. `SandboxApp` and `IntegrationTestCase` serve +**both** tiers; `ProxyManager`, `ProxySession`, and `ProxyTestCase` are **proxy-only**. + +- **`SandboxApp`** (`infra/integration/SandboxApp.swift`) — provisions/deletes a sandbox test app from the + shared `test-app-setup.json` in ably-common. `SandboxApp.create()` returns a `SandboxApp` with `appId`, + `defaultKey`, and `keys` (`defaultKey` is a full-capability `appId.keyId:keySecret`); `app.delete()` + tears it down. Also owns the single upstream sandbox host constant `SandboxApp.sandboxHost` + (`sandbox.realtime.ably-nonprod.net`, the resolved `nonprod:sandbox` endpoint) — the default target of + every `ProxySession`, and what direct-sandbox clients set `realtimeHost` / `restHost` from. +- **`IntegrationTestCase`** (`infra/integration/IntegrationTestCase.swift`) — the direct-sandbox base + class: `withSandboxApp`, `withRealtimeClient`, and the `runThenCleanUp(_:body:cleanup:)` engine they're + built on (run body, always run cleanup, rethrow the body's error) — subclasses use it for their own + scopes. +- **`ProxyManager`** (`infra/integration/proxy/ProxyManager.swift`) — actor that syncs (downloads, + checksum-verifies, caches) and starts the shared `uts-proxy` process; `ensureProxy()` is idempotent. + Spawning a local process makes the proxy tier **macOS-only** — wrap proxy test files in + `#if os(macOS)`. (`UTS_PROXY_LOCAL_PATH` points it at a locally built binary instead.) +- **`ProxySession`** (`infra/integration/proxy/ProxySession.swift`) — one programmable session wrapping + the proxy control API; also defines the `connectThroughProxy` extension, the rule-factory helpers, and + the typed `ProxyEvent`. +- **`ProxyTestCase`** (`infra/integration/proxy/ProxyTestCase.swift`) — the proxy base class (extends + `IntegrationTestCase`): `withProxySession(rules:)` and `proxyClientOptions(for:through:)`. + +The `SandboxApp` and `ProxySession` methods are all **`async`** — call them with `try await` inside the +scoped bodies. Imports are simple: `import Testing`, `import Foundation`, `import Ably` (+ +`import Ably.Private` only if the spec asserts on SDK internals) — the whole UTS target is one module, so +the infra helpers need no imports. + +### Proxy test class docstring + +Give every proxy integration test class this doc comment: ```swift -// Proxy tests spawn a local uts-proxy process — macOS-only. -#if os(macOS) +/// Proxy integration test against Ably Sandbox endpoint. +/// +/// Uses the programmable uts-proxy to inject transport-level faults while the +/// SDK communicates with the real Ably backend. See +/// `uts/docs/proxy.md` for proxy infrastructure details. +``` -import Testing -import Foundation -import Ably +### Session lifecycle -/// () -/// Derived from -@Suite(.serialized) -final class Tests: ProxyTestCase { +`create_proxy_session(endpoint: "nonprod:sandbox", rules: [...])` → the base class's +`withProxySession(rules:)` — it ensures the proxy is running, provisions the sandbox app, creates the +session, and **always** closes the session and deletes the app afterwards (the sandbox is already the +session's default target, so the endpoint argument drops). Never hand-roll the teardown: - // UTS: - @Test - func test__() async throws { - try await withProxySession(rules: []) { app, session in // rule-less start = late fault injection - let options = proxyClientOptions(for: app, through: session) // token auth; JSON forced - options.autoConnect = false - try await withRealtimeClient(options) { client in - client.connect() - guard await awaitState(client, .connected) else { return } - // Late imperative fault injection, then poll — never sleep — and verify via the log: - try await session.triggerAction(["type": "inject_to_client", "message": ["action": 17]]) - await pollUntil("client reacted to the injected frame") { /* observable state */ true } - let log = try await session.getLog() - #expect(log.contains { $0.type == "ws_frame" && $0.direction == "client_to_server" }) - } - } // client closed, session closed, app deleted — even on failure +```swift +try await withProxySession(rules: [wsConnectRule(action: ["type": "refuse_connection"], count: 2)]) { app, session in + let options = proxyClientOptions(for: app, through: session) + options.autoConnect = false + try await withRealtimeClient(options) { client in + client.connect() + guard await awaitState(client, .connected, timeout: 15) else { return } + // … scenario … } -} - -#endif +} // client closed, session closed, app deleted — even on failure ``` ---- - -## Step 5 — Compile +| Pseudocode | Swift | +|---|---| +| `create_proxy_session(endpoint: "nonprod:sandbox", rules: [...])` | `withProxySession(rules: [...]) { app, session in … }` | +| `session.add_rules(rules, position: "prepend")` | `try await session.addRules(rules, position: "prepend")` | +| `session.trigger_action({ type: "disconnect" })` | `try await session.triggerAction(["type": "disconnect"])` | +| `session.get_log()` | `try await session.getLog()` | +| `session.close()` | handled by `withProxySession` — don't call it yourself | +| `session.proxy_port` / `session.proxy_host` | `session.proxyPort` / `session.proxyHost` | -```bash -swift build --build-tests -``` +### Connecting through the proxy -Fix any compilation errors and recompile until clean. +Call `options.connectThroughProxy(session)` on the client options. It is an `ARTClientOptions` extension +(in `ProxySession.swift`) that wires the SDK through the proxy: ---- +| Proxy-def option | What `connectThroughProxy` sets | +|---|---| +| `endpoint: "localhost"` | `realtimeHost` **and** `restHost` = `session.proxyHost` (`"localhost"`) | +| `port: proxy_port` | `port = session.proxyPort` | +| `tls: false` | `tls = false` | +| `useBinaryProtocol: false` | `useBinaryProtocol = false` — set explicitly (the proxy can only inspect text frames; the UTS proxy specs require JSON), so proxy tests are **always JSON** — never parameterise the protocol here | -## Step 6 — Run tests +It does **not** touch `autoConnect`, so set that yourself. Setting explicit hosts disables fallback hosts +automatically (REC2c2), so don't add `fallbackHosts`. -```bash -swift test --filter -# or a single test (Swift Testing matches the function name): -swift test --filter / -``` +### Auth in proxy tests -(`--filter` takes a regex over test names; `swift test --filter UTS` runs the whole suite.) +The proxy serves plain ws (`tls = false`) and basic (key) auth is TLS-only (**RSA1**), so a proxied client +can't just use the sandbox key. Where the pseudocode "generates a JWT from the key parts", the idiomatic +ably-cocoa equivalent is a **locally-signed `TokenRequest`** from the same sandbox key — no JWT library +required: a separate TLS "token signer" `ARTRest` calls `auth.createTokenRequest(params, options:)` inside +an `authCallback`, and the realtime client exchanges it for a token through the proxy. -Handle failures using this decision tree (see [reference doc](https://github.com/ably/specification/blob/main/uts/docs/writing-derived-tests.md)): +The base class packages all of this: **`proxyClientOptions(for: app, through: session)`** returns +`ARTClientOptions` with the token-signer `authCallback` and `connectThroughProxy` already wired. Its body +(in `ProxyTestCase.swift`) is the pattern to inline when a spec needs to count or intercept the auth +callbacks itself: -``` -Test fails - | - +-- Does the UTS spec match the features spec? - | NO → fix the test, record the UTS spec error in deviations.md - | YES - | +-- Does the test accurately translate the UTS spec? - | NO → fix the test (no deviation entry) - | YES → SDK deviation — adapt the test, record in deviations.md +```swift +let signerOptions = ARTClientOptions(key: app.defaultKey) +signerOptions.restHost = SandboxApp.sandboxHost +let tokenSigner = ARTRest(options: signerOptions) // TLS, real host — local signing plus one token fetch + +let options = ARTClientOptions() +options.authCallback = { params, callback in + tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } +} +options.connectThroughProxy(session) ``` -### Deviation patterns +(The auth types are top-level classes in cocoa — `ARTTokenParams`, `ARTTokenRequest`, `ARTAuthDetails` — +unlike ably-java, where `TokenParams`/`TokenRequest` are nested in `Auth` and `AuthDetails` in +`ProtocolMessage`.) -**Env-gated skip (preferred)** — test contains spec-correct assertions but is disabled by default via the Swift Testing `.enabled(if:)` trait, so it only runs when `RUN_DEVIATIONS` is set: +### Rule factory helpers -```swift -// DEVIATION: see deviations.md -@Test(.enabled(if: ProcessInfo.processInfo.environment["RUN_DEVIATIONS"] != nil)) -func test__...() throws { - // ... spec-correct setup and assertions ... -} -``` +Build rules with the factory helpers in `ProxySession.swift` rather than raw dictionary literals. Rules +are evaluated in order, first match wins, unmatched traffic passes through, and `times` auto-removes a +rule after N firings. -Reproduce with `RUN_DEVIATIONS=1 swift test --filter /`. +| Match condition | Swift | +|---|---| +| `{ "type": "ws_connect", "count": 2 }` | `wsConnectRule(action: …, count: 2)` | +| `{ "type": "ws_connect", "queryContains": { "resume": "*" } }` | `wsConnectRule(action: …, queryContains: ["resume": "*"])` | +| `{ "type": "ws_frame_to_client", "action": "ATTACHED", "channel": "c" }` | `wsFrameToClientRule(action: …, messageAction: 11, channel: "c")` | +| `{ "type": "ws_frame_to_server", "action": "ATTACH", "channel": "c" }` | `wsFrameToServerRule(action: …, messageAction: 10, channel: "c")` | +| `{ "type": "http_request", "method": "POST", "pathContains": "/keys/" }` | `httpRequestRule(action: …, method: "POST", pathContains: "/keys/")` | + +`messageAction` is the protocol action **number** (e.g. `4` CONNECTED, `6` DISCONNECTED, `9` ERROR, `10` +ATTACH, `11` ATTACHED) — see the action-number table in `uts/docs/proxy.md` (the doc you fetched at the +top of this section). -**Adapted assertion** — assert the SDK's actual behaviour to prevent regressions: +Actions are passed as `ProxyRule` dictionary literals, e.g.: ```swift -// DEVIATION: spec requires code 40106, SDK returns 40160 — see deviations.md -#expect(error.code == 40160) +["type": "refuse_connection"] +["type": "disconnect"] +["type": "suppress"] +["type": "delay", "delayMs": 2000] +["type": "inject_to_client", "message": ["action": 6]] +["type": "http_respond", "status": 401, "body": ["...": "..."]] ``` -**Never use the accommodate-both pattern** Every test must assert either spec behaviour or the SDK's actual behaviour — never both at once. +### Verifying the event log -Note: an *infra-driving* difference (e.g. using fake timers where the spec uses real ones, or a queue-ordering workaround) is **not** an SDK deviation — explain it in a code comment, not `deviations.md`. `deviations.md` is only for SDK non-compliance and mock-capability gaps. +`getLog()` returns a typed `[ProxyEvent]`. Access fields via dot notation (`$0.type`, `$0.direction`, +`$0.queryParams`, `$0.status`); numeric fields (`status`, `closeCode`) are already `Int?`. The raw protocol +message is exposed as `ProxyEvent.message` (a `[String: Any]?`) — introspect it with +`event.message?["action"] as? Int`. -### Deviations file +```swift +let log = try await session.getLog() +let wsConnects = log.filter { $0.type == "ws_connect" } +#expect(wsConnects.count >= 2) +let queryParams = try #require(wsConnects.first?.queryParams) +#expect(queryParams["resume"] != nil) +``` -Append to `Test/UTS/deviations.md` under the matching section (Failing Tests / Adapted Tests / Mock Infrastructure Limitations). Each entry needs: +### Conventions -1. The spec point (e.g. `RSA4c2`) -2. What the spec says -3. What the SDK does -4. Which test is affected and how it was adapted +1. Each test references the spec point and (where it exists) the corresponding unit test. +2. Setup/teardown (proxy launch, sandbox app, session, client close) is the **base class's** job — wrap + scenarios in `withProxySession` / `withSandboxApp` / `withRealtimeClient`; never hand-roll it. +3. **Guard every wait** — `guard await awaitState(…) else { return }`: a timeout already records an + `Issue`, so stop instead of driving a client in the wrong state (the scopes still tear down). +4. Use `awaitState` / `awaitChannelState` for state assertions; verify via SDK state **and** the proxy log + where useful. +5. Use generous timeouts (10–30s) — real network is involved: `await awaitState(client, .connected, timeout: 15)`. +6. Don't set `fallbackHosts`; explicit hosts already disable fallbacks. Wrap proxy test files in + `#if os(macOS)`. ---- +Step 5 (compile) still applies; Step 6 (run) applies only in evaluate mode (Step E). Note that proxy tests +hit the live sandbox and download the proxy binary on first run, so they are slower and require network +access. -## Step 7 — Review generated output against the spec +**File template — proxy** (`integration/proxy//`): -Re-read the original spec and the generated Swift test side-by-side. Fix anything that fails a check before declaring the task done. +```swift +// Proxy tests spawn a local uts-proxy process — macOS-only. +#if os(macOS) -- **Coverage** — every spec test-case ID has a `func` with a matching `// UTS:` comment and a descriptive name. -- **Assertion completeness** — every `ASSERT`/`AWAIT`/observable outcome has a direct `#expect` / `#require` / `awaitConnectionState` / `awaitChannelState` / `poll`; none silently dropped or weakened to a comment. -- **Setup fidelity** — client options, mock responses, timer setup, and the order of channel operations match the spec. -- **Spec comments copied** — the pseudocode's inline `#` comments are carried over as `//` comments at the matching steps verbatim. +import Testing +import Foundation +import Ably -### Deviation honesty +/// Proxy integration test against Ably Sandbox endpoint. +/// +/// Uses the programmable uts-proxy to inject transport-level faults while the +/// SDK communicates with the real Ably backend. See +/// `uts/docs/proxy.md` for proxy infrastructure details. +@Suite(.serialized) +final class : ProxyTestCase { -For any place where the generated test diverges from the spec pseudocode (adapted assertion, env-gated skip, or omitted step): -- A `// DEVIATION:` comment explains why -- The deviation is recorded in `deviations.md` + // UTS: + @Test + func test__() async throws { + try await withProxySession(rules: []) { app, session in // rule-less start = late fault injection + let options = proxyClientOptions(for: app, through: session) // token auth; JSON forced + options.autoConnect = false + try await withRealtimeClient(options) { client in + client.connect() + guard await awaitState(client, .connected) else { return } + // Late imperative fault injection, then poll — never sleep — and verify via the log: + try await session.triggerAction(["type": "inject_to_client", "message": ["action": 17]]) + await pollUntil("client reacted to the injected frame") { /* observable state */ true } + let log = try await session.getLog() + #expect(log.contains { $0.type == "ws_frame" && $0.direction == "client_to_server" }) + } + } // client closed, session closed, app deleted — even on failure + } +} -If you find gaps, fix them and re-run Steps 5–6 before finishing. +#endif +``` diff --git a/.claude/skills/uts-to-swift/references/objects-mapping.md b/.claude/skills/uts-to-swift/references/objects-mapping.md new file mode 100644 index 000000000..715a4a91e --- /dev/null +++ b/.claude/skills/uts-to-swift/references/objects-mapping.md @@ -0,0 +1,7 @@ +# `objects` UTS → ably-cocoa: ably-js ⇄ ably-cocoa type/interface map + +> **Placeholder — intentionally empty.** The `objects` UTS specs are written against the +> ably-js LiveObjects API; the mapping to the Swift SDK's surface is SDK-specific and has +> not been authored yet. Fill this in (mirroring the structure of ably-java's +> `uts-to-kotlin/references/objects-mapping.md`) before translating any `objects` module +> spec for ably-cocoa. diff --git a/.claude/skills/uts-to-swift/scripts/audit_translation.py b/.claude/skills/uts-to-swift/scripts/audit_translation.py new file mode 100755 index 000000000..005fe930c --- /dev/null +++ b/.claude/skills/uts-to-swift/scripts/audit_translation.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +""" +audit_translation.py — deterministic faithfulness audit of a UTS spec against its +generated Swift test, so the per-spec review (SKILL.md Step 7) reconciles a concrete +extracted ledger instead of eyeballing two files. + +Usage: + python3 audit_translation.py + +It does ZERO semantic judgement — only mechanical extraction with regex, so the same +inputs always give the same report: + + 1. Test-ID coverage. Every `**Test ID**: \`\`` in the spec vs every `// UTS: ` + comment in the Swift file. + - missingInSwift: a spec Test ID with no matching // UTS: tag → a whole test + case is absent; implement it (or consciously exclude it and explain why). + - orphanInSwift: a // UTS: tag with no matching spec Test ID → a stale or + hand-edited tag. Investigate. + + 2. Per-test line ledger. Within each spec test block (from one Test ID to the next), + every non-blank, non-comment code line inside the ```pseudo fences is extracted + verbatim, grouped by its section (Setup / Test Steps / Assertions / …) — so setup, + operations AND assertions are all enumerated, nothing escapes the ledger. Each line + is tagged: "assert" (ASSERT*), "await" (AWAIT* / EXPECT), or "step" (everything else + — setup, mock construction, operations). For convenience the ASSERT* and AWAIT* + lines are also surfaced flat as specAsserts / specAwaits. + Alongside them, the count of assertion / await / poll calls in the matching Swift + method is reported as a tripwire: Swift assertions < spec ASSERTs for a test is a + strong signal an assertion was silently dropped. + +Robustness contract: this tool must never crash mid-run on any spec/test pair. It is a +review aid — if it can't extract something it degrades to "couldn't verify" (fewer lines +in the ledger) rather than throwing. Whatever happens it emits ONE parseable JSON object, +never a traceback, so a caller can always rely on the output shape. + +Exit status: + 0 — clean audit (no missing/orphan Test IDs) + 2 — audit ran, but there are missing/orphan Test IDs (gateable) + 64 — could not run: bad usage, unreadable file, or an internal error (JSON carries `error`) +""" + +import json +import re +import sys +from pathlib import Path + +# Spec markers ------------------------------------------------------------- +TEST_ID_RE = re.compile(r"\*\*Test ID\*\*:\s*`([^`]+)`") +HEADING_RE = re.compile(r"^#{1,4}\s+(.*\S)\s*$") +FENCE_RE = re.compile(r"^\s*```") +# Imperative spec keywords. Order matters: longest / most specific first so AWAIT_STATE +# is classified before the bare AWAIT. +DIRECTIVE_RE = re.compile(r"\b(ASSERT_[A-Z_]+|ASSERT|AWAIT_STATE|AWAIT_ERROR|AWAIT_ALL|AWAIT|EXPECT)\b") + +# Swift markers ------------------------------------------------------------ +UTS_TAG_RE = re.compile(r"//\s*UTS:\s*(\S+)") +# Swift Testing macros + the UTS infra's wait/poll helpers, both tiers. Longest +# alternatives first so `pollUntil` isn't half-matched by `poll`. +SWIFT_ASSERT_RE = re.compile( + r"(#expect|#require|" + r"\bawaitConnectionState\b|\bawaitChannelState\b|\bawaitState\b|" + r"\bpollUntil\b|\bpoll\b)" +) + + +def fail(code, message, status=64, **extra): + """Emit a structured error (same JSON shape as resolve_uts.py) and exit. `status` + defaults to 64 — "couldn't run" — kept distinct from the audit's own 0 (clean) and + 2 (missing/orphan IDs) outcomes so callers can tell the two apart.""" + print(json.dumps({"ok": False, "error": code, "message": message, **extra}, indent=2)) + sys.exit(status) + + +def read_lines(path): + """Read a file into lines, tolerant of encoding issues — a stray non-UTF-8 byte in a + spec must never crash the audit, so undecodable bytes are replaced, not raised on.""" + return Path(path).read_text(encoding="utf-8", errors="replace").splitlines() + + +def classify(keyword): + if keyword.startswith("ASSERT"): + return "assert" + if keyword.startswith("AWAIT") or keyword == "EXPECT": + return "await" + return "other" + + +def line_kind(stripped): + """Tag a pseudocode line: assert / await / step.""" + m = DIRECTIVE_RE.search(stripped) + if m: + k = classify(m.group(1)) + if k in ("assert", "await"): + return k + return "step" + + +def parse_spec(path): + """Return an ordered list of test-id dicts, each: + {testId, title, sections:[{heading, lines:[{text, kind}]}], asserts[], awaits[], codeLineTotal}.""" + lines = read_lines(path) + + # locate each Test ID line and the nearest preceding heading (its human title) + tests = [] + last_heading = "" + for i, line in enumerate(lines): + h = HEADING_RE.match(line) + if h: + last_heading = h.group(1) + m = TEST_ID_RE.search(line) + if m: + tests.append({"testId": m.group(1), "title": last_heading, "_start": i}) + + # block boundaries: each test runs to the next test's start (or EOF) + for idx, t in enumerate(tests): + start = t["_start"] + end = tests[idx + 1]["_start"] if idx + 1 < len(tests) else len(lines) + + sections = [] # [{heading, lines:[{text, kind}]}] + cur = None # current section being filled + in_fence = False + for line in lines[start:end]: + h = HEADING_RE.match(line) + if h and not in_fence: + cur = {"heading": h.group(1), "lines": []} + sections.append(cur) + continue + if FENCE_RE.match(line): + in_fence = not in_fence + continue + if not in_fence: + continue + stripped = line.strip() + if not stripped or stripped.startswith("#"): # blank / pseudocode comment + continue + if cur is None: # code before any heading in the block (rare) + cur = {"heading": "(preamble)", "lines": []} + sections.append(cur) + cur["lines"].append({"text": stripped, "kind": line_kind(stripped)}) + + # drop sections with no code (prose-only headings, requirement tables, the title line) + sections = [s for s in sections if s["lines"]] + asserts = [ln["text"] for s in sections for ln in s["lines"] if ln["kind"] == "assert"] + awaits = [ln["text"] for s in sections for ln in s["lines"] if ln["kind"] == "await"] + t["sections"] = sections + t["asserts"] = asserts + t["awaits"] = awaits + t["codeLineTotal"] = sum(len(s["lines"]) for s in sections) + del t["_start"] + return tests + + +def parse_swift(path): + """Return dict: uts-id -> {assertionCalls[], assertionCount}. Method block for a tag + runs from its // UTS: line to the next // UTS: line (or EOF). + + Comment-only lines are skipped when counting assertion calls — a commented-out + `// #expect(...)` (or an annotated-omission comment) is NOT an assertion, and counting + it would silently mask a dropped one. (The spec-side parser skips `#` pseudocode + comments for the same reason.)""" + lines = read_lines(path) + tags = [(i, m.group(1)) for i, line in enumerate(lines) for m in [UTS_TAG_RE.search(line)] if m] + out = {} + for idx, (start, tag) in enumerate(tags): + end = tags[idx + 1][0] if idx + 1 < len(tags) else len(lines) + calls = [] + for line in lines[start:end]: + if line.strip().startswith("//"): # comment-only line — not executable + continue + for m in SWIFT_ASSERT_RE.finditer(line): + calls.append(m.group(1)) + out[tag] = {"assertionCalls": calls, "assertionCount": len(calls)} + return out + + +def main(): + if len(sys.argv) != 3: + fail("USAGE", "Usage: audit_translation.py ") + + spec_path, swift_path = sys.argv[1], sys.argv[2] + for p in (spec_path, swift_path): + if not Path(p).is_file(): + fail("FILE_NOT_FOUND", f"{p!r} not found.") + + # Robustness backstop: never let an unexpected parsing/IO error surface as a traceback. + # fail() emits structured JSON and exits 64 (distinct from the 0/2 audit outcomes) so + # callers and the model can tell "couldn't run" apart from "ran, found gaps". + try: + spec_tests = parse_spec(spec_path) + swift = parse_swift(swift_path) + report = build_report(spec_path, swift_path, spec_tests, swift) + except Exception as exc: # noqa: BLE001 — deliberate catch-all; this tool must not crash + fail("INTERNAL_ERROR", f"{type(exc).__name__}: {exc}", spec=spec_path, swift=swift_path) + + print(json.dumps(report, indent=2)) + cov = report["idCoverage"] + sys.exit(2 if (cov["missingInSwift"] or cov["orphanInSwift"]) else 0) + + +def build_report(spec_path, swift_path, spec_tests, swift): + spec_ids = [t["testId"] for t in spec_tests] + swift_ids = list(swift.keys()) + spec_id_set, swift_id_set = set(spec_ids), set(swift_ids) + missing = [i for i in spec_ids if i not in swift_id_set] + orphan = [i for i in swift_ids if i not in spec_id_set] + + per_test = [] + for t in spec_tests: + sw = swift.get(t["testId"]) + per_test.append({ + "testId": t["testId"], + "title": t["title"], + # every code line of the spec test, grouped by section (setup / steps / assertions), + # each tagged assert | await | step — the full "did I translate every line?" ledger + "sections": t["sections"], + "specCodeLineTotal": t["codeLineTotal"], + # flat convenience views of the observable lines + "specAsserts": t["asserts"], + "specAwaits": t["awaits"], + "specAssertCount": len(t["asserts"]), + "specAwaitCount": len(t["awaits"]), + "swiftPresent": sw is not None, + "swiftAssertionCalls": sw["assertionCalls"] if sw else [], + "swiftAssertionCount": sw["assertionCount"] if sw else 0, + # tripwire: fewer Swift assertions than spec ASSERTs => likely a dropped assertion + "assertionShortfall": (len(t["asserts"]) - sw["assertionCount"]) if sw else len(t["asserts"]), + }) + + report = { + "ok": True, + "spec": spec_path, + "swift": swift_path, + "idCoverage": { + "specCount": len(spec_ids), + "swiftCount": len(swift_ids), + "missingInSwift": missing, + "orphanInSwift": orphan, + }, + "perTest": per_test, + "summary": { + "specAssertTotal": sum(len(t["asserts"]) for t in spec_tests), + "specAwaitTotal": sum(len(t["awaits"]) for t in spec_tests), + "swiftAssertionTotal": sum(k["assertionCount"] for k in swift.values()), + "testsWithShortfall": [p["testId"] for p in per_test if p["assertionShortfall"] > 0], + }, + } + return report + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/uts-to-swift/scripts/resolve_uts.py b/.claude/skills/uts-to-swift/scripts/resolve_uts.py new file mode 100755 index 000000000..01d408180 --- /dev/null +++ b/.claude/skills/uts-to-swift/scripts/resolve_uts.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Resolve a UTS spec module directory to its ably-cocoa test targets. + +Deterministic helper for the uts-to-swift skill. Given a UTS spec *module* +directory (a directory directly under .../specification/uts/), it: + + - validates the path and the module's tier structure, + - reads uts-package-mapping.json (next to this script's skill dir), + - resolves, per tier, the target output directory, and + - lists the candidate spec files with their derived Swift class names. + +Doing this in code (rather than asking the model to eyeball regexes, join +paths, and hand-convert snake_case -> PascalCase every run) keeps the skill's +selection phase byte-for-byte deterministic. + +Usage: + resolve_uts.py # validate + resolve + list specs + resolve_uts.py --create NAME # add a mapping entry for this + # module (target base name NAME), + # then resolve + +Always prints a single JSON object to stdout. On failure: {"ok": false, +"error": , "message": ...} and a non-zero exit. +""" +import argparse +import json +import re +import sys +from pathlib import Path + +SKILL_DIR = Path(__file__).resolve().parent.parent +MAPPING = SKILL_DIR / "uts-package-mapping.json" +TIERS = ("unit", "integration", "proxy") + + +def fail(code, message): + print(json.dumps({"ok": False, "error": code, "message": message}, indent=2)) + sys.exit(1) + + +def class_name(md_path: Path) -> str: + """objects_batch_test.md -> ObjectsBatchTests; instance.md -> InstanceTests.""" + stem = md_path.stem + if stem.endswith("_test"): + stem = stem[: -len("_test")] + return "".join(part.capitalize() for part in stem.split("_") if part) + "Tests" + + +def is_nonspec_doc(name: str) -> bool: + """README/PLAN/*_SUMMARY markdown are docs, not test specs.""" + if re.fullmatch(r"(README|PLAN)\.md", name, re.IGNORECASE): + return True + return name.upper().endswith("_SUMMARY.MD") + + +def list_specs(base: Path, exclude_proxy: bool = False): + """List spec .md files under `base`, deterministically. + + All exclusions are checked against the path **relative to base**, so they + can't be tripped by an ancestor directory in the checkout path (e.g. a + clone living under some `.../helpers/...` path). + """ + if not base.is_dir(): + return [] + specs = [] + for p in sorted(base.rglob("*.md")): + rel_parts = p.relative_to(base).parts + if "helpers" in rel_parts: + continue + if exclude_proxy and "proxy" in rel_parts: + continue + if is_nonspec_doc(p.name): + continue + specs.append(p) + return specs + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("module_dir") + ap.add_argument( + "--create", + metavar="NAME", + help="add a mapping for this source module using NAME as the ably-cocoa " + "module base name, then resolve", + ) + args = ap.parse_args() + + # Validate via Path.parts so this works regardless of separator (Windows '\' as well as + # POSIX '/'); a hard-coded "/uts/$" regex would reject otherwise-valid Windows paths. + module_dir = Path(args.module_dir) + raw = str(module_dir) + parts = module_dir.parts + if len(parts) < 2 or parts[-2] != "uts": + fail("NOT_A_UTS_MODULE_PATH", + f"{raw!r} is not a module directory directly under uts/ " + f"(expected .../uts/).") + if not module_dir.is_dir(): + fail("DIR_NOT_FOUND", f"{raw!r} does not exist or is not a directory.") + if not (module_dir / "unit").is_dir() and not (module_dir / "integration").is_dir(): + fail("NO_TIER_DIRS", + f"{raw!r} has no unit/ or integration/ sub-directory; " + f"not a valid UTS module.") + + source_module = module_dir.name + + if not MAPPING.is_file(): + fail("MAPPING_NOT_FOUND", f"mapping file not found at {MAPPING}") + data = json.loads(MAPPING.read_text(encoding="utf-8")) + packages = data.setdefault("packages", {}) + test_root = data.get("testRoot", "") + + if args.create: + target = args.create + if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", target): + fail("BAD_TARGET_NAME", + f"--create target {target!r} must be a simple module base name " + f"(letters/digits/underscore, e.g. 'objects') so it forms a " + f"valid path.") + new_entry = { + "unit": f"unit/{target}", + "integration": f"integration/standard/{target}", + "proxy": f"integration/proxy/{target}", + } + # preserve a hand-maintained "notes" pointer when re-creating an existing entry + notes = packages.get(source_module, {}).get("notes") + if notes: + new_entry["notes"] = notes + packages[source_module] = new_entry + # Write bytes (not write_text): explicit utf-8, and binary mode does zero newline + # translation on any OS or Python version — so this git-tracked file stays LF on + # Windows too. (write_text(newline=...) would need Python 3.10+.) + MAPPING.write_bytes((json.dumps(data, indent=2) + "\n").encode("utf-8")) + + mapped = source_module in packages + entry = packages.get(source_module, {}) + + # Per-module translation notes (ably-js -> ably-cocoa type map etc.), declared by + # the module's "notes" field in the mapping (a path relative to this skill dir). + # Read it before translating when present. None when the module declares no notes, + # or the declared file is missing. + notes_rel = entry.get("notes") + notes_path = SKILL_DIR / notes_rel if notes_rel else None + translation_notes = str(notes_path) if (notes_path and notes_path.is_file()) else None + + src = { + "unit": module_dir / "unit", + "integration": module_dir / "integration", + "proxy": module_dir / "integration" / "proxy", + } + specs = { + "unit": list_specs(src["unit"]), + "integration": list_specs(src["integration"], exclude_proxy=True), + "proxy": list_specs(src["proxy"]), + } + + tiers_out = {} + for tier in TIERS: + target_dir = f"{test_root}/{entry[tier]}" if (mapped and tier in entry) else None + tiers_out[tier] = { + "present": src[tier].is_dir(), + "sourceDir": str(src[tier]), + "targetDir": target_dir, + "specs": [{"file": str(p), "className": class_name(p)} for p in specs[tier]], + } + + print(json.dumps({ + "ok": True, + "sourceModule": source_module, + "mapped": mapped, + "testRoot": test_root, + "translationNotes": translation_notes, + "tiers": tiers_out, + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/uts-to-swift/uts-package-mapping.json b/.claude/skills/uts-to-swift/uts-package-mapping.json new file mode 100644 index 000000000..5b3973c92 --- /dev/null +++ b/.claude/skills/uts-to-swift/uts-package-mapping.json @@ -0,0 +1,22 @@ +{ + "_comment": "Maps each UTS spec module (a dir under specification/uts/) to its target test directories. Output dir = testRoot + '/' + tier entry. Swift has no package concept, so the directory is the full story. An optional 'notes' field points (relative to this skill dir) to a per-module ably-js -> ably-cocoa translation reference, read before translating that module. Used by the uts-to-swift skill.", + "testRoot": "Test/UTS", + "packages": { + "realtime": { + "unit": "unit/realtime", + "integration": "integration/standard/realtime", + "proxy": "integration/proxy/realtime" + }, + "objects": { + "unit": "unit/objects", + "integration": "integration/standard/objects", + "proxy": "integration/proxy/objects", + "notes": "references/objects-mapping.md" + }, + "rest": { + "unit": "unit/rest", + "integration": "integration/standard/rest", + "proxy": "integration/proxy/rest" + } + } +} From 7e383be38b6ef2b2d3a39f7bb6136bbacb28ed3a Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Wed, 15 Jul 2026 01:01:47 +0530 Subject: [PATCH 4/6] =?UTF-8?q?uts:=20address=20skill=20review=20=E2=80=94?= =?UTF-8?q?=20split=20assert/await=20audit=20counts,=20pseudo-only=20fence?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply four fixes from the PR #2224 review: - audit_translation.py: count Swift assertions (#expect/#require) and infra wait/poll calls SEPARATELY, mirroring the spec side's specAssertCount/ specAwaitCount — under the previous combined count, a surplus of waits could mask dropped assertions (demonstrably: RTN16f has 2 spec asserts and 3 wait calls, so deleting both assertions kept the shortfall negative). assertionShortfall stays the primary tripwire; awaitShortfall is a softer secondary signal since continuation-bridged helpers legitimately replace infra wait calls. Adds swiftAwaitCalls/Count, awaitShortfall, swiftAwaitTotal and testsWithAwaitShortfall to the report. - audit_translation.py: only ```pseudo fences open ledger extraction — other labelled and unlabelled fences (json fixtures, examples) are skipped, making the code match its own docstring. Output-neutral on the current corpus (146 spec files, zero ledger diffs): every non-pseudo fence today sits in a file preamble, outside the test blocks. - audit_translation.py: drop invalid backslash-backtick escapes from the module docstring (Ruff W605; SyntaxWarning on Python 3.12+). - resolve_uts.py: expanduser() on the module path so ~-prefixed paths resolve instead of failing with a misleading DIR_NOT_FOUND. SKILL.md Step 7 updated to describe the split counts and the new awaitShortfall secondary-signal checklist item. Verified: both real spec/test pairs clean under the split with no false positives; mutation test still trips the assert tripwire; whole-corpus audit sweep unchanged (146 files, 0 crashes, 0 spec-ledger diffs vs the previous revision). --- .claude/skills/uts-to-swift/SKILL.md | 17 ++-- .../uts-to-swift/scripts/audit_translation.py | 78 ++++++++++++------- .../uts-to-swift/scripts/resolve_uts.py | 7 +- 3 files changed, 64 insertions(+), 38 deletions(-) diff --git a/.claude/skills/uts-to-swift/SKILL.md b/.claude/skills/uts-to-swift/SKILL.md index 8b4e999aa..cfac21beb 100644 --- a/.claude/skills/uts-to-swift/SKILL.md +++ b/.claude/skills/uts-to-swift/SKILL.md @@ -798,8 +798,10 @@ semantic judgement — it only extracts, deterministically, what you must then r **grouped by section** (`Setup` / `Test Steps` / `Assertions` / …) in `sections[]` and each line tagged `assert` (an `ASSERT*` outcome), `await` (an `AWAIT*` / `EXPECT` wait), or `step` (setup, mock/client construction, an operation). `specAsserts` / `specAwaits` are flat convenience views of the first two; - `specCodeLineTotal` is the size of the spec test. The matching Swift method's assertion/await/poll calls - and their count come alongside. + `specCodeLineTotal` is the size of the spec test. The matching Swift method's calls come alongside, + counted **separately** so a surplus of waits can't hide a dropped assertion: assertions + (`#expect`/`#require` → `swiftAssertionCount`) and waits (`awaitConnectionState`/`awaitChannelState`/ + `awaitState`/`pollUntil`/`poll` → `swiftAwaitCount`). The audit is a review aid, not a gate — it never crashes, it degrades. It always prints one JSON object (an `error` field instead of a report if a file is unreadable). If `idCoverage.specCount` is `0` for a file @@ -833,10 +835,13 @@ This is the guarantee that no setup step, operation, or assertion was dropped - [ ] A spec line with no Swift equivalent is annotated per "Comments and assertion fidelity" (spec line kept + `//` note), never silently dropped - [ ] `assertionShortfall > 0` for a test (`summary.testsWithShortfall`) is a **tripwire** — fewer Swift - assertions than spec `ASSERT`s strongly suggests a dropped assertion; open that test and account for - each one. (A negative shortfall is fine — the SDK mapping often needs *more* Swift lines per spec - `ASSERT`; an annotated omission — e.g. a type check satisfied by a helper's return type — is a valid - account.) + `#expect`/`#require` calls than spec `ASSERT`s strongly suggests a dropped assertion; open that test + and account for each one. (A negative shortfall is fine — the SDK mapping often needs *more* Swift + lines per spec `ASSERT`; an annotated omission — e.g. a type check satisfied by a helper's return + type — is a valid account.) +- [ ] `awaitShortfall > 0` (`summary.testsWithAwaitShortfall`) is the **softer secondary signal** — fewer + infra wait calls than spec `AWAIT`s. A custom continuation-bridged helper (like `awaitTime`) + legitimately replaces a wait call, so account for those rather than treating this as a gate. ### Setup fidelity — preconditions match the spec diff --git a/.claude/skills/uts-to-swift/scripts/audit_translation.py b/.claude/skills/uts-to-swift/scripts/audit_translation.py index 005fe930c..f7e499d3f 100755 --- a/.claude/skills/uts-to-swift/scripts/audit_translation.py +++ b/.claude/skills/uts-to-swift/scripts/audit_translation.py @@ -10,23 +10,27 @@ It does ZERO semantic judgement — only mechanical extraction with regex, so the same inputs always give the same report: - 1. Test-ID coverage. Every `**Test ID**: \`\`` in the spec vs every `// UTS: ` - comment in the Swift file. + 1. Test-ID coverage. Every `**Test ID**: ` marker in the spec vs every + `// UTS: ` comment in the Swift file. - missingInSwift: a spec Test ID with no matching // UTS: tag → a whole test case is absent; implement it (or consciously exclude it and explain why). - orphanInSwift: a // UTS: tag with no matching spec Test ID → a stale or hand-edited tag. Investigate. 2. Per-test line ledger. Within each spec test block (from one Test ID to the next), - every non-blank, non-comment code line inside the ```pseudo fences is extracted - verbatim, grouped by its section (Setup / Test Steps / Assertions / …) — so setup, + every non-blank, non-comment code line inside the ```pseudo fences (only pseudo — + other fenced languages like json fixtures are ignored) is extracted verbatim, + grouped by its section (Setup / Test Steps / Assertions / …) — so setup, operations AND assertions are all enumerated, nothing escapes the ledger. Each line is tagged: "assert" (ASSERT*), "await" (AWAIT* / EXPECT), or "step" (everything else — setup, mock construction, operations). For convenience the ASSERT* and AWAIT* lines are also surfaced flat as specAsserts / specAwaits. - Alongside them, the count of assertion / await / poll calls in the matching Swift - method is reported as a tripwire: Swift assertions < spec ASSERTs for a test is a - strong signal an assertion was silently dropped. + Alongside them, the matching Swift method's assertion calls (#expect / #require) + and wait/poll calls are counted SEPARATELY — so a surplus of waits can't hide a + dropped assertion. Swift assertions < spec ASSERTs for a test (assertionShortfall) + is a strong signal an assertion was silently dropped; awaitShortfall is the softer + secondary signal (custom continuation-bridged helpers legitimately replace the + infra wait calls, so account for those rather than treating it as a gate). Robustness contract: this tool must never crash mid-run on any spec/test pair. It is a review aid — if it can't extract something it degrades to "couldn't verify" (fewer lines @@ -47,18 +51,21 @@ # Spec markers ------------------------------------------------------------- TEST_ID_RE = re.compile(r"\*\*Test ID\*\*:\s*`([^`]+)`") HEADING_RE = re.compile(r"^#{1,4}\s+(.*\S)\s*$") -FENCE_RE = re.compile(r"^\s*```") +# An opening fence carries its language label (```pseudo, ```json, …); the closer is a +# bare ```. Only pseudo blocks are spec code — other languages are fixtures/examples. +FENCE_RE = re.compile(r"^\s*```(\S*)") # Imperative spec keywords. Order matters: longest / most specific first so AWAIT_STATE # is classified before the bare AWAIT. DIRECTIVE_RE = re.compile(r"\b(ASSERT_[A-Z_]+|ASSERT|AWAIT_STATE|AWAIT_ERROR|AWAIT_ALL|AWAIT|EXPECT)\b") # Swift markers ------------------------------------------------------------ UTS_TAG_RE = re.compile(r"//\s*UTS:\s*(\S+)") -# Swift Testing macros + the UTS infra's wait/poll helpers, both tiers. Longest -# alternatives first so `pollUntil` isn't half-matched by `poll`. -SWIFT_ASSERT_RE = re.compile( - r"(#expect|#require|" - r"\bawaitConnectionState\b|\bawaitChannelState\b|\bawaitState\b|" +# Assertions and waits are counted separately (mirroring the spec side's +# specAssertCount / specAwaitCount), so a surplus of waits can't mask a dropped +# assertion. Longest alternatives first so `pollUntil` isn't half-matched by `poll`. +SWIFT_ASSERT_RE = re.compile(r"(#expect|#require)") +SWIFT_AWAIT_RE = re.compile( + r"(\bawaitConnectionState\b|\bawaitChannelState\b|\bawaitState\b|" r"\bpollUntil\b|\bpoll\b)" ) @@ -118,17 +125,18 @@ def parse_spec(path): sections = [] # [{heading, lines:[{text, kind}]}] cur = None # current section being filled - in_fence = False + fence = None # None = prose; else the open fence's label ("pseudo", "json", "") for line in lines[start:end]: h = HEADING_RE.match(line) - if h and not in_fence: + if h and fence is None: cur = {"heading": h.group(1), "lines": []} sections.append(cur) continue - if FENCE_RE.match(line): - in_fence = not in_fence + m = FENCE_RE.match(line) + if m: + fence = m.group(1) if fence is None else None # open with label / close continue - if not in_fence: + if fence != "pseudo": # prose, or a non-pseudo fence (json fixtures, examples) continue stripped = line.strip() if not stripped or stripped.startswith("#"): # blank / pseudocode comment @@ -151,25 +159,30 @@ def parse_spec(path): def parse_swift(path): - """Return dict: uts-id -> {assertionCalls[], assertionCount}. Method block for a tag - runs from its // UTS: line to the next // UTS: line (or EOF). - - Comment-only lines are skipped when counting assertion calls — a commented-out - `// #expect(...)` (or an annotated-omission comment) is NOT an assertion, and counting - it would silently mask a dropped one. (The spec-side parser skips `#` pseudocode - comments for the same reason.)""" + """Return dict: uts-id -> {assertionCalls[], assertionCount, awaitCalls[], awaitCount}. + Method block for a tag runs from its // UTS: line to the next // UTS: line (or EOF). + Assertions (#expect / #require) and waits (await*/poll* helpers) are counted + separately, mirroring the spec side's assert/await split. + + Comment-only lines are skipped when counting — a commented-out `// #expect(...)` + (or an annotated-omission comment) is NOT an assertion, and counting it would + silently mask a dropped one. (The spec-side parser skips `#` pseudocode comments + for the same reason.)""" lines = read_lines(path) tags = [(i, m.group(1)) for i, line in enumerate(lines) for m in [UTS_TAG_RE.search(line)] if m] out = {} for idx, (start, tag) in enumerate(tags): end = tags[idx + 1][0] if idx + 1 < len(tags) else len(lines) - calls = [] + asserts, awaits = [], [] for line in lines[start:end]: if line.strip().startswith("//"): # comment-only line — not executable continue for m in SWIFT_ASSERT_RE.finditer(line): - calls.append(m.group(1)) - out[tag] = {"assertionCalls": calls, "assertionCount": len(calls)} + asserts.append(m.group(1)) + for m in SWIFT_AWAIT_RE.finditer(line): + awaits.append(m.group(1)) + out[tag] = {"assertionCalls": asserts, "assertionCount": len(asserts), + "awaitCalls": awaits, "awaitCount": len(awaits)} return out @@ -222,8 +235,13 @@ def build_report(spec_path, swift_path, spec_tests, swift): "swiftPresent": sw is not None, "swiftAssertionCalls": sw["assertionCalls"] if sw else [], "swiftAssertionCount": sw["assertionCount"] if sw else 0, - # tripwire: fewer Swift assertions than spec ASSERTs => likely a dropped assertion + "swiftAwaitCalls": sw["awaitCalls"] if sw else [], + "swiftAwaitCount": sw["awaitCount"] if sw else 0, + # primary tripwire: fewer Swift assertions than spec ASSERTs => likely a dropped assertion "assertionShortfall": (len(t["asserts"]) - sw["assertionCount"]) if sw else len(t["asserts"]), + # secondary signal: fewer Swift waits than spec AWAITs — softer, since custom + # continuation-bridged helpers legitimately replace the infra wait calls + "awaitShortfall": (len(t["awaits"]) - sw["awaitCount"]) if sw else len(t["awaits"]), }) report = { @@ -241,7 +259,9 @@ def build_report(spec_path, swift_path, spec_tests, swift): "specAssertTotal": sum(len(t["asserts"]) for t in spec_tests), "specAwaitTotal": sum(len(t["awaits"]) for t in spec_tests), "swiftAssertionTotal": sum(k["assertionCount"] for k in swift.values()), + "swiftAwaitTotal": sum(k["awaitCount"] for k in swift.values()), "testsWithShortfall": [p["testId"] for p in per_test if p["assertionShortfall"] > 0], + "testsWithAwaitShortfall": [p["testId"] for p in per_test if p["awaitShortfall"] > 0], }, } return report diff --git a/.claude/skills/uts-to-swift/scripts/resolve_uts.py b/.claude/skills/uts-to-swift/scripts/resolve_uts.py index 01d408180..b6838bd28 100755 --- a/.claude/skills/uts-to-swift/scripts/resolve_uts.py +++ b/.claude/skills/uts-to-swift/scripts/resolve_uts.py @@ -86,9 +86,10 @@ def main(): ) args = ap.parse_args() - # Validate via Path.parts so this works regardless of separator (Windows '\' as well as - # POSIX '/'); a hard-coded "/uts/$" regex would reject otherwise-valid Windows paths. - module_dir = Path(args.module_dir) + # Expand a leading ~ first (like any CLI tool), then validate via Path.parts so this + # works regardless of separator (Windows '\' as well as POSIX '/'); a hard-coded + # "/uts/$" regex would reject otherwise-valid Windows paths. + module_dir = Path(args.module_dir).expanduser() raw = str(module_dir) parts = module_dir.parts if len(parts) < 2 or parts[-2] != "uts": From 455f4ca7051154ca886f722318be26e3824d62c5 Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Wed, 15 Jul 2026 12:05:28 +0530 Subject: [PATCH 5/6] =?UTF-8?q?uts:=20address=20second=20skill=20review=20?= =?UTF-8?q?round=20=E2=80=94=20mapping=20validation,=20duplicate=20tags,?= =?UTF-8?q?=20neutral=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply three fixes from the PR #2224 re-review: - resolve_uts.py: fail fast with a new BAD_MAPPING error when the mapping's testRoot is blank/missing/non-string — previously a corrupted mapping silently emitted absolute-looking nonsense like "/unit/realtime". Validated before --create so a corrupted file is never written back; trailing slash normalised. - audit_translation.py: detect duplicated // UTS: tags — the per-tag dict keeps only the last method's block, so a copy-paste-duplicated tag made a method invisible to the audit while coverage read clean. Reported as idCoverage.duplicateInSwift and included in the exit-2 gate alongside missing/orphan IDs (a duplicate is a finding, not an inability to run, so no hard failure; blocks are deliberately not merged). - SKILL.md: replace the personal example paths with /uts/objects — explicit that the spec repo must be cloned first (usage guard now links to ably/specification), and uts/ sits at the spec repo root. Verified: all three BAD_MAPPING variants fire and --create leaves a broken mapping untouched; a duplicate-tag mutation exits 2 flagging both the shared tag and the displaced missing id; corpus-wide Test IDs confirm no legitimate duplicates exist within any spec file (same spec *point* across many tests is common and unaffected — tags compare full structured IDs); both real spec/test pairs regression-clean. --- .claude/skills/uts-to-swift/SKILL.md | 17 ++++++---- .../uts-to-swift/scripts/audit_translation.py | 32 +++++++++++++------ .../uts-to-swift/scripts/resolve_uts.py | 10 +++++- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/.claude/skills/uts-to-swift/SKILL.md b/.claude/skills/uts-to-swift/SKILL.md index cfac21beb..2c02af856 100644 --- a/.claude/skills/uts-to-swift/SKILL.md +++ b/.claude/skills/uts-to-swift/SKILL.md @@ -1,6 +1,6 @@ --- name: uts-to-swift -description: "Translate the UTS pseudocode test specs in a whole module directory into runnable Swift tests in the ably-cocoa UTS test target. Takes a UTS module directory (e.g. .../specification/uts/objects), validates its structure, resolves the target test directories, lets you pick a tier (unit/integration/proxy) and which specs, then derives a Swift test per spec. Usage: /uts-to-swift " +description: "Translate the UTS pseudocode test specs in a whole module directory into runnable Swift tests in the ably-cocoa UTS test target. Takes a UTS module directory (e.g. /uts/objects), validates its structure, resolves the target test directories, lets you pick a tier (unit/integration/proxy) and which specs, then derives a Swift test per spec. Usage: /uts-to-swift " license: proprietary allowed-tools: Bash, Read, Edit, Write, WebFetch metadata: @@ -13,8 +13,8 @@ metadata: Translate the UTS pseudocode test specs under the **module directory** `$ARGUMENTS` into runnable Swift tests in the ably-cocoa `UTS` test target (`Test/UTS`). -`$ARGUMENTS` is a UTS *module* directory — a directory sitting directly under `.../specification/uts/`, -e.g. `/Users/sachinsh/ably-specification/specification/uts/objects`. Its name (`objects`, `realtime`, +`$ARGUMENTS` is a UTS *module* directory — a directory sitting directly under the spec repo's `uts/`, +e.g. `/uts/objects`. Its name (`objects`, `realtime`, `rest`, …) is the **source module**. A module directory holds many spec files, organised into tiers (`unit/`, `integration/`, and `integration/proxy/`). @@ -38,7 +38,9 @@ bundled script does them — that keeps selection byte-for-byte deterministic in to re-eyeball regexes, join paths, and hand-convert `snake_case` → `PascalCase` each run. > **If `$ARGUMENTS` is empty or blank**, stop and show: `Usage: /uts-to-swift ` -> — with the example `/uts-to-swift /Users/sachinsh/ably-specification/specification/uts/objects`. +> — with the example `/uts-to-swift /uts/objects` +> (the path to a module directory inside a local clone of +> [`ably/specification`](https://github.com/ably/specification)). ## Step A — Resolve the module @@ -794,6 +796,8 @@ semantic judgement — it only extracts, deterministically, what you must then r - `missingInSwift` **must be empty.** Each entry is a spec test case with no `@Test` method — implement it. - `orphanInSwift` **must be empty** (or every entry explained) — a `// UTS:` tag that no longer matches any spec Test ID means a stale or hand-edited tag. + - `duplicateInSwift` **must be empty** — a tag carried by more than one `@Test` means the audit only + examined the last method with that tag; deduplicate the tags first. - **`perTest[]`** — for each spec test case, every non-comment code line inside the spec's `pseudo` blocks, **grouped by section** (`Setup` / `Test Steps` / `Assertions` / …) in `sections[]` and each line tagged `assert` (an `ASSERT*` outcome), `await` (an `AWAIT*` / `EXPECT` wait), or `step` (setup, mock/client @@ -817,6 +821,7 @@ verify that file by hand. From the audit's `idCoverage`: - [ ] `missingInSwift` is empty (every spec Test ID has an `@Test` with that ID in its `// UTS:` comment) - [ ] `orphanInSwift` is empty, or each orphan is a deliberate, explained rename +- [ ] `duplicateInSwift` is empty (no `// UTS:` tag is shared by two test methods) - [ ] Each method name contains the spec ID and a meaningful description ### Line-by-line completeness — every spec line is translated @@ -859,8 +864,8 @@ generated test diverges from the spec pseudocode (adapted assertion, env-gated s - [ ] The deviation is recorded in `Test/UTS/deviations.md` If you find gaps during this review, fix them, then **re-run the audit script** until `missingInSwift` / -`orphanInSwift` are empty and every `perTest` entry reconciles, and re-run Step 5 (compile) — and, in -evaluate mode, Step 6 — before finishing. +`orphanInSwift` / `duplicateInSwift` are empty and every `perTest` entry reconciles, and re-run Step 5 +(compile) — and, in evaluate mode, Step 6 — before finishing. --- diff --git a/.claude/skills/uts-to-swift/scripts/audit_translation.py b/.claude/skills/uts-to-swift/scripts/audit_translation.py index f7e499d3f..e3386cd25 100755 --- a/.claude/skills/uts-to-swift/scripts/audit_translation.py +++ b/.claude/skills/uts-to-swift/scripts/audit_translation.py @@ -16,6 +16,9 @@ case is absent; implement it (or consciously exclude it and explain why). - orphanInSwift: a // UTS: tag with no matching spec Test ID → a stale or hand-edited tag. Investigate. + - duplicateInSwift: a // UTS: tag carried by more than one @Test → the audit + only examines the last method with that tag, so deduplicate before trusting + the ledger. 2. Per-test line ledger. Within each spec test block (from one Test ID to the next), every non-blank, non-comment code line inside the ```pseudo fences (only pseudo — @@ -159,10 +162,16 @@ def parse_spec(path): def parse_swift(path): - """Return dict: uts-id -> {assertionCalls[], assertionCount, awaitCalls[], awaitCount}. - Method block for a tag runs from its // UTS: line to the next // UTS: line (or EOF). - Assertions (#expect / #require) and waits (await*/poll* helpers) are counted - separately, mirroring the spec side's assert/await split. + """Return (methods, duplicates). `methods` is a dict uts-id -> + {assertionCalls[], assertionCount, awaitCalls[], awaitCount}; a method block for a + tag runs from its // UTS: line to the next // UTS: line (or EOF). Assertions + (#expect / #require) and waits (await*/poll* helpers) are counted separately, + mirroring the spec side's assert/await split. + + `duplicates` lists tags carried by more than one method — the dict keeps only the + LAST block for a duplicated tag, making earlier methods invisible, so duplicates are + reported (not merged: merging would paper over the defect) and gate the exit status + like missing/orphan IDs do. Comment-only lines are skipped when counting — a commented-out `// #expect(...)` (or an annotated-omission comment) is NOT an assertion, and counting it would @@ -170,6 +179,10 @@ def parse_swift(path): for the same reason.)""" lines = read_lines(path) tags = [(i, m.group(1)) for i, line in enumerate(lines) for m in [UTS_TAG_RE.search(line)] if m] + tag_counts = {} + for _, tag in tags: + tag_counts[tag] = tag_counts.get(tag, 0) + 1 + duplicates = [tag for tag, n in tag_counts.items() if n > 1] out = {} for idx, (start, tag) in enumerate(tags): end = tags[idx + 1][0] if idx + 1 < len(tags) else len(lines) @@ -183,7 +196,7 @@ def parse_swift(path): awaits.append(m.group(1)) out[tag] = {"assertionCalls": asserts, "assertionCount": len(asserts), "awaitCalls": awaits, "awaitCount": len(awaits)} - return out + return out, duplicates def main(): @@ -200,17 +213,17 @@ def main(): # callers and the model can tell "couldn't run" apart from "ran, found gaps". try: spec_tests = parse_spec(spec_path) - swift = parse_swift(swift_path) - report = build_report(spec_path, swift_path, spec_tests, swift) + swift, duplicates = parse_swift(swift_path) + report = build_report(spec_path, swift_path, spec_tests, swift, duplicates) except Exception as exc: # noqa: BLE001 — deliberate catch-all; this tool must not crash fail("INTERNAL_ERROR", f"{type(exc).__name__}: {exc}", spec=spec_path, swift=swift_path) print(json.dumps(report, indent=2)) cov = report["idCoverage"] - sys.exit(2 if (cov["missingInSwift"] or cov["orphanInSwift"]) else 0) + sys.exit(2 if (cov["missingInSwift"] or cov["orphanInSwift"] or cov["duplicateInSwift"]) else 0) -def build_report(spec_path, swift_path, spec_tests, swift): +def build_report(spec_path, swift_path, spec_tests, swift, duplicates): spec_ids = [t["testId"] for t in spec_tests] swift_ids = list(swift.keys()) spec_id_set, swift_id_set = set(spec_ids), set(swift_ids) @@ -253,6 +266,7 @@ def build_report(spec_path, swift_path, spec_tests, swift): "swiftCount": len(swift_ids), "missingInSwift": missing, "orphanInSwift": orphan, + "duplicateInSwift": duplicates, }, "perTest": per_test, "summary": { diff --git a/.claude/skills/uts-to-swift/scripts/resolve_uts.py b/.claude/skills/uts-to-swift/scripts/resolve_uts.py index b6838bd28..1dc822b45 100755 --- a/.claude/skills/uts-to-swift/scripts/resolve_uts.py +++ b/.claude/skills/uts-to-swift/scripts/resolve_uts.py @@ -109,7 +109,15 @@ def main(): fail("MAPPING_NOT_FOUND", f"mapping file not found at {MAPPING}") data = json.loads(MAPPING.read_text(encoding="utf-8")) packages = data.setdefault("packages", {}) - test_root = data.get("testRoot", "") + # testRoot anchors every targetDir; a blank/missing one would silently emit + # absolute-looking nonsense like "/unit/realtime". Fail fast — and before --create + # can write the corrupted mapping back to disk. + test_root = data.get("testRoot") or "" + if not isinstance(test_root, str) or not test_root.strip(): + fail("BAD_MAPPING", + f"{MAPPING} has no usable 'testRoot' (expected e.g. 'Test/UTS'); " + f"fix the mapping file before resolving.") + test_root = test_root.rstrip("/") if args.create: target = args.create From 58fa0f5ea67ac9a400fcb36b45cee8be8eeac050 Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Wed, 15 Jul 2026 15:59:07 +0530 Subject: [PATCH 6/6] uts: replace infra smoke tests with spec-derived integration tests, drop env gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first spec-derived integration tests now exist for both real-backend tiers, translated from the UTS specs via the uts-to-swift skill: - integration/standard/realtime/ChannelHistoryTests.swift — RTL10d (channel_history_test.md): cross-client history, JSON + msgpack variants. - integration/standard/realtime/TokenRequestTests.swift — RSA9/RSA9a/RSA9g (token_request_test.md): locally signed TokenRequest accepted by the server. - integration/proxy/realtime/AuthReauthTests.swift — RTN22/RTC8a (auth_reauth.md): injected server-initiated AUTH triggers re-auth without disrupting the connection; verified via the proxy event log. macOS-only. Per their TODOs, the interim infra acceptance ("smoke") tests are removed — the spec-derived tests cover their ground end-to-end. All env-var gating is dropped from the integration suites so every tier runs directly from swift test and the Xcode test runner UI; tier selection is by --filter, mirroring ably-java's per-tier Gradle tasks. UTS_PROXY_LOCAL_PATH (config override) and RUN_DEVIATIONS (deviation mechanism) remain. Test/UTS/README.md is brought to block-level parity with ably-java's uts/README.md: §2 tier table now carries "Example in this repo" file paths; §3 lists all five derived specs and uses the current fail-fast decision-tree wording; §7 unit walkthrough restructured to numbered sub-sections; §10 rewritten around ungated per-suite runs (+ the Notes block); §11.4/§11.5 walkthroughs now walk the real ChannelHistoryTests/AuthReauthTests with code excerpts at the same points as ably-java's §10/§11; §11.3 gains the control API endpoints and common-actions list; §12 cheat-sheet gains the build-client snippets, proxy-log inspect, Test ID format and decision-tree entries. SKILL.md: reference examples now point at the spec-derived suites (matching uts-to-kotlin), generated tests are instructed to be ungated, and the Step 7 checklist restores kotlin's setup_synced_channel awaited-operation example. Verified: all three suites pass ungated against the real sandbox (TokenRequest 2/2, ChannelHistory 2/2 JSON+msgpack, AuthReauth 1/1); unit suites 11/11; swift build --build-tests clean; make lint clean; zero smoke/env-gate references remain in code or docs. --- .claude/skills/uts-to-swift/SKILL.md | 23 +- Test/UTS/README.md | 451 ++++++++++++------ .../proxy/ProxyInfraSmokeTests.swift | 67 --- .../proxy/realtime/AuthReauthTests.swift | 138 ++++++ .../standard/IntegrationSmokeTest.swift | 72 --- .../realtime/ChannelHistoryTests.swift | 118 +++++ .../standard/realtime/TokenRequestTests.swift | 100 ++++ 7 files changed, 683 insertions(+), 286 deletions(-) delete mode 100644 Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift create mode 100644 Test/UTS/integration/proxy/realtime/AuthReauthTests.swift delete mode 100644 Test/UTS/integration/standard/IntegrationSmokeTest.swift create mode 100644 Test/UTS/integration/standard/realtime/ChannelHistoryTests.swift create mode 100644 Test/UTS/integration/standard/realtime/TokenRequestTests.swift diff --git a/.claude/skills/uts-to-swift/SKILL.md b/.claude/skills/uts-to-swift/SKILL.md index 2c02af856..8c6c31bb3 100644 --- a/.claude/skills/uts-to-swift/SKILL.md +++ b/.claude/skills/uts-to-swift/SKILL.md @@ -691,8 +691,9 @@ swift test --filter "UTS." swift test --filter "UTS./" ``` -(`swift test --filter UTS` still runs the whole UTS target. The integration and proxy tiers additionally -need network access and the suite's env gate — see `Test/UTS/README.md` §10 for the current gating.) +(`swift test --filter UTS` still runs the whole UTS target — including the integration tiers. There is +no env-var gating: integration and proxy suites run directly from `swift test` or the Xcode test runner; +they just need network access, and the proxy tier needs macOS — see `Test/UTS/README.md` §10.) Handle test failures using this decision tree (the **Required reading** doc you fetched up front has the full detail): @@ -830,9 +831,10 @@ Walk the audit's `perTest[].sections` and reconcile **each** extracted spec line This is the guarantee that no setup step, operation, or assertion was dropped — the whole point of this step: - [ ] Every `assert` line maps to a concrete Swift assertion (`#expect`, `try #require`, …) — not a comment, not a weaker check -- [ ] Every `await` line (state waits **and** awaited operations like `AWAIT channel.attach()`) is - performed via `awaitConnectionState` / `awaitChannelState` / `poll` (unit tier) or `awaitState` / - `awaitChannelState` / `pollUntil` (integration tiers) or the corresponding SDK call +- [ ] Every `await` line (state waits **and** awaited operations like `AWAIT channel.attach()` / + `setup_synced_channel(...)`) is performed via `awaitConnectionState` / `awaitChannelState` / + `poll` (unit tier) or `awaitState` / `awaitChannelState` / `pollUntil` (integration tiers) or + the corresponding SDK call - [ ] Every `step` line — client/mock construction, `ClientOptions`, installed mocks, channel ops — is reflected in the test setup or body. Multi-line spec constructs split across several `step` lines; reconcile them as a group. @@ -908,10 +910,11 @@ Recognise a **proxy** spec by a reference to `create_proxy_session()`, proxy `ru A **direct-sandbox** spec (no `create_proxy_session`, no rules — just happy-path interop against `nonprod:sandbox`) uses the same `SandboxApp` provisioning as a proxy test but **drops all proxy wiring**: no `ProxyManager`, no `ProxySession`, no `connectThroughProxy`. The client connects straight to the -sandbox host. `IntegrationSmokeTest.swift` (under `Test/UTS/integration/standard/`) is the reference -example — read it before translating a direct-sandbox spec. (It's an infra acceptance test slated for -removal once spec-derived integration suites exist — the reference then migrates to the first of those; -see `Test/UTS/README.md` §11.7.) +sandbox host. `Test/UTS/integration/standard/realtime/ChannelHistoryTests.swift` (RTL10d — ably-java's +`ChannelHistoryTest` is its sibling) is the reference example — read it before translating a +direct-sandbox spec. For the **proxy** tier the reference is +`Test/UTS/integration/proxy/realtime/AuthReauthTests.swift` (RTN22/RTC8a). Don't add env-var gating to +generated tests — integration suites run ungated (they just need network; see `Test/UTS/README.md` §10). Suites subclass `IntegrationTestCase` and wrap the scenario in the scoped-resource methods: `withSandboxApp { app in … }` provisions a throwaway sandbox app and always deletes it; @@ -960,7 +963,7 @@ or poll: |---|---| | `AWAIT channel.attach()` | `channel.attach()` then `guard await awaitChannelState(channel, .attached, timeout: 10) else { return }` | | `AWAIT channel.publish(name, data)` (await the ack) | bridge the callback overload (`publish(_:data:callback:)`) with a continuation, resuming on the callback and recording an `Issue` on error — same helper pattern as Step 4's `awaitTime` | -| `poll_until(() => AWAIT channel.history().items.length == N, …)` | `await pollUntil("history has N items") { await historyCount(channel) == N }` — with a continuation-bridged `history` helper | +| `poll_until(() => AWAIT channel.history().items.length == N, …)` | `await pollUntil("history has N items") { await historyItems(of: channel).count == N }` — with a continuation-bridged `history` helper (see `ChannelHistoryTests` and `Test/UTS/README.md` §11.4 for the concrete helpers) | Use generous timeouts (10–30s) — real network is involved. Everything else is the shared foundation described at the top of this section; a direct-sandbox test just skips the proxy-only subsections diff --git a/Test/UTS/README.md b/Test/UTS/README.md index 7a35463b6..aab9853c1 100644 --- a/Test/UTS/README.md +++ b/Test/UTS/README.md @@ -81,11 +81,11 @@ test method, linking it back to its spec (e.g. `// UTS: realtime/unit/RTN16g/rec UTS divides tests into three tiers by *what infrastructure they need* and *what confidence they give*: -| Tier | Transport | Backend | Purpose | Status in ably-cocoa | +| Tier | Transport | Backend | Purpose | Example in this repo | |------|-----------|---------|---------|----------------------| -| **Unit** | **Mocked** (`MockWebSocket`, `MockHTTPClient`) | none | Client-side logic: state machines, request formation, response parsing, timer behaviour. Fast & deterministic. | ✅ Implemented — `unit/realtime/`, `unit/rest/` | -| **Direct sandbox integration** | Real network | Real Ably sandbox | Happy-path interop: connect, publish, subscribe. No fault injection. | 🚧 Infra ready + smoke-tested (§11) — spec-derived tests TODO | -| **Proxy integration** | Real network **through a programmable proxy** | Real Ably sandbox | Fault behaviour: dropped connections, injected errors, timeouts, re-auth. | 🚧 Infra ready + smoke-tested (§11) — spec-derived tests TODO | +| **Unit** | **Mocked** (`MockWebSocket`, `MockHTTPClient`) | none | Client-side logic: state machines, request formation, response parsing, timer behaviour. Fast & deterministic. | `unit/realtime/ConnectionRecoveryTests.swift` | +| **Direct sandbox integration** | Real network | Real Ably sandbox | Happy-path interop: connect, publish, subscribe. No fault injection. | `integration/standard/realtime/ChannelHistoryTests.swift` (§11.4) | +| **Proxy integration** | Real network **through a programmable proxy** | Real Ably sandbox | Fault behaviour: dropped connections, injected errors, timeouts, re-auth. | `integration/proxy/realtime/AuthReauthTests.swift` (§11.5) | Each tier folder is organised **by module** (`realtime`, `rest`, …), so a feature's tests sit together by SDK area — e.g. `unit/realtime/ConnectionRecoveryTests.swift`. @@ -121,9 +121,9 @@ describe: `AWAIT_STATE`, polling, and fake timers). - [`writing-derived-tests.md`](https://github.com/ably/specification/blob/main/uts/docs/writing-derived-tests.md) — how to translate a spec into a real SDK test, and the **decision tree** when a translated test - fails: spec wrong → fix test + record a UTS spec error; translation wrong → fix test; - SDK non-compliant → gate the spec-correct assertion behind `RUN_DEVIATIONS` and record a - **deviation** (§9). + fails: spec wrong → fix the spec at source + a fail-fast test + record a UTS spec error; + translation wrong → fix the test; SDK non-compliant → gate the spec-correct assertion behind + `RUN_DEVIATIONS` and record a **deviation** (§9). - [`integration-testing.md`](https://github.com/ably/specification/blob/main/uts/docs/integration-testing.md) — the policy for the two integration tiers (directory layout, sandbox provisioning, proxy session lifecycle, late fault injection). Implemented by the §11 infrastructure. @@ -145,6 +145,15 @@ The specs currently derived here: - `RSC16` (server time) → unit spec [`time.md`](https://github.com/ably/specification/blob/main/uts/rest/unit/time.md) → **`unit/rest/TimeTests.swift`**. +- `RTL10d` (channel history) → integration spec + [`channel_history_test.md`](https://github.com/ably/specification/blob/main/uts/realtime/integration/channel_history_test.md) + → **`integration/standard/realtime/ChannelHistoryTests.swift`**. +- `RSA9`/`RSA9a`/`RSA9g` (token request) → integration spec + [`token_request_test.md`](https://github.com/ably/specification/blob/main/uts/realtime/integration/auth/token_request_test.md) + → **`integration/standard/realtime/TokenRequestTests.swift`**. +- `RTN22`/`RTC8a` (server-initiated re-auth) → proxy spec + [`auth_reauth.md`](https://github.com/ably/specification/blob/main/uts/realtime/integration/proxy/auth_reauth.md) + → **`integration/proxy/realtime/AuthReauthTests.swift`**. --- @@ -211,9 +220,12 @@ Test/UTS/ │ └── integration/ # ── INTEGRATION TESTS (real backend) ── · per module ├── standard/ # direct sandbox: happy-path, no fault injection - │ └── IntegrationSmokeTest.swift # env-gated acceptance test: SandboxApp + real TLS client (JSON + msgpack) + │ └── realtime/ + │ ├── ChannelHistoryTests.swift# RTL10d (cross-client history, JSON + msgpack) — §11.4 + │ └── TokenRequestTests.swift # RSA9/RSA9a/RSA9g (server-accepted TokenRequest) └── proxy/ # sandbox through the fault-injecting uts-proxy - └── ProxyInfraSmokeTests.swift # env-gated acceptance test: ProxyManager + ProxySession + └── realtime/ + └── AuthReauthTests.swift # RTN22/RTC8a (injected AUTH → re-auth) — §11.5 ``` The mental model (same as ably-java): **`infra/unit/` powers the unit tests, `infra/integration/` @@ -464,28 +476,39 @@ The integration-tier counterpart of this picture — real network through the pr Six tests, each carrying a `// UTS: realtime/unit/RTN16…/…` tag: -- **`RTN16g` (+`RTN16g1`) — recovery-key structure (incl. Unicode).** Connects (the - `onConnectionAttempt` handler responds with success + CONNECTED carrying a known key), attaches - two channels — one ASCII, one Unicode (`channel-éàü-世界`) — feeding each an ATTACHED with a - `channelSerial` via `sendToClient`. Asserts `createRecoveryKey()` contains the connection key, - `msgSerial == 0`, and both channel serials — including an encode→decode round-trip proving the - Unicode name survives. -- **`RTN16g2` — `createRecoveryKey()` returns nil in inactive states.** Walks the connection - through INITIALIZED → CONNECTED → CLOSING/CLOSED → FAILED → SUSPENDED asserting nil in every - inactive state. The SUSPENDED leg uses `enableFakeTimers()` + `advanceTime` to expire the - connection-state TTL deterministically — no real sleeping. -- **`RTN16k` — `recover` adds the `recover` query param.** Builds the client with - `options.recover = ` and asserts via `MockWebSocket.queryParams` (real production URL - building) that the first attempt carries `recover=` and a post-disconnect reconnect switches to - `resume=` — recover is a one-shot bootstrap. -- **`RTN16f` — `recover` initialises `msgSerial` from the recovery key**, verified through an ACK - round-trip (`.ack(msgSerial:count:)`). -- **`RTN16f1` — malformed `recover` key degrades gracefully.** A garbage key must be logged - (asserted via `CapturingLog`) and ignored: the client connects normally, with no - `recover`/`resume` params. -- **`RTN16j` (+`RTN16i`) — `recover` instantiates channels with their serials**, not auto-attached; - a manual `attach()` sends an ATTACH frame carrying the recovered serial (verified via - `sentMessages`). +### 7.1 `RTN16g, RTN16g1` — recovery-key structure (incl. Unicode) + +Connects (the `onConnectionAttempt` handler responds with success + CONNECTED carrying a known +key), attaches two channels — one ASCII, one Unicode (`channel-éàü-世界`) — feeding each an +ATTACHED with a `channelSerial` via `sendToClient`. Asserts `createRecoveryKey()` contains the +connection key, `msgSerial == 0`, and both channel serials — including an encode→decode +round-trip proving the Unicode name survives. + +### 7.2 `RTN16g2` — `createRecoveryKey()` returns nil in inactive states + +Walks the connection through INITIALIZED → CONNECTED → CLOSING/CLOSED → FAILED → SUSPENDED +asserting nil in every inactive state. The SUSPENDED leg uses `enableFakeTimers()` + +`advanceTime` to expire the connection-state TTL deterministically — no real sleeping. + +### 7.3 `RTN16k` — `recover` adds the `recover` query param + +Builds the client with `options.recover = ` and asserts via `MockWebSocket.queryParams` +(real production URL building) that the first attempt carries `recover=` and a post-disconnect +reconnect switches to `resume=` — recover is a one-shot bootstrap. + +### 7.4 `RTN16f` — `recover` initialises `msgSerial` from the recovery key + +Verified through an ACK round-trip (`.ack(msgSerial:count:)`). + +### 7.5 `RTN16f1` — malformed `recover` key degrades gracefully + +A garbage key must be logged (asserted via `CapturingLog`) and ignored: the client connects +normally, with no `recover`/`resume` params. + +### 7.6 `RTN16j` — `recover` instantiates channels with their serials (RTN16i too) + +Not auto-attached; a manual `attach()` sends an ATTACH frame carrying the recovered serial +(verified via `sentMessages`). **What this file teaches about the infra:** the handler-style mock, `sendToClient` vs a real close, fake-timer-driven SUSPENDED, `queryParams`/`sentMessages` as the two inspection surfaces, and @@ -533,10 +556,19 @@ and the gated test becomes the acceptance test for the fix. ## 10. How to Run the Tests +There is **no env-var gating** — every suite (unit, integration, proxy) runs directly from +`swift test` or the Xcode test runner UI, just like ably-java's Gradle tasks. Pick the tier by +**which suites you run**: unit suites are sub-second and fully mocked; integration/proxy suites +hit the real Ably sandbox (and the proxy suites additionally sync + spawn the local `uts-proxy`, +macOS-only — on iOS/tvOS destinations they compile to nothing, §11.2). + ```bash -# The whole UTS target (fast — fully mocked, no network): +# The whole UTS target — unit AND integration tiers (integration needs network; proxy needs macOS): swift test --filter UTS +# Unit suites only (fast, fully mocked, no network): +swift test --filter "UTS.ConnectionRecoveryTests|UTS.TimeTests" + # One suite / one test: swift test --filter UTS.ConnectionRecoveryTests swift test --filter UTS.ConnectionRecoveryTests/test_RTN16k_recover_option_adds_recover_query_param_to_WebSocket_URL @@ -544,33 +576,42 @@ swift test --filter UTS.ConnectionRecoveryTests/test_RTN16k_recover_option_adds_ # Turn on the spec-correct (currently failing) deviation assertions: RUN_DEVIATIONS=1 swift test --filter UTS./ -# The integration-infra acceptance tests (need network; skipped without the env var). -# IntegrationSmokeTest hits the Ably sandbox directly (once per protocol variant: JSON + msgpack); -# ProxyInfraSmokeTests (JSON-only, as the proxy requires) additionally syncs -# the uts-proxy binary from GitHub releases on first run and is macOS-only: -UTS_INTEGRATION_SMOKE=1 swift test --filter "UTS.IntegrationSmokeTest|UTS.ProxyInfraSmokeTests" +# Spec-derived integration & proxy suites (real sandbox; AuthReauthTests is macOS-only): +swift test --filter UTS.ChannelHistoryTests +swift test --filter UTS.AuthReauthTests -# Run the proxy infra against a locally built uts-proxy instead of a GitHub release: -UTS_PROXY_LOCAL_PATH=/path/to/uts-proxy UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.ProxyInfraSmokeTests +# Run the proxy tier against a locally built uts-proxy instead of a GitHub release +# (a config override, not a gate): +UTS_PROXY_LOCAL_PATH=/path/to/uts-proxy swift test --filter UTS.AuthReauthTests ``` **Where CI runs them:** there is currently **no UTS-specific CI job** — the UTS target runs as part of the full test suite (the `ably-cocoa` scheme driven by the fastlane lanes in -`.github/workflows/integration-test.yaml`). Since `swift test --filter UTS` is sub-second and -network-free, a dedicated fast PR-gate step (e.g. in `check-spm.yaml`, mirroring ably-java's -`runUtsUnitTests` gate in its `check.yml`) is a cheap improvement worth making — especially once -the integration tier exists and the unit/integration split needs separate CI cadences. +`.github/workflows/integration-test.yaml`, which already has sandbox network access). A dedicated +fast PR-gate step running just the unit suites (e.g. in `check-spm.yaml`, mirroring ably-java's +`runUtsUnitTests` gate in its `check.yml`) is a cheap improvement worth making — the +unit/integration split now runs on suite selection rather than env vars, so the lanes only need +different `--filter` lists (ably-java splits the same way with `runUtsUnitTests` / +`runUtsIntegrationTests`). + +Notes: +- `ProxyManager` **advises** running proxy suites from one test process at a time — they share the + control port (10100), see §11.2. +- Proxy/sandbox tests need outbound network (sandbox + GitHub releases on first run; the binary is + then cached under `~/.cache/uts-proxy/`). --- ## 11. Integration & Proxy Infrastructure The `infra/integration/` layer mirrors ably-java's `infra/integration/` (see its `uts/README.md` -§7 for the reference design) and is verified end-to-end by two env-gated acceptance tests (§10): -`integration/standard/IntegrationSmokeTest.swift` (SandboxApp + a real TLS client, no proxy; run once per protocol variant — JSON + msgpack) and -`integration/proxy/ProxyInfraSmokeTests.swift` (the full proxy chain). Three components -(§11.1–11.3), the base test cases that own setup/teardown (below), then a walkthrough of each -tier's reference test (§11.4–11.5) and the request-flow picture (§11.6). +§7 for the reference design) and is exercised end-to-end by the spec-derived tests (§10): +`integration/standard/realtime/ChannelHistoryTests.swift` (SandboxApp + real TLS clients, no +proxy; run once per protocol variant — JSON + msgpack) and +`integration/proxy/realtime/AuthReauthTests.swift` (the full proxy chain, fault injection +included). Three components (§11.1–11.3), the base test cases that own setup/teardown (below), +then a walkthrough of each tier's reference test (§11.4–11.5) and the request-flow picture +(§11.6). **The base test cases.** Swift Testing has no `setUp()`/`tearDown()` hooks and `deinit` cannot `await`, so integration suites subclass a base case whose **scoped-resource methods** own the @@ -598,7 +639,7 @@ across the integration specs; both `ProxySession` targets and direct-sandbox cli `SandboxApp` is the shared backbone of *both* integration kinds: **proxy** tests pair it with a `ProxySession`, while **direct sandbox** tests use it alone — connecting straight to `SandboxApp.sandboxHost` over TLS, where plain key (basic) auth works; -`IntegrationSmokeTest.swift` is the reference shape. +`integration/standard/realtime/ChannelHistoryTests.swift` is the reference shape. ### 11.2 `proxy/ProxyManager.swift` — syncs and launches the proxy binary @@ -608,8 +649,9 @@ owns the *process*: - `try await ProxyManager.shared.ensureProxy()` (call in suite setup) is **idempotent**: if a proxy is already healthy on the control port (**10100**), it's a no-op. Otherwise it **downloads** the - pinned release (`v0.3.0`) archive for the current arch from GitHub releases, **verifies its - SHA-256 checksum**, extracts the binary (via the system `tar` — one deliberate simplification + pinned release (`v0.3.0`) archive (`uts-proxy___.tar.gz`) for the current arch + from GitHub releases, **verifies its SHA-256 checksum**, extracts the binary (via the system + `tar` — one deliberate simplification over ably-java's hand-rolled JDK-only tar reader), caches it under `~/.cache/uts-proxy//` — the **same cache ably-java uses**, so the two SDKs share one download — and launches it with `--port 10100`. @@ -636,24 +678,28 @@ One session per test: - `ProxySession.create(rules:)` → `POST /sessions` with a `target` (the sandbox hosts) and an initial **rule list**; the proxy assigns a `sessionId` and a fresh **listening port**. -- `addRules(_:position:)` → add rules mid-test; `triggerAction(_:)` → fire an **imperative** action - right now (late fault injection, e.g. `["type": "inject_to_client", "message": ["action": 17]]`). -- `getLog()` → the session's ordered, typed `[ProxyEvent]` — each carries `type` (`ws_connect`, - `ws_frame`, `http_request`, …), `direction`, `queryParams`, and the parsed protocol `message` - (introspect via `message?["action"] as? Int`). Proxy-log assertions are a proxy test's primary - verification. +- `addRules(_:position:)` → add rules mid-test (`POST /sessions/{id}/rules`). +- `triggerAction(_:)` → fire an **imperative** action right now (`POST /sessions/{id}/actions`) — + late fault injection, e.g. `["type": "inject_to_client", "message": ["action": 17]]`. +- `getLog()` → `GET /sessions/{id}/log`, returning the session's ordered, typed `[ProxyEvent]` — + each carries `type` (`ws_connect`, `ws_frame`, `http_request`, …), `direction`, `queryParams`, + and the parsed protocol `message` (introspect via `message?["action"] as? Int`). Proxy-log + assertions are a proxy test's primary verification. - `close()` → `DELETE /sessions/{id}` — always call it in teardown. **Rules** = `match` + `action` (+ optional `times`), built with `wsConnectRule` / `wsFrameToClientRule` / `wsFrameToServerRule` / `httpRequestRule`. Rules evaluate in order, first -match wins, unmatched traffic passes through, `times: N` auto-removes after N firings. +match wins, unmatched traffic passes through, `times: N` auto-removes after N firings. Common +actions: `refuse_connection`, `suppress`, `replace`, `inject_to_client[_and_close]`, `disconnect`, +`http_respond`. **Wiring a client through the proxy** — `options.connectThroughProxy(session)` sets `realtimeHost`/`restHost` = localhost, `port` = the session's port, `tls = false`, and `useBinaryProtocol = false` (the proxy only understands text frames). ⚠️ Because the proxy serves plain ws, **basic (key) auth is rejected** (RSA1: basic auth is TLS-only) — authenticate through -the proxy with an `authCallback` that signs a `TokenRequest` locally using the sandbox key (see -`ProxyInfraSmokeTests` for the shape; ably-java's `AuthReauthTest` does the same). +the proxy with an `authCallback` that signs a `TokenRequest` locally using the sandbox key +(`ProxyTestCase.proxyClientOptions` packages this; `AuthReauthTests` inlines it to count the +callbacks — ably-java's `AuthReauthTest` does the same). **The shared async helpers** both integration walkthroughs below lean on (from `infra/Utils.swift` — java's `infra/Utils.kt`, see §6.9). All are wall-clock and poll-based @@ -666,81 +712,183 @@ rather than throwing: | `awaitChannelState` | `await awaitChannelState(channel, .attached, timeout: 15)` | same, for a channel's state | | `pollUntil` | `await pollUntil("what you wait for", timeout: 15, interval: 0.1) { condition }` | suspend until an arbitrary predicate holds — e.g. `pollUntil("re-auth") { count.count > original }` | -### 11.4 Walkthrough: a direct-sandbox test (`IntegrationSmokeTest`) +### 11.4 Walkthrough: the direct-sandbox test (`ChannelHistoryTests`) -**File:** `integration/standard/IntegrationSmokeTest.swift` +**File:** `integration/standard/realtime/ChannelHistoryTests.swift` (ably-java's +`ChannelHistoryTest.kt` is its sibling) **Tier:** Direct-sandbox integration (real network, real Ably sandbox, **no** proxy, **no** fault injection). +**Spec point:** RTL10d — messages published by one realtime client are retrievable from a +*separate* client's `history()`. +**Spec:** [`channel_history_test.md`](https://github.com/ably/specification/blob/main/uts/realtime/integration/channel_history_test.md) + +This is the reference for the **middle tier**. Like a proxy test it talks to the real backend, but +it connects *straight* to `SandboxApp.sandboxHost` — there is no `ProxyManager`, no `ProxySession`, +and no `connectThroughProxy` wiring. It's the shape every happy-path interop spec +(connect/publish/subscribe/presence) follows. + +#### 11.4.1 Suite setup/teardown + +Swift Testing has no `@BeforeAll`/`@AfterAll` (§6.9), so the suite subclasses +`IntegrationTestCase` and the **scoped-resource methods** own the lifecycle — provisioning +**`SandboxApp` only**, no `ProxyManager.ensureProxy()`: + +```swift +try await withSandboxApp { app in // SandboxApp.create() → body → app.delete() + try await withRealtimeClient(publisherOptions) { publisher in + try await withRealtimeClient(subscriberOptions) { subscriber in + // … scenario … + } // close() + await CLOSED — always + } +} +``` + +#### 11.4.2 The client — wired straight to the sandbox + +The options point the **real** transport at the sandbox host (no proxy in between). Setting +explicit hosts auto-disables fallback hosts (REC2c2), so there's nothing else to configure: + +```swift +let publisherOptions = ARTClientOptions(key: app.defaultKey) +publisherOptions.realtimeHost = SandboxApp.sandboxHost // sandbox.realtime.ably-nonprod.net +publisherOptions.restHost = SandboxApp.sandboxHost +publisherOptions.useBinaryProtocol = useBinaryProtocol +publisherOptions.autoConnect = false +``` + +(Plain `ARTClientOptions` with no `installMock` — TLS stays on, so basic key auth works here, +and the SDK drives its real `ARTWebSocketTransport` instead of a `MockWebSocket`.) + +#### 11.4.3 Protocol variants — the parameterised-test pattern -This is the reference for the **middle tier** — the shape every happy-path interop spec -(connect/publish/subscribe/presence/history) follows. Step by step: - -1. **Setup/teardown via the base class** — the suite subclasses `IntegrationTestCase` and wraps - the scenario in `withSandboxApp { app in … withRealtimeClient(options) { client in … } }`: - the app is provisioned up front, and app deletion + client close always run afterwards, even - when the scenario throws or a wait fails. -2. **Protocol variants** — the test takes `useBinaryProtocol: Bool` via - `@Test(arguments: [false, true])`, the cocoa realisation of the spec's `PROTOCOL` dimension - (§2): each case runs the whole scenario once over JSON and once over msgpack. -3. **A real client, wired straight to the sandbox** — plain `ARTClientOptions(key: app.defaultKey)` - with `realtimeHost`/`restHost` = `SandboxApp.sandboxHost`. TLS stays on, so basic key auth is - fine here (unlike through the proxy), and explicit hosts auto-disable fallback hosts (REC2c2). - No mocks anywhere — this drives the SDK's real `ARTWebSocketTransport`. -4. **Connect and wait, never sleep** — `client.connect()` then - `await awaitState(client, .connected)`: real network, so the async wall-clock waits from - `infra/Utils.swift` (§6.9), not the unit tier's fake timers. -5. **A fresh channel per run** — `"smoke-\(UUID().uuidString)"`, so variants and retries never - collide on server-side channel state. -6. **The round-trip** — subscribe first (capturing into a `Captured` — the callback - arrives on the SDK's queue, §6.6), publish, then - `await pollUntil("published message is echoed back…") { received.count == 1 }` — the real - backend is eventually consistent, so poll on observable state rather than assuming timing. -7. **Guarded waits** — the scenario lives in a helper where every wait is `guard`-ed: a timeout - has already recorded its `Issue`, so the scenario stops instead of cascading into secondary - failures, and the base class's teardown still runs. - -**What this teaches about the infra:** `IntegrationTestCase`'s scoped setup/teardown, -`SandboxApp`-only provisioning, direct-sandbox client wiring, protocol-variant parameterisation, -`Captured` for cross-queue capture, and `pollUntil` over real network state. - -### 11.5 Walkthrough: a proxy test (`ProxyInfraSmokeTests`) - -**File:** `integration/proxy/ProxyInfraSmokeTests.swift` (macOS-only, §11.2) +The spec declares a `PROTOCOL` dimension (`json` / `msgpack`) and says *each test runs once per +variant*. Cocoa realises that with a Swift Testing **parameterised test** over +`useBinaryProtocol` (built in — no extra dependency, unlike ably-java's `junit-jupiter-params`): + +```swift +// UTS: realtime/integration/RTL10d/history-cross-client-0 +@Test(arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack +func test_RTL10d_history_contains_messages_published_by_another_client(useBinaryProtocol: Bool) async throws { + … +} +``` + +A plain `@Test` test (no protocol dimension) stays a `@Test` — reach for `arguments:` only when +the spec actually declares variants. + +#### 11.4.4 The scenario — real publish, real history + +Two independent clients on the same app: the publisher's *confirmed* messages must appear in the +subscriber's history. The integration-specific techniques on show: + +- **Awaiting a publish ack.** Realtime publish is fire-and-forget, so to honour the spec's + `AWAIT publish` the test wraps the callback overload `publish(_:data:callback:)` in a + `withCheckedContinuation` (`awaitPublish`, in the suite's extension), resuming on the callback + and recording an `Issue` on error. This is the integration analogue of the unit test's + `sentMessages` inspection. +- **`AWAIT attach()`** → `attach()` then `await awaitChannelState(channel, .attached, timeout: 10)`. +- **Polling real REST state.** `history()` is a callback-based REST call against the sandbox and + the message store is eventually-consistent, so the test bridges it with a continuation + (`historyItems`) and `await pollUntil("subscriber history contains all 3 messages", timeout: 10, + interval: 0.5) { … == 3 }` — never a fixed sleep (the same anti-flake rule as the other tiers). +- **Order assertion.** History defaults to newest-first, so `items[0]` is `event3` … `items[2]` + is `event1`. + +**What this test teaches about the infra:** `SandboxApp`-only provisioning, the direct-sandbox +client wiring (`realtimeHost`/`restHost` from `SandboxApp.sandboxHost`, no proxy), the +protocol-variant parameterised test, awaiting a publish ack via the callback overload, and +`pollUntil` over a real `history()` call. + +### 11.5 Walkthrough: the proxy test (`AuthReauthTests`) + +**File:** `integration/proxy/realtime/AuthReauthTests.swift` (ably-java's `AuthReauthTest.kt` is +its sibling; macOS-only, §11.2) **Tier:** Proxy integration (real sandbox, traffic routed through the local `uts-proxy`). +**Spec points:** RTN22 (server-initiated re-authentication) and RTC8a (the client sends an AUTH +frame with renewed auth details). Unit-tier spec counterparts: +`server_initiated_reauth_test.md`, `realtime_authorize.md`. +**Spec:** [`auth_reauth.md`](https://github.com/ably/specification/blob/main/uts/realtime/integration/proxy/auth_reauth.md) -Step by step: - -1. **Setup/teardown via the base class** — the suite subclasses `ProxyTestCase` and wraps the - scenario in `withProxySession(rules: []) { app, session in … }`: the proxy is ensured running - (§11.2), the app is provisioned **directly** against the sandbox (not through the proxy, so - provisioning is independent of any fault rules), and session close + app deletion always run - afterwards. -2. **A session with no rules** — starting rule-less is the **late-fault-injection** principle - (§2): the connect handshake runs against the real server unmodified; a spec test injects its - fault *afterwards*, as the final interaction. -3. **Token auth, not the key** — `proxyClientOptions(for: app, through: session)`: the proxy - serves plain ws (`tls = false`), and basic key auth is TLS-only (RSA1), so the client - authenticates via an `authCallback` that signs a `TokenRequest` locally using the sandbox key - (through a separate TLS `ARTRest` "token signer"). The options come back already wired through - the proxy (§11.3). -4. **Run the client in a scope** — `withRealtimeClient(options) { client in … }`, then - `client.connect()` and `await awaitState(client, .connected)` — the SDK believes it is talking - to Ably; every byte actually flows through the proxy. -5. **The proxy log is the primary verification** — `try await session.getLog()` and filter the - typed events: the smoke asserts a `ws_connect` event and a server→client `ws_frame` whose - `message?["action"] as? Int == 4` (CONNECTED). Spec tests assert on exactly this log — e.g. - "the client sent an AUTH frame (17) carrying non-nil `auth` details". -6. **Teardown is automatic** — the scopes unwind in order: client closed and awaited CLOSED, +#### 11.5.1 Suite setup/teardown + +Swift Testing has no `@BeforeAll`/`@AfterAll` (§6.9), so the suite subclasses `ProxyTestCase` and +one scope owns everything the java `@BeforeAll`/`@AfterAll` pair does — ensure the proxy is +running (§11.2), provision the app **directly** against the sandbox (not through the proxy, so +provisioning is independent of any fault rules), and always close the session + delete the app: + +```swift +try await withProxySession(rules: []) { app, session in + // ensureProxy() ran, app provisioned, session created against the sandbox target + // … scenario … +} // session.close() + app.delete() — always, even on failure +``` + +#### 11.5.2 The test, step by step + +1. **A session with no rules** (the `rules: []` above) — the fault will be injected + *imperatively* later (late injection, §2 — the connect handshake runs against the real server + unmodified). Declarative faults would use `addRules` / the rule builders instead (§11.3). +2. **Auth via `authCallback`** — the spec generates a JWT from the sandbox key; the idiomatic + cocoa equivalent is a locally-signed `TokenRequest` from the same key (no external JWT + library). A `Captured` counter records how many times the callback is invoked + (`proxyClientOptions(for:through:)` packages this wiring; the test inlines it to count): + + ```swift + let authCallbackInvocations = Captured() + let signerOptions = ARTClientOptions(key: app.defaultKey) + signerOptions.restHost = SandboxApp.sandboxHost + let tokenSigner = ARTRest(options: signerOptions) + + let options = ARTClientOptions() + options.authCallback = { params, callback in + authCallbackInvocations.append(params) + tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } + } + ``` +3. **Build the client through the proxy** and connect (JSON stays on so the proxy can inspect + frames): + + ```swift + options.connectThroughProxy(session) // localhost + session port, tls = false, JSON + options.autoConnect = false + try await withRealtimeClient(options) { client in + client.connect() + guard await awaitState(client, .connected, timeout: 15) else { return } + // … + } + ``` +4. **Snapshot identity** — `connection.id` (`try #require` — later lines depend on it) and the + callback count, and assert the callback already ran ≥ 1 (initial auth). +5. **Start recording state changes** (a `Captured` fed by `connection.on`), then **inject a + server-initiated AUTH** (protocol action 17) imperatively — simulating Ably asking the client + to re-authenticate: + + ```swift + try await session.triggerAction(["type": "inject_to_client", "message": ["action": 17]]) + ``` +6. **Wait for the re-auth round-trip** with `pollUntil` (real network, so poll — don't sleep): + first on the callback count, then until the client→server AUTH frame appears in the proxy log + (the cocoa callback returns a `TokenRequest` the SDK still exchanges for a token — a REST + round-trip through the proxy — before it sends AUTH, a timing adaptation the test documents + inline). +7. **Assertions** prove RTN22 + RTC8a: + - `authCallback` was invoked **again** (count incremented) → re-auth was triggered. + - Connection is still **CONNECTED** and `connection.id` is **unchanged** → re-auth does not + reconnect. + - **No** transitions away from CONNECTED were recorded. + - The **proxy event log** contains a client→server **AUTH frame (action 17) carrying non-nil + `auth` details** (RTC8a) — verified by filtering `session.getLog()`. +8. **Teardown is automatic** — the scopes unwind in order: client closed and awaited CLOSED, then `session.close()` (leaked sessions hold proxy listeners), then `app.delete()` — even when the scenario failed or threw. -**What a full spec-derived proxy test adds** (ably-java's `AuthReauthTest`, RTN22/RTC8a, is the -reference): snapshot `connection.id` and a callback counter after connecting; inject the fault -imperatively — `try await session.triggerAction(["type": "inject_to_client", "message": -["action": 17]])` (a server-initiated AUTH); `await pollUntil("re-auth round-trip") { … }` on the -counter; then assert the callback re-fired, the connection stayed CONNECTED with an **unchanged** -`connection.id`, and the log contains the client→server AUTH frame. Declarative faults use -`addRules` / the rule builders instead (§11.3). +**What this test teaches about the infra:** `ProxyTestCase`'s scoped setup/teardown +(`ensureProxy` + `SandboxApp` + session), `connectThroughProxy` with token auth, **late +imperative fault injection** via `triggerAction`, real-network waiting with `pollUntil`, and +**proxy-log assertions** as the primary verification (`getLog()` → filter by +`type`/`direction`/`message?["action"]`). ### 11.6 How the pieces connect (request flow) @@ -778,14 +926,15 @@ and the test's only side channel is `SandboxApp` provisioning. ### 11.7 What remains TODO -- **`integration/standard//`** — spec-derived direct-sandbox happy-path tests (SandboxApp - only, real transport, protocol-variant parameterisation via Swift Testing - `@Test(arguments: [false, true])` over `useBinaryProtocol` — `IntegrationSmokeTest` demonstrates - the shape). -- **`integration/proxy//`** — spec-derived fault-injection tests (SandboxApp + - ProxySession). +The first spec-derived tests for both tiers now exist (`ChannelHistoryTests`, +`TokenRequestTests`, `AuthReauthTests`). What's left: + +- **More spec coverage** — translate the remaining `uts//integration/` and + `…/integration/proxy/` specs into `integration/standard//` and + `integration/proxy//` (via the `uts-to-swift` skill; §11.4–11.5 are the reference + shapes). - **CI wiring** — a dedicated job/lane for the integration tier (macOS, network access), separate - from the fast unit gate. + from the fast unit gate (§10). Design constraints to carry over: late fault injection, JSON-only through the proxy, poll — never sleep — on real-network state (`pollUntil`), and every proxy test also keeping a unit-tier @@ -799,6 +948,24 @@ counterpart. `import Ably.Private`): `transportFactory` (WS) · `httpExecutor` (HTTP) · `timeProvider` (time) · `reachabilityClass` (network monitor) — plus `options.logHandler` (log assertions). +**Build a unit-test client:** +```swift +let wsProvider = MockWebSocketProvider(onConnectionAttempt: { $0.respondWithSuccess(.connectedMessage) }) +installMock(wsProvider) +let client = makeRealtime { $0.autoConnect = false } +client.connect() +awaitConnectionState(client, .connected) +``` + +**Build a proxy-test client:** +```swift +try await withProxySession(rules: []) { app, session in // ensureProxy + SandboxApp + session + let options = proxyClientOptions(for: app, through: session) // token auth + proxy wiring + options.autoConnect = false + try await withRealtimeClient(options) { client in /* … */ } +} +``` + **Writing a new test?** This guide documents the *existing* setup — the actionable authoring material (file templates for every tier, pseudocode→Swift translation tables, deviation patterns) lives in the `uts-to-swift` skill (`.claude/skills/uts-to-swift/SKILL.md`), which carries out UTS @@ -809,7 +976,8 @@ spec translation and evaluation. The reference tests to crib from are listed in `simulateDisconnect` (1001 drop). **Inspect what the SDK did:** `mockWebSocket.queryParams` / `.sentMessages` (WS) · -`PendingHTTPRequest.url/method/headers/body/queryParams` (HTTP) · `CapturingLog.contains(…)` (logs). +`PendingHTTPRequest.url/method/headers/body/queryParams` (HTTP) · `CapturingLog.contains(…)` (logs) · +`session.getLog()` (proxy). **Wait (never sleep):** unit tier — `awaitConnectionState` · `awaitChannelState` · `poll("…") { … }` · `enableFakeTimers()` + `advanceTime(byMilliseconds:)`; integration tier @@ -819,6 +987,14 @@ spec translation and evaluation. The reference tests to crib from are listed in **Protocol action numbers** (used in proxy rules & log assertions): CONNECTED=4, DISCONNECTED=6, ERROR=9, ATTACH=10, ATTACHED=11, DETACH=12, DETACHED=13, **AUTH=17**. +**Test ID format:** `//-` → +`// UTS: realtime/proxy/RTN22/server-initiated-reauth-0`. + +**The decision tree when a translated test fails:** spec wrong → fix the spec at source + a +fail-fast test + record under *UTS Spec Errors*; translation wrong → fix the test; SDK +non-compliant → gate the spec-correct assertion behind `RUN_DEVIATIONS` and record in +`deviations.md`. + **Test ID format:** `//-` → `// UTS: realtime/unit/RTN16g/recovery-key-structure-0` (comment immediately above each test). @@ -862,8 +1038,9 @@ DISCONNECTED=6, ERROR=9, ATTACH=10, ATTACHED=11, DETACH=12, DETACHED=13, **AUTH= |------|----------|-------| | `unit/realtime/ConnectionRecoveryTests.swift` | 6 `@Test`s: RTN16g/g1, RTN16g2, RTN16k, RTN16f, RTN16f1, RTN16j/i | Mocked WS + fake timers; see §7. | | `unit/rest/TimeTests.swift` | 5 `@Test`s: RSC16 ×5 | Mocked HTTP; see §8. | -| `integration/standard/IntegrationSmokeTest.swift` | 1 `@Test` × {JSON, msgpack} (`arguments: [false, true]`), gated behind `UTS_INTEGRATION_SMOKE` | Acceptance test for the direct-sandbox tier (not spec-derived): sandbox app → real TLS client → publish/subscribe round-trip, once per protocol variant. | -| `integration/proxy/ProxyInfraSmokeTests.swift` | 1 `@Test`, gated behind `UTS_INTEGRATION_SMOKE` | End-to-end acceptance test for the proxy infra (not spec-derived): binary sync → proxy launch → sandbox app → real client through the proxy → log assertions. macOS-only. | +| `integration/proxy/realtime/AuthReauthTests.swift` | 1 `@Test`: RTN22/RTC8a (needs network; macOS-only) | Spec-derived proxy test (`uts/realtime/integration/proxy/auth_reauth.md`): injects a server-initiated AUTH (17), asserts re-auth via authCallback + client→server AUTH frame, connection undisturbed. Walkthrough: §11.5. | +| `integration/standard/realtime/TokenRequestTests.swift` | 2 `@Test`s: RSA9a/RSA9g, RSA9 (needs network) | Spec-derived direct-sandbox tests (`uts/realtime/integration/auth/token_request_test.md`): a locally signed TokenRequest is accepted by the server via another client's authCallback, with and without clientId. | +| `integration/standard/realtime/ChannelHistoryTests.swift` | 1 `@Test` × {JSON, msgpack}: RTL10d (needs network) | Spec-derived direct-sandbox test (`uts/realtime/integration/channel_history_test.md`): messages published by one client appear (newest-first) in another client's channel history. Walkthrough: §11.4. | | `deviations.md` | none recorded yet | Catalogue of SDK-vs-spec divergences + the `RUN_DEVIATIONS` pattern. | > **Coverage note:** the infrastructure is built out beyond what the current suites exercise @@ -888,6 +1065,6 @@ DISCONNECTED=6, ERROR=9, ATTACH=10, ATTACHED=11, DETACH=12, DETACHED=13, **AUTH= | Unit mocks | `Test/UTS/infra/unit/*` | | Shared helpers | `Test/UTS/infra/Utils.swift` | | Integration helpers | `Test/UTS/infra/integration/*` (+ `proxy/*`) | -| The reference tests | `unit/realtime/ConnectionRecoveryTests.swift`, `unit/rest/TimeTests.swift`, `integration/standard/IntegrationSmokeTest.swift`, `integration/proxy/ProxyInfraSmokeTests.swift` | +| The reference tests | `unit/realtime/ConnectionRecoveryTests.swift`, `unit/rest/TimeTests.swift`, `integration/standard/realtime/ChannelHistoryTests.swift`, `integration/proxy/realtime/AuthReauthTests.swift` | | Deviations | `Test/UTS/deviations.md` | | The ably-java counterpart this guide mirrors | `uts/README.md` in the `ably-java` repository | diff --git a/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift b/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift deleted file mode 100644 index d0b5eebf0..000000000 --- a/Test/UTS/integration/proxy/ProxyInfraSmokeTests.swift +++ /dev/null @@ -1,67 +0,0 @@ -// Proxy integration tests spawn a local uts-proxy process — macOS-only (see ProxyManager). -#if os(macOS) - -import Testing -import Foundation -import Ably - -/// Acceptance test for the integration infrastructure itself (`SandboxApp`, `ProxyManager`, -/// `ProxySession`, `ProxyTestCase`) — not derived from a UTS spec. It proves the full chain works -/// end-to-end: binary sync → proxy launch → sandbox provisioning → a real client connecting -/// through the proxy → typed event-log assertions → teardown (handled by the base class). -/// -/// Needs outbound network (GitHub releases on first run, then the Ably sandbox), so it is gated -/// behind an env var and skipped by default: -/// -/// ```bash -/// UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.ProxyInfraSmokeTests -/// ``` -// TODO: Remove this infra acceptance (smoke) test once spec-derived integration/proxy -// tests exist and cover this ground (see Test/UTS/README.md §11.7). -@Suite(.serialized) -final class ProxyInfraSmokeTests: ProxyTestCase { - - @Test(.enabled(if: ProcessInfo.processInfo.environment["UTS_INTEGRATION_SMOKE"] != nil)) - func proxy_and_sandbox_infra_work_end_to_end() async throws { - // Session with no rules — traffic passes through to the real sandbox. The base class - // ensures the proxy is running and always closes the session + deletes the app. - try await withProxySession(rules: []) { app, session in - #expect(app.defaultKey.hasPrefix(app.appId + ".")) - #expect(session.proxyPort > 0) - - // A REAL client (no mocks) wired through the proxy with token auth (see - // proxyClientOptions — basic key auth is TLS-only, RSA1). - let options = proxyClientOptions(for: app, through: session) - options.autoConnect = false - - try await withRealtimeClient(options) { client in - await runScenario(client, session) - } - } - } - - /// The happy-path scenario. Each wait is guarded (a timeout already records an `Issue`), and - /// the control-plane `getLog()` failure is recorded rather than thrown — the base class's - /// teardown runs regardless. - private func runScenario(_ client: ARTRealtime, _ session: ProxySession) async { - client.connect() - guard await awaitState(client, .connected) else { return } - #expect(client.connection.state == .connected) - - // The proxy recorded the handshake: a ws_connect and a server→client CONNECTED frame (4). - let log: [ProxyEvent] - do { - log = try await session.getLog() - } catch { - Issue.record("Proxy getLog() failed: \(error)") - return - } - #expect(log.contains { $0.type == "ws_connect" }) - #expect(log.contains { event in - event.type == "ws_frame" && event.direction == "server_to_client" - && event.message?["action"] as? Int == 4 - }) - } -} - -#endif diff --git a/Test/UTS/integration/proxy/realtime/AuthReauthTests.swift b/Test/UTS/integration/proxy/realtime/AuthReauthTests.swift new file mode 100644 index 000000000..1ca4ce2cc --- /dev/null +++ b/Test/UTS/integration/proxy/realtime/AuthReauthTests.swift @@ -0,0 +1,138 @@ +// Proxy tests spawn a local uts-proxy process — macOS-only. +#if os(macOS) + +import Testing +import Foundation +import Ably + +/// Auth re-authorization (RTN22, RTC8a) +/// Derived from ably/specification `uts/realtime/integration/proxy/auth_reauth.md` +/// +/// Proxy integration test against Ably Sandbox endpoint. +/// +/// Uses the programmable uts-proxy to inject transport-level faults while the +/// SDK communicates with the real Ably backend. See +/// `uts/docs/proxy.md` for proxy infrastructure details. +/// +/// Needs outbound network (the uts-proxy binary on first run, then the Ably sandbox): +/// +/// ```bash +/// swift test --filter UTS.AuthReauthTests +/// ``` +@Suite(.serialized) +final class AuthReauthTests: ProxyTestCase { + + // UTS: realtime/proxy/RTN22/server-initiated-reauth-0 + @Test + func test_RTN22_RTC8a_server_initiated_reauthentication() async throws { + // Setup + // Proxy rules: None (passthrough). The AUTH injection is triggered imperatively after the + // SDK connects. (The spec's BEFORE/AFTER ALL sandbox app provisioning and the session + // teardown are owned by the withProxySession scope.) + try await withProxySession(rules: []) { app, session in + // SDK config: Use authCallback so re-authentication can be observed. + // Generate a JWT token signed with the sandbox key + // (cocoa equivalent: a TokenRequest signed locally with the sandbox key by a separate + // TLS "token signer" ARTRest — the proxyClientOptions pattern, inlined here so the + // test can count the authCallback invocations itself) + let authCallbackInvocations = Captured() + let signerOptions = ARTClientOptions(key: app.defaultKey) + signerOptions.restHost = SandboxApp.sandboxHost + let tokenSigner = ARTRest(options: signerOptions) + + let options = ARTClientOptions() + options.authCallback = { params, callback in + authCallbackInvocations.append(params) + tokenSigner.auth.createTokenRequest(params, options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } + } + // endpoint "localhost" + proxy port + tls: false + useBinaryProtocol: false + options.connectThroughProxy(session) + options.autoConnect = false + + try await withRealtimeClient(options) { client in + // Test Steps + // Connect through proxy + client.connect() + guard await awaitState(client, .connected, timeout: 15) else { return } + + // Record identity and auth state before injection + // ASSERT original_connection_id IS NOT null + let originalConnectionId = try #require(client.connection.id) + let originalAuthCallbackCount = authCallbackInvocations.count + #expect(originalAuthCallbackCount >= 1) + + // Record state changes from this point + let stateChanges = Captured() + client.connection.on { change in + stateChanges.append(change.current) + } + + // Inject a server-initiated AUTH ProtocolMessage (action 17) + // This simulates Ably requesting re-authentication + try await session.triggerAction(["type": "inject_to_client", "message": ["action": 17]]) + + // Wait for the SDK to process the AUTH and send its response + // The authCallback should be invoked, and the SDK should send AUTH back. + // Allow time for the token request round-trip to the sandbox. + guard await pollUntil("authCallback re-invoked after injected AUTH", timeout: 15, { + authCallbackInvocations.count > originalAuthCallbackCount + }) else { return } + // Timing adaptation (not a deviation): the spec's callback returns a ready JWT, so + // the AUTH frame follows the callback immediately. The cocoa callback returns a + // TokenRequest that the SDK still exchanges for a token (a REST round-trip through + // the proxy) before sending AUTH — so also wait for the AUTH frame to reach the + // proxy log before asserting on it. + guard await pollUntil("client-to-server AUTH frame recorded in the proxy log", timeout: 15, { + let log = (try? await session.getLog()) ?? [] + return !self.clientToServerAuthFrames(in: log).isEmpty + }) else { return } + + // Assertions + // authCallback was called again (re-authentication triggered) + #expect(authCallbackInvocations.count == originalAuthCallbackCount + 1) + + // Connection remains CONNECTED (re-auth does not disrupt the connection) + #expect(client.connection.state == .connected) + + // Connection ID is unchanged (no reconnection occurred) + #expect(client.connection.id == originalConnectionId) + + // No state transitions away from CONNECTED occurred + let nonConnectedChanges = stateChanges.all.filter { state in + state != .connected + } + #expect(nonConnectedChanges.count == 0) + + // Proxy log shows the SDK sent an AUTH frame (action 17) from client to server + let log = try await session.getLog() + let clientAuthFrames = clientToServerAuthFrames(in: log) + #expect(clientAuthFrames.count >= 1) + + // Spec note: after the SDK sends the AUTH response, the server may respond with a + // CONNECTED message (connection update per RTN24). Since the injected AUTH was not + // a genuine server request, the real Ably server may not respond as expected — the + // key assertions are that the SDK's auth machinery was triggered (authCallback + // invoked, AUTH frame sent) and that the connection was not disrupted. + } + // Cleanup (per spec): client.connection.close() + AWAIT_STATE closed (10s) is handled + // by the withRealtimeClient scope; session.close() by the withProxySession scope. + } + } +} + +extension AuthReauthTests { + /// Client→server AUTH frames (action 17) carrying non-nil `auth` details — the spec's + /// `client_auth_frames` filter over the proxy event log. + func clientToServerAuthFrames(in log: [ProxyEvent]) -> [ProxyEvent] { + log.filter { event in + event.type == "ws_frame" + && event.direction == "client_to_server" + && (event.message?["action"] as? Int == 17 || event.message?["action"] as? String == "AUTH") + && event.message?["auth"] != nil + } + } +} + +#endif diff --git a/Test/UTS/integration/standard/IntegrationSmokeTest.swift b/Test/UTS/integration/standard/IntegrationSmokeTest.swift deleted file mode 100644 index 72242654a..000000000 --- a/Test/UTS/integration/standard/IntegrationSmokeTest.swift +++ /dev/null @@ -1,72 +0,0 @@ -import Testing -import Foundation -import Ably - -/// Acceptance test for the direct-sandbox integration infrastructure (`SandboxApp` + -/// `IntegrationTestCase` — no proxy, no fault rules) — not derived from a UTS spec. It proves the -/// middle tier's shape works end-to-end: sandbox provisioning → a real client connecting straight -/// to the sandbox over TLS (basic key auth is fine here, unlike through the proxy) → a -/// publish/subscribe round-trip → teardown (handled by the base class). -/// -/// Runs once per protocol variant (the UTS integration specs' `PROTOCOL` dimension, ably-java's -/// `@ParameterizedTest` over `useBinaryProtocol`): `false` = JSON, `true` = msgpack. Only the -/// proxy tier is JSON-only (the proxy can't inspect binary frames) — direct-sandbox tests must -/// exercise both. -/// -/// Needs outbound network (the Ably sandbox), so it is gated behind an env var and skipped by -/// default: -/// -/// ```bash -/// UTS_INTEGRATION_SMOKE=1 swift test --filter UTS.IntegrationSmokeTest -/// ``` -// TODO: Remove this infra acceptance (smoke) test once spec-derived integration/standard -// tests exist and cover this ground (see Test/UTS/README.md §11.7). -@Suite(.serialized) -final class IntegrationSmokeTest: IntegrationTestCase { - - @Test(.enabled(if: ProcessInfo.processInfo.environment["UTS_INTEGRATION_SMOKE"] != nil), - arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack - func sandbox_infra_works_end_to_end(useBinaryProtocol: Bool) async throws { - try await withSandboxApp { app in - #expect(app.defaultKey.hasPrefix(app.appId + ".")) - - // A REAL client (no mocks) wired straight to the sandbox — TLS stays on, so the plain - // sandbox key works (RSA1). Explicit hosts auto-disable fallback hosts (REC2c2). - let options = ARTClientOptions(key: app.defaultKey) - options.realtimeHost = SandboxApp.sandboxHost - options.restHost = SandboxApp.sandboxHost - options.useBinaryProtocol = useBinaryProtocol - options.autoConnect = false - - try await withRealtimeClient(options) { client in - await runScenario(client) - } - } - } - - /// The happy-path scenario. Each wait is guarded: on timeout an `Issue` is already recorded - /// by the helper, so just stop instead of driving a client in the wrong state — the base - /// class's teardown runs regardless. - private func runScenario(_ client: ARTRealtime) async { - client.connect() - guard await awaitState(client, .connected) else { return } - #expect(client.connection.state == .connected) - - // Publish/subscribe round-trip on a fresh channel (fresh name per variant/retry, so runs - // never collide on server-side channel state). - let channel = client.channels.get("smoke-\(UUID().uuidString)") - channel.attach() - guard await awaitChannelState(channel, .attached, timeout: 10) else { return } - - let received = Captured() - channel.subscribe { message in - received.append(message) - } - channel.publish("event", data: "payload") - guard await pollUntil("published message is echoed back to the subscriber", timeout: 10, { - received.count == 1 - }) else { return } - #expect(received.first?.name == "event") - #expect(received.first?.data as? String == "payload") - } -} diff --git a/Test/UTS/integration/standard/realtime/ChannelHistoryTests.swift b/Test/UTS/integration/standard/realtime/ChannelHistoryTests.swift new file mode 100644 index 000000000..d1e7ed1f4 --- /dev/null +++ b/Test/UTS/integration/standard/realtime/ChannelHistoryTests.swift @@ -0,0 +1,118 @@ +import Testing +import Foundation +import Ably + +/// RealtimeChannel history (RTL10d) +/// Derived from ably/specification `uts/realtime/integration/channel_history_test.md` +/// +/// Direct-sandbox integration test against the Ably Sandbox (`sandbox.realtime.ably-nonprod.net`, +/// via SandboxApp.sandboxHost) — no proxy, no fault injection. Provisions a throwaway SandboxApp +/// and connects real clients straight to the sandbox. Needs outbound network: +/// +/// ```bash +/// swift test --filter UTS.ChannelHistoryTests +/// ``` +@Suite(.serialized) +final class ChannelHistoryTests: IntegrationTestCase { + + // UTS: realtime/integration/RTL10d/history-cross-client-0 + @Test(arguments: [false, true]) // useBinaryProtocol: false = JSON, true = msgpack + func test_RTL10d_history_contains_messages_published_by_another_client(useBinaryProtocol: Bool) async throws { + // The spec's BEFORE/AFTER ALL sandbox app provisioning is owned by the withSandboxApp scope. + try await withSandboxApp { app in + // Setup + let channelName = "history-RTL10d-\(UUID().uuidString)" + + let publisherOptions = ARTClientOptions(key: app.defaultKey) + publisherOptions.realtimeHost = SandboxApp.sandboxHost + publisherOptions.restHost = SandboxApp.sandboxHost + publisherOptions.useBinaryProtocol = useBinaryProtocol + publisherOptions.autoConnect = false + + let subscriberOptions = ARTClientOptions(key: app.defaultKey) + subscriberOptions.realtimeHost = SandboxApp.sandboxHost + subscriberOptions.restHost = SandboxApp.sandboxHost + subscriberOptions.useBinaryProtocol = useBinaryProtocol + subscriberOptions.autoConnect = false + + try await withRealtimeClient(publisherOptions) { publisher in + try await withRealtimeClient(subscriberOptions) { subscriber in + publisher.connect() + subscriber.connect() + + guard await awaitState(publisher, .connected, timeout: 10) else { return } + guard await awaitState(subscriber, .connected, timeout: 10) else { return } + + let pubChannel = publisher.channels.get(channelName) + let subChannel = subscriber.channels.get(channelName) + + pubChannel.attach() + guard await awaitChannelState(pubChannel, .attached, timeout: 10) else { return } + subChannel.attach() + guard await awaitChannelState(subChannel, .attached, timeout: 10) else { return } + + // Test Steps + // Publish messages from publisher client and await confirmation + await awaitPublish(pubChannel, name: "event1", data: "data1") + await awaitPublish(pubChannel, name: "event2", data: "data2") + await awaitPublish(pubChannel, name: "event3", data: "data3") + + // Retrieve history from subscriber client + // Poll until all messages appear + guard await pollUntil("subscriber history contains all 3 messages", timeout: 10, interval: 0.5, { + await self.historyItems(of: subChannel).count == 3 + }) else { return } + let historyItems = await historyItems(of: subChannel) + + // Assertions + try #require(historyItems.count == 3) + + // Default order is backwards (newest first) + #expect(historyItems[0].name == "event3") + #expect(historyItems[0].data as? String == "data3") + + #expect(historyItems[1].name == "event2") + #expect(historyItems[1].data as? String == "data2") + + #expect(historyItems[2].name == "event1") + #expect(historyItems[2].data as? String == "data1") + + // Cleanup (per spec): publisher.close() / subscriber.close() are handled by the + // withRealtimeClient scopes (close + await CLOSED). + } + } + } + } +} + +extension ChannelHistoryTests { + /// Awaits the publish acknowledgement (the spec's `AWAIT pub_channel.publish(...)`), recording + /// an issue on error. + private func awaitPublish(_ channel: ARTRealtimeChannel, + name: String, + data: String, + sourceLocation: SourceLocation = #_sourceLocation) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + channel.publish(name, data: data) { error in + if let error { + Issue.record("publish(\(name)) failed: \(error)", sourceLocation: sourceLocation) + } + continuation.resume() + } + } + } + + /// Fetches the channel's history (default query — backwards order) and returns its items, + /// recording an issue on error. + private func historyItems(of channel: ARTRealtimeChannel, + sourceLocation: SourceLocation = #_sourceLocation) async -> [ARTMessage] { + await withCheckedContinuation { (continuation: CheckedContinuation<[ARTMessage], Never>) in + channel.history { result, error in + if let error { + Issue.record("history() failed: \(error)", sourceLocation: sourceLocation) + } + continuation.resume(returning: result?.items ?? []) + } + } + } +} diff --git a/Test/UTS/integration/standard/realtime/TokenRequestTests.swift b/Test/UTS/integration/standard/realtime/TokenRequestTests.swift new file mode 100644 index 000000000..60d550fda --- /dev/null +++ b/Test/UTS/integration/standard/realtime/TokenRequestTests.swift @@ -0,0 +1,100 @@ +import Testing +import Foundation +import Ably + +/// Realtime token request (RSA9, RSA9a, RSA9g) +/// Derived from ably/specification `uts/realtime/integration/auth/token_request_test.md` +/// +/// Direct-sandbox integration test against the Ably Sandbox (`sandbox.realtime.ably-nonprod.net`, +/// via SandboxApp.sandboxHost) — no proxy, no fault injection. Provisions a throwaway SandboxApp +/// and connects real clients straight to the sandbox. +/// +/// End-to-end verification that `auth.createTokenRequest` produces a signed TokenRequest that the +/// Ably service accepts — validating that the HMAC signature computation (RSA9g) is compatible +/// with the server. +/// +/// Needs outbound network (the Ably sandbox): +/// +/// ```bash +/// swift test --filter UTS.TokenRequestTests +/// ``` +@Suite(.serialized) +final class TokenRequestTests: IntegrationTestCase { + + // UTS: realtime/integration/RSA9a/token-request-server-accepted-0 + @Test + func test_RSA9a_RSA9g_createTokenRequest_produces_server_accepted_token() async throws { + // The spec's BEFORE/AFTER ALL sandbox app provisioning is owned by the withSandboxApp scope. + try await withSandboxApp { app in + // Setup + // Client A creates TokenRequests using the API key + let creatorOptions = ARTClientOptions(key: app.defaultKey) + creatorOptions.restHost = SandboxApp.sandboxHost + let creator = ARTRest(options: creatorOptions) + + // Client B connects using TokenRequests from client A + let options = ARTClientOptions() + options.authCallback = { _, callback in + creator.auth.createTokenRequest(nil, options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } + } + options.realtimeHost = SandboxApp.sandboxHost + options.restHost = SandboxApp.sandboxHost + options.autoConnect = false + options.useBinaryProtocol = false + + try await withRealtimeClient(options) { client in + // Test Steps + client.connect() + guard await awaitState(client, .connected, timeout: 15) else { return } + + // Assertions + #expect(client.connection.state == .connected) + #expect(client.connection.id != nil) + #expect(client.connection.errorReason == nil) + + // CLOSE_CLIENT(client) — handled by the withRealtimeClient scope + // (close + await CLOSED). + } + } + } + + // UTS: realtime/integration/RSA9/token-request-with-clientid-0 + @Test + func test_RSA9_createTokenRequest_with_clientId() async throws { + try await withSandboxApp { app in + // Setup + let testClientId = "token-request-client-\(UUID().uuidString)" + + let creatorOptions = ARTClientOptions(key: app.defaultKey) + creatorOptions.restHost = SandboxApp.sandboxHost + let creator = ARTRest(options: creatorOptions) + + let options = ARTClientOptions() + options.authCallback = { _, callback in + creator.auth.createTokenRequest(ARTTokenParams(clientId: testClientId), options: nil) { tokenRequest, error in + callback(tokenRequest, error) + } + } + options.clientId = testClientId + options.realtimeHost = SandboxApp.sandboxHost + options.restHost = SandboxApp.sandboxHost + options.autoConnect = false + options.useBinaryProtocol = false + + try await withRealtimeClient(options) { client in + // Test Steps + client.connect() + guard await awaitState(client, .connected, timeout: 15) else { return } + + // Assertions + #expect(client.connection.state == .connected) + #expect(client.auth.clientId == testClientId) + + // CLOSE_CLIENT(client) — handled by the withRealtimeClient scope + // (close + await CLOSED). + } + } + } +}