diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 35240f8..f3b21d7 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -11,15 +11,24 @@ This directory contains multiple fixture suites with different contracts. ## API contract fixture -[`conformance/api/public-api-v1.json`](conformance/api/public-api-v1.json) defines a small public API presence contract for the Python 0.7.x surface that ports must expose. +[`conformance/api/public-api-v1.json`](conformance/api/public-api-v1.json) defines the current portable core public API contract for Python and ports. Ports may sync this artifact with conformance fixtures. -Ports should check equivalent public exports and methods using language-appropriate names where casing differs. +The contract encodes: + +* exact exported `context_compiler.__all__` names +* export kinds (`callable`, `constant`, `type_alias`, `type`, `class`) +* exact public `Engine` members +* stable callable signatures where parameter shape is part of the contract +* forbidden package-root names that must not become public exports +* lightweight deterministic return-shape probes for selected stable helpers + +Ports should check equivalent public exports, members, and signatures using language-appropriate names where casing differs. Behavioral semantics remain covered by conformance and structured fixtures. -The API presence contract includes the public controller helper accessors: +The API contract includes the public controller helper accessors: * `get_step_decision` * `get_step_state` diff --git a/tests/fixtures/conformance/api/public-api-v1.json b/tests/fixtures/conformance/api/public-api-v1.json index 8df070f..7f2e533 100644 --- a/tests/fixtures/conformance/api/public-api-v1.json +++ b/tests/fixtures/conformance/api/public-api-v1.json @@ -3,44 +3,655 @@ "kind": "api-contract", "target": "context-compiler-ports", "module": "context_compiler", - "required_exports": [ - "create_engine", - "compile_transcript", - "get_premise_value", - "get_policy_items", - "is_update", - "is_clarify", - "is_passthrough", - "get_clarify_prompt", - "get_decision_state", - "get_step_decision", - "get_step_state", - "get_preview_decision", - "get_preview_state_after", - "preview_would_mutate", - "diff_has_changes", - "DECISION_PASSTHROUGH", - "DECISION_UPDATE", - "DECISION_CLARIFY", - "POLICY_USE", - "POLICY_PROHIBIT", - "TranscriptMessage", - "Transcript", - "ApplyResult" + "forbidden_exports": [ + "DecisionKind", + "PolicyValue", + "CheckpointPending", + "CheckpointPendingReplacement", + "PendingReplacement", + "_PASSTHROUGH" ], + "exports": { + "mode": "exact", + "names": [ + "ApplyResult", + "Checkpoint", + "Decision", + "DECISION_CLARIFY", + "DECISION_PASSTHROUGH", + "DECISION_UPDATE", + "Engine", + "POLICY_PROHIBIT", + "POLICY_USE", + "PreviewResult", + "State", + "StepResult", + "StructuralDiff", + "Transcript", + "TranscriptMessage", + "diff_has_changes", + "compile_transcript", + "create_engine", + "get_clarify_prompt", + "get_decision_state", + "get_preview_decision", + "get_preview_state_after", + "get_premise_value", + "get_policy_items", + "get_step_decision", + "get_step_state", + "is_clarify", + "is_passthrough", + "is_update", + "preview", + "preview_would_mutate", + "state_diff", + "step" + ], + "members": { + "ApplyResult": { + "kind": "type_alias" + }, + "Checkpoint": { + "kind": "type" + }, + "Decision": { + "kind": "type" + }, + "DECISION_CLARIFY": { + "kind": "constant", + "value": "clarify" + }, + "DECISION_PASSTHROUGH": { + "kind": "constant", + "value": "passthrough" + }, + "DECISION_UPDATE": { + "kind": "constant", + "value": "update" + }, + "Engine": { + "kind": "class" + }, + "POLICY_PROHIBIT": { + "kind": "constant", + "value": "prohibit" + }, + "POLICY_USE": { + "kind": "constant", + "value": "use" + }, + "PreviewResult": { + "kind": "type" + }, + "State": { + "kind": "type" + }, + "StepResult": { + "kind": "type" + }, + "StructuralDiff": { + "kind": "type" + }, + "Transcript": { + "kind": "type_alias" + }, + "TranscriptMessage": { + "kind": "type" + }, + "diff_has_changes": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "diff", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "diff": { + "changed": true, + "premise": { + "before": null, + "after": "concise", + "changed": true + }, + "policies": { + "added": {}, + "removed": {}, + "changed": {} + } + } + }, + "return_shape": { + "type": "boolean", + "const": true + } + } + ] + }, + "compile_transcript": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "messages", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "create_engine": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "state", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": true + } + ] + }, + "shape_probes": [ + { + "kwargs": {}, + "return_shape": { + "kind": "engine_instance" + } + } + ] + }, + "get_clarify_prompt": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "decision", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "decision": { + "kind": "clarify", + "state": null, + "prompt_to_user": "confirm?" + } + }, + "return_shape": { + "type": "string", + "const": "confirm?" + } + } + ] + }, + "get_decision_state": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "decision", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "decision": { + "kind": "update", + "prompt_to_user": null, + "state": { + "premise": null, + "policies": { + "docker": "use" + }, + "version": 2 + } + } + }, + "return_shape": { + "type": "object", + "required_keys": [ + "premise", + "policies", + "version" + ], + "properties": { + "policies": { + "type": "object" + } + } + } + } + ] + }, + "get_preview_decision": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "preview_result", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "get_preview_state_after": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "preview_result", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "get_premise_value": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "state", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "get_policy_items": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "state", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + }, + { + "name": "value", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": true + } + ] + } + }, + "get_step_decision": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "step_result", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "get_step_state": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "step_result", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "is_clarify": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "decision", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "decision": { + "kind": "clarify", + "state": null, + "prompt_to_user": "confirm?" + } + }, + "return_shape": { + "type": "boolean", + "const": true + } + } + ] + }, + "is_passthrough": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "decision", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "decision": { + "kind": "passthrough", + "state": null, + "prompt_to_user": null + } + }, + "return_shape": { + "type": "boolean", + "const": true + } + } + ] + }, + "is_update": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "decision", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "decision": { + "kind": "update", + "prompt_to_user": null, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } + }, + "return_shape": { + "type": "boolean", + "const": true + } + } + ] + }, + "preview": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + }, + { + "name": "user_input", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "engine": { + "fixture": "empty_engine" + }, + "user_input": "use docker" + }, + "return_shape": { + "type": "object", + "required_keys": [ + "output_version", + "mode", + "decision", + "state_before", + "state_after", + "diff", + "would_mutate" + ], + "properties": { + "output_version": { + "type": "number", + "const": 1 + }, + "mode": { + "type": "string", + "const": "preview" + }, + "would_mutate": { + "type": "boolean", + "const": true + } + } + } + } + ] + }, + "preview_would_mutate": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "preview_result", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "state_diff": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "before", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + }, + { + "name": "after", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "before": { + "premise": null, + "policies": {}, + "version": 2 + }, + "after": { + "premise": null, + "policies": { + "docker": "use" + }, + "version": 2 + } + }, + "return_shape": { + "type": "object", + "required_keys": [ + "changed", + "premise", + "policies" + ], + "properties": { + "changed": { + "type": "boolean", + "const": true + } + } + } + } + ] + }, + "step": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + }, + { + "name": "user_input", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "engine": { + "fixture": "empty_engine" + }, + "user_input": "use docker" + }, + "return_shape": { + "type": "object", + "required_keys": [ + "output_version", + "mode", + "decision", + "state" + ], + "properties": { + "output_version": { + "type": "number", + "const": 1 + }, + "mode": { + "type": "string", + "const": "step" + } + } + } + } + ] + } + } + }, "engine": { "type": "Engine", - "required_members": [ - "step", - "state", - "export_json", - "import_json", - "apply_transcript", - "export_checkpoint", - "import_checkpoint", - "export_checkpoint_json", - "import_checkpoint_json", - "has_pending_clarification" - ] + "public_members": { + "mode": "exact", + "members": { + "apply_transcript": { + "kind": "method", + "signature": { + "params": [ + { + "name": "messages", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "export_checkpoint": { + "kind": "method", + "signature": { + "params": [] + } + }, + "export_checkpoint_json": { + "kind": "method", + "signature": { + "params": [] + } + }, + "export_json": { + "kind": "method", + "signature": { + "params": [] + } + }, + "has_pending_clarification": { + "kind": "method", + "signature": { + "params": [] + } + }, + "import_checkpoint": { + "kind": "method", + "signature": { + "params": [ + { + "name": "payload", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "import_checkpoint_json": { + "kind": "method", + "signature": { + "params": [ + { + "name": "payload", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "import_json": { + "kind": "method", + "signature": { + "params": [ + { + "name": "payload", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + }, + "state": { + "kind": "property" + }, + "step": { + "kind": "method", + "signature": { + "params": [ + { + "name": "user_input", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + } + } + } + } } } diff --git a/tests/test_api_contract_fixture.py b/tests/test_api_contract_fixture.py index a16afb1..7101a70 100644 --- a/tests/test_api_contract_fixture.py +++ b/tests/test_api_contract_fixture.py @@ -1,3 +1,4 @@ +import inspect import json from pathlib import Path @@ -12,22 +13,167 @@ def _load_contract() -> dict[str, object]: return json.loads(_CONTRACT_PATH.read_text(encoding="utf-8")) +def _json_type_matches(value: object, expected: str) -> bool: + return { + "null": value is None, + "string": isinstance(value, str), + "object": isinstance(value, dict), + "array": isinstance(value, list), + "boolean": isinstance(value, bool), + "number": isinstance(value, int | float) and not isinstance(value, bool), + }[expected] + + +def _resolve_probe_value(value: object) -> object: + if not isinstance(value, dict) or "fixture" not in value: + return value + + fixture = value["fixture"] + if fixture == "empty_engine": + return context_compiler.create_engine() + + raise AssertionError(f"Unknown probe fixture: {fixture}") + + +def _assert_shape(value: object, shape: dict[str, object], contract: dict[str, object]) -> None: + if "kind" in shape and shape["kind"] == "engine_instance": + assert isinstance(value, context_compiler.Engine) + expected_members = contract["engine"]["public_members"]["members"] + actual_members = sorted(name for name in dir(value) if not name.startswith("_")) + assert actual_members == sorted(expected_members.keys()) + return + + expected_types = shape["type"] + if isinstance(expected_types, str): + expected_types = [expected_types] + assert any(_json_type_matches(value, expected_type) for expected_type in expected_types) + + if "const" in shape: + assert value == shape["const"] + + if isinstance(value, dict): + required_keys = shape.get("required_keys", []) + assert set(required_keys).issubset(value) + properties = shape.get("properties", {}) + for key, property_shape in properties.items(): + if key in value: + _assert_shape(value[key], property_shape, contract) + + +def _assert_signature_matches(obj: object, expected: dict[str, object], label: str) -> None: + signature = inspect.signature(obj) + params = list(signature.parameters.values()) + expected_params = expected["params"] + + assert len(params) == len(expected_params), label + for actual, expected_param in zip(params, expected_params, strict=True): + assert actual.name == expected_param["name"], label + assert actual.kind.name == expected_param["kind"], label + assert (actual.default is not inspect.Signature.empty) is expected_param["has_default"], ( + label + ) + + +def _assert_export_kind(name: str, exported: object, expected_kind: str) -> None: + if expected_kind == "callable": + assert inspect.isroutine(exported), name + return + if expected_kind == "constant": + assert not inspect.isroutine(exported) and not inspect.isclass(exported), name + return + if expected_kind == "type_alias": + assert not inspect.isroutine(exported) and not inspect.isclass(exported), name + return + if expected_kind == "type": + assert inspect.isclass(exported), name + return + assert expected_kind == "class", name + assert inspect.isclass(exported), name + + def test_api_contract_fixture_matches_python_public_surface() -> None: contract = _load_contract() assert contract["kind"] == "api-contract" - for name in contract["required_exports"]: + exports = contract["exports"] + expected_exports = exports["names"] + export_members = exports["members"] + + assert context_compiler.__all__ == expected_exports + for name in expected_exports: assert hasattr(context_compiler, name), name assert name in context_compiler.__all__, name + for name, export_contract in export_members.items(): + exported = getattr(context_compiler, name) + _assert_export_kind(name, exported, export_contract["kind"]) + if "value" in export_contract: + assert exported == export_contract["value"], name + if "signature" in export_contract: + _assert_signature_matches(exported, export_contract["signature"], name) + for probe in export_contract.get("shape_probes", []): + kwargs = { + key: _resolve_probe_value(value) for key, value in probe.get("kwargs", {}).items() + } + result = exported(**kwargs) + _assert_shape(result, probe["return_shape"], contract) + engine = context_compiler.create_engine() - for name in contract["engine"]["required_members"]: + engine_contract = contract["engine"]["public_members"] + expected_members = engine_contract["members"] + + actual_public_members = sorted(name for name in dir(engine) if not name.startswith("_")) + assert actual_public_members == sorted(expected_members.keys()) + + engine_type = type(engine) + for name, member_contract in expected_members.items(): assert hasattr(engine, name), name + kind = member_contract["kind"] + + if kind == "property": + assert isinstance(inspect.getattr_static(engine_type, name), property), name + continue + + assert callable(getattr(engine, name)), name + _assert_signature_matches(getattr(engine, name), member_contract["signature"], name) + + +def test_api_contract_fixture_forbidden_exports_are_not_present() -> None: + contract = _load_contract() + + for name in contract.get("forbidden_exports", []): + assert name not in context_compiler.__all__, name + assert not hasattr(context_compiler, name), name def test_api_contract_fixture_has_unique_entries() -> None: contract = _load_contract() - required_exports = contract["required_exports"] - assert len(required_exports) == len(set(required_exports)) - required_members = contract["engine"]["required_members"] - assert len(required_members) == len(set(required_members)) + export_names = contract["exports"]["names"] + assert len(export_names) == len(set(export_names)) + + forbidden_exports = contract.get("forbidden_exports", []) + assert len(forbidden_exports) == len(set(forbidden_exports)) + assert not (set(forbidden_exports) & set(export_names)) + + export_member_names = list(contract["exports"]["members"].keys()) + assert len(export_member_names) == len(set(export_member_names)) + assert set(export_member_names) == set(export_names) + + for export_name, export_contract in contract["exports"]["members"].items(): + kind = export_contract["kind"] + assert kind in {"callable", "constant", "type_alias", "type", "class"}, export_name + if kind == "callable": + assert "signature" in export_contract, export_name + else: + assert "signature" not in export_contract, export_name + + engine_members = list(contract["engine"]["public_members"]["members"].keys()) + assert len(engine_members) == len(set(engine_members)) + + for member_name, member_contract in contract["engine"]["public_members"]["members"].items(): + kind = member_contract["kind"] + assert kind in {"method", "property"}, member_name + if kind == "property": + assert "signature" not in member_contract, member_name + else: + assert "signature" in member_contract, member_name