diff --git a/ace/intelligence/__init__.py b/ace/intelligence/__init__.py index 51a6c8a..3928262 100644 --- a/ace/intelligence/__init__.py +++ b/ace/intelligence/__init__.py @@ -7,6 +7,7 @@ deterministic functions over contract values only. """ +from ace.intelligence.agent_memory_evaluation import compare_memory_conditions from ace.intelligence.contracts import * # noqa: F403 from ace.intelligence.contracts import __all__ as _CONTRACTS_ALL from ace.intelligence.derivation import ( @@ -122,6 +123,7 @@ "eligible_live_signal_routes", "eligible_signal_routes", "evaluate_measured_impact", + "compare_memory_conditions", "compare_measured_composition", "interpret_live_source_mapping", "interpret_prepared_source_mapping", diff --git a/ace/intelligence/agent_memory_evaluation.py b/ace/intelligence/agent_memory_evaluation.py new file mode 100644 index 0000000..7e80ff2 --- /dev/null +++ b/ace/intelligence/agent_memory_evaluation.py @@ -0,0 +1,175 @@ +"""Deterministic comparison of preregistered Agent Memory conditions.""" + +from __future__ import annotations + +from ace.intelligence.contracts.agent_memory_evaluation import ( + BenefitDisposition, + CorrectnessDisposition, + EvaluationCaseGate, + MaterialInfluenceDisposition, + MeasureAvailability, + MemoryConditionAssignmentV1Alpha1, + MemoryEvaluationCondition, + MemoryEvaluationCorpusV1Alpha1, + MemoryEvaluationProtocolV1Alpha1, + MemoryMatchedComparisonV1Alpha1, + MemoryMeasure, + MemoryRunObservationV1Alpha1, + memory_evaluation_reference, +) + + +def _measurement( + observation: MemoryRunObservationV1Alpha1, + measure: MemoryMeasure, +): + values = [item for item in observation.measurements if item.measure is measure and item.stratum is None] + if len(values) != 1: + return None + return values[0] + + +def _available_value(observation: MemoryRunObservationV1Alpha1, measure: MemoryMeasure) -> int | None: + item = _measurement(observation, measure) + if item is None or item.availability is not MeasureAvailability.AVAILABLE: + return None + return item.value + + +def compare_memory_conditions( + *, + corpus: MemoryEvaluationCorpusV1Alpha1, + protocol: MemoryEvaluationProtocolV1Alpha1, + assignment: MemoryConditionAssignmentV1Alpha1, + observations: tuple[MemoryRunObservationV1Alpha1, ...], + compared_at, +) -> MemoryMatchedComparisonV1Alpha1: + """Close one exact matched trio without proposing or applying a policy change.""" + + corpus_ref = memory_evaluation_reference(corpus) + protocol_ref = memory_evaluation_reference(protocol) + assignment_ref = memory_evaluation_reference(assignment) + if protocol.corpus != corpus_ref: + raise ValueError("protocol does not bind the exact frozen corpus") + if assignment.protocol != protocol_ref or assignment.corpus != corpus_ref: + raise ValueError("assignment crossed the exact protocol or corpus coordinate") + if assignment.assigned_at < protocol.preregistered_at or protocol.preregistered_at < corpus.frozen_at: + raise ValueError("corpus, preregistration, and assignment time order is invalid") + case = next((item for item in corpus.cases if item.case_id == assignment.case_id), None) + if case is None: + raise ValueError("assignment names a case outside the frozen corpus") + if len(observations) != 3 or {item.condition for item in observations} != set(MemoryEvaluationCondition): + raise ValueError("comparison requires exactly one memory, no-memory, and full-context observation") + ordered = tuple(sorted(observations, key=lambda item: item.condition.value)) + if compared_at < max(item.observed_at for item in ordered): + raise ValueError("comparison cannot predate an observation") + for item in ordered: + if ( + item.protocol != protocol_ref + or item.assignment != assignment_ref + or item.case_id != case.case_id + or item.observed_at < assignment.assigned_at + ): + raise ValueError("observation crossed exact protocol, assignment, case, or time closure") + + by_condition = {item.condition: item for item in ordered} + memory = by_condition[MemoryEvaluationCondition.MEMORY] + no_memory = by_condition[MemoryEvaluationCondition.NO_MEMORY] + full_context = by_condition[MemoryEvaluationCondition.FULL_CONTEXT] + reasons: set[str] = set() + limitations = { + "fixture_outcome_labels_validate_the_evaluator_and_are_not_agent_memory_benefit_evidence", + "material_influence_benefit_correctness_and_causality_are_separate_dispositions", + "no_rank_retention_consolidation_promotion_roster_authority_delivery_or_effect_policy_changes", + "causality_is_not_established_by_this_provider_free_preparation_fixture", + } + + missing: set[str] = set() + for observation in ordered: + for measure in case.required_measures: + item = _measurement(observation, measure) + if item is None or item.availability is not MeasureAvailability.AVAILABLE: + condition = observation.condition.value + reason = item.unavailable_reason if item is not None else "not_observed" + missing.add(f"{condition}:{measure.value}:{reason}") + + if case.gate is EvaluationCaseGate.FUTURE_ACCEPTED_AM4: + missing.add("future_accepted_am4_coordinate:required") + limitations.add("am4_runtime_semantics_are_not_invented_or_executed_by_this_am3_runnable_suite") + + unauthorized = tuple(_available_value(item, MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT) for item in ordered) + stale = _available_value(memory, MemoryMeasure.STALE_INFLUENCE_COUNT) + superseded = _available_value(memory, MemoryMeasure.SUPERSEDED_INFLUENCE_COUNT) + memory_correctness = _available_value(memory, MemoryMeasure.TASK_CORRECTNESS_BPS) + no_memory_correctness = _available_value(no_memory, MemoryMeasure.TASK_CORRECTNESS_BPS) + full_context_correctness = _available_value(full_context, MemoryMeasure.TASK_CORRECTNESS_BPS) + + if missing: + material = MaterialInfluenceDisposition.UNDERPOWERED + benefit = BenefitDisposition.UNDERPOWERED + correctness = CorrectnessDisposition.UNDERPOWERED + paired = False + reasons.add("required_measurement_or_future_am4_coordinate_is_unavailable") + else: + paired = True + material = ( + MaterialInfluenceDisposition.OBSERVED + if memory.decision_digest != no_memory.decision_digest + else MaterialInfluenceDisposition.NOT_OBSERVED + ) + if any(value is not None and value > 0 for value in unauthorized): + benefit = BenefitDisposition.HARMFUL + correctness = CorrectnessDisposition.INCORRECT + reasons.add("zero_tolerance_unauthorized_retrieval_was_observed") + elif (stale or 0) > 0 or (superseded or 0) > 0: + benefit = BenefitDisposition.HARMFUL + correctness = CorrectnessDisposition.INCORRECT + reasons.add("stale_or_superseded_memory_materially_influenced_the_task") + else: + assert memory_correctness is not None + assert no_memory_correctness is not None + assert full_context_correctness is not None + if memory_correctness == 10_000: + correctness = CorrectnessDisposition.CORRECT + elif memory_correctness == 0: + correctness = CorrectnessDisposition.INCORRECT + else: + correctness = CorrectnessDisposition.MIXED + gain = memory_correctness - no_memory_correctness + full_gap = full_context_correctness - memory_correctness + if ( + material is MaterialInfluenceDisposition.OBSERVED + and gain >= protocol.minimum_beneficial_gain_bps + and full_gap <= protocol.maximum_full_context_correctness_gap_bps + ): + benefit = BenefitDisposition.BENEFICIAL + reasons.add("synthetic_bounded_outcome_cleared_the_preregistered_matched_rule") + elif gain < 0: + benefit = BenefitDisposition.HARMFUL + reasons.add("memory_condition_reduced_the_preregistered_bounded_task_score") + else: + benefit = BenefitDisposition.NEUTRAL + reasons.add("memory_condition_did_not_change_the_preregistered_bounded_task_score") + + if material is MaterialInfluenceDisposition.OBSERVED: + reasons.add("memory_and_no_memory_decision_digests_differed_under_held_constants") + elif material is MaterialInfluenceDisposition.NOT_OBSERVED: + reasons.add("memory_and_no_memory_decision_digests_were_identical_under_held_constants") + + return MemoryMatchedComparisonV1Alpha1( + protocol=protocol_ref, + assignment=assignment_ref, + case_id=case.case_id, + observations=tuple(memory_evaluation_reference(item) for item in ordered), + paired_and_controlled=paired, + material_influence=material, + benefit=benefit, + correctness=correctness, + missing_measurements=tuple(sorted(missing)), + reasons=tuple(sorted(reasons)), + limitations=tuple(sorted(limitations)), + compared_at=compared_at, + ) + + +__all__ = ["compare_memory_conditions"] diff --git a/ace/intelligence/contracts/__init__.py b/ace/intelligence/contracts/__init__.py index 8463cec..7e16b43 100644 --- a/ace/intelligence/contracts/__init__.py +++ b/ace/intelligence/contracts/__init__.py @@ -130,6 +130,34 @@ SourceAuthorityKind, SourceIndependence, ) +from ace.intelligence.contracts.agent_memory_evaluation import ( + MEMORY_CONDITION_ASSIGNMENT_VERSION, + MEMORY_EVALUATION_CORPUS_VERSION, + MEMORY_EVALUATION_PROTOCOL_VERSION, + MEMORY_MATCHED_COMPARISON_VERSION, + MEMORY_RUN_OBSERVATION_VERSION, + BenefitDisposition, + CausalityDisposition, + CorrectnessDisposition, + EvaluationCaseGate, + MaterialInfluenceDisposition, + MeasureAvailability, + MeasureDirection, + MeasureUnit, + MemoryConditionAssignmentV1Alpha1, + MemoryConditionPlanV1Alpha1, + MemoryEvaluationCaseV1Alpha1, + MemoryEvaluationCondition, + MemoryEvaluationCorpusV1Alpha1, + MemoryEvaluationProtocolV1Alpha1, + MemoryMatchedComparisonV1Alpha1, + MemoryMatchedCoordinatesV1Alpha1, + MemoryMeasure, + MemoryMeasureDefinitionV1Alpha1, + MemoryMeasureObservationV1Alpha1, + MemoryRunObservationV1Alpha1, + memory_evaluation_reference, +) from ace.intelligence.contracts.agent_memory_recall import ( CANDIDATE_EVIDENCE_VERSION, CONDITION_ASSIGNMENT_VERSION, @@ -712,6 +740,32 @@ "OutcomeAvailability", "TelemetryAvailability", "measured_composition_reference", + "MEMORY_CONDITION_ASSIGNMENT_VERSION", + "MEMORY_EVALUATION_CORPUS_VERSION", + "MEMORY_EVALUATION_PROTOCOL_VERSION", + "MEMORY_MATCHED_COMPARISON_VERSION", + "MEMORY_RUN_OBSERVATION_VERSION", + "BenefitDisposition", + "CausalityDisposition", + "CorrectnessDisposition", + "EvaluationCaseGate", + "MaterialInfluenceDisposition", + "MeasureAvailability", + "MeasureDirection", + "MeasureUnit", + "MemoryConditionAssignmentV1Alpha1", + "MemoryConditionPlanV1Alpha1", + "MemoryEvaluationCaseV1Alpha1", + "MemoryEvaluationCondition", + "MemoryEvaluationCorpusV1Alpha1", + "MemoryEvaluationProtocolV1Alpha1", + "MemoryMatchedComparisonV1Alpha1", + "MemoryMatchedCoordinatesV1Alpha1", + "MemoryMeasure", + "MemoryMeasureDefinitionV1Alpha1", + "MemoryMeasureObservationV1Alpha1", + "MemoryRunObservationV1Alpha1", + "memory_evaluation_reference", "COMPOSITION_POLICY_ADMISSION_PLAN_VERSION", "COMPOSITION_POLICY_ADMISSION_RECEIPT_VERSION", "COMPOSITION_POLICY_ADMISSION_REQUEST_VERSION", diff --git a/ace/intelligence/contracts/agent_memory_evaluation.py b/ace/intelligence/contracts/agent_memory_evaluation.py new file mode 100644 index 0000000..dbb2b19 --- /dev/null +++ b/ace/intelligence/contracts/agent_memory_evaluation.py @@ -0,0 +1,609 @@ +"""Provider-neutral contracts for preregistered Agent Memory evaluation. + +These contracts describe measurement evidence only. They do not change memory, +ranking, retention, consolidation, promotion, composition, authority, delivery, +or external effects. AM4-dependent cases are inert gated coordinates until an +accepted AM4 artifact is supplied by its owning lane. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from enum import StrEnum +from typing import Literal, Self + +from pydantic import ConfigDict, Field, field_validator, model_validator + +from ace.core.agent_composition import ExactArtifactReferenceV1Alpha1 +from ace.core.contracts import FrozenContract, canonical_hash + +MEMORY_EVALUATION_CORPUS_VERSION = "ace.intelligence.agent-memory-evaluation-corpus/v1alpha1" +MEMORY_EVALUATION_PROTOCOL_VERSION = "ace.intelligence.agent-memory-evaluation-protocol/v1alpha1" +MEMORY_CONDITION_ASSIGNMENT_VERSION = "ace.intelligence.agent-memory-condition-assignment/v1alpha1" +MEMORY_RUN_OBSERVATION_VERSION = "ace.intelligence.agent-memory-run-observation/v1alpha1" +MEMORY_MATCHED_COMPARISON_VERSION = "ace.intelligence.agent-memory-matched-comparison/v1alpha1" + +MAX_ITEMS = 256 +EXPECTED_CONDITIONS = {"memory", "no_memory", "full_context"} + + +class _Contract(FrozenContract): + model_config = ConfigDict( + extra="forbid", + frozen=True, + strict=True, + revalidate_instances="always", + validate_default=True, + allow_inf_nan=False, + ) + + +def _bounded(value: str, *, name: str, maximum: int = 240) -> str: + if not value or value != value.strip() or len(value) > maximum: + raise ValueError(f"{name} must be non-empty, trimmed, and at most {maximum} characters") + return value + + +def _aware(value: datetime, *, name: str) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{name} must include a timezone") + return value.astimezone(UTC) + + +def _digest(value: str, *, name: str) -> str: + if len(value) != 71 or not value.startswith("sha256:") or value != value.lower(): + raise ValueError(f"{name} must use lowercase sha256:<64-hex> syntax") + try: + int(value[7:], 16) + except ValueError as exc: + raise ValueError(f"{name} must use lowercase sha256:<64-hex> syntax") from exc + return value + + +def _unique_strings(values: tuple[str, ...], *, name: str, minimum: int = 0) -> tuple[str, ...]: + if not minimum <= len(values) <= MAX_ITEMS: + raise ValueError(f"{name} must contain between {minimum} and {MAX_ITEMS} values") + normalized = tuple(sorted(_bounded(item, name=name) for item in values)) + if len(normalized) != len(set(normalized)): + raise ValueError(f"{name} must be unique") + return normalized + + +def _unique_refs( + values: tuple[ExactArtifactReferenceV1Alpha1, ...], *, name: str, minimum: int = 0 +) -> tuple[ExactArtifactReferenceV1Alpha1, ...]: + if not minimum <= len(values) <= MAX_ITEMS: + raise ValueError(f"{name} must contain between {minimum} and {MAX_ITEMS} values") + keys = [(item.artifact_contract, item.artifact_id, item.artifact_digest) for item in values] + if len(keys) != len(set(keys)): + raise ValueError(f"{name} must be unique") + return tuple(sorted(values, key=lambda item: (item.artifact_contract, item.artifact_id, item.artifact_digest))) + + +def _identity(instance: _Contract, *, prefix: str, id_field: str, digest_field: str) -> None: + material = instance.model_dump(mode="json", exclude={id_field, digest_field}) + digest = canonical_hash(material) + expected_id = f"{prefix}:{digest[:32]}" + expected_digest = f"sha256:{digest}" + if getattr(instance, id_field) not in {None, expected_id}: + raise ValueError(f"{id_field} does not match exact contract material") + if getattr(instance, digest_field) not in {None, expected_digest}: + raise ValueError(f"{digest_field} does not match exact contract material") + object.__setattr__(instance, id_field, expected_id) + object.__setattr__(instance, digest_field, expected_digest) + + +class MemoryEvaluationCondition(StrEnum): + MEMORY = "memory" + NO_MEMORY = "no_memory" + FULL_CONTEXT = "full_context" + + +class EvaluationCaseGate(StrEnum): + RUNNABLE_AM3 = "runnable_am3" + FUTURE_ACCEPTED_AM4 = "future_accepted_am4" + + +class MeasureAvailability(StrEnum): + AVAILABLE = "available" + UNAVAILABLE = "unavailable" + NOT_APPLICABLE = "not_applicable" + + +class MeasureDirection(StrEnum): + HIGHER_IS_BETTER = "higher_is_better" + LOWER_IS_BETTER = "lower_is_better" + DESCRIPTIVE = "descriptive" + ZERO_TOLERANCE = "zero_tolerance" + + +class MeasureUnit(StrEnum): + BASIS_POINTS = "basis_points" + COUNT = "count" + TOKENS = "tokens" + MILLISECONDS = "milliseconds" + MICROUNITS = "microunits" + + +class MemoryMeasure(StrEnum): + INGESTION_COMPLETENESS_BPS = "ingestion_completeness_bps" + REPLAY_CORRECTNESS_BPS = "replay_correctness_bps" + EXTRACTION_PRECISION_BPS = "extraction_precision_bps" + EXTRACTION_RECALL_BPS = "extraction_recall_bps" + SOURCE_SPAN_ACCURACY_BPS = "source_span_accuracy_bps" + IDENTITY_ERROR_BPS = "identity_error_bps" + UNRESOLVED_IDENTITY_BPS = "unresolved_identity_bps" + CORRECTION_RECALL_BPS = "correction_recall_bps" + CONTRADICTION_RECALL_BPS = "contradiction_recall_bps" + UNCERTAINTY_RECALL_BPS = "uncertainty_recall_bps" + INSTRUCTION_POLICY_RECALL_BPS = "instruction_policy_recall_bps" + RETRIEVAL_PRECISION_BPS = "retrieval_precision_bps" + RETRIEVAL_RECALL_BPS = "retrieval_recall_bps" + RANK_QUALITY_BPS = "rank_quality_bps" + CITATION_CORRECTNESS_BPS = "citation_correctness_bps" + OMISSION_COVERAGE_BPS = "omission_coverage_bps" + UNAUTHORIZED_RETRIEVAL_COUNT = "unauthorized_retrieval_count" + STALE_INFLUENCE_COUNT = "stale_influence_count" + SUPERSEDED_INFLUENCE_COUNT = "superseded_influence_count" + CONTEXT_TOKENS = "context_tokens" + RESIDUAL_WINDOW_TOKENS = "residual_window_tokens" + LATENCY_MS = "latency_ms" + PROVIDER_CALLS = "provider_calls" + CACHE_REUSE_COUNT = "cache_reuse_count" + COST_MICROUNITS = "cost_microunits" + DEPENDENCY_INVALIDATION_BPS = "dependency_invalidation_bps" + SELECTED_RATE_BPS = "selected_rate_bps" + INJECTED_RATE_BPS = "injected_rate_bps" + REFLECTED_RATE_BPS = "reflected_rate_bps" + DECISION_MATERIAL_RATE_BPS = "decision_material_rate_bps" + TASK_CORRECTNESS_BPS = "task_correctness_bps" + + +class MaterialInfluenceDisposition(StrEnum): + OBSERVED = "observed" + NOT_OBSERVED = "not_observed" + UNDERPOWERED = "underpowered" + + +class BenefitDisposition(StrEnum): + BENEFICIAL = "beneficial" + HARMFUL = "harmful" + NEUTRAL = "neutral" + UNDERPOWERED = "underpowered" + + +class CorrectnessDisposition(StrEnum): + CORRECT = "correct" + INCORRECT = "incorrect" + MIXED = "mixed" + UNDERPOWERED = "underpowered" + + +class CausalityDisposition(StrEnum): + NOT_ESTABLISHED = "not_established" + + +class MemoryMeasureDefinitionV1Alpha1(_Contract): + measure: MemoryMeasure + unit: MeasureUnit + direction: MeasureDirection + missing_yields_underpowered: bool = True + + +class MemoryEvaluationCaseV1Alpha1(_Contract): + case_id: str + title: str + coverage_tags: tuple[str, ...] = Field(min_length=1, max_length=MAX_ITEMS) + required_measures: tuple[MemoryMeasure, ...] = Field(min_length=1, max_length=len(MemoryMeasure)) + gate: EvaluationCaseGate + future_required_coordinate: str | None = None + + @field_validator("case_id", "title") + @classmethod + def validate_text(cls, value: str, info) -> str: + return _bounded(value, name=info.field_name) + + @field_validator("coverage_tags") + @classmethod + def normalize_tags(cls, value: tuple[str, ...]) -> tuple[str, ...]: + return _unique_strings(value, name="coverage_tags", minimum=1) + + @field_validator("required_measures") + @classmethod + def normalize_measures(cls, value: tuple[MemoryMeasure, ...]) -> tuple[MemoryMeasure, ...]: + if len(value) != len(set(value)): + raise ValueError("required measures must be unique") + return tuple(sorted(value, key=lambda item: item.value)) + + @model_validator(mode="after") + def validate_gate(self) -> Self: + if self.gate is EvaluationCaseGate.FUTURE_ACCEPTED_AM4: + if self.future_required_coordinate != "future_accepted_am4_coordinate": + raise ValueError("AM4-gated cases require only the exact future accepted AM4 coordinate placeholder") + elif self.future_required_coordinate is not None: + raise ValueError("AM3-runnable cases cannot require an AM4 coordinate") + return self + + +class MemoryEvaluationCorpusV1Alpha1(_Contract): + contract: Literal["ace.intelligence.agent-memory-evaluation-corpus/v1alpha1"] = MEMORY_EVALUATION_CORPUS_VERSION + corpus_key: str + synthetic_only: Literal[True] = True + source_artifacts: tuple[ExactArtifactReferenceV1Alpha1, ...] = Field(min_length=4, max_length=MAX_ITEMS) + cases: tuple[MemoryEvaluationCaseV1Alpha1, ...] = Field(min_length=1, max_length=MAX_ITEMS) + frozen_at: datetime + corpus_id: str | None = None + corpus_digest: str | None = None + + @field_validator("corpus_key") + @classmethod + def validate_key(cls, value: str) -> str: + return _bounded(value, name="corpus_key") + + @field_validator("source_artifacts") + @classmethod + def normalize_sources( + cls, value: tuple[ExactArtifactReferenceV1Alpha1, ...] + ) -> tuple[ExactArtifactReferenceV1Alpha1, ...]: + return _unique_refs(value, name="source_artifacts", minimum=4) + + @field_validator("cases") + @classmethod + def normalize_cases( + cls, value: tuple[MemoryEvaluationCaseV1Alpha1, ...] + ) -> tuple[MemoryEvaluationCaseV1Alpha1, ...]: + ids = [item.case_id for item in value] + if len(ids) != len(set(ids)): + raise ValueError("corpus case identities must be unique") + return tuple(sorted(value, key=lambda item: item.case_id)) + + @field_validator("frozen_at") + @classmethod + def normalize_time(cls, value: datetime) -> datetime: + return _aware(value, name="frozen_at") + + @field_validator("corpus_digest") + @classmethod + def validate_digest(cls, value: str | None) -> str | None: + return _digest(value, name="corpus_digest") if value is not None else None + + @model_validator(mode="after") + def derive_identity(self) -> Self: + _identity(self, prefix="memory_evaluation_corpus", id_field="corpus_id", digest_field="corpus_digest") + return self + + +class MemoryMatchedCoordinatesV1Alpha1(_Contract): + task: ExactArtifactReferenceV1Alpha1 + provider: ExactArtifactReferenceV1Alpha1 + model: ExactArtifactReferenceV1Alpha1 + prompt_contract: ExactArtifactReferenceV1Alpha1 + decision_schema: ExactArtifactReferenceV1Alpha1 + toolset: ExactArtifactReferenceV1Alpha1 + configuration: ExactArtifactReferenceV1Alpha1 + + +class MemoryEvaluationProtocolV1Alpha1(_Contract): + contract: Literal["ace.intelligence.agent-memory-evaluation-protocol/v1alpha1"] = MEMORY_EVALUATION_PROTOCOL_VERSION + protocol_key: str + corpus: ExactArtifactReferenceV1Alpha1 + conditions: tuple[MemoryEvaluationCondition, ...] = Field(min_length=3, max_length=3) + measure_definitions: tuple[MemoryMeasureDefinitionV1Alpha1, ...] = Field( + min_length=len(MemoryMeasure), max_length=len(MemoryMeasure) + ) + minimum_beneficial_gain_bps: int = Field(ge=1, le=10_000) + maximum_full_context_correctness_gap_bps: int = Field(ge=0, le=10_000) + preregistered_at: datetime + provider_required: Literal[False] = False + network_required: Literal[False] = False + changes_rank_policy: Literal[False] = False + changes_retention_policy: Literal[False] = False + changes_consolidation_policy: Literal[False] = False + changes_promotion_policy: Literal[False] = False + changes_roster_or_authority: Literal[False] = False + delivers_or_sends_effect: Literal[False] = False + protocol_id: str | None = None + protocol_digest: str | None = None + + @field_validator("protocol_key") + @classmethod + def validate_key(cls, value: str) -> str: + return _bounded(value, name="protocol_key") + + @field_validator("conditions") + @classmethod + def normalize_conditions( + cls, value: tuple[MemoryEvaluationCondition, ...] + ) -> tuple[MemoryEvaluationCondition, ...]: + if {item.value for item in value} != EXPECTED_CONDITIONS or len(set(value)) != 3: + raise ValueError("protocol must freeze memory, no-memory, and full-context exactly once") + return tuple(sorted(value, key=lambda item: item.value)) + + @field_validator("measure_definitions") + @classmethod + def normalize_definitions( + cls, value: tuple[MemoryMeasureDefinitionV1Alpha1, ...] + ) -> tuple[MemoryMeasureDefinitionV1Alpha1, ...]: + if {item.measure for item in value} != set(MemoryMeasure) or len(value) != len(MemoryMeasure): + raise ValueError("protocol must freeze the complete AM6 measure registry exactly once") + unauthorized = next(item for item in value if item.measure is MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT) + if unauthorized.direction is not MeasureDirection.ZERO_TOLERANCE: + raise ValueError("unauthorized retrieval must remain a zero-tolerance measure") + return tuple(sorted(value, key=lambda item: item.measure.value)) + + @field_validator("preregistered_at") + @classmethod + def normalize_time(cls, value: datetime) -> datetime: + return _aware(value, name="preregistered_at") + + @field_validator("protocol_digest") + @classmethod + def validate_digest(cls, value: str | None) -> str | None: + return _digest(value, name="protocol_digest") if value is not None else None + + @model_validator(mode="after") + def validate_corpus_and_identity(self) -> Self: + if self.corpus.artifact_contract != MEMORY_EVALUATION_CORPUS_VERSION: + raise ValueError("protocol requires one exact frozen AM6 corpus") + _identity(self, prefix="memory_evaluation_protocol", id_field="protocol_id", digest_field="protocol_digest") + return self + + +class MemoryConditionPlanV1Alpha1(_Contract): + condition: MemoryEvaluationCondition + memory_mode: Literal["authorized_selected", "disabled", "full_authorized_context"] + + @model_validator(mode="after") + def validate_mode(self) -> Self: + expected = { + MemoryEvaluationCondition.MEMORY: "authorized_selected", + MemoryEvaluationCondition.NO_MEMORY: "disabled", + MemoryEvaluationCondition.FULL_CONTEXT: "full_authorized_context", + } + if self.memory_mode != expected[self.condition]: + raise ValueError("condition plan changed the frozen memory treatment") + return self + + +class MemoryConditionAssignmentV1Alpha1(_Contract): + contract: Literal["ace.intelligence.agent-memory-condition-assignment/v1alpha1"] = ( + MEMORY_CONDITION_ASSIGNMENT_VERSION + ) + protocol: ExactArtifactReferenceV1Alpha1 + corpus: ExactArtifactReferenceV1Alpha1 + case_id: str + matched_coordinates: MemoryMatchedCoordinatesV1Alpha1 + condition_plans: tuple[MemoryConditionPlanV1Alpha1, ...] = Field(min_length=3, max_length=3) + assigned_at: datetime + assignment_id: str | None = None + assignment_digest: str | None = None + + @field_validator("case_id") + @classmethod + def validate_case_id(cls, value: str) -> str: + return _bounded(value, name="case_id") + + @field_validator("condition_plans") + @classmethod + def normalize_plans(cls, value: tuple[MemoryConditionPlanV1Alpha1, ...]) -> tuple[MemoryConditionPlanV1Alpha1, ...]: + if {item.condition.value for item in value} != EXPECTED_CONDITIONS or len( + set(item.condition for item in value) + ) != 3: + raise ValueError("assignment must bind all three frozen conditions exactly once") + return tuple(sorted(value, key=lambda item: item.condition.value)) + + @field_validator("assigned_at") + @classmethod + def normalize_time(cls, value: datetime) -> datetime: + return _aware(value, name="assigned_at") + + @field_validator("assignment_digest") + @classmethod + def validate_digest(cls, value: str | None) -> str | None: + return _digest(value, name="assignment_digest") if value is not None else None + + @model_validator(mode="after") + def derive_identity(self) -> Self: + if self.protocol.artifact_contract != MEMORY_EVALUATION_PROTOCOL_VERSION: + raise ValueError("assignment requires the exact preregistered AM6 protocol") + if self.corpus.artifact_contract != MEMORY_EVALUATION_CORPUS_VERSION: + raise ValueError("assignment requires the exact frozen AM6 corpus") + _identity( + self, prefix="memory_condition_assignment", id_field="assignment_id", digest_field="assignment_digest" + ) + return self + + +class MemoryMeasureObservationV1Alpha1(_Contract): + measure: MemoryMeasure + availability: MeasureAvailability + value: int | None = Field(default=None, ge=0) + stratum: str | None = None + unavailable_reason: str | None = None + + @field_validator("stratum", "unavailable_reason") + @classmethod + def validate_optional_text(cls, value: str | None, info) -> str | None: + return _bounded(value, name=info.field_name) if value is not None else None + + @model_validator(mode="after") + def validate_availability(self) -> Self: + if self.availability is MeasureAvailability.AVAILABLE: + if self.value is None or self.unavailable_reason is not None: + raise ValueError("available measurement requires a value and no unavailable reason") + elif self.value is not None or self.unavailable_reason is None: + raise ValueError("unavailable or not-applicable measurement requires a reason and no value") + if self.measure.value.endswith("_bps") and self.value is not None and self.value > 10_000: + raise ValueError("basis-point measurements cannot exceed 10,000") + return self + + +class MemoryRunObservationV1Alpha1(_Contract): + contract: Literal["ace.intelligence.agent-memory-run-observation/v1alpha1"] = MEMORY_RUN_OBSERVATION_VERSION + protocol: ExactArtifactReferenceV1Alpha1 + assignment: ExactArtifactReferenceV1Alpha1 + case_id: str + condition: MemoryEvaluationCondition + decision_digest: str | None + route_ref: str + tier_ref: str + evidence_artifacts: tuple[ExactArtifactReferenceV1Alpha1, ...] = Field(max_length=MAX_ITEMS) + measurements: tuple[MemoryMeasureObservationV1Alpha1, ...] = Field(min_length=1, max_length=MAX_ITEMS) + observed_at: datetime + observation_id: str | None = None + observation_digest: str | None = None + + @field_validator("case_id", "route_ref", "tier_ref") + @classmethod + def validate_text(cls, value: str, info) -> str: + return _bounded(value, name=info.field_name) + + @field_validator("decision_digest") + @classmethod + def validate_decision_digest(cls, value: str | None) -> str | None: + return _digest(value, name="decision_digest") if value is not None else None + + @field_validator("evidence_artifacts") + @classmethod + def normalize_evidence( + cls, value: tuple[ExactArtifactReferenceV1Alpha1, ...] + ) -> tuple[ExactArtifactReferenceV1Alpha1, ...]: + return _unique_refs(value, name="evidence_artifacts") + + @field_validator("measurements") + @classmethod + def normalize_measurements( + cls, value: tuple[MemoryMeasureObservationV1Alpha1, ...] + ) -> tuple[MemoryMeasureObservationV1Alpha1, ...]: + keys = [(item.measure, item.stratum) for item in value] + if len(keys) != len(set(keys)): + raise ValueError("measurement and stratum coordinates must be unique") + return tuple(sorted(value, key=lambda item: (item.measure.value, item.stratum or ""))) + + @field_validator("observed_at") + @classmethod + def normalize_time(cls, value: datetime) -> datetime: + return _aware(value, name="observed_at") + + @field_validator("observation_digest") + @classmethod + def validate_digest(cls, value: str | None) -> str | None: + return _digest(value, name="observation_digest") if value is not None else None + + @model_validator(mode="after") + def derive_identity(self) -> Self: + if self.protocol.artifact_contract != MEMORY_EVALUATION_PROTOCOL_VERSION: + raise ValueError("observation requires the exact AM6 protocol") + if self.assignment.artifact_contract != MEMORY_CONDITION_ASSIGNMENT_VERSION: + raise ValueError("observation requires the exact matched assignment") + _identity(self, prefix="memory_run_observation", id_field="observation_id", digest_field="observation_digest") + return self + + +class MemoryMatchedComparisonV1Alpha1(_Contract): + contract: Literal["ace.intelligence.agent-memory-matched-comparison/v1alpha1"] = MEMORY_MATCHED_COMPARISON_VERSION + protocol: ExactArtifactReferenceV1Alpha1 + assignment: ExactArtifactReferenceV1Alpha1 + case_id: str + observations: tuple[ExactArtifactReferenceV1Alpha1, ...] = Field(min_length=3, max_length=3) + paired_and_controlled: bool + material_influence: MaterialInfluenceDisposition + benefit: BenefitDisposition + correctness: CorrectnessDisposition + causality: Literal[CausalityDisposition.NOT_ESTABLISHED] = CausalityDisposition.NOT_ESTABLISHED + missing_measurements: tuple[str, ...] = Field(default_factory=tuple, max_length=MAX_ITEMS) + reasons: tuple[str, ...] = Field(min_length=1, max_length=MAX_ITEMS) + limitations: tuple[str, ...] = Field(min_length=1, max_length=MAX_ITEMS) + compared_at: datetime + changes_any_policy: Literal[False] = False + changes_authority_or_roster: Literal[False] = False + comparison_id: str | None = None + comparison_digest: str | None = None + + @field_validator("case_id") + @classmethod + def validate_case_id(cls, value: str) -> str: + return _bounded(value, name="case_id") + + @field_validator("observations") + @classmethod + def normalize_observations( + cls, value: tuple[ExactArtifactReferenceV1Alpha1, ...] + ) -> tuple[ExactArtifactReferenceV1Alpha1, ...]: + return _unique_refs(value, name="observations", minimum=3) + + @field_validator("missing_measurements", "reasons", "limitations") + @classmethod + def normalize_text(cls, value: tuple[str, ...], info) -> tuple[str, ...]: + minimum = 0 if info.field_name == "missing_measurements" else 1 + return _unique_strings(value, name=info.field_name, minimum=minimum) + + @field_validator("compared_at") + @classmethod + def normalize_time(cls, value: datetime) -> datetime: + return _aware(value, name="compared_at") + + @field_validator("comparison_digest") + @classmethod + def validate_digest(cls, value: str | None) -> str | None: + return _digest(value, name="comparison_digest") if value is not None else None + + @model_validator(mode="after") + def validate_claim_boundary(self) -> Self: + underpowered = self.benefit is BenefitDisposition.UNDERPOWERED + if underpowered != bool(self.missing_measurements): + raise ValueError("underpowered benefit disposition must match explicit missing measurements") + if underpowered and self.material_influence is not MaterialInfluenceDisposition.UNDERPOWERED: + raise ValueError("underpowered comparison cannot claim material influence") + if self.benefit is BenefitDisposition.HARMFUL and self.correctness is CorrectnessDisposition.CORRECT: + raise ValueError("harmful comparison cannot be labeled fully correct") + _identity(self, prefix="memory_matched_comparison", id_field="comparison_id", digest_field="comparison_digest") + return self + + +def memory_evaluation_reference(value: object) -> ExactArtifactReferenceV1Alpha1: + layouts = { + MemoryEvaluationCorpusV1Alpha1: ("corpus_id", "corpus_digest"), + MemoryEvaluationProtocolV1Alpha1: ("protocol_id", "protocol_digest"), + MemoryConditionAssignmentV1Alpha1: ("assignment_id", "assignment_digest"), + MemoryRunObservationV1Alpha1: ("observation_id", "observation_digest"), + MemoryMatchedComparisonV1Alpha1: ("comparison_id", "comparison_digest"), + } + for model, (id_field, digest_field) in layouts.items(): + if isinstance(value, model): + return ExactArtifactReferenceV1Alpha1( + artifact_id=str(getattr(value, id_field)), + artifact_digest=str(getattr(value, digest_field)), + artifact_contract=value.contract, + ) + raise TypeError("unsupported Agent Memory evaluation artifact") + + +__all__ = [ + "MEMORY_CONDITION_ASSIGNMENT_VERSION", + "MEMORY_EVALUATION_CORPUS_VERSION", + "MEMORY_EVALUATION_PROTOCOL_VERSION", + "MEMORY_MATCHED_COMPARISON_VERSION", + "MEMORY_RUN_OBSERVATION_VERSION", + "BenefitDisposition", + "CausalityDisposition", + "CorrectnessDisposition", + "EvaluationCaseGate", + "MaterialInfluenceDisposition", + "MeasureAvailability", + "MeasureDirection", + "MeasureUnit", + "MemoryConditionAssignmentV1Alpha1", + "MemoryConditionPlanV1Alpha1", + "MemoryEvaluationCaseV1Alpha1", + "MemoryEvaluationCondition", + "MemoryEvaluationCorpusV1Alpha1", + "MemoryEvaluationProtocolV1Alpha1", + "MemoryMatchedComparisonV1Alpha1", + "MemoryMatchedCoordinatesV1Alpha1", + "MemoryMeasure", + "MemoryMeasureDefinitionV1Alpha1", + "MemoryMeasureObservationV1Alpha1", + "MemoryRunObservationV1Alpha1", + "memory_evaluation_reference", +] diff --git a/docs/README.md b/docs/README.md index da82c89..40871fd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -97,3 +97,6 @@ support the public roadmap but do not compete with it for outcome state or dispa - [Agent Memory AM3 work packet](design/agent-memory-am3-work-packet-v1.md) — authorized recall, frozen provider-neutral ranking, Context Planner and Manifest, composition/I3 lineage, matched materiality, durability, privacy, and AM4 stop boundary. +- [Agent Memory AM6 evaluation-preparation work packet](design/agent-memory-am6-evaluation-prep-work-packet-v1.md) + — provider-free corpus, matched memory/no-memory/full-context controls, complete measurement + registry, bounded outcome labels, and exact AM4 convergence gates without lifecycle invention. diff --git a/docs/design/agent-memory-am6-evaluation-prep-work-packet-v1.md b/docs/design/agent-memory-am6-evaluation-prep-work-packet-v1.md new file mode 100644 index 0000000..9127897 --- /dev/null +++ b/docs/design/agent-memory-am6-evaluation-prep-work-packet-v1.md @@ -0,0 +1,221 @@ +# ACE Agent Memory AM6 evaluation preparation work packet v1 + +- Date: 2026-08-12 +- Status: provider-free evaluation-preparation candidate; not memory-benefit evidence +- Exact base: `f761a682164d10e2ff81ba38cd2d0c987b4f8efd` +- Base branch: `codex/v0.7-cumulative-integration-acceptance` +- Candidate branch: `codex/v0.7-agent-memory-am6-evaluation-prep` +- Parallel boundary: AM3-runnable; AM4 lifecycle cases are gated placeholders only + +## Outcome and claim boundary + +This packet freezes the AM6 measurement vocabulary, synthetic corpus, matched conditions, +content-addressed artifact chain, deterministic comparator, result shape, and verification entry +point that can run over the existing AM0–AM3 boundary. + +The provider-free fixture proves that the evaluator reproduces exact coordinates and classifies +synthetic bounded outcomes as beneficial, harmful, neutral, or underpowered under a preregistered +rule. It does **not** prove that Agent Memory is beneficial, correct in general, causally effective, +production-ready, or eligible for a policy change. + +The evaluator: + +- writes no Agent Memory; +- changes no rank, retention, consolidation, promotion, roster, authority, delivery, or effect + policy; +- trains or updates no model; +- requires no provider, credential, network, database, schema, or migration; +- adds no public TaskCreate field, endpoint, command, or MCP tool; and +- invents no AM4 retention, export, import, expiry, or erasure runtime contract. + +## Exact artifact chain + +The provider-neutral Intelligence contracts freeze five content-addressed artifacts: + +1. `ace.intelligence.agent-memory-evaluation-corpus/v1alpha1`; +2. `ace.intelligence.agent-memory-evaluation-protocol/v1alpha1`; +3. `ace.intelligence.agent-memory-condition-assignment/v1alpha1`; +4. `ace.intelligence.agent-memory-run-observation/v1alpha1`; and +5. `ace.intelligence.agent-memory-matched-comparison/v1alpha1`. + +The corpus is frozen before the protocol, the protocol before assignment, every assignment before +its three observations, and every observation before comparison. Exact identities derive from the +complete canonical material. Reconstructing the fixture in a fresh process must reproduce the same +corpus, protocol, assignment, observation, and comparison coordinates. + +These are evaluation artifacts, not memory records, governed policy proposals, authority receipts, +retention receipts, or delivery/effect artifacts. + +## Preregistered matched conditions + +Every case has exactly three conditions: + +| Condition | Treatment | Boundary | +|---|---|---| +| `memory` | Existing AM3 authorized selection and smallest eligible context | May reference exact recall, Context Manifest, and I3 artifacts; cannot widen their semantics | +| `no_memory` | Memory disabled | Holds every non-memory coordinate constant | +| `full_context` | Complete authorized context baseline | Holds every non-memory coordinate constant; does not bypass scope or privacy | + +Every assignment binds one exact task plus the same provider, model, prompt contract, decision +schema, toolset, and configuration across all three conditions. The deterministic fixture freezes +`provider:deterministic-fixture` and `model:none`; a provider run is optional and cannot replace the +provider-free conformance authority. + +## Frozen corpus + +The v1 corpus contains 18 synthetic cases. Fifteen run honestly over AM0–AM3; three are explicit +AM4 convergence gates. + +| Case | Current status | Required coverage | +|---|---|---| +| `am1_ingestion_replay_restart` | runnable | ingestion completeness, exact replay, restart, episodic experience | +| `am2_family_extraction_and_spans` | runnable | all seven AM2 assertion families, precision/recall, exact source spans | +| `am2_identity_and_unresolved` | runnable | identity error, unresolved identity, unknown stays unknown | +| `am2_conflict_correction_uncertainty` | runnable | conflict, correction, contradiction, uncertainty | +| `am2_instruction_isolation` | runnable | instruction recall and prompt-shaped source-data isolation | +| `am2_independent_time_axes` | runnable | ledger, knowledge, and world time, including unknown time | +| `am1_scope_privacy_isolation` | runnable | product/principal scope, cross-scope privacy, non-disclosure | +| `am3_authorization_denial` | runnable | authorization before signals, graph, cache, and body; zero leakage | +| `am3_stale_superseded_safety` | runnable | correction priority and zero stale/superseded influence | +| `am3_harmful_influence_probe` | runnable negative probe | evaluator detects stale/superseded harmful influence | +| `am3_manifest_selection_omission` | runnable | retrieval, rank, citation, omission, selected/injected/reflected/material states | +| `am3_degraded_retrieval_signal` | runnable degraded case | missing optional signal is explicit and underpowered | +| `am3_later_restart_material_use` | runnable | independent later invocation after restart and material-use distinction | +| `am3_material_but_neutral` | runnable | material influence remains distinct from benefit | +| `am3_missing_resource_telemetry` | runnable degraded case | missing token/latency/call/cache/cost telemetry is explicit and underpowered | +| `am4_retention_expiry_placeholder` | gated | future retention/expiry behavior; no current execution | +| `am4_export_import_placeholder` | gated | future export/import identity/lifecycle behavior; no current execution | +| `am4_hard_erasure_placeholder` | gated | future dependency-complete erasure/restart behavior; no current execution | + +The harmful case is a deliberately injected negative observation proving the comparator fails the +unsafe result. It is not evidence that the current AM3 implementation exhibited harmful behavior. + +## Measurement registry + +The protocol freezes 31 measures rather than interpreting absent data as zero or success. + +### Admission, extraction, identity, and reconciliation + +- ingestion completeness and replay correctness; +- extraction precision and recall, with family strata; +- exact source-span accuracy; +- identity error and unresolved-identity rate; and +- correction, contradiction, uncertainty, and instruction-policy recall. + +### Retrieval, safety, and omission + +- retrieval precision and recall; +- rank quality; +- citation correctness; +- omission coverage; +- unauthorized retrieval count, with zero tolerance; +- stale influence count; +- superseded influence count; and +- dependency invalidation rate. + +### Resources, routes, and use states + +- context tokens and residual window tokens; +- latency, provider calls, cache reuse, and cost; +- observed route and tier coordinates, aggregated as deterministic frequencies; and +- selected, injected, reflected, and decision-material rates. + +### Bounded task outcome + +- task correctness in basis points; and +- beneficial, harmful, neutral, or underpowered comparison disposition. + +An unavailable required measure contains no numeric value and names its reason. The comparison then +sets paired/control status false and material influence, benefit, and correctness to underpowered. +Optional retrieval-signal loss and required resource-telemetry loss therefore cannot silently look +like a zero-cost or fully evaluated success. + +## Material influence, benefit, correctness, and causality + +The comparison keeps four independent dimensions: + +| Dimension | v1 meaning | +|---|---| +| Material influence | Whether the exact memory and no-memory decision digests differ under held constants | +| Benefit disposition | Synthetic bounded label: beneficial, harmful, neutral, or underpowered | +| Correctness | Correct, incorrect, mixed, or underpowered against the frozen synthetic oracle | +| Causality | Always `not_established` in this preparation fixture | + +A material difference is not automatically beneficial or correct. A neutral result can be +material. A correct result can be neutral. Missing evidence is underpowered rather than neutral. +The synthetic benefit label proves the evaluator's rule only; it is not an Agent Memory benefit +claim. + +## Comparison rule + +The deterministic comparator: + +1. verifies exact corpus, protocol, assignment, case, condition, and time closure; +2. requires exactly one memory, no-memory, and full-context observation; +3. verifies every case-required measurement is available in every condition; +4. makes any AM4-gated case underpowered until `future_accepted_am4_coordinate` is replaced through + a separately accepted convergence change; +5. labels any unauthorized retrieval, stale influence, or superseded influence as harmful; +6. records material influence only from a changed memory/no-memory decision digest; +7. labels a synthetic result beneficial only when memory beats no-memory by the preregistered + threshold and remains within the frozen full-context correctness gap; +8. labels a negative task-score delta harmful and an unchanged score neutral; and +9. emits no proposal or policy mutation for any disposition. + +## AC6 reuse and separation + +AM6 reuses semantically valid AC6 patterns: + +- freeze before assignment and observation; +- exact matched coordinates; +- provider-free acceptance authority; +- content-addressed artifact identity; +- visible failures, missing telemetry, negative results, and limitations; +- deterministic fresh-process reproduction; and +- claim bounding. + +AM6 does not reuse AC6's composition conditions, participant materiality thresholds, dynamic +composition proposal, roster semantics, or policy-admission path. Composition policy and memory +policy remain separate owners. + +## AM4 convergence gate + +The three AM4 cases carry only the literal `future_accepted_am4_coordinate`. They do not name or +construct a retention policy, export service, erasure service, derivative index implementation, +receipt grammar, lifecycle transition, or persistence behavior. + +After AM4 is independently accepted, the minimal convergence change is: + +1. record the exact accepted AM4 commit and owning contract coordinates as corpus source artifacts; +2. replace each placeholder gate with exact existing AM4 observation inputs; +3. add the AM4-required measures without changing the AM3 case coordinates or matched constants; +4. rerun the same provider-free verifier and focused privacy/package boundaries; and +5. publish the new result as AM4-converged evidence without rewriting this v1 preparation record. + +No AM4 branch, PR, or runtime implementation is a dependency of the current executable suite. + +## Artifacts + +- contracts: `ace/intelligence/contracts/agent_memory_evaluation.py`; +- deterministic comparator: `ace/intelligence/agent_memory_evaluation.py`; +- frozen corpus/protocol fixture: `evaluations/fixtures/agent_memory_am6_evaluation_prep_v1.json`; +- provider-free runner: `evaluations/source/agent_memory_am6_evaluation.py`; +- frozen result: `evaluations/results/agent_memory_am6_evaluation_prep_v1.json`; +- verifier: `scripts/verify_agent_memory_am6_evaluation.py`; and +- conformance: `tests/agent_memory/am6/test_evaluation_prep.py`. + +## Verification and publication gate + +Before publication, this lane must record: + +- focused AM6 conformance; +- AM0–AM3 focused regression and AC6 provider-free verifier; +- privacy, Core/Intelligence import, package, naked-kernel, and exact eleven-tool MCP boundaries; +- fixture/result deterministic diff and fresh-process reproduction; +- Ruff, format, lock, diff, secret, authority, privacy, AM4-invention, and domain scans; +- installed-wheel checkout-free verifier reproduction because runtime code is included; and +- effective diff against exact base `f761a682164d10e2ff81ba38cd2d0c987b4f8efd`. + +Publication may be a stacked draft against `codex/v0.7-cumulative-integration-acceptance` only if +the effective diff remains AM6-only and independently reviewable. Merge, release, tag, package +publication, policy activation, and downstream dispatch remain prohibited. diff --git a/docs/evidence/README.md b/docs/evidence/README.md index f32ee7f..81cbdaa 100644 --- a/docs/evidence/README.md +++ b/docs/evidence/README.md @@ -133,6 +133,9 @@ Two earlier local checkpoints predate the P1/P2 sequence and are superseded by i - [AM3 authorized recall and Context Planner candidate](agent-memory-am3-candidate-v1.md) — stacked candidate; provider-free selection, Context Manifest/I3 lineage, matched material influence, privacy, restart/rebuild, and installed-wheel evidence +- [AM6 evaluation-preparation candidate](agent-memory-am6-evaluation-prep-candidate-v1.md) — stacked + candidate; provider-free corpus, matched memory/no-memory/full-context controls, 31 measures, + explicit negative/underpowered outcomes, and AM4-gated placeholders without lifecycle invention ## Extension invocation diff --git a/docs/evidence/agent-memory-am6-evaluation-prep-candidate-v1.md b/docs/evidence/agent-memory-am6-evaluation-prep-candidate-v1.md new file mode 100644 index 0000000..ffa1157 --- /dev/null +++ b/docs/evidence/agent-memory-am6-evaluation-prep-candidate-v1.md @@ -0,0 +1,159 @@ +# ACE Agent Memory AM6 evaluation-preparation candidate evidence v1 + +## Candidate coordinates + +- Date: 2026-08-12 +- Exact base: `f761a682164d10e2ff81ba38cd2d0c987b4f8efd` +- Base branch: `codex/v0.7-cumulative-integration-acceptance` +- Candidate branch: `codex/v0.7-agent-memory-am6-evaluation-prep` +- Exact implementation artifact: `948f452cc691e68c599dbe8ee4b57b61f0c95710` +- Pre-convergence AM6 head: `a6c6dc736648e58d378bf6813551982faafe9486` +- Control-tower-supplied main coordinate: `9e0a9d248c073b6a7883451cb1d219eb7c15999b` +- Actual current main at convergence: `11b44e84d92e0674fa433103779f4050eeca2725` +- Two-parent convergence merge: `1eb30999b5082a4c07d2abd2af08132226bde939` +- Merge parents, in order: AM6 `a6c6dc736648e58d378bf6813551982faafe9486` and main `11b44e84d92e0674fa433103779f4050eeca2725` +- Cumulative review authority: merged PR #122; release source and closeout are preserved from main +- Unpublished converged wheel: `ace_core-0.7.0-py3-none-any.whl` +- Wheel SHA-256: `1aad86d2b7d609aa3e3c7cbfe66932f93bcefc1b87ee52883d89a577f6471487` +- Status: main-converged draft candidate; not accepted, merged, released, or supported + +## Candidate claim and limit + +This candidate freezes a provider-neutral AM6 corpus, measurement protocol, matched condition +assignment, observation, comparison, report shape, deterministic fixture, verifier, and conformance +suite that can run over the existing AM0–AM3 boundary. + +It does not prove Agent Memory benefit, general correctness, causal effect, production maturity, or +an eligible policy change. Its beneficial, harmful, neutral, and underpowered results are synthetic +oracle labels that prove the evaluator recognizes the preregistered cases. Causality remains +`not_established` for every result. + +No rank, retention, consolidation, promotion, roster, authority, delivery, or effect policy changes +are emitted or applied. No provider, credential, network, schema, migration, database, package +identity, public TaskCreate field, or new MCP tool is required. + +## Frozen corpus and protocol + +The frozen provider-free coordinates are: + +- corpus: + `memory_evaluation_corpus:57b629504eac4f10af39f7943be4143f` + (`sha256:57b629504eac4f10af39f7943be4143fac31d483e9029769aabfcea3024f02be`); +- protocol: + `memory_evaluation_protocol:434a03e50ca5c3c43c3eaa9430600ac9` + (`sha256:434a03e50ca5c3c43c3eaa9430600ac95bf00e26101400795df030db4b045729`); +- 18 cases; +- three matched conditions per case: memory, no-memory, and full authorized context; +- 54 condition observations; +- 31 frozen measures; and +- 15 AM0–AM3-runnable cases plus three future-AM4 gated placeholders. + +The assignments hold exact task, provider, model, prompt contract, decision schema, toolset, and +configuration constant. The acceptance fixture explicitly uses `model:none`; provider credentials +and network access are not required. + +Coverage includes AM1 ingestion/replay/restart, every AM2 family, extraction and source spans, +identity/unresolved state, conflict/correction/contradiction/uncertainty, instruction isolation, +the three time axes, scope/privacy, AM3 authorization denial, stale/superseded influence, Context +Manifest selection and omission, degraded signals, later restart/material use, resource telemetry, +and the separation of material influence from benefit. + +The deterministic outcome distribution is: + +| Disposition | Count | Meaning | +|---|---:|---| +| beneficial | 8 | Synthetic bounded rule cleared; not an Agent Memory benefit claim | +| harmful | 1 | Deliberately injected stale/superseded negative probe was detected | +| neutral | 4 | No bounded score gain, including one materially different result | +| underpowered | 5 | Missing signal/telemetry or future AM4 coordinate was explicit | + +## AM4 boundary + +The current runnable suite imports no AM4 implementation and invents no AM4 runtime contract. The +retention/expiry, export/import, and hard-erasure cases require only the literal +`future_accepted_am4_coordinate`; all their required measurements are unavailable and their result +is underpowered. + +The minimal later convergence is to bind exact accepted AM4 artifact coordinates, replace only the +three placeholder observation inputs with existing AM4 evidence, and rerun the same matched +protocol. No current AM3 case, held constant, or v1 evidence record needs rewriting. + +## Verification ledger + +| Gate | Result | +|---|---| +| Exact base, detached-head cleanliness, and requested sibling branch | Passed before changes; base was exact `f761a682164d10e2ff81ba38cd2d0c987b4f8efd` | +| Focused AM6 conformance | 14 passed | +| Focused AM0–AM3 plus AC6 and AM6 matrix | 177 passed | +| Privacy, package, naked-kernel, evidence-index, public Core/Intelligence and exact-MCP boundaries | 60 passed | +| Full supported extension-disabled non-E2E lane | 7,820 passed, 50 skipped, 261 marker-deselected | +| Provider-free AM6 verifier | 18 cases, 54 observations, three AM4 placeholders, deterministic restart reconstruction | +| Existing provider-free AC6 verifier | 14 matched cases, one inert AC6 proposal, deterministic restart replay | +| Whole-repository Ruff and changed-file format | Passed | +| Lock and diff integrity | `uv lock --check` and `git diff --check` passed; no dependency or lock change | +| Secret scan | Passed across every AM6-owned path | +| Provider/host/storage/import boundary | Passed; pure Intelligence contracts/evaluator import no host, provider, extension, MCP, or SurrealDB runtime | +| Authority/privacy/AM4-invention scan | Passed in focused negative conformance; no policy-applying type or AM4 service/contract exists in the diff | +| Checkout-free installed-wheel reproduction | Passed in two clean targets; both loaded the evaluator from the installed wheel and reproduced exact corpus/protocol IDs, 18 outcomes, restart determinism, and 11 thin MCP tools | +| Main convergence | Normal two-parent merge passed with no conflicts; actual current main was one release-closeout commit beyond the supplied coordinate | +| Converged package/release/evaluation/boundary matrix | 98 passed; every package identity remained `0.7.0` and both AM6 and AC6 provider-free verifiers passed | + +The full supported lane emitted existing dependency deprecation, weak synthetic JWT test-key, and +test-collection/runtime warnings. It had no failure. No local database, live provider, or external +network was used by AM6 acceptance. + +## Installed-wheel reproduction + +The wheel was built without isolation from the exact locked repository environment and was not +published. It was installed without dependencies into: + +- `/tmp/ace-am6-v070-target-one.BhUak8`; and +- `/tmp/ace-am6-v070-target-two.gJM9C0`. + +Both targets loaded `ace.intelligence.agent_memory_evaluation` from the installed target rather +than the checkout and reproduced: + +- corpus `memory_evaluation_corpus:57b629504eac4f10af39f7943be4143f`; +- protocol `memory_evaluation_protocol:434a03e50ca5c3c43c3eaa9430600ac9`; +- 18 cases with outcome counts 8 beneficial, 1 harmful, 4 neutral, and 5 underpowered; +- fresh reconstruction identity equality; and +- exactly eleven unique public thin-MCP tools; and +- installed package version `0.7.0`. + +## Artifacts and effective diff + +The effective diff changes 14 paths against converged current `main`: + +- additive Intelligence contract and pure comparator modules; +- additive exports in the existing Intelligence initializers; +- frozen fixture and result; +- provider-free source runner and verifier; +- focused AM6 conformance plus one public-surface boundary expectation; and +- AM6 work packet and documentation index. + +It changes no AM0–AM3 runtime implementation, AM4 branch or contract, AC6 composition policy, +schema, migration, dependency, package version, MCP server, TaskCreate contract, release metadata, +or external repository. The merge preserves main's complete 0.7.0 package and release state without +making it part of the effective AM6 PR diff. + +## Publication and convergence limit + +The earlier publication hold was accurate when PR #122 had merged and GitHub had deleted the +requested stacked base. The control tower later explicitly authorized a normal main convergence +and a draft PR targeting `main`, superseding that hold without rewriting the existing AM6 commits. + +Before merge, the branch was clean at exact pre-convergence head +`a6c6dc736648e58d378bf6813551982faafe9486`. A fresh fetch found that the supplied main coordinate +`9e0a9d248c073b6a7883451cb1d219eb7c15999b` was the sole parent of newer current main +`11b44e84d92e0674fa433103779f4050eeca2725`, which adds the 0.7 release closeout. The normal merge +used that actual current head so publication would not begin stale. Merge commit +`1eb30999b5082a4c07d2abd2af08132226bde939` has exactly the two recorded parents and had no +conflicts. No rebase, force-push, history rewrite, main mutation, release, or tag occurred. + +The effective diff against `main` remains independently reviewable and contains no AM4 invention. +It is eligible only for the control-tower-authorized draft PR. Merge, release, tag, package +publication, policy activation, and downstream dispatch remain prohibited. + +After AM4 acceptance, the minimal convergence step is a new additive commit that supplies the exact +accepted AM4 coordinate and observation evidence for the three gated cases, then reruns this frozen +protocol and publishes a new point-in-time result without rewriting this record. diff --git a/evaluations/fixtures/agent_memory_am6_evaluation_prep_v1.json b/evaluations/fixtures/agent_memory_am6_evaluation_prep_v1.json new file mode 100644 index 0000000..fcbefac --- /dev/null +++ b/evaluations/fixtures/agent_memory_am6_evaluation_prep_v1.json @@ -0,0 +1,181 @@ +{ + "contract": "ace.evaluation.agent-memory-am6-evaluation-prep/v1", + "claim_scope": "provider_free_measurement_contract_and_evaluator_conformance_only", + "synthetic_only": true, + "exact_base": "f761a682164d10e2ff81ba38cd2d0c987b4f8efd", + "source_artifacts": [ + ["fixture:agent-memory-am0-contract-v1", "ace.evaluation.agent-memory-am0-contract/v1"], + ["fixture:agent-memory-am1-session-normalization-v1", "ace.evaluation.agent-memory-am1-session-normalization-spec/v1"], + ["fixture:agent-memory-am2-assertion-reconciliation-v1", "ace.evaluation.agent-memory-am2-assertion-reconciliation/v1"], + ["fixture:agent-memory-am3-context-planner-v1", "ace.evaluation.agent-memory-am3-context-planner/v1"], + ["evidence:ac6-measured-composition-v1", "ace.evidence.agent-measured-composition-ac6-candidate/v1"], + ["commit:cumulative-v0.7-integration-acceptance", "ace.repository.commit/v1"] + ], + "protocol": { + "minimum_beneficial_gain_bps": 1, + "maximum_full_context_correctness_gap_bps": 0, + "held_constants": { + "provider": ["provider:deterministic-fixture", "ace.evaluation.provider/v1"], + "model": ["model:none", "ace.evaluation.model/v1"], + "prompt_contract": ["prompt:am6-frozen-v1", "ace.evaluation.prompt-contract/v1"], + "decision_schema": ["decision-schema:bounded-choice-v1", "ace.evaluation.decision-schema/v1"], + "toolset": ["toolset:eleven-thin-mcp-unchanged", "ace.evaluation.toolset/v1"], + "configuration": ["configuration:am6-provider-free-v1", "ace.evaluation.configuration/v1"] + } + }, + "cases": [ + { + "case_id": "am1_ingestion_replay_restart", + "title": "AM1 ingestion completeness, exact replay, and restart", + "coverage_tags": ["am1", "ingestion", "replay", "restart", "episodic_experience"], + "required_measures": ["ingestion_completeness_bps", "replay_correctness_bps", "task_correctness_bps", "context_tokens", "latency_ms", "provider_calls", "cost_microunits"], + "expected": "beneficial" + }, + { + "case_id": "am2_family_extraction_and_spans", + "title": "AM2 seven-family extraction and exact source spans", + "coverage_tags": ["am2", "identity", "learned_fact", "active_context", "preference", "instruction_policy_proposal", "uncertainty", "correction", "source_span"], + "required_measures": ["extraction_precision_bps", "extraction_recall_bps", "source_span_accuracy_bps", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am2_identity_and_unresolved", + "title": "AM2 identity error and explicit unresolved state", + "coverage_tags": ["am2", "identity", "unresolved_identity", "unknown_stays_unknown"], + "required_measures": ["identity_error_bps", "unresolved_identity_bps", "task_correctness_bps"], + "decision_mode": "same", + "scores": {"memory": 10000, "no_memory": 10000, "full_context": 10000}, + "expected": "neutral" + }, + { + "case_id": "am2_conflict_correction_uncertainty", + "title": "AM2 conflict, correction, contradiction, and uncertainty", + "coverage_tags": ["am2", "conflict", "correction", "contradiction", "uncertainty"], + "required_measures": ["correction_recall_bps", "contradiction_recall_bps", "uncertainty_recall_bps", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am2_instruction_isolation", + "title": "AM2 prompt-shaped source text remains outside instruction authority", + "coverage_tags": ["am2", "instruction_policy", "instruction_isolation", "prompt_injection"], + "required_measures": ["instruction_policy_recall_bps", "unauthorized_retrieval_count", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am2_independent_time_axes", + "title": "AM2 ledger, knowledge, and world-time selectors remain independent", + "coverage_tags": ["am2", "ledger_time", "knowledge_time", "world_time", "unknown_time"], + "required_measures": ["citation_correctness_bps", "omission_coverage_bps", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am1_scope_privacy_isolation", + "title": "AM1 product and principal scope privacy isolation", + "coverage_tags": ["am1", "scope", "privacy", "cross_product", "cross_principal"], + "required_measures": ["unauthorized_retrieval_count", "task_correctness_bps"], + "decision_mode": "same", + "scores": {"memory": 10000, "no_memory": 10000, "full_context": 10000}, + "expected": "neutral" + }, + { + "case_id": "am3_authorization_denial", + "title": "AM3 authorization denial precedes signal, graph, cache, and body access", + "coverage_tags": ["am3", "authorization_denial", "zero_tolerance", "non_disclosure"], + "required_measures": ["unauthorized_retrieval_count", "provider_calls", "task_correctness_bps"], + "decision_mode": "same", + "scores": {"memory": 10000, "no_memory": 10000, "full_context": 10000}, + "expected": "neutral" + }, + { + "case_id": "am3_stale_superseded_safety", + "title": "AM3 current correction excludes stale and superseded influence", + "coverage_tags": ["am3", "stale", "superseded", "correction_priority"], + "required_measures": ["stale_influence_count", "superseded_influence_count", "correction_recall_bps", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am3_harmful_influence_probe", + "title": "Synthetic negative probe detects stale or superseded harmful influence", + "coverage_tags": ["am3", "negative_probe", "stale", "superseded", "harmful"], + "required_measures": ["stale_influence_count", "superseded_influence_count", "task_correctness_bps"], + "scores": {"memory": 0, "no_memory": 10000, "full_context": 10000}, + "overrides": {"memory": {"stale_influence_count": 1, "superseded_influence_count": 1}}, + "expected": "harmful" + }, + { + "case_id": "am3_manifest_selection_omission", + "title": "AM3 Context Manifest selection, omission, and material-use distinctions", + "coverage_tags": ["am3", "context_manifest", "selection", "omission", "injection", "reflection", "decision_material"], + "required_measures": ["retrieval_precision_bps", "retrieval_recall_bps", "rank_quality_bps", "citation_correctness_bps", "omission_coverage_bps", "selected_rate_bps", "injected_rate_bps", "reflected_rate_bps", "decision_material_rate_bps", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am3_degraded_retrieval_signal", + "title": "AM3 unavailable optional retrieval signal remains explicit", + "coverage_tags": ["am3", "degraded_retrieval", "missing_signal", "underpowered"], + "required_measures": ["rank_quality_bps", "task_correctness_bps"], + "unavailable": {"memory": {"rank_quality_bps": "vector_signal_unavailable"}, "full_context": {"rank_quality_bps": "vector_signal_unavailable"}, "no_memory": {"rank_quality_bps": "not_applicable_without_memory"}}, + "expected": "underpowered" + }, + { + "case_id": "am3_later_restart_material_use", + "title": "AM3 independent later invocation after restart distinguishes material use", + "coverage_tags": ["am3", "restart", "later_invocation", "selected", "injected", "reflected", "decision_material"], + "required_measures": ["selected_rate_bps", "injected_rate_bps", "reflected_rate_bps", "decision_material_rate_bps", "task_correctness_bps"], + "expected": "beneficial" + }, + { + "case_id": "am3_material_but_neutral", + "title": "AM3 material influence remains distinct from bounded benefit", + "coverage_tags": ["am3", "material_influence", "neutral", "benefit_separation"], + "required_measures": ["decision_material_rate_bps", "task_correctness_bps"], + "scores": {"memory": 10000, "no_memory": 10000, "full_context": 10000}, + "expected": "neutral" + }, + { + "case_id": "am3_missing_resource_telemetry", + "title": "Missing token, latency, provider-call, or cost telemetry is underpowered", + "coverage_tags": ["am3", "telemetry", "tokens", "latency", "provider_calls", "cost", "underpowered"], + "required_measures": ["context_tokens", "residual_window_tokens", "latency_ms", "provider_calls", "cache_reuse_count", "cost_microunits", "task_correctness_bps"], + "unavailable": {"memory": {"cost_microunits": "cost_telemetry_unavailable"}}, + "expected": "underpowered" + }, + { + "case_id": "am4_retention_expiry_placeholder", + "title": "Future AM4 retention and expiry evaluation coordinate", + "coverage_tags": ["am4", "retention", "expiry", "gated_placeholder"], + "required_measures": ["retrieval_recall_bps", "omission_coverage_bps", "task_correctness_bps"], + "gate": "future_accepted_am4", + "expected": "underpowered" + }, + { + "case_id": "am4_export_import_placeholder", + "title": "Future AM4 export and import evaluation coordinate", + "coverage_tags": ["am4", "export", "import", "identity", "lifecycle", "gated_placeholder"], + "required_measures": ["replay_correctness_bps", "citation_correctness_bps", "task_correctness_bps"], + "gate": "future_accepted_am4", + "expected": "underpowered" + }, + { + "case_id": "am4_hard_erasure_placeholder", + "title": "Future AM4 dependency-complete hard-erasure evaluation coordinate", + "coverage_tags": ["am4", "hard_erasure", "derivatives", "restart", "gated_placeholder"], + "required_measures": ["unauthorized_retrieval_count", "omission_coverage_bps", "task_correctness_bps"], + "gate": "future_accepted_am4", + "expected": "underpowered" + } + ], + "invariants": { + "provider_required": false, + "network_required": false, + "public_mcp_tool_count": 11, + "changes_rank_policy": false, + "changes_retention_policy": false, + "changes_consolidation_policy": false, + "changes_promotion_policy": false, + "changes_roster_or_authority": false, + "delivers_or_sends_effect": false, + "am4_runtime_contracts_implemented": false, + "fixture_claims_agent_memory_benefit": false + } +} diff --git a/evaluations/results/agent_memory_am6_evaluation_prep_v1.json b/evaluations/results/agent_memory_am6_evaluation_prep_v1.json new file mode 100644 index 0000000..0b00d57 --- /dev/null +++ b/evaluations/results/agent_memory_am6_evaluation_prep_v1.json @@ -0,0 +1,419 @@ +{ + "am3_runnable_cases": 15, + "am4_gated_placeholders": 3, + "case_count": 18, + "condition_count": 3, + "corpus_digest": "sha256:57b629504eac4f10af39f7943be4143fac31d483e9029769aabfcea3024f02be", + "corpus_id": "memory_evaluation_corpus:57b629504eac4f10af39f7943be4143f", + "exact_base": "f761a682164d10e2ff81ba38cd2d0c987b4f8efd", + "fixture_contract": "ace.evaluation.agent-memory-am6-evaluation-prep/v1", + "measure_count": 31, + "network_used": false, + "observation_count": 54, + "outcome_counts": { + "beneficial": 8, + "harmful": 1, + "neutral": 4, + "underpowered": 5 + }, + "policy_changes_emitted": 0, + "protocol_digest": "sha256:434a03e50ca5c3c43c3eaa9430600ac95bf00e26101400795df030db4b045729", + "protocol_id": "memory_evaluation_protocol:434a03e50ca5c3c43c3eaa9430600ac9", + "provider_credentials_used": false, + "restart_reconstruction_identical": true, + "results": [ + { + "assignment_digest": "sha256:fcc89ac82e8091bbe3a453a55721c9b597139ff3a4ad05d3f87ed5ea55bf8990", + "assignment_id": "memory_condition_assignment:fcc89ac82e8091bbe3a453a55721c9b5", + "benefit_disposition": "beneficial", + "case_id": "am1_ingestion_replay_restart", + "causality": "not_established", + "comparison_digest": "sha256:e1f82491fba8439dd7d9eebf411957c6a626ef206083d554eeade55166ad1e6e", + "comparison_id": "memory_matched_comparison:e1f82491fba8439dd7d9eebf411957c6", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:d77267a8ba383fbd83dad721bb162fd1", + "memory_run_observation:1aa1f90bcfb3d4bdbc261a7c3af189f3", + "memory_run_observation:9183701deb7bc47de8732d49f0f97bc4" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:a68f6d35434058e27108a6a90f564547510f1c16e3795d7d373d65dd678f74a8", + "assignment_id": "memory_condition_assignment:a68f6d35434058e27108a6a90f564547", + "benefit_disposition": "beneficial", + "case_id": "am2_family_extraction_and_spans", + "causality": "not_established", + "comparison_digest": "sha256:da5bc1a65828ae92e7369933ea9f4104bcca6e8e309085c8f233e42129541dfc", + "comparison_id": "memory_matched_comparison:da5bc1a65828ae92e7369933ea9f4104", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:cbd750323c3e9da6724234fd278167ce", + "memory_run_observation:2321bf0b12d0b16b7c724fc2e6c30993", + "memory_run_observation:c4d7f42f0d94f4ac4adac70760abd66c" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:ff1c2df030f0ab20213035231f6524bec596e55e59029d685bcce79ed38acde4", + "assignment_id": "memory_condition_assignment:ff1c2df030f0ab20213035231f6524be", + "benefit_disposition": "neutral", + "case_id": "am2_identity_and_unresolved", + "causality": "not_established", + "comparison_digest": "sha256:e3548e6461f483d3de048244fabbc21ce4c0a483049ad86fae0d7a0156f7405c", + "comparison_id": "memory_matched_comparison:e3548e6461f483d3de048244fabbc21c", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "not_observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:07e27e24b1aec189f9c689ba8eee3c8b", + "memory_run_observation:a5221c55f431fbc5e9615cc1106d8e8b", + "memory_run_observation:026bfa54222cba16b516f2ef9205e9c9" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:f6ac4b26e5bbd75c9133170eeabcdff5d8c6b7feeff7f953673e311ef2d0c603", + "assignment_id": "memory_condition_assignment:f6ac4b26e5bbd75c9133170eeabcdff5", + "benefit_disposition": "beneficial", + "case_id": "am2_conflict_correction_uncertainty", + "causality": "not_established", + "comparison_digest": "sha256:32f281850ca144de18d375757dc9953756e306a16e328be1581a02d8bef126c9", + "comparison_id": "memory_matched_comparison:32f281850ca144de18d375757dc99537", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:49d50df8de570b74918c2d17924887c4", + "memory_run_observation:fc8c24203d9c9153e0fb25200cc61edb", + "memory_run_observation:b2f14ea42cfc68100bf3e53e8922f8ce" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:1d8ef8d1a187ff503643f6d1a40c992493f75cb4c2bd66601f55341a30e0f32a", + "assignment_id": "memory_condition_assignment:1d8ef8d1a187ff503643f6d1a40c9924", + "benefit_disposition": "beneficial", + "case_id": "am2_instruction_isolation", + "causality": "not_established", + "comparison_digest": "sha256:2757916772fd0f30aec6bae446a2a831f3a4efc0a45738701b23c63b17b96969", + "comparison_id": "memory_matched_comparison:2757916772fd0f30aec6bae446a2a831", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:75863d76e480606daf934a8b3c6d772f", + "memory_run_observation:700f2effef477bedbd98e19b26fec22b", + "memory_run_observation:23eeba5b6b8242a6efc72014e884506c" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:86a301a72a58d87370890318bf6d2a59a29ea57d5732b13a099c03ee4be81cdb", + "assignment_id": "memory_condition_assignment:86a301a72a58d87370890318bf6d2a59", + "benefit_disposition": "beneficial", + "case_id": "am2_independent_time_axes", + "causality": "not_established", + "comparison_digest": "sha256:ba88be8306a13a6f10046630084d636737d8e97669577adb46b6e5c5234789bb", + "comparison_id": "memory_matched_comparison:ba88be8306a13a6f10046630084d6367", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:6d9762259ae111f4e34cfa52ffedfa12", + "memory_run_observation:a8792511b5b24a09aecb6254358e096c", + "memory_run_observation:f56b1e627dd4cfb30f6c2cc267df9bab" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:af72193d4154506bafb485248ca604c5dcfbe49e5c5166dfdd168aae7f73319e", + "assignment_id": "memory_condition_assignment:af72193d4154506bafb485248ca604c5", + "benefit_disposition": "neutral", + "case_id": "am1_scope_privacy_isolation", + "causality": "not_established", + "comparison_digest": "sha256:1b606fafc5467bc6cbc71cbc26405e12f2776d44b66006ced4d0c8a7ce20c82c", + "comparison_id": "memory_matched_comparison:1b606fafc5467bc6cbc71cbc26405e12", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "not_observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:be13be9ecf31a0db71d50b23c31ed441", + "memory_run_observation:d9cd800b1813c347bb87ecd48c3458a0", + "memory_run_observation:45b504821335f29c7fb2238790708b9f" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:155a2fd6883fbaf7109ce5475f43d6359d60610610a04d189aa49835ba48b6b8", + "assignment_id": "memory_condition_assignment:155a2fd6883fbaf7109ce5475f43d635", + "benefit_disposition": "neutral", + "case_id": "am3_authorization_denial", + "causality": "not_established", + "comparison_digest": "sha256:017b57a9213f1fd0db748f0b580753c6f8ad8a4f554515be29fe219380752098", + "comparison_id": "memory_matched_comparison:017b57a9213f1fd0db748f0b580753c6", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "not_observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:a58cd443bd9d30e435924c6144a426ae", + "memory_run_observation:4684c674b9d942e4dcda443ae62c9ec4", + "memory_run_observation:2b6361e9fe098aa83466d909e97d1d5d" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:39f2f271ea3fe513b28d0a4fb1361005143dce3003b82bd8abc10da5b15fb2db", + "assignment_id": "memory_condition_assignment:39f2f271ea3fe513b28d0a4fb1361005", + "benefit_disposition": "beneficial", + "case_id": "am3_stale_superseded_safety", + "causality": "not_established", + "comparison_digest": "sha256:3b6fd6b8e6ac1bcfc7ffb10def13a8087065e72b3ce3e90853cf204a1a5e6448", + "comparison_id": "memory_matched_comparison:3b6fd6b8e6ac1bcfc7ffb10def13a808", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:3a0465794c24e9a58f3d8351a6d859f0", + "memory_run_observation:87503ea7ee1dbabefed048c9e002e453", + "memory_run_observation:e373f4f0e3fd835078a25da159823980" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:9e1d76f018a7a253d715e357a7097799620a295cf3d70f65a9223a1960b46fe6", + "assignment_id": "memory_condition_assignment:9e1d76f018a7a253d715e357a7097799", + "benefit_disposition": "harmful", + "case_id": "am3_harmful_influence_probe", + "causality": "not_established", + "comparison_digest": "sha256:97da7593602cef25e0cb500e3eed1953b1ac072ee97d5c14fc0be4fc1b189a8f", + "comparison_id": "memory_matched_comparison:97da7593602cef25e0cb500e3eed1953", + "correctness": "incorrect", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:e629d0dd0ead7257a1ccdc047e5cbf4d", + "memory_run_observation:117bcc53337e88baee4fb7ab86684db0", + "memory_run_observation:153b4eb5c17e9989a200b542d4bbe001" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:58f2fb9be0d867d2dca893afea42cf53ef465407a8dd7a7e6808b30d3e191899", + "assignment_id": "memory_condition_assignment:58f2fb9be0d867d2dca893afea42cf53", + "benefit_disposition": "beneficial", + "case_id": "am3_manifest_selection_omission", + "causality": "not_established", + "comparison_digest": "sha256:50599871ebd15faeb65210afaca8f22e38e14fdbd9f04f234ead46e4d4c4be13", + "comparison_id": "memory_matched_comparison:50599871ebd15faeb65210afaca8f22e", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:24a19aa95e67e9271e7a393d15887177", + "memory_run_observation:4a34310ce54230cebac64ba19b1720da", + "memory_run_observation:82b078063720dfe257b133bbb85d6678" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:f26eb8ca1dd226f80dc03f96a8cccd3d439079de86f2aef1ba8267745cb26bec", + "assignment_id": "memory_condition_assignment:f26eb8ca1dd226f80dc03f96a8cccd3d", + "benefit_disposition": "underpowered", + "case_id": "am3_degraded_retrieval_signal", + "causality": "not_established", + "comparison_digest": "sha256:48d3353b1cf03a53b3688e7ac6156b8f365fabdf288a3c55d26f9107aaeb0022", + "comparison_id": "memory_matched_comparison:48d3353b1cf03a53b3688e7ac6156b8f", + "correctness": "underpowered", + "gate": "runnable_am3", + "material_influence": "underpowered", + "missing_measurements": [ + "full_context:rank_quality_bps:vector_signal_unavailable", + "memory:rank_quality_bps:vector_signal_unavailable", + "no_memory:rank_quality_bps:not_applicable_without_memory" + ], + "observation_ids": [ + "memory_run_observation:0edc18d000e8cdae07c17fb1f0f38b07", + "memory_run_observation:8fd12781f840d56e9e439b9f3153300c", + "memory_run_observation:0a28d7ad2f0db33f9e0e75322acd2ad4" + ], + "paired_and_controlled": false + }, + { + "assignment_digest": "sha256:f9828ea49124f0beb1bf328ae492ecf7692930770fdacb06007b4300b73b6a2f", + "assignment_id": "memory_condition_assignment:f9828ea49124f0beb1bf328ae492ecf7", + "benefit_disposition": "beneficial", + "case_id": "am3_later_restart_material_use", + "causality": "not_established", + "comparison_digest": "sha256:4d38f752d449bace58e48c96b8500d28af250bc2127fdcd8986f000b2664dce0", + "comparison_id": "memory_matched_comparison:4d38f752d449bace58e48c96b8500d28", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:f3a57c9290de840c77470020c78e3fba", + "memory_run_observation:88e19700abc4e6631374c0c6e40184a0", + "memory_run_observation:784cd6dd5617c4cf2fd50c338a336ffe" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:5f9be4cee47b9e0bf31e07473e5b34f2bd4fb9e49b8a8b9db22b6be964e21a5a", + "assignment_id": "memory_condition_assignment:5f9be4cee47b9e0bf31e07473e5b34f2", + "benefit_disposition": "neutral", + "case_id": "am3_material_but_neutral", + "causality": "not_established", + "comparison_digest": "sha256:f4e73dee10ca9f6a289102d56976285bd92160c626aaca04d7ae525b1b0aa909", + "comparison_id": "memory_matched_comparison:f4e73dee10ca9f6a289102d56976285b", + "correctness": "correct", + "gate": "runnable_am3", + "material_influence": "observed", + "missing_measurements": [], + "observation_ids": [ + "memory_run_observation:42422744a49fe02c399790547d557764", + "memory_run_observation:27d8cc5686f1c637cf520397eed07bd4", + "memory_run_observation:2fb1da45f0888dc9b34b5db760808195" + ], + "paired_and_controlled": true + }, + { + "assignment_digest": "sha256:5940bfe597ac67253d5706fdc0e8b25c099415c6cf4b48d75ae0d3e03d078068", + "assignment_id": "memory_condition_assignment:5940bfe597ac67253d5706fdc0e8b25c", + "benefit_disposition": "underpowered", + "case_id": "am3_missing_resource_telemetry", + "causality": "not_established", + "comparison_digest": "sha256:4d114e9afbeb3cbfaa4bfa2143c11c4ebbbc4310c159cc4df82d54dcff8a2fba", + "comparison_id": "memory_matched_comparison:4d114e9afbeb3cbfaa4bfa2143c11c4e", + "correctness": "underpowered", + "gate": "runnable_am3", + "material_influence": "underpowered", + "missing_measurements": [ + "memory:cost_microunits:cost_telemetry_unavailable" + ], + "observation_ids": [ + "memory_run_observation:20334e8ee7c0cd69ee2e4726d2a56b59", + "memory_run_observation:1017cc5a8c783ab54f0d4c55f43a5bc9", + "memory_run_observation:292b8e29e58de9cb0dbf6793ce549ecb" + ], + "paired_and_controlled": false + }, + { + "assignment_digest": "sha256:9dfe82fc11eae9e2a50b44636b126dd7e7ef00556b4d99ea1eea1514df9a4a98", + "assignment_id": "memory_condition_assignment:9dfe82fc11eae9e2a50b44636b126dd7", + "benefit_disposition": "underpowered", + "case_id": "am4_retention_expiry_placeholder", + "causality": "not_established", + "comparison_digest": "sha256:6befe9e2c7db2b7770e272af5b735c0007cc5fbd5b8d03cb5a90506161dfb51c", + "comparison_id": "memory_matched_comparison:6befe9e2c7db2b7770e272af5b735c00", + "correctness": "underpowered", + "gate": "future_accepted_am4", + "material_influence": "underpowered", + "missing_measurements": [ + "full_context:omission_coverage_bps:future_accepted_am4_coordinate_required", + "full_context:retrieval_recall_bps:future_accepted_am4_coordinate_required", + "full_context:task_correctness_bps:future_accepted_am4_coordinate_required", + "future_accepted_am4_coordinate:required", + "memory:omission_coverage_bps:future_accepted_am4_coordinate_required", + "memory:retrieval_recall_bps:future_accepted_am4_coordinate_required", + "memory:task_correctness_bps:future_accepted_am4_coordinate_required", + "no_memory:omission_coverage_bps:future_accepted_am4_coordinate_required", + "no_memory:retrieval_recall_bps:future_accepted_am4_coordinate_required", + "no_memory:task_correctness_bps:future_accepted_am4_coordinate_required" + ], + "observation_ids": [ + "memory_run_observation:755fc9c50abee66b42f365cceedc0ff7", + "memory_run_observation:72c46504851d87028e7579caff5d8851", + "memory_run_observation:dce306a6414458ec4d698a757a45b3d0" + ], + "paired_and_controlled": false + }, + { + "assignment_digest": "sha256:9f1cfa79fa60866c7a60a04fe1acfe07e2c5caef0d602a1bdc9fe7345e602732", + "assignment_id": "memory_condition_assignment:9f1cfa79fa60866c7a60a04fe1acfe07", + "benefit_disposition": "underpowered", + "case_id": "am4_export_import_placeholder", + "causality": "not_established", + "comparison_digest": "sha256:a7e2d966cb2dc4f2813e1e2d40bdd0970ff307266837a56f9027df2cd0856271", + "comparison_id": "memory_matched_comparison:a7e2d966cb2dc4f2813e1e2d40bdd097", + "correctness": "underpowered", + "gate": "future_accepted_am4", + "material_influence": "underpowered", + "missing_measurements": [ + "full_context:citation_correctness_bps:future_accepted_am4_coordinate_required", + "full_context:replay_correctness_bps:future_accepted_am4_coordinate_required", + "full_context:task_correctness_bps:future_accepted_am4_coordinate_required", + "future_accepted_am4_coordinate:required", + "memory:citation_correctness_bps:future_accepted_am4_coordinate_required", + "memory:replay_correctness_bps:future_accepted_am4_coordinate_required", + "memory:task_correctness_bps:future_accepted_am4_coordinate_required", + "no_memory:citation_correctness_bps:future_accepted_am4_coordinate_required", + "no_memory:replay_correctness_bps:future_accepted_am4_coordinate_required", + "no_memory:task_correctness_bps:future_accepted_am4_coordinate_required" + ], + "observation_ids": [ + "memory_run_observation:e259ad712717bd2508f9789be7c43c1b", + "memory_run_observation:44d4f40e0f48f9a7ebd32e98a295ef98", + "memory_run_observation:53001b3d0ef1262c09031106e02d4cb1" + ], + "paired_and_controlled": false + }, + { + "assignment_digest": "sha256:4f77cfd8908f6989da3d46c0ee0d35ee292c942dc526f09872a626e82962a452", + "assignment_id": "memory_condition_assignment:4f77cfd8908f6989da3d46c0ee0d35ee", + "benefit_disposition": "underpowered", + "case_id": "am4_hard_erasure_placeholder", + "causality": "not_established", + "comparison_digest": "sha256:3e923c96b466c7c7ebc3fea57e85183114d471a003e80aed248f076b38ff92ee", + "comparison_id": "memory_matched_comparison:3e923c96b466c7c7ebc3fea57e851831", + "correctness": "underpowered", + "gate": "future_accepted_am4", + "material_influence": "underpowered", + "missing_measurements": [ + "full_context:omission_coverage_bps:future_accepted_am4_coordinate_required", + "full_context:task_correctness_bps:future_accepted_am4_coordinate_required", + "full_context:unauthorized_retrieval_count:future_accepted_am4_coordinate_required", + "future_accepted_am4_coordinate:required", + "memory:omission_coverage_bps:future_accepted_am4_coordinate_required", + "memory:task_correctness_bps:future_accepted_am4_coordinate_required", + "memory:unauthorized_retrieval_count:future_accepted_am4_coordinate_required", + "no_memory:omission_coverage_bps:future_accepted_am4_coordinate_required", + "no_memory:task_correctness_bps:future_accepted_am4_coordinate_required", + "no_memory:unauthorized_retrieval_count:future_accepted_am4_coordinate_required" + ], + "observation_ids": [ + "memory_run_observation:1b1ef305a37efd202cd4742240059e69", + "memory_run_observation:39e998e4025cbedbfa7a89451a8d214a", + "memory_run_observation:ba7a686d0797250e76d0f8c596507790" + ], + "paired_and_controlled": false + } + ], + "route_frequency": { + "route:full-context-control": 15, + "route:gated-future-am4": 9, + "route:no-memory-control": 15, + "route:structured-or-fused-am3": 15 + }, + "tier_frequency": { + "tier:disabled": 15, + "tier:full-context": 15, + "tier:structured_or_fused": 15, + "tier:unavailable": 9 + } +} diff --git a/evaluations/source/agent_memory_am6_evaluation.py b/evaluations/source/agent_memory_am6_evaluation.py new file mode 100644 index 0000000..54b6a28 --- /dev/null +++ b/evaluations/source/agent_memory_am6_evaluation.py @@ -0,0 +1,410 @@ +"""Provider-free deterministic AM6 Agent Memory evaluation-preparation fixture.""" + +from __future__ import annotations + +import json +from collections import Counter +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from ace.core.agent_composition import ExactArtifactReferenceV1Alpha1 +from ace.core.contracts import canonical_hash +from ace.intelligence.agent_memory_evaluation import compare_memory_conditions +from ace.intelligence.contracts.agent_memory_evaluation import ( + BenefitDisposition, + EvaluationCaseGate, + MeasureAvailability, + MeasureDirection, + MeasureUnit, + MemoryConditionAssignmentV1Alpha1, + MemoryConditionPlanV1Alpha1, + MemoryEvaluationCaseV1Alpha1, + MemoryEvaluationCondition, + MemoryEvaluationCorpusV1Alpha1, + MemoryEvaluationProtocolV1Alpha1, + MemoryMatchedCoordinatesV1Alpha1, + MemoryMeasure, + MemoryMeasureDefinitionV1Alpha1, + MemoryMeasureObservationV1Alpha1, + MemoryRunObservationV1Alpha1, + memory_evaluation_reference, +) + +FIXTURE_PATH = Path(__file__).parents[1] / "fixtures" / "agent_memory_am6_evaluation_prep_v1.json" +BASE = datetime(2026, 8, 12, 18, 0, tzinfo=UTC) + +_BPS_LOWER = { + MemoryMeasure.IDENTITY_ERROR_BPS, + MemoryMeasure.UNRESOLVED_IDENTITY_BPS, +} +_COUNTS = { + MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT, + MemoryMeasure.STALE_INFLUENCE_COUNT, + MemoryMeasure.SUPERSEDED_INFLUENCE_COUNT, + MemoryMeasure.PROVIDER_CALLS, + MemoryMeasure.CACHE_REUSE_COUNT, +} +_RESOURCE_UNITS = { + MemoryMeasure.CONTEXT_TOKENS: MeasureUnit.TOKENS, + MemoryMeasure.RESIDUAL_WINDOW_TOKENS: MeasureUnit.TOKENS, + MemoryMeasure.LATENCY_MS: MeasureUnit.MILLISECONDS, + MemoryMeasure.COST_MICROUNITS: MeasureUnit.MICROUNITS, +} + + +def _ref(key: str, contract: str) -> ExactArtifactReferenceV1Alpha1: + return ExactArtifactReferenceV1Alpha1( + artifact_id=key, + artifact_digest=f"sha256:{canonical_hash([key, contract])}", + artifact_contract=contract, + ) + + +def _definition(measure: MemoryMeasure) -> MemoryMeasureDefinitionV1Alpha1: + if measure is MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT: + direction = MeasureDirection.ZERO_TOLERANCE + elif measure in _BPS_LOWER or measure in { + MemoryMeasure.STALE_INFLUENCE_COUNT, + MemoryMeasure.SUPERSEDED_INFLUENCE_COUNT, + MemoryMeasure.CONTEXT_TOKENS, + MemoryMeasure.LATENCY_MS, + MemoryMeasure.PROVIDER_CALLS, + MemoryMeasure.COST_MICROUNITS, + }: + direction = MeasureDirection.LOWER_IS_BETTER + elif measure in {MemoryMeasure.RESIDUAL_WINDOW_TOKENS, MemoryMeasure.CACHE_REUSE_COUNT}: + direction = MeasureDirection.DESCRIPTIVE + else: + direction = MeasureDirection.HIGHER_IS_BETTER + if measure in _RESOURCE_UNITS: + unit = _RESOURCE_UNITS[measure] + elif measure in _COUNTS: + unit = MeasureUnit.COUNT + else: + unit = MeasureUnit.BASIS_POINTS + return MemoryMeasureDefinitionV1Alpha1(measure=measure, unit=unit, direction=direction) + + +def _build_corpus(fixture: dict) -> MemoryEvaluationCorpusV1Alpha1: + cases = [] + for raw in fixture["cases"]: + gate = EvaluationCaseGate(raw.get("gate", EvaluationCaseGate.RUNNABLE_AM3.value)) + cases.append( + MemoryEvaluationCaseV1Alpha1( + case_id=raw["case_id"], + title=raw["title"], + coverage_tags=tuple(raw["coverage_tags"]), + required_measures=tuple(MemoryMeasure(item) for item in raw["required_measures"]), + gate=gate, + future_required_coordinate=( + "future_accepted_am4_coordinate" if gate is EvaluationCaseGate.FUTURE_ACCEPTED_AM4 else None + ), + ) + ) + return MemoryEvaluationCorpusV1Alpha1( + corpus_key="agent-memory-am6-evaluation-prep-v1", + source_artifacts=tuple(_ref(key, contract) for key, contract in fixture["source_artifacts"]), + cases=tuple(cases), + frozen_at=BASE, + ) + + +def _build_protocol(fixture: dict, corpus: MemoryEvaluationCorpusV1Alpha1) -> MemoryEvaluationProtocolV1Alpha1: + raw = fixture["protocol"] + return MemoryEvaluationProtocolV1Alpha1( + protocol_key="agent-memory-am6-measurement-v1", + corpus=memory_evaluation_reference(corpus), + conditions=tuple(MemoryEvaluationCondition), + measure_definitions=tuple(_definition(item) for item in MemoryMeasure), + minimum_beneficial_gain_bps=raw["minimum_beneficial_gain_bps"], + maximum_full_context_correctness_gap_bps=raw["maximum_full_context_correctness_gap_bps"], + preregistered_at=BASE + timedelta(seconds=1), + ) + + +def _matched_coordinates(fixture: dict, case_id: str) -> MemoryMatchedCoordinatesV1Alpha1: + held = fixture["protocol"]["held_constants"] + return MemoryMatchedCoordinatesV1Alpha1( + task=_ref(f"task:{case_id}", "ace.evaluation.agent-memory-task/v1"), + provider=_ref(*held["provider"]), + model=_ref(*held["model"]), + prompt_contract=_ref(*held["prompt_contract"]), + decision_schema=_ref(*held["decision_schema"]), + toolset=_ref(*held["toolset"]), + configuration=_ref(*held["configuration"]), + ) + + +def _build_assignment( + fixture: dict, + corpus: MemoryEvaluationCorpusV1Alpha1, + protocol: MemoryEvaluationProtocolV1Alpha1, + raw_case: dict, + *, + assigned_at: datetime, +) -> MemoryConditionAssignmentV1Alpha1: + modes = { + MemoryEvaluationCondition.MEMORY: "authorized_selected", + MemoryEvaluationCondition.NO_MEMORY: "disabled", + MemoryEvaluationCondition.FULL_CONTEXT: "full_authorized_context", + } + return MemoryConditionAssignmentV1Alpha1( + protocol=memory_evaluation_reference(protocol), + corpus=memory_evaluation_reference(corpus), + case_id=raw_case["case_id"], + matched_coordinates=_matched_coordinates(fixture, raw_case["case_id"]), + condition_plans=tuple( + MemoryConditionPlanV1Alpha1(condition=condition, memory_mode=modes[condition]) + for condition in MemoryEvaluationCondition + ), + assigned_at=assigned_at, + ) + + +def _default_value(measure: MemoryMeasure, condition: MemoryEvaluationCondition, raw_case: dict) -> int: + if measure is MemoryMeasure.TASK_CORRECTNESS_BPS: + scores = raw_case.get("scores", {"memory": 10000, "no_memory": 5000, "full_context": 10000}) + return int(scores[condition.value]) + if measure in _BPS_LOWER or measure in { + MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT, + MemoryMeasure.STALE_INFLUENCE_COUNT, + MemoryMeasure.SUPERSEDED_INFLUENCE_COUNT, + }: + return 0 + resources = { + MemoryEvaluationCondition.MEMORY: { + MemoryMeasure.CONTEXT_TOKENS: 120, + MemoryMeasure.RESIDUAL_WINDOW_TOKENS: 3880, + MemoryMeasure.LATENCY_MS: 10, + MemoryMeasure.PROVIDER_CALLS: 0, + MemoryMeasure.CACHE_REUSE_COUNT: 0, + MemoryMeasure.COST_MICROUNITS: 0, + }, + MemoryEvaluationCondition.NO_MEMORY: { + MemoryMeasure.CONTEXT_TOKENS: 0, + MemoryMeasure.RESIDUAL_WINDOW_TOKENS: 4000, + MemoryMeasure.LATENCY_MS: 8, + MemoryMeasure.PROVIDER_CALLS: 0, + MemoryMeasure.CACHE_REUSE_COUNT: 0, + MemoryMeasure.COST_MICROUNITS: 0, + }, + MemoryEvaluationCondition.FULL_CONTEXT: { + MemoryMeasure.CONTEXT_TOKENS: 1000, + MemoryMeasure.RESIDUAL_WINDOW_TOKENS: 3000, + MemoryMeasure.LATENCY_MS: 12, + MemoryMeasure.PROVIDER_CALLS: 0, + MemoryMeasure.CACHE_REUSE_COUNT: 0, + MemoryMeasure.COST_MICROUNITS: 0, + }, + } + if measure in resources[condition]: + return resources[condition][measure] + if measure in { + MemoryMeasure.SELECTED_RATE_BPS, + MemoryMeasure.INJECTED_RATE_BPS, + MemoryMeasure.REFLECTED_RATE_BPS, + MemoryMeasure.DECISION_MATERIAL_RATE_BPS, + }: + return 0 if condition is MemoryEvaluationCondition.NO_MEMORY else 10_000 + return 10_000 + + +def _decision_digest(raw_case: dict, condition: MemoryEvaluationCondition) -> str: + if raw_case.get("decision_mode") == "same": + material = [raw_case["case_id"], "same-decision"] + elif raw_case["case_id"] == "am3_harmful_influence_probe": + material = [raw_case["case_id"], "bad" if condition is MemoryEvaluationCondition.MEMORY else "good"] + else: + material = [ + raw_case["case_id"], + "bounded-alpha" if condition is not MemoryEvaluationCondition.NO_MEMORY else "bounded-beta", + ] + return f"sha256:{canonical_hash(material)}" + + +def _build_observation( + corpus: MemoryEvaluationCorpusV1Alpha1, + protocol: MemoryEvaluationProtocolV1Alpha1, + assignment: MemoryConditionAssignmentV1Alpha1, + raw_case: dict, + condition: MemoryEvaluationCondition, + *, + observed_at: datetime, +) -> MemoryRunObservationV1Alpha1: + del corpus + gate = raw_case.get("gate") == EvaluationCaseGate.FUTURE_ACCEPTED_AM4.value + unavailable = raw_case.get("unavailable", {}).get(condition.value, {}) + overrides = raw_case.get("overrides", {}).get(condition.value, {}) + measurements = [] + for measure in MemoryMeasure: + if gate: + measurements.append( + MemoryMeasureObservationV1Alpha1( + measure=measure, + availability=MeasureAvailability.UNAVAILABLE, + unavailable_reason="future_accepted_am4_coordinate_required", + ) + ) + elif measure.value in unavailable: + measurements.append( + MemoryMeasureObservationV1Alpha1( + measure=measure, + availability=MeasureAvailability.UNAVAILABLE, + unavailable_reason=unavailable[measure.value], + ) + ) + else: + measurements.append( + MemoryMeasureObservationV1Alpha1( + measure=measure, + availability=MeasureAvailability.AVAILABLE, + value=int(overrides.get(measure.value, _default_value(measure, condition, raw_case))), + ) + ) + if raw_case["case_id"] == "am2_family_extraction_and_spans": + for family in ( + "identity", + "learned_fact", + "active_context", + "preference", + "instruction_policy_proposal", + "uncertainty", + "correction", + ): + for measure in ( + MemoryMeasure.EXTRACTION_PRECISION_BPS, + MemoryMeasure.EXTRACTION_RECALL_BPS, + MemoryMeasure.SOURCE_SPAN_ACCURACY_BPS, + ): + measurements.append( + MemoryMeasureObservationV1Alpha1( + measure=measure, + availability=MeasureAvailability.AVAILABLE, + value=10_000 if condition is not MemoryEvaluationCondition.NO_MEMORY else 0, + stratum=f"family:{family}", + ) + ) + if gate: + route_ref = "route:gated-future-am4" + tier_ref = "tier:unavailable" + evidence = () + elif condition is MemoryEvaluationCondition.NO_MEMORY: + route_ref = "route:no-memory-control" + tier_ref = "tier:disabled" + evidence = (_ref(f"assignment:{raw_case['case_id']}:no-memory", "ace.evaluation.condition/v1"),) + elif condition is MemoryEvaluationCondition.FULL_CONTEXT: + route_ref = "route:full-context-control" + tier_ref = "tier:full-context" + evidence = (_ref(f"context:{raw_case['case_id']}:full", "ace.context.manifest/v1"),) + else: + route_ref = "route:structured-or-fused-am3" + tier_ref = "tier:structured_or_fused" + evidence = ( + _ref(f"recall:{raw_case['case_id']}", "ace.intelligence.memory-recall-receipt/v1alpha1"), + _ref(f"manifest:{raw_case['case_id']}", "ace.context.manifest/v1"), + _ref(f"use:{raw_case['case_id']}", "intelligence-use-receipt-v1"), + ) + return MemoryRunObservationV1Alpha1( + protocol=memory_evaluation_reference(protocol), + assignment=memory_evaluation_reference(assignment), + case_id=raw_case["case_id"], + condition=condition, + decision_digest=_decision_digest(raw_case, condition), + route_ref=route_ref, + tier_ref=tier_ref, + evidence_artifacts=evidence, + measurements=tuple(measurements), + observed_at=observed_at, + ) + + +def run_provider_free_fixture() -> dict: + fixture = json.loads(FIXTURE_PATH.read_text()) + corpus = _build_corpus(fixture) + protocol = _build_protocol(fixture, corpus) + results = [] + route_counts: Counter[str] = Counter() + tier_counts: Counter[str] = Counter() + outcome_counts: Counter[str] = Counter() + for index, raw_case in enumerate(fixture["cases"], start=1): + assigned_at = protocol.preregistered_at + timedelta(minutes=index) + assignment = _build_assignment(fixture, corpus, protocol, raw_case, assigned_at=assigned_at) + observations = tuple( + _build_observation( + corpus, + protocol, + assignment, + raw_case, + condition, + observed_at=assigned_at + timedelta(seconds=condition_index + 1), + ) + for condition_index, condition in enumerate(MemoryEvaluationCondition) + ) + comparison = compare_memory_conditions( + corpus=corpus, + protocol=protocol, + assignment=assignment, + observations=observations, + compared_at=assigned_at + timedelta(seconds=10), + ) + expected = BenefitDisposition(raw_case["expected"]) + assert comparison.benefit is expected + route_counts.update(item.route_ref for item in observations) + tier_counts.update(item.tier_ref for item in observations) + outcome_counts.update((comparison.benefit.value,)) + results.append( + { + "case_id": raw_case["case_id"], + "gate": raw_case.get("gate", EvaluationCaseGate.RUNNABLE_AM3.value), + "assignment_id": assignment.assignment_id, + "assignment_digest": assignment.assignment_digest, + "observation_ids": [item.observation_id for item in observations], + "comparison_id": comparison.comparison_id, + "comparison_digest": comparison.comparison_digest, + "paired_and_controlled": comparison.paired_and_controlled, + "material_influence": comparison.material_influence.value, + "benefit_disposition": comparison.benefit.value, + "correctness": comparison.correctness.value, + "causality": comparison.causality.value, + "missing_measurements": list(comparison.missing_measurements), + } + ) + rerun_corpus = _build_corpus(fixture) + rerun_protocol = _build_protocol(fixture, rerun_corpus) + return { + "fixture_contract": fixture["contract"], + "exact_base": fixture["exact_base"], + "corpus_id": corpus.corpus_id, + "corpus_digest": corpus.corpus_digest, + "protocol_id": protocol.protocol_id, + "protocol_digest": protocol.protocol_digest, + "case_count": len(results), + "condition_count": len(MemoryEvaluationCondition), + "observation_count": len(results) * len(MemoryEvaluationCondition), + "measure_count": len(MemoryMeasure), + "outcome_counts": dict(sorted(outcome_counts.items())), + "route_frequency": dict(sorted(route_counts.items())), + "tier_frequency": dict(sorted(tier_counts.items())), + "am3_runnable_cases": sum(item.gate is EvaluationCaseGate.RUNNABLE_AM3 for item in corpus.cases), + "am4_gated_placeholders": sum(item.gate is EvaluationCaseGate.FUTURE_ACCEPTED_AM4 for item in corpus.cases), + "restart_reconstruction_identical": ( + memory_evaluation_reference(rerun_corpus) == memory_evaluation_reference(corpus) + and memory_evaluation_reference(rerun_protocol) == memory_evaluation_reference(protocol) + ), + "network_used": False, + "provider_credentials_used": False, + "policy_changes_emitted": 0, + "results": results, + } + + +__all__ = [ + "BASE", + "FIXTURE_PATH", + "_build_assignment", + "_build_corpus", + "_build_observation", + "_build_protocol", + "_ref", + "run_provider_free_fixture", +] diff --git a/scripts/verify_agent_memory_am6_evaluation.py b/scripts/verify_agent_memory_am6_evaluation.py new file mode 100644 index 0000000..2659ad5 --- /dev/null +++ b/scripts/verify_agent_memory_am6_evaluation.py @@ -0,0 +1,29 @@ +"""Run the provider-free AM6 Agent Memory evaluation-preparation fixture.""" + +from __future__ import annotations + +import argparse +import json + +from evaluations.source.agent_memory_am6_evaluation import run_provider_free_fixture + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--json", action="store_true", help="emit canonical machine-readable output") + args = parser.parse_args() + result = run_provider_free_fixture() + if args.json: + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + else: + print( + "AM6 evaluation prep: " + f"{result['case_count']} cases, {result['observation_count']} observations, " + f"{result['am4_gated_placeholders']} AM4 placeholders, " + f"restart reconstruction={result['restart_reconstruction_identical']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/agent_memory/am6/test_evaluation_prep.py b/tests/agent_memory/am6/test_evaluation_prep.py new file mode 100644 index 0000000..5d634e5 --- /dev/null +++ b/tests/agent_memory/am6/test_evaluation_prep.py @@ -0,0 +1,322 @@ +"""AM6 provider-free measurement contracts and deterministic harness conformance.""" + +from __future__ import annotations + +import ast +import json +import os +import subprocess +import sys +from datetime import timedelta +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ace.intelligence.agent_memory_evaluation import compare_memory_conditions +from ace.intelligence.contracts.agent_memory_evaluation import ( + BenefitDisposition, + CausalityDisposition, + EvaluationCaseGate, + MaterialInfluenceDisposition, + MeasureAvailability, + MeasureDirection, + MemoryEvaluationCondition, + MemoryEvaluationProtocolV1Alpha1, + MemoryMatchedComparisonV1Alpha1, + MemoryMeasure, + MemoryRunObservationV1Alpha1, +) +from evaluations.source.agent_memory_am6_evaluation import ( + FIXTURE_PATH, + _build_assignment, + _build_corpus, + _build_observation, + _build_protocol, + run_provider_free_fixture, +) + +pytestmark = pytest.mark.unit +REPO = Path(__file__).resolve().parents[3] +RESULT_PATH = REPO / "evaluations/results/agent_memory_am6_evaluation_prep_v1.json" +CONTRACT_PATH = REPO / "ace/intelligence/contracts/agent_memory_evaluation.py" +EVALUATOR_PATH = REPO / "ace/intelligence/agent_memory_evaluation.py" +THIN_MCP_PATH = REPO / "ace_mcp_client/server.py" + + +def _fixture() -> dict: + return json.loads(FIXTURE_PATH.read_text()) + + +def _scenario(case_id: str): + fixture = _fixture() + corpus = _build_corpus(fixture) + protocol = _build_protocol(fixture, corpus) + raw_case = next(item for item in fixture["cases"] if item["case_id"] == case_id) + assigned_at = protocol.preregistered_at + timedelta(minutes=1) + assignment = _build_assignment(fixture, corpus, protocol, raw_case, assigned_at=assigned_at) + observations = tuple( + _build_observation( + corpus, + protocol, + assignment, + raw_case, + condition, + observed_at=assigned_at + timedelta(seconds=index + 1), + ) + for index, condition in enumerate(MemoryEvaluationCondition) + ) + return corpus, protocol, assignment, observations + + +def _imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(), filename=str(path)) + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(item.name for item in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + names.add(node.module) + return names + + +def test_corpus_covers_am0_am3_measurement_families_and_exact_am4_placeholders() -> None: + corpus = _build_corpus(_fixture()) + tags = {tag for case in corpus.cases for tag in case.coverage_tags} + assert { + "episodic_experience", + "identity", + "learned_fact", + "active_context", + "preference", + "instruction_policy_proposal", + "uncertainty", + "correction", + "conflict", + "ledger_time", + "knowledge_time", + "world_time", + "scope", + "privacy", + "authorization_denial", + "stale", + "superseded", + "context_manifest", + "degraded_retrieval", + "restart", + "decision_material", + }.issubset(tags) + gated = [case for case in corpus.cases if case.gate is EvaluationCaseGate.FUTURE_ACCEPTED_AM4] + assert {case.case_id for case in gated} == { + "am4_export_import_placeholder", + "am4_hard_erasure_placeholder", + "am4_retention_expiry_placeholder", + } + assert all(case.future_required_coordinate == "future_accepted_am4_coordinate" for case in gated) + + +def test_protocol_freezes_complete_measure_registry_controls_and_policy_inertness() -> None: + corpus = _build_corpus(_fixture()) + protocol = _build_protocol(_fixture(), corpus) + assert set(protocol.conditions) == set(MemoryEvaluationCondition) + assert {item.measure for item in protocol.measure_definitions} == set(MemoryMeasure) + unauthorized = next( + item for item in protocol.measure_definitions if item.measure is MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT + ) + assert unauthorized.direction is MeasureDirection.ZERO_TOLERANCE + assert protocol.provider_required is False + assert protocol.network_required is False + assert not any( + ( + protocol.changes_rank_policy, + protocol.changes_retention_policy, + protocol.changes_consolidation_policy, + protocol.changes_promotion_policy, + protocol.changes_roster_or_authority, + protocol.delivers_or_sends_effect, + ) + ) + + +def test_assignment_holds_provider_model_prompt_task_schema_toolset_and_configuration_constant() -> None: + _, _, assignment, _ = _scenario("am3_manifest_selection_omission") + assert {item.condition for item in assignment.condition_plans} == set(MemoryEvaluationCondition) + coordinates = assignment.matched_coordinates + assert coordinates.provider.artifact_id == "provider:deterministic-fixture" + assert coordinates.model.artifact_id == "model:none" + assert coordinates.task.artifact_id == "task:am3_manifest_selection_omission" + assert coordinates.prompt_contract.artifact_id == "prompt:am6-frozen-v1" + assert coordinates.decision_schema.artifact_id == "decision-schema:bounded-choice-v1" + assert coordinates.toolset.artifact_id == "toolset:eleven-thin-mcp-unchanged" + assert coordinates.configuration.artifact_id == "configuration:am6-provider-free-v1" + + +def test_provider_free_fixture_reports_beneficial_harmful_neutral_and_underpowered_without_causality() -> None: + result = run_provider_free_fixture() + assert result["outcome_counts"] == {"beneficial": 8, "harmful": 1, "neutral": 4, "underpowered": 5} + assert result["case_count"] == 18 + assert result["condition_count"] == 3 + assert result["observation_count"] == 54 + assert result["measure_count"] == len(MemoryMeasure) + assert result["network_used"] is False + assert result["provider_credentials_used"] is False + assert result["policy_changes_emitted"] == 0 + assert result["restart_reconstruction_identical"] is True + assert {item["causality"] for item in result["results"]} == {CausalityDisposition.NOT_ESTABLISHED.value} + + +def test_material_influence_is_distinct_from_neutral_benefit_and_correctness() -> None: + corpus, protocol, assignment, observations = _scenario("am3_material_but_neutral") + comparison = compare_memory_conditions( + corpus=corpus, + protocol=protocol, + assignment=assignment, + observations=observations, + compared_at=assignment.assigned_at + timedelta(seconds=10), + ) + assert comparison.material_influence is MaterialInfluenceDisposition.OBSERVED + assert comparison.benefit is BenefitDisposition.NEUTRAL + assert comparison.correctness.value == "correct" + assert comparison.causality is CausalityDisposition.NOT_ESTABLISHED + + +def test_zero_tolerance_unauthorized_retrieval_is_harmful_and_never_policy_applying() -> None: + corpus, protocol, assignment, observations = _scenario("am3_authorization_denial") + memory = next(item for item in observations if item.condition is MemoryEvaluationCondition.MEMORY) + changed = memory.model_dump(mode="python") + changed["observation_id"] = None + changed["observation_digest"] = None + measurements = [] + for item in changed["measurements"]: + if item["measure"] is MemoryMeasure.UNAUTHORIZED_RETRIEVAL_COUNT and item["stratum"] is None: + item["value"] = 1 + measurements.append(item) + changed["measurements"] = tuple(measurements) + unauthorized = MemoryRunObservationV1Alpha1.model_validate(changed) + mutated = tuple( + unauthorized if item.condition is MemoryEvaluationCondition.MEMORY else item for item in observations + ) + comparison = compare_memory_conditions( + corpus=corpus, + protocol=protocol, + assignment=assignment, + observations=mutated, + compared_at=assignment.assigned_at + timedelta(seconds=10), + ) + assert comparison.benefit is BenefitDisposition.HARMFUL + assert comparison.correctness.value == "incorrect" + assert comparison.changes_any_policy is False + assert comparison.changes_authority_or_roster is False + + +def test_missing_telemetry_and_optional_signals_are_explicitly_underpowered() -> None: + result = run_provider_free_fixture() + by_case = {item["case_id"]: item for item in result["results"]} + for case_id in ("am3_degraded_retrieval_signal", "am3_missing_resource_telemetry"): + item = by_case[case_id] + assert item["benefit_disposition"] == "underpowered" + assert item["material_influence"] == "underpowered" + assert item["paired_and_controlled"] is False + assert item["missing_measurements"] + + +def test_am4_placeholders_are_unavailable_and_do_not_mint_runtime_contracts() -> None: + result = run_provider_free_fixture() + gated = [item for item in result["results"] if item["gate"] == "future_accepted_am4"] + assert len(gated) == 3 + assert all(item["benefit_disposition"] == "underpowered" for item in gated) + assert all("future_accepted_am4_coordinate:required" in item["missing_measurements"] for item in gated) + source = CONTRACT_PATH.read_text() + EVALUATOR_PATH.read_text() + assert "agent_memory_am4" not in source + assert "RetentionPolicy" not in source + assert "ErasureService" not in source + assert "ExportService" not in source + + +def test_contracts_are_provider_host_extension_and_am4_runtime_free() -> None: + forbidden = ("ace_mcp_client", "core.engine", "extensions", "fastapi", "httpx", "surrealdb") + offenders = [ + f"{path.relative_to(REPO)}:{name}" + for path in (CONTRACT_PATH, EVALUATOR_PATH) + for name in _imports(path) + if name.startswith(forbidden) + ] + assert offenders == [] + + +def test_unknown_or_mutated_protocol_material_fails_strictly() -> None: + corpus = _build_corpus(_fixture()) + protocol = _build_protocol(_fixture(), corpus) + changed = protocol.model_dump(mode="python") + changed["contract"] = "ace.intelligence.agent-memory-evaluation-protocol/v9" + changed["protocol_id"] = None + changed["protocol_digest"] = None + with pytest.raises(ValidationError): + MemoryEvaluationProtocolV1Alpha1.model_validate(changed) + + +def test_frozen_result_matches_fresh_fixture_and_subprocess_reconstruction() -> None: + expected = json.loads(RESULT_PATH.read_text()) + assert run_provider_free_fixture() == expected + environment = {**os.environ, "ACE_DISABLE_EXTENSIONS": "1"} + command = ( + "import json; " + "from evaluations.source.agent_memory_am6_evaluation import run_provider_free_fixture; " + "print(json.dumps(run_provider_free_fixture(),sort_keys=True,separators=(',',':')))" + ) + completed = subprocess.run( + [sys.executable, "-c", command], + cwd=REPO, + env=environment, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + assert json.loads(completed.stdout) == expected + + +def test_route_tier_frequency_and_exact_eleven_mcp_boundary_remain_visible() -> None: + result = run_provider_free_fixture() + assert sum(result["route_frequency"].values()) == result["observation_count"] + assert sum(result["tier_frequency"].values()) == result["observation_count"] + tree = ast.parse(THIN_MCP_PATH.read_text(), filename=str(THIN_MCP_PATH)) + names = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call) or not isinstance(decorator.func, ast.Attribute): + continue + if decorator.func.attr == "tool": + for keyword in decorator.keywords: + if keyword.arg == "name" and isinstance(keyword.value, ast.Constant): + names.append(str(keyword.value.value)) + assert len(names) == 11 + assert len(names) == len(set(names)) + + +def test_underpowered_contract_requires_explicit_missing_coordinates() -> None: + corpus, protocol, assignment, observations = _scenario("am3_missing_resource_telemetry") + comparison = compare_memory_conditions( + corpus=corpus, + protocol=protocol, + assignment=assignment, + observations=observations, + compared_at=assignment.assigned_at + timedelta(seconds=10), + ) + changed = comparison.model_dump(mode="python") + changed["missing_measurements"] = () + changed["comparison_id"] = None + changed["comparison_digest"] = None + with pytest.raises(ValidationError, match="underpowered benefit disposition"): + MemoryMatchedComparisonV1Alpha1.model_validate(changed) + + +def test_unavailable_measurement_cannot_silently_default_to_zero() -> None: + _, _, _, observations = _scenario("am3_missing_resource_telemetry") + memory = next(item for item in observations if item.condition is MemoryEvaluationCondition.MEMORY) + cost = next(item for item in memory.measurements if item.measure is MemoryMeasure.COST_MICROUNITS) + assert cost.availability is MeasureAvailability.UNAVAILABLE + assert cost.value is None + assert cost.unavailable_reason == "cost_telemetry_unavailable" diff --git a/tests/intelligence/test_contract_boundaries.py b/tests/intelligence/test_contract_boundaries.py index 018ca03..2cdf3d6 100644 --- a/tests/intelligence/test_contract_boundaries.py +++ b/tests/intelligence/test_contract_boundaries.py @@ -53,6 +53,7 @@ def test_contracts_never_import_application_compiler_or_host_dependencies() -> N def test_intelligence_initializer_exports_contracts_and_pure_interpreters_only() -> None: imports = _imports(INTELLIGENCE / "__init__.py") assert imports == { + "ace.intelligence.agent_memory_evaluation", "ace.intelligence.contracts", "ace.intelligence.derivation", "ace.intelligence.detection",