Skip to content

Commit acb5cd7

Browse files
Pigbibiclaude
andcommitted
feat: shadow audit feedback hook — report AI vs deterministic disagreements
After each shadow audit completes with a non-"agree" verdict, fire-and-forget report to AiGateway\'s POST /v1/ai/feedback/shadow. The gateway tracks cumulative disagreements per plugin. When a threshold (5 consecutive) is reached, it auto-escalates for deterministic logic review. Co-Authored-By: Claude <noreply@anthropic.com>
1 parent fba0f93 commit acb5cd7

1 file changed

Lines changed: 66 additions & 0 deletions

File tree

src/quant_strategy_plugins/ai_audit.py

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -758,6 +758,62 @@ def _failure_text(exc: BaseException) -> str:
758758
return _bounded_text(f"{type(exc).__name__}: {exc}", limit=300)
759759

760760

761+
def _report_shadow_disagreement(
762+
*,
763+
audit_kind: str,
764+
ai_verdict: str,
765+
ai_confidence: float,
766+
deterministic_route: str,
767+
) -> None:
768+
"""Fire-and-forget report of AI vs deterministic disagreement to AiGateway.
769+
770+
When AI shadow audit disagrees with the deterministic route, report it
771+
so the gateway can track cumulative disagreements and auto-escalate.
772+
Only sends if CODEX_AUDIT_SERVICE_URL is configured.
773+
"""
774+
import urllib.request as _ur
775+
service_url = os.environ.get("CODEX_AUDIT_SERVICE_URL", "").strip()
776+
if not service_url:
777+
return
778+
# Only report if AI disagrees (verdict is not "agree")
779+
if ai_verdict == "agree":
780+
return
781+
try:
782+
# Map audit kind to plugin name
783+
plugin_map = {
784+
"crisis_response_shadow": "crisis_response",
785+
"taco_rebound_shadow": "taco_rebound",
786+
}
787+
plugin = plugin_map.get(audit_kind, audit_kind)
788+
token = _env(
789+
"ACTIONS_ID_TOKEN_REQUEST_TOKEN",
790+
"CODEX_AUDIT_SERVICE_TOKEN",
791+
) or ""
792+
if not token:
793+
token = os.environ.get("CODEX_AUDIT_SERVICE_TOKEN", "")
794+
if not token:
795+
return # No auth available, skip silently
796+
payload = json.dumps({
797+
"plugin": plugin,
798+
"ai_verdict": ai_verdict,
799+
"ai_confidence": ai_confidence,
800+
"deterministic_route": deterministic_route,
801+
"source_repository": os.environ.get("AI_GATEWAY_SOURCE_REPO", ""),
802+
}).encode("utf-8")
803+
req = _ur.Request(
804+
f"{service_url.rstrip('/')}/v1/ai/feedback/shadow",
805+
data=payload, method="POST",
806+
headers={
807+
"Authorization": f"Bearer {token}",
808+
"Content-Type": "application/json",
809+
"User-Agent": "quant-strategy-plugins",
810+
},
811+
)
812+
_ur.urlopen(req, timeout=5)
813+
except Exception:
814+
pass # Fire-and-forget — never block the main audit flow
815+
816+
761817
def build_disabled_ai_audit(*, audit_kind: str = "strategy_plugin") -> dict[str, Any]:
762818
return {
763819
"schema_version": AI_AUDIT_SCHEMA_VERSION,
@@ -839,6 +895,16 @@ def _run_ai_audit(
839895
raw_response = client(endpoint, messages, float(timeout_seconds))
840896
audit_response = _normalize_ai_audit_response(_extract_json_object(raw_response))
841897
attempts.append({**endpoint.report(), "status": "ok"})
898+
899+
# Phase 3: report AI vs deterministic disagreement to AiGateway
900+
_report_shadow_disagreement(
901+
audit_kind=audit_kind,
902+
ai_verdict=audit_response.get("verdict", ""),
903+
ai_confidence=audit_response.get("confidence") or 0.0,
904+
deterministic_route=str(deterministic_payload.get("canonical_route") or
905+
deterministic_payload.get("suggested_action") or ""),
906+
)
907+
842908
return {
843909
**base_payload,
844910
"status": "ok",

0 commit comments

Comments
 (0)