From 1d0e7cdf32d162aaa1993d771348c36b61fd4387 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:48:47 +0000 Subject: [PATCH] refactor(model-routing): centralize explicit foreground fallback policy Make foreground fallback an explicit per-user, availability-only policy shared by streaming Chat, non-stream Chat, and Agent runs. Preserve strict defaults, owner/model and credential boundaries, pinned Agent routes, and truthful per-round provenance/accounting. Carry provider-reported model identifiers through native streaming adapters, non-stream responses, and caches, and keep legacy default_model_fallbacks as tombstoned raw storage that generic settings APIs and agent tools cannot expose or mutate. --- core/database.py | 21 +- mcp_servers/email_server.py | 8 - routes/auth_routes.py | 8 +- routes/chat_helpers.py | 147 +- routes/chat_routes.py | 587 +++- routes/email_routes.py | 13 +- routes/model_routes.py | 8 +- routes/prefs_routes.py | 62 +- src/agent_loop.py | 1277 +++++-- src/agent_runs.py | 97 +- src/agent_tools/admin_tools.py | 24 +- src/context_compactor.py | 64 +- src/endpoint_resolver.py | 214 +- src/foreground_model_routing.py | 174 +- src/llm_core.py | 969 ++++- src/settings.py | 20 +- src/task_endpoint.py | 5 - static/index.html | 7 - static/js/chat.js | 407 ++- static/js/chatModelProvenance.js | 104 + static/js/chatRenderer.js | 266 +- static/js/chatStreamErrors.js | 23 + static/js/settings.js | 73 - static/js/slashCommands.js | 12 +- tests/test_agent_loop.py | 4 + tests/test_agent_round_model_provenance_ui.py | 237 ++ tests/test_chat_model_provenance_js.py | 203 ++ tests/test_chat_stream_errors_js.py | 47 + tests/test_compare_stop_disconnect_poll.py | 175 +- tests/test_context_compactor.py | 61 + tests/test_foreground_model_routing.py | 3126 ++++++++++++++++- tests/test_legacy_default_fallback_ui.py | 8 +- tests/test_live_fallback_round_attribution.py | 284 ++ tests/test_llm_core_fallback.py | 1135 +++++- tests/test_llm_core_usage_finish_delta.py | 65 +- tests/test_model_routes.py | 22 +- tests/test_prefs_routes.py | 24 +- tests/test_prefs_single_user_no_clobber.py | 58 + tests/test_resolve_endpoint_fallbacks.py | 104 +- tests/test_resolve_session_auth_chatgpt.py | 50 - tests/test_retired_settings_interfaces.py | 119 + tests/test_tool_support_heuristic.py | 63 +- tests/test_user_time.py | 25 + 43 files changed, 9385 insertions(+), 1015 deletions(-) create mode 100644 static/js/chatModelProvenance.js create mode 100644 static/js/chatStreamErrors.js create mode 100644 tests/test_agent_round_model_provenance_ui.py create mode 100644 tests/test_chat_model_provenance_js.py create mode 100644 tests/test_chat_stream_errors_js.py create mode 100644 tests/test_live_fallback_round_attribution.py create mode 100644 tests/test_retired_settings_interfaces.py diff --git a/core/database.py b/core/database.py index a9ad90b8b7..afb9e820bc 100644 --- a/core/database.py +++ b/core/database.py @@ -1404,8 +1404,25 @@ def _migrate_assign_legacy_owner(): with open(prefs_path, "r", encoding="utf-8") as f: prefs = _json.load(f) if "_users" not in prefs and prefs: - # Flat format → nest under admin user - new_prefs = {"_users": {admin_user: prefs}} + # Flat format → nest ordinary preferences under the admin + # user. Foreground fallback is an explicit per-owner opt-in, + # so auth-disabled consent must remain inert at the flat root + # rather than becoming consent for the first named owner. + foreground_keys = { + "foreground_fallback_enabled", + "foreground_model_fallbacks", + } + named_prefs = { + key: value + for key, value in prefs.items() + if key not in foreground_keys + } + new_prefs = { + key: prefs[key] + for key in foreground_keys + if key in prefs + } + new_prefs["_users"] = {admin_user: named_prefs} with open(prefs_path, "w", encoding="utf-8") as f: _json.dump(new_prefs, f, indent=2) logger.info(f"Migrated user_prefs.json to per-user format under '{admin_user}'") diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index e2bccfbfb2..767f0e4b91 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -1540,7 +1540,6 @@ async def _ai_draft_reply_to_email(uid, folder="INBOX", reply_all=False, account from src.endpoint_resolver import ( resolve_endpoint, resolve_utility_fallback_candidates, - resolve_chat_fallback_candidates, ) from src.llm_core import llm_call_async_with_fallback except Exception as exc: @@ -1582,13 +1581,6 @@ def _add(url, model, headers): utility_fallbacks = resolve_utility_fallback_candidates() or [] for cand in utility_fallbacks: _add(*cand) - try: - chat_fallbacks = resolve_chat_fallback_candidates(owner=None) or [] - except TypeError: - chat_fallbacks = resolve_chat_fallback_candidates() or [] - for cand in chat_fallbacks: - _add(*cand) - if not candidates: return {"error": "No LLM endpoint configured for AI reply"} diff --git a/routes/auth_routes.py b/routes/auth_routes.py index 5c7a4e04ab..1f6dc67381 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -22,6 +22,8 @@ load_features as _load_features, save_features as _save_features, DEFAULT_SETTINGS, + RETIRED_SETTING_KEYS, + without_retired_settings, ) from src.integrations import ( load_integrations, @@ -637,7 +639,7 @@ async def get_settings(request: Request): a scrubbed copy with secret keys blanked. The frontend uses this for keybinds + TTS prefs, so it stays callable without admin.""" user = _get_current_user(request) - settings = _load_settings() + settings = without_retired_settings(_load_settings()) if user and auth_manager.is_admin(user): return settings return scrub_settings(settings) @@ -657,6 +659,8 @@ async def set_settings(request: Request): "agent_max_tool_calls": (0, 1000), # 0 = unlimited } for key in DEFAULT_SETTINGS: + if key in RETIRED_SETTING_KEYS: + continue if key not in body: continue val = body[key] @@ -669,7 +673,7 @@ async def set_settings(request: Request): val = max(lo, min(val, hi)) current[key] = val _save_settings(current) - return current + return without_retired_settings(current) # ---- Integrations CRUD ---- diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index 63c8abc8f8..d34084a30a 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -14,7 +14,7 @@ from src.llm_core import normalize_model_id from src.endpoint_resolver import normalize_base from src.context_compactor import maybe_compact, trim_for_context -from src.model_context import estimate_tokens +from src.model_context import estimate_tokens, get_context_length from src.auth_helpers import effective_user from src.prompt_security import untrusted_context_message from src.attachment_refs import attachment_ref @@ -114,10 +114,38 @@ class ChatContext: # Uploads attached to this user turn, resolved and owner-checked for the # agent's private context. This is not emitted to the browser. uploaded_files: list = field(default_factory=list) + # Route-neutral prompt before any model-window compaction/trimming. This is + # retained only when explicit foreground fallbacks are enabled so each + # concrete candidate can apply its own context budget independently. + route_messages: list = field(default_factory=list) # ── Helpers ────────────────────────────────────────────────────────────── # +def _allowed_models_from_privileges(privs: dict) -> Optional[frozenset[str]]: + if privs.get("block_all_models"): + return frozenset() + allowed_raw = privs.get("allowed_models") + allowed = allowed_raw if isinstance(allowed_raw, list) else [] + restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed) + return frozenset(model for model in allowed if isinstance(model, str)) if restricted else None + + +def _allowed_models_for_request(request) -> Optional[frozenset[str]]: + """Return the caller's model allowlist, or ``None`` when unrestricted.""" + + try: + user = effective_user(request) + except Exception: + user = None + if not user: + return None + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + if not auth_manager: + return None + privs = auth_manager.get_privileges(user) or {} + return _allowed_models_from_privileges(privs) + def _enforce_chat_privileges(request, sess) -> None: """Apply the per-user privilege gates (allowed_models + max_messages_per_day) that both /api/chat and /api/chat_stream must enforce BEFORE any LLM work. @@ -147,10 +175,8 @@ def _enforce_chat_privileges(request, sess) -> None: if privs.get("block_all_models"): raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.") - allowed_raw = privs.get("allowed_models") - allowed = allowed_raw if isinstance(allowed_raw, list) else [] - restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed) - if restricted and sess.model and sess.model not in allowed: + allowed_models = _allowed_models_from_privileges(privs) + if allowed_models is not None and sess.model and sess.model not in allowed_models: raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.") cap = int(privs.get("max_messages_per_day") or 0) @@ -249,96 +275,6 @@ async def auto_name_session(session_manager, sess): logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}") -def try_fallback_endpoint(sess, session_id: str) -> dict | None: - """Find an alternative working endpoint when the current one fails. - - Returns {"model": ..., "endpoint_url": ..., "endpoint_name": ...} or None. - """ - import requests as _req - from src.endpoint_resolver import ( - build_chat_url, - build_headers, - build_models_url, - normalize_base, - resolve_endpoint_runtime, - ) - from src.chatgpt_subscription import is_chatgpt_subscription_base - - current_url = sess.endpoint_url or "" - owner = getattr(sess, "owner", None) - db = SessionLocal() - try: - q = db.query(ModelEndpoint).filter( - ModelEndpoint.is_enabled == True - ) - if owner: - from src.auth_helpers import owner_filter - q = owner_filter(q, ModelEndpoint, owner) - endpoints = q.all() - finally: - db.close() - - for ep in endpoints: - base = normalize_base(ep.base_url) - # Skip current endpoint - if current_url and base in current_url: - continue - try: - base, api_key = resolve_endpoint_runtime(ep, owner=owner) - except Exception: - continue - ping_url = build_models_url(base) - headers = build_headers(api_key, base) - try: - if ping_url: - r = _req.get(ping_url, headers=headers, timeout=5) - r.raise_for_status() - data = r.json() - models = [m.get("id") for m in (data.get("data") or []) if m.get("id")] - if not models: - models = [ - m.get("name") or m.get("model") - for m in (data.get("models") or []) - if m.get("name") or m.get("model") - ] - else: - models = json.loads(ep.cached_models or "[]") - if not models: - continue - # Found a working endpoint — update session - new_model = models[0] - chat_url = build_chat_url(base) - new_headers = build_headers(api_key, base) - persisted_headers = {} if is_chatgpt_subscription_base(base) else new_headers - - sess.model = new_model - sess.endpoint_url = chat_url - sess.headers = new_headers - - # Persist - _db = SessionLocal() - try: - _db.query(DBSession).filter(DBSession.id == session_id).update({ - "model": new_model, - "endpoint_url": chat_url, - "headers": persisted_headers, - }) - _db.commit() - finally: - _db.close() - - logger.info(f"Fallback: switched session {session_id} from {current_url} to {ep.name} ({new_model})") - return { - "model": new_model, - "endpoint_url": chat_url, - "endpoint_name": ep.name, - } - except Exception: - continue - - return None - - def extract_preset(chat_handler, preset_id) -> PresetInfo: """Extract preset parameters via chat_handler.""" temperature, max_tokens, system_prompt, char_name = ( @@ -648,6 +584,7 @@ async def build_chat_context( use_enhanced_message: bool = False, agent_mode: bool = False, allow_tool_preprocessing: bool = True, + defer_context_shaping: bool = False, ) -> ChatContext: """Build the full context (preface + messages) for an LLM call. @@ -783,13 +720,22 @@ async def build_chat_context( except Exception: logger.debug("Failed to add current date/time context", exc_info=True) - # Auto-compact - messages, context_length, was_compacted = await maybe_compact( - sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user, - ) + route_messages = list(messages) + # Explicit fallback routing must shape from the same route-neutral prompt + # for every candidate. Running selected-model compaction here would mutate + # session history before we know which route can answer and would make a + # later larger-context candidate unable to recover discarded history. + if defer_context_shaping: + context_length = get_context_length(sess.endpoint_url, sess.model) + was_compacted = False + else: + messages, context_length, was_compacted = await maybe_compact( + sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user, + ) _before_trim_messages = len(messages) _before_trim_tokens = estimate_tokens(messages) - messages = trim_for_context(messages, context_length) + if not defer_context_shaping: + messages = trim_for_context(messages, context_length) _after_trim_messages = len(messages) _after_trim_tokens = estimate_tokens(messages) _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens @@ -813,6 +759,7 @@ async def build_chat_context( context_tokens_after_trim=_after_trim_tokens, auto_opened_docs=auto_opened_docs, uploaded_files=uploaded_files, + route_messages=route_messages, ) diff --git a/routes/chat_routes.py b/routes/chat_routes.py index e674c70417..1cbc87c927 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -15,13 +15,28 @@ from core.models import ChatMessage from src.request_models import ChatRequest -from src.llm_core import llm_call_async, stream_llm, stream_llm_with_fallback +from src.llm_core import ( + _normalize_http_status, + llm_call_async, + llm_call_async_with_route_fallback, + stream_llm, + stream_llm_with_fallback, +) from src.agent_loop import stream_agent_loop from src import agent_runs from src.model_context import estimate_tokens +from src.context_compactor import ( + apply_compaction_state, + maybe_compact, + trim_for_context, +) from src.chat_helpers import coerce_message_and_session from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_url -from src.foreground_model_routing import build_foreground_model_candidates +from src.foreground_model_routing import ( + build_foreground_model_candidates, + build_foreground_route_descriptors, + resolve_foreground_model_policy, +) from src.session_search import search_session_messages from src.prompt_security import untrusted_context_message from core.exceptions import SessionNotFoundError @@ -39,7 +54,9 @@ build_chat_context, save_assistant_response, run_post_response_tasks, + accumulate_token_usage, clean_thinking_for_save, + _allowed_models_for_request, _enforce_chat_privileges, ) from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent @@ -57,6 +74,74 @@ _IMAGE_MODEL_PREFIXES = ("gpt-image", "dall-e", "chatgpt-image") +def _stream_failure_status(chunk: str) -> Optional[int]: + """Extract a provider status without retaining provider-supplied detail.""" + + try: + for line in str(chunk or "").splitlines(): + if not line.startswith("data: "): + continue + status = json.loads(line[6:]).get("status") + return _normalize_http_status(status) + except json.JSONDecodeError: + return None + return None + + +def _chat_candidate_request_factory( + messages, + fallback_context_length: int = 0, + *, + session=None, + owner: Optional[str] = None, +): + """Shape one route-neutral Chat prompt for each candidate window.""" + + state = { + "requests": {}, + "context_lengths": {}, + "trim_stats": {}, + "compactions": {}, + "was_compacted": {}, + } + + async def factory(index, candidate_url, candidate_model, candidate_headers): + compaction_state = {} + candidate_messages, context_length, was_compacted = await maybe_compact( + session, + candidate_url, + candidate_model, + list(messages), + candidate_headers, + owner=owner, + persist=False, + compaction_state=compaction_state, + ) + if not context_length: + context_length = fallback_context_length + request_messages = trim_for_context(candidate_messages, context_length) + state["requests"][index] = request_messages + state["context_lengths"][index] = context_length + state["compactions"][index] = compaction_state + state["was_compacted"][index] = was_compacted + state["trim_stats"][index] = { + "messages_before": len(messages), + "messages_after": len(request_messages), + "tokens_before": estimate_tokens(messages), + "tokens_after": estimate_tokens(request_messages), + } + return {"messages": request_messages} + + return factory, state + + +def _candidate_index(candidates, actual_candidate) -> int: + for index, candidate in enumerate(candidates): + if candidate == actual_candidate: + return index + return 0 + + def _stream_set(session_id: str, **fields) -> None: """Update fields on the active-stream entry for `session_id`, or no-op if the entry has already been popped. Using .get() avoids a @@ -423,8 +508,8 @@ def setup_chat_routes( # ------------------------------------------------------------------ # # POST /api/chat (non-streaming) # ------------------------------------------------------------------ # - @router.post("/api/chat", response_model=Dict[str, str]) - async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str, str]: + @router.post("/api/chat", response_model=Dict[str, Any]) + async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str, Any]: _set_user_time_from_request(request) message = chat_request.message @@ -456,6 +541,8 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str 400, "No model selected for this chat. Open the model picker and choose one before sending.", ) + if not getattr(sess, "endpoint_url", "").strip(): + raise HTTPException(400, "Selected model endpoint is not configured") # Same allowed_models + daily-cap gate as chat_stream (mirror so the # non-streaming path can't be used to bypass). @@ -471,6 +558,11 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str if memory_response: return {"response": memory_response} + foreground_policy = resolve_foreground_model_policy( + owner=owner, + allowed_models=_allowed_models_for_request(request), + ) + # Build shared context (preset, preprocess, preface, compact) ctx = await build_chat_context( sess, request, chat_handler, chat_processor, @@ -482,6 +574,7 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str time_filter=time_filter, webhook_manager=webhook_manager, allow_tool_preprocessing=allow_tool_preprocessing, + defer_context_shaping=foreground_policy.enabled, ) # Research injection @@ -495,24 +588,87 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str research_ctx = await research_handler.call_research_service( message, _r_ep, _r_model, llm_headers=_r_headers ) - ctx.messages.insert( - len(ctx.preface), - untrusted_context_message("research context", research_ctx), - ) + research_message = untrusted_context_message("research context", research_ctx) + ctx.messages.insert(len(ctx.preface), research_message) + if foreground_policy.enabled: + getattr(ctx, "route_messages", ctx.messages).insert( + len(ctx.preface), + research_message, + ) except Exception as e: logger.error(f"Research failed: {e}") - reply = await llm_call_async( + foreground_candidates = build_foreground_model_candidates( sess.endpoint_url, sess.model, - ctx.messages, - headers=sess.headers, + sess.headers, + owner=owner, + policy=foreground_policy, + ) + route_descriptors = build_foreground_route_descriptors( + sess.endpoint_url, + sess.model, + sess.headers, + owner=owner, + policy=foreground_policy, + ) + candidate_request_factory = None + selected_context_length = getattr(ctx, "context_length", 0) + candidate_request_state = { + "context_lengths": {0: selected_context_length}, + "requests": {0: ctx.messages}, + "trim_stats": {}, + } + request_messages = ctx.messages + if foreground_policy.enabled: + request_messages = getattr(ctx, "route_messages", ctx.messages) + candidate_request_factory, candidate_request_state = _chat_candidate_request_factory( + request_messages, + selected_context_length, + session=sess, + owner=owner, + ) + requested_model = sess.model + reply, actual_candidate, actual_model = await llm_call_async_with_route_fallback( + foreground_candidates, + request_messages, + fallback_statuses=foreground_policy.eligible_statuses, + candidate_request_factory=candidate_request_factory, temperature=ctx.preset.temperature, max_tokens=ctx.preset.max_tokens, prompt_type=preset_id, session_id=session, ) - _clean_reply, _clean_md = clean_thinking_for_save(reply, {"model": sess.model}) + actual_index = _candidate_index(foreground_candidates, actual_candidate) + apply_compaction_state( + sess, + candidate_request_state.get("compactions", {}).get(actual_index), + ) + requested_route = route_descriptors[0] + actual_route = route_descriptors[actual_index] + actual_trim = candidate_request_state.get("trim_stats", {}).get(actual_index, {}) + _clean_reply, _clean_md = clean_thinking_for_save( + reply, + { + "model": actual_model, + "requested_model": requested_model, + "endpoint_id": actual_route.get("endpoint_id"), + "endpoint_label": actual_route.get("endpoint_label"), + "requested_endpoint_id": requested_route.get("endpoint_id"), + "requested_endpoint_label": requested_route.get("endpoint_label"), + "context_length": candidate_request_state["context_lengths"].get( + actual_index, + selected_context_length, + ), + "context_trimmed": bool( + actual_trim + and ( + actual_trim.get("messages_after") < actual_trim.get("messages_before") + or actual_trim.get("tokens_after") < actual_trim.get("tokens_before") + ) + ), + }, + ) sess.add_message(ChatMessage("assistant", _clean_reply, metadata=_clean_md)) from core.database import update_session_last_accessed @@ -528,7 +684,15 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str allow_background_extraction=not tool_policy.block_all_tool_calls, ) - return {"response": reply} + return { + "response": reply, + "requested_model": requested_model, + "model": actual_model, + "requested_endpoint_id": requested_route.get("endpoint_id"), + "requested_endpoint_label": requested_route.get("endpoint_label"), + "endpoint_id": actual_route.get("endpoint_id"), + "endpoint_label": actual_route.get("endpoint_label"), + } # ------------------------------------------------------------------ # # POST /api/chat_stream @@ -703,6 +867,8 @@ async def chat_stream(request: Request) -> StreamingResponse: 400, "No model selected for this chat. Open the model picker and choose one before sending.", ) + if not getattr(sess, "endpoint_url", "").strip(): + raise HTTPException(400, "Selected model endpoint is not configured") if ( chat_mode == "chat" and isinstance(message, str) @@ -756,6 +922,10 @@ async def chat_stream(request: Request) -> StreamingResponse: last_user_message=message, ) allow_tool_preprocessing = not pre_context_tool_policy.block_all_tool_calls + foreground_policy = resolve_foreground_model_policy( + owner=owner, + allowed_models=_allowed_models_for_request(request), + ) # Build shared context (stream path uses enhanced_message for context preface) ctx = await build_chat_context( @@ -778,6 +948,7 @@ async def chat_stream(request: Request) -> StreamingResponse: # index would be useless / unwanted noise. agent_mode=(chat_mode == "agent"), allow_tool_preprocessing=allow_tool_preprocessing, + defer_context_shaping=foreground_policy.enabled, ) _research_flags = {"do": do_research} # Mutable container for generator scope @@ -1070,6 +1241,8 @@ async def stream_with_save() -> AsyncGenerator[str, None]: "what aspects matter most, are they comparing to something, what's their context " "(moving, traveling, curiosity). Be conversational. Keep it short." }) + if foreground_policy.enabled: + getattr(ctx, "route_messages", ctx.messages).insert(0, dict(ctx.messages[0])) _skip_research = True else: _skip_research = False @@ -1166,7 +1339,12 @@ def _on_research_done(_sid, _result, _sources, _findings): _active_streams.pop(session, None) return - messages = _ensure_current_request_is_latest_user(ctx.messages, message) + context_source = ( + getattr(ctx, "route_messages", ctx.messages) + if foreground_policy.enabled + else ctx.messages + ) + messages = _ensure_current_request_is_latest_user(context_source, message) # Auto-compact notification if ctx.was_compacted: @@ -1178,24 +1356,54 @@ def _on_research_done(_sid, _result, _sources, _findings): thinking_response = "" last_metrics = None - # Foreground Chat and Agent requests use one owner-aware policy - # boundary. Legacy `default_model_fallbacks` data is not eligible. + # Foreground Chat and Agent requests share one explicit owner-aware + # policy. Strict mode is the default; legacy values are unrelated. + _foreground_policy = foreground_policy _foreground_candidates = build_foreground_model_candidates( sess.endpoint_url, sess.model, sess.headers, owner=_user, + policy=_foreground_policy, + ) + _foreground_route_descriptors = build_foreground_route_descriptors( + sess.endpoint_url, + sess.model, + sess.headers, + owner=_user, + policy=_foreground_policy, ) + _chat_request_factory = None + _selected_context_length = getattr(ctx, "context_length", 0) + _chat_request_state = { + "context_lengths": {0: _selected_context_length}, + "requests": {0: messages}, + "trim_stats": {}, + } + if _foreground_policy.enabled: + _chat_request_factory, _chat_request_state = _chat_candidate_request_factory( + messages, + _selected_context_length, + session=sess, + owner=_user, + ) # Send model name early so the frontend can show it during streaming _model_suffix = "Research" if effective_do_research else None - _model_info = {"type": "model_info", "model": sess.model} + _selected_route = _foreground_route_descriptors[0] + _model_info = { + "type": "model_info", + "model": sess.model, + "endpoint_id": _selected_route.get("endpoint_id"), + "endpoint_label": _selected_route.get("endpoint_label"), + } if _model_suffix: _model_info["suffix"] = _model_suffix if ctx.preset.character_name: _model_info["character_name"] = ctx.preset.character_name yield f'data: {json.dumps(_model_info)}\n\n' + _terminal_saved = False if _is_image_generation_session(sess, owner=_user): from src.settings import get_setting if tool_policy.blocks("generate_image"): @@ -1240,6 +1448,16 @@ def _on_research_done(_sid, _result, _sources, _findings): _answered_by = None # set if the selected model failed and a fallback answered _requested_model = sess.model _actual_model = None + _requested_route = _foreground_route_descriptors[0] + _actual_route = _requested_route + _actual_candidate_index = 0 + _chat_terminal_saved = False + def _commit_chat_compaction(candidate_index: int) -> bool: + return apply_compaction_state( + sess, + _chat_request_state.get("compactions", {}).get(candidate_index), + ) + # ── Chat mode: call stream_llm directly, NO tools, NO document access ── try: async for chunk in stream_llm_with_fallback( @@ -1255,11 +1473,21 @@ def _on_research_done(_sid, _result, _sources, _findings): prompt_type=preset_id, tools=None, session_id=session, + fallback_statuses=_foreground_policy.eligible_statuses, + fallback_on_empty=_foreground_policy.fallback_on_empty, + candidate_request_factory=_chat_request_factory, + candidate_route_descriptors=_foreground_route_descriptors, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: data = json.loads(chunk[6:]) if "delta" in data: + if _commit_chat_compaction(_actual_candidate_index): + _compacted_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n' # Reasoning tokens arrive flagged thinking:true. # Forward them so the client can show a thinking # indicator, but don't fold them into the saved @@ -1275,27 +1503,82 @@ def _on_research_done(_sid, _result, _sources, _findings): # Forward the notice and remember the real model. _answered_by = data.get("answered_by") or _answered_by _actual_model = _actual_model or _answered_by + _actual_candidate_index = data.get("candidate_index", 0) + if not isinstance(_actual_candidate_index, int): + _actual_candidate_index = 0 + if 0 <= _actual_candidate_index < len(_foreground_route_descriptors): + _actual_route = _foreground_route_descriptors[_actual_candidate_index] + if _commit_chat_compaction(_actual_candidate_index): + _compacted_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n' data["selected_model"] = data.get("selected_model") or _requested_model - yield chunk + yield f'data: {json.dumps(data)}\n\n' elif data.get("type") == "model_actual": + if _commit_chat_compaction(_actual_candidate_index): + _compacted_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n' _actual_model = data.get("model") or _actual_model data["requested_model"] = _requested_model + data["requested_endpoint_id"] = _requested_route.get("endpoint_id") + data["requested_endpoint_label"] = _requested_route.get("endpoint_label") + data["endpoint_id"] = _actual_route.get("endpoint_id") + data["endpoint_label"] = _actual_route.get("endpoint_label") yield f'data: {json.dumps(data)}\n\n' elif data.get("type") == "usage": + if _commit_chat_compaction(_actual_candidate_index): + _compacted_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n' last_metrics = data.get("data", {}) _reported_model = last_metrics.get("model") last_metrics["requested_model"] = _requested_model last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model - if ctx.context_trimmed: + last_metrics["requested_endpoint_id"] = _requested_route.get("endpoint_id") + last_metrics["requested_endpoint_label"] = _requested_route.get("endpoint_label") + last_metrics["endpoint_id"] = _actual_route.get("endpoint_id") + last_metrics["endpoint_label"] = _actual_route.get("endpoint_label") + if isinstance( + _actual_route.get("endpoint_cost_tracked"), + bool, + ): + last_metrics["endpoint_cost_tracked"] = _actual_route.get( + "endpoint_cost_tracked" + ) + _actual_context_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + _route_trim = _chat_request_state.get("trim_stats", {}).get( + _actual_candidate_index, + {}, + ) + if _route_trim and ( + _route_trim.get("messages_after") < _route_trim.get("messages_before") + or _route_trim.get("tokens_after") < _route_trim.get("tokens_before") + ): + last_metrics["context_trimmed"] = True + last_metrics["context_messages_before_trim"] = _route_trim.get("messages_before") + last_metrics["context_messages_after_trim"] = _route_trim.get("messages_after") + last_metrics["context_tokens_before_trim"] = _route_trim.get("tokens_before") + last_metrics["context_tokens_after_trim"] = _route_trim.get("tokens_after") + elif ctx.context_trimmed: last_metrics["context_trimmed"] = True last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim - if ctx.context_length and last_metrics.get("input_tokens"): - pct = min(round((last_metrics["input_tokens"] / ctx.context_length) * 100, 1), 100.0) + if _actual_context_length and last_metrics.get("input_tokens"): + pct = min(round((last_metrics["input_tokens"] / _actual_context_length) * 100, 1), 100.0) last_metrics["context_percent"] = pct - last_metrics["context_length"] = ctx.context_length + last_metrics["context_length"] = _actual_context_length # The frontend reads `tokens_per_second`; the raw usage event # carries the backend's true gen speed as `gen_tps` (llama.cpp # timings). Map it through so this direct-chat path shows real @@ -1310,30 +1593,146 @@ def _on_research_done(_sid, _result, _sources, _findings): yield chunk elif chunk.startswith("event: error"): logger.warning(f"Stream error for {sess.model} on {sess.endpoint_url}: {chunk!r}") + if ( + not _chat_terminal_saved + and (full_response.strip() or thinking_response.strip()) + ): + _failure_status = _stream_failure_status(chunk) + _failure_message = ( + f"Model request failed (HTTP {_failure_status})" + if _failure_status is not None + else "Model request failed" + ) + _terminal_content = full_response.strip() + _failure_note = f"[Response stopped: {_failure_message}]" + _terminal_content = ( + f"{_terminal_content}\n\n{_failure_note}" + if _terminal_content + else _failure_note + ) + _had_terminal_usage = bool(last_metrics) + _terminal_metrics = dict(last_metrics or {}) + if not _had_terminal_usage: + _actual_request_messages = _chat_request_state["requests"].get( + _actual_candidate_index, + messages, + ) + _actual_context_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + _estimated_input = estimate_tokens(_actual_request_messages) + _estimated_output = max( + len(full_response + thinking_response) // 4, + 0, + ) + _terminal_metrics.update({ + "input_tokens": _estimated_input, + "output_tokens": _estimated_output, + "total_tokens": _estimated_input + _estimated_output, + "usage_source": "estimated", + "response_time": round(time.time() - _chat_start, 2), + "context_length": _actual_context_length, + "context_percent": ( + min( + round( + (_estimated_input / _actual_context_length) * 100, + 1, + ), + 100.0, + ) + if _actual_context_length + else 0 + ), + }) + _terminal_metrics.update({ + "failed": True, + "failure": { + "status": _failure_status, + "message": _failure_message, + }, + "model": _actual_model or _answered_by or _requested_model, + "requested_model": _requested_model, + "endpoint_id": _actual_route.get("endpoint_id"), + "endpoint_label": _actual_route.get("endpoint_label"), + "requested_endpoint_id": _requested_route.get("endpoint_id"), + "requested_endpoint_label": _requested_route.get("endpoint_label"), + }) + if isinstance( + _actual_route.get("endpoint_cost_tracked"), + bool, + ): + _terminal_metrics["endpoint_cost_tracked"] = _actual_route.get( + "endpoint_cost_tracked" + ) + if thinking_response.strip(): + _terminal_metrics["thinking"] = thinking_response.strip() + _commit_chat_compaction(_actual_candidate_index) + _saved_id = save_assistant_response( + sess, + session_manager, + session, + _terminal_content, + _terminal_metrics, + character_name=ctx.preset.character_name, + incognito=incognito, + ) + accumulate_token_usage(session, _terminal_metrics) + _chat_terminal_saved = True + _stream_set(session, status="error") + if _saved_id: + yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n' + yield f'data: {json.dumps({"type": "chat_terminal", "data": _terminal_metrics})}\n\n' yield chunk elif chunk.startswith("event: "): yield chunk elif chunk == "data: [DONE]\n\n": + if _chat_terminal_saved: + # Some providers append DONE after a terminal + # error. The failed partial is already saved; + # never re-save/post-process it as a success or + # advertise successful completion to the client. + continue # Generate fallback metrics if LLM didn't send usage if not last_metrics and full_response: _elapsed = time.time() - _chat_start - _est_in = estimate_tokens(messages) _est_out = len(full_response) // 4 _tps = round(_est_out / _elapsed, 2) if _elapsed > 0 else 0 - _ctx_pct = min(round((_est_in / ctx.context_length) * 100, 1), 100.0) if ctx.context_length else 0 + _actual_context_length = _chat_request_state["context_lengths"].get( + _actual_candidate_index, + _selected_context_length, + ) + _actual_request_messages = _chat_request_state["requests"].get( + _actual_candidate_index, + messages, + ) + _est_in = estimate_tokens(_actual_request_messages) + _ctx_pct = min(round((_est_in / _actual_context_length) * 100, 1), 100.0) if _actual_context_length else 0 last_metrics = { "response_time": round(_elapsed, 2), "input_tokens": _est_in, "output_tokens": _est_out, "tokens_per_second": _tps, "context_percent": _ctx_pct, - "context_length": ctx.context_length, + "context_length": _actual_context_length, "model": _actual_model or _answered_by or _requested_model, "requested_model": _requested_model, + "requested_endpoint_id": _requested_route.get("endpoint_id"), + "requested_endpoint_label": _requested_route.get("endpoint_label"), + "endpoint_id": _actual_route.get("endpoint_id"), + "endpoint_label": _actual_route.get("endpoint_label"), "usage_source": "estimated", } + if isinstance( + _actual_route.get("endpoint_cost_tracked"), + bool, + ): + last_metrics["endpoint_cost_tracked"] = _actual_route.get( + "endpoint_cost_tracked" + ) yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n' if full_response: + _commit_chat_compaction(_actual_candidate_index) _metrics_to_save = dict(last_metrics or {}) if thinking_response.strip() and not _metrics_to_save.get("thinking"): _metrics_to_save["thinking"] = thinking_response.strip() @@ -1368,6 +1767,10 @@ def _on_research_done(_sid, _result, _sources, _findings): "stopped": True, "model": _actual_model or _answered_by or _requested_model, "requested_model": _requested_model, + "endpoint_id": _actual_route.get("endpoint_id"), + "endpoint_label": _actual_route.get("endpoint_label"), + "requested_endpoint_id": _requested_route.get("endpoint_id"), + "requested_endpoint_label": _requested_route.get("endpoint_label"), }, ) sess.add_message(ChatMessage("assistant", _stopped_content, metadata=_stopped_md)) @@ -1383,6 +1786,12 @@ def _on_research_done(_sid, _result, _sources, _findings): _answered_by = None # set if the selected model failed and a fallback answered _requested_model = sess.model _actual_model = None + _agent_requested_route = _foreground_route_descriptors[0] + _agent_actual_endpoint_id = _agent_requested_route.get("endpoint_id") + _agent_actual_endpoint_label = _agent_requested_route.get("endpoint_label") + _agent_round_models = {1: _requested_model} + _agent_round_endpoint_ids = {1: _agent_actual_endpoint_id} + _agent_round_endpoint_labels = {1: _agent_actual_endpoint_label} try: from src.settings import get_setting from src.agent_tools import MAX_AGENT_ROUNDS as _DEFAULT_ROUNDS @@ -1416,19 +1825,24 @@ def _on_research_done(_sid, _result, _sources, _findings): prompt_type=preset_id, max_tool_calls=_tool_budget, max_rounds=_max_rounds, - context_length=ctx.context_length, + context_length=_selected_context_length, active_document=active_doc, active_email=active_email_ctx, session_id=session, + history_session=sess, disabled_tools=disabled_tools if disabled_tools else None, tool_policy=tool_policy, owner=_user, fallbacks=_foreground_candidates[1:], + route_descriptors=_foreground_route_descriptors, + fallback_statuses=_foreground_policy.eligible_statuses, + fallback_on_empty=_foreground_policy.fallback_on_empty, plan_mode=plan_mode, approved_plan=approved_plan or None, workspace=workspace or None, forced_tools=_forced_tools, uploaded_files=ctx.uploaded_files, + defer_context_shaping=_foreground_policy.enabled, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: @@ -1457,7 +1871,20 @@ def _on_research_done(_sid, _result, _sources, _findings): "plan_update", ): if data.get("type") == "agent_step": - _agent_rounds = max(_agent_rounds, data.get("round", 1)) + _event_round = data.get("round", 1) + _agent_rounds = max(_agent_rounds, _event_round) + _agent_round_models.setdefault( + _event_round, + _actual_model or _answered_by or _requested_model, + ) + _agent_round_endpoint_ids.setdefault( + _event_round, + _agent_actual_endpoint_id, + ) + _agent_round_endpoint_labels.setdefault( + _event_round, + _agent_actual_endpoint_label, + ) elif data.get("type") == "tool_start": _agent_tool_calls += 1 yield chunk @@ -1467,13 +1894,70 @@ def _on_research_done(_sid, _result, _sources, _findings): # model so metrics reflect it, not the masked # selected model. _answered_by = data.get("answered_by") or _answered_by - _actual_model = _actual_model or _answered_by + _actual_model = _answered_by or _actual_model + if "answered_by_endpoint_id" in data: + _agent_actual_endpoint_id = data.get("answered_by_endpoint_id") + if data.get("answered_by_endpoint_label"): + _agent_actual_endpoint_label = data.get("answered_by_endpoint_label") + _event_round = data.get("round") or max(_agent_rounds, 1) + _agent_round_models[_event_round] = _answered_by or _requested_model + _agent_round_endpoint_ids[_event_round] = _agent_actual_endpoint_id + _agent_round_endpoint_labels[_event_round] = _agent_actual_endpoint_label data["selected_model"] = data.get("selected_model") or _requested_model yield chunk elif data.get("type") == "model_actual": _actual_model = data.get("model") or _actual_model + if "endpoint_id" in data: + _agent_actual_endpoint_id = data.get("endpoint_id") + if data.get("endpoint_label"): + _agent_actual_endpoint_label = data.get("endpoint_label") + _event_round = data.get("round") or max(_agent_rounds, 1) + _agent_round_models[_event_round] = _actual_model or _requested_model + _agent_round_endpoint_ids[_event_round] = _agent_actual_endpoint_id + _agent_round_endpoint_labels[_event_round] = _agent_actual_endpoint_label data["requested_model"] = _requested_model yield f'data: {json.dumps(data)}\n\n' + elif data.get("type") == "agent_terminal": + terminal_metadata = dict(data.get("data") or {}) + last_metrics = terminal_metadata + failure = terminal_metadata.get("failure") or {} + failure_status = _normalize_http_status( + failure.get("status") + ) + failure_message = ( + f"Model request failed (HTTP {failure_status})" + if failure_status is not None + else "Model request failed" + ) + terminal_metadata["failure"] = { + "status": failure_status, + "message": failure_message, + } + terminal_content = full_response.strip() + failure_note = f"[Agent stopped: {failure_message}]" + if terminal_content: + terminal_content = f"{terminal_content}\n\n{failure_note}" + else: + terminal_content = failure_note + if not _terminal_saved: + _saved_id = save_assistant_response( + sess, + session_manager, + session, + terminal_content, + terminal_metadata, + character_name=ctx.preset.character_name, + web_sources=web_sources, + rag_sources=ctx.rag_sources, + used_memories=ctx.used_memories, + incognito=incognito, + ) + _terminal_saved = True + accumulate_token_usage(session, terminal_metadata) + _stream_set(session, status="error") + if _saved_id: + yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n' + yield chunk elif data.get("type") == "metrics": last_metrics = data.get("data", {}) _reported_model = last_metrics.get("model") @@ -1485,7 +1969,16 @@ def _on_research_done(_sid, _result, _sources, _findings): last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim - yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n' + _metrics_event = {"type": "metrics", "data": last_metrics} + # Inline teacher escalation marks its + # recursively emitted events at the SSE + # envelope. Preserve that non-secret marker + # when normalizing metrics so the browser's + # replay-stable ledger keeps primary and + # teacher segments distinct. + if data.get("teacher") is True: + _metrics_event["teacher"] = True + yield f'data: {json.dumps(_metrics_event)}\n\n' except json.JSONDecodeError: yield chunk elif chunk.startswith("event: "): @@ -1537,6 +2030,22 @@ def _on_research_done(_sid, _result, _sources, _findings): "stopped": True, "model": _actual_model or _answered_by or _requested_model, "requested_model": _requested_model, + "endpoint_id": _agent_actual_endpoint_id, + "endpoint_label": _agent_actual_endpoint_label, + "requested_endpoint_id": _agent_requested_route.get("endpoint_id"), + "requested_endpoint_label": _agent_requested_route.get("endpoint_label"), + "round_models": [ + _agent_round_models.get(i, _actual_model or _requested_model) + for i in range(1, max(_agent_round_models, default=1) + 1) + ], + "round_endpoint_ids": [ + _agent_round_endpoint_ids.get(i) + for i in range(1, max(_agent_round_models, default=1) + 1) + ], + "round_endpoint_labels": [ + _agent_round_endpoint_labels.get(i) + for i in range(1, max(_agent_round_models, default=1) + 1) + ], }, ) sess.add_message(ChatMessage("assistant", _stopped_content2, metadata=_stopped_md2)) @@ -1580,8 +2089,12 @@ async def _safe_stream() -> AsyncGenerator[str, None]: if compare_mode: return StreamingResponse(_safe_stream(), media_type="text/event-stream") - agent_runs.start(session, _safe_stream()) - return StreamingResponse(agent_runs.subscribe(session), media_type="text/event-stream") + _detached_run = agent_runs.start(session, _safe_stream()) + return StreamingResponse( + agent_runs.subscribe(session, _detached_run), + media_type="text/event-stream", + headers={"X-Odysseus-Run-Id": _detached_run.run_id}, + ) # ------------------------------------------------------------------ # # GET /api/chat/resume — reconnect to a detached run that's still going @@ -1590,9 +2103,14 @@ async def _safe_stream() -> AsyncGenerator[str, None]: @router.get("/api/chat/resume/{session_id}") async def chat_resume(request: Request, session_id: str) -> StreamingResponse: _verify_session_owner(request, session_id) - if not agent_runs.is_active(session_id): + _active_run = agent_runs.get_active_run(session_id) + if _active_run is None: raise HTTPException(404, "No active run for this session") - return StreamingResponse(agent_runs.subscribe(session_id), media_type="text/event-stream") + return StreamingResponse( + agent_runs.subscribe(session_id, _active_run), + media_type="text/event-stream", + headers={"X-Odysseus-Run-Id": _active_run.run_id}, + ) # ------------------------------------------------------------------ # # POST /api/chat/stop — cancel a detached run (Stop button). Closing the SSE @@ -1601,7 +2119,8 @@ async def chat_resume(request: Request, session_id: str) -> StreamingResponse: @router.post("/api/chat/stop/{session_id}") async def chat_stop(request: Request, session_id: str) -> Dict[str, Any]: _verify_session_owner(request, session_id) - stopped = agent_runs.stop(session_id) + _expected_run_id = request.headers.get("X-Odysseus-Run-Id") + stopped = agent_runs.stop(session_id, _expected_run_id) return {"stopped": stopped} # ------------------------------------------------------------------ # diff --git a/routes/email_routes.py b/routes/email_routes.py index ad13ae504c..241e141b3e 100644 --- a/routes/email_routes.py +++ b/routes/email_routes.py @@ -4244,7 +4244,6 @@ async def translate_email(data: dict, owner: str = Depends(require_owner)): from src.endpoint_resolver import ( resolve_endpoint, resolve_utility_fallback_candidates, - resolve_chat_fallback_candidates, ) from src.llm_core import llm_call_async_with_fallback @@ -4306,8 +4305,6 @@ def _add(url, model, headers): pass for cand in resolve_utility_fallback_candidates(owner=owner) or []: _add(*cand) - for cand in resolve_chat_fallback_candidates(owner=owner) or []: - _add(*cand) if not candidates: return {"success": False, "error": "No LLM endpoint configured"} @@ -4561,13 +4558,11 @@ async def ai_reply(data: dict, owner: str = Depends(require_owner)): # Build a candidate chain so a stale session-stored API key # (the most common cause of "authentication failed" here) # doesn't kill AI Reply outright — fall through to the - # user's Utility / Default endpoints and the active Utility - # fallback chain. The retired default-fallback hook stays empty. - # Dedupe by url+model so we don't retry the same broken endpoint. + # user's Utility / Default endpoints and active Utility fallback + # chain. Dedupe by url+model so we don't retry the same endpoint. from src.llm_core import llm_call_async_with_fallback from src.endpoint_resolver import ( resolve_utility_fallback_candidates, - resolve_chat_fallback_candidates, ) _seen = set() _candidates = [] @@ -4592,11 +4587,9 @@ def _add(_url, _model, _headers): _add(_d_url, _d_model, _d_headers) except Exception: pass - # Active Utility fallbacks, then the retired default hook. + # Active Utility fallbacks last. for cand in resolve_utility_fallback_candidates(owner=owner) or []: _add(*cand) - for cand in resolve_chat_fallback_candidates(owner=owner) or []: - _add(*cand) _messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_msg}, diff --git a/routes/model_routes.py b/routes/model_routes.py index 62bac56907..57f36c9b76 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -46,6 +46,7 @@ } _ENDPOINT_FALLBACK_FIELDS = { + "foreground_model_fallbacks": "Foreground Model Fallbacks", "utility_model_fallbacks": "Utility Model Fallbacks", "vision_model_fallbacks": "Vision Model Fallbacks", } @@ -180,7 +181,12 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int: if not isinstance(all_prefs, dict): return 0 users = all_prefs.get("_users") - pref_sets = users.values() if isinstance(users, dict) else [all_prefs] + # A mixed store can contain auth-disabled foreground policy at the root + # alongside named-owner preferences. Both are active namespaces; legacy + # `default_model_fallbacks` remains untouched by the field allowlist. + pref_sets = [all_prefs] + if isinstance(users, dict): + pref_sets.extend(users.values()) cleared_users = 0 for prefs in pref_sets: if isinstance(prefs, dict) and _clear_endpoint_settings_for_endpoint(prefs, ep_id): diff --git a/routes/prefs_routes.py b/routes/prefs_routes.py index f2a778c2dc..740af8cb64 100644 --- a/routes/prefs_routes.py +++ b/routes/prefs_routes.py @@ -7,6 +7,10 @@ from src.constants import USER_PREFS_FILE PREFS_FILE = USER_PREFS_FILE +_FOREGROUND_POLICY_KEYS = ( + "foreground_fallback_enabled", + "foreground_model_fallbacks", +) def _load(): @@ -32,14 +36,27 @@ def _save(prefs): def _load_for_user(user: Optional[str] = None) -> dict: """Load preferences for a specific user.""" all_prefs = _load() - if "_users" in all_prefs: + users = all_prefs.get("_users") + if isinstance(users, dict): if user is None: # Auth disabled — return first user's prefs for backward compat - users = all_prefs["_users"] - return dict(next(iter(users.values()), {})) - return dict(all_prefs["_users"].get(user, {})) - # Legacy flat format — return as-is - return dict(all_prefs) + prefs = dict(next(iter(users.values()), {})) + # Foreground fallback consent is never borrowed from a named + # owner. Auth-disabled operation has a separate flat/root opt-in + # that remains inert when authentication is enabled again. + for key in _FOREGROUND_POLICY_KEYS: + prefs.pop(key, None) + if key in all_prefs: + prefs[key] = all_prefs[key] + return prefs + prefs = users.get(user, {}) + return dict(prefs) if isinstance(prefs, dict) else {} + # A legacy flat store belongs only to auth-disabled single-user mode. + # Copying it into the first named user's new `_users` record during an + # auth transition would silently transfer another user's preferences and, + # critically, foreground fallback consent. Named owners therefore start + # with an empty record and must write their own preferences explicitly. + return dict(all_prefs) if user is None else {} def _save_for_user(user: Optional[str], prefs: dict): @@ -51,17 +68,40 @@ def _save_for_user(user: Optional[str], prefs: dict): # `prefs` flat would overwrite the whole `_users` map and destroy every # other user's preferences. Instead write back into the same (first) # slot _load_for_user(None) reads from, preserving the others. - if "_users" in all_prefs: - users = all_prefs["_users"] + users = all_prefs.get("_users") + if isinstance(users, dict): first_key = next(iter(users), None) if first_key is not None: - users[first_key] = prefs + existing_named = users.get(first_key) + existing_named = ( + dict(existing_named) + if isinstance(existing_named, dict) + else {} + ) + named_foreground = { + key: existing_named[key] + for key in _FOREGROUND_POLICY_KEYS + if key in existing_named + } + users[first_key] = { + key: value + for key, value in prefs.items() + if key not in _FOREGROUND_POLICY_KEYS + } + users[first_key].update(named_foreground) + for key in _FOREGROUND_POLICY_KEYS: + if key in prefs: + all_prefs[key] = prefs[key] _save(all_prefs) return _save(prefs) return - if "_users" not in all_prefs: - all_prefs = {"_users": {}} + if not isinstance(all_prefs.get("_users"), dict): + # Preserve the flat single-user object as inert legacy data while + # creating the first named-owner namespace. In particular, historical + # fallback values must not be deleted or copied into the new owner. + all_prefs = dict(all_prefs) + all_prefs["_users"] = {} all_prefs["_users"][user] = prefs _save(all_prefs) diff --git a/src/agent_loop.py b/src/agent_loop.py index 6f7dde6054..03c9dc6c59 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -16,11 +16,19 @@ from urllib.parse import urlparse from src.llm_core import ( + dedupe_model_candidates, stream_llm, stream_llm_with_fallback, _is_ollama_native_url, + _normalize_http_status, + _normalize_usage_counts, ) from src.model_context import estimate_tokens +from src.context_compactor import ( + apply_compaction_state, + apply_compaction_state_for_session, + maybe_compact, +) from src.settings import get_setting from src.prompt_security import untrusted_context_message from src.tool_security import blocked_tools_for_owner, plan_mode_disabled_tools @@ -757,6 +765,88 @@ def add(value: str): pass return keys + +def _agent_route_tool_mode( + endpoint_url: str, + model: str, + owner: Optional[str] = None, + headers: Optional[Dict] = None, +) -> tuple[bool, bool, bool]: + """Resolve tool transport behavior for the currently active model route.""" + + model_lc = (model or "").lower() + endpoint_supports: Optional[bool] = None + try: + from core.database import SessionLocal as _SL, ModelEndpoint as _ME + + db = _SL() + try: + endpoints = [] + seen_ids = set() + for key in _endpoint_lookup_keys(endpoint_url): + query = db.query(_ME).filter(_ME.base_url == key) + if owner: + from src.auth_helpers import owner_filter + + query = owner_filter(query, _ME, owner) + rows = query.all() if hasattr(query, "all") else [query.first()] + for row in rows: + row_id = getattr(row, "id", None) + if row is not None and row_id not in seen_ids: + seen_ids.add(row_id) + endpoints.append(row) + endpoint = None + if headers is not None: + from src.endpoint_resolver import build_headers, resolve_endpoint_runtime + + expected_headers = { + str(key).lower(): str(value) + for key, value in (headers or {}).items() + } + for candidate in endpoints: + runtime_base, api_key = resolve_endpoint_runtime(candidate, owner=owner) + candidate_headers = { + str(key).lower(): str(value) + for key, value in build_headers(api_key, runtime_base).items() + } + if candidate_headers == expected_headers: + endpoint = candidate + break + elif endpoints: + endpoint = endpoints[0] + if endpoint is not None: + endpoint_supports = endpoint.supports_tools + finally: + db.close() + except Exception as exc: + logger.debug("endpoint supports_tools lookup failed: %s", exc) + + model_supports_tools = any(kw in model_lc for kw in ( + "gpt-4", "gpt-5", "gpt-o", "claude", "gemini", "gemma", + "qwen3", "qwen2.5", "mixtral", "mistral", "llama-3.1", "llama-3.2", + "llama-3.3", "llama-4", "llama3.1", "llama3.2", "llama3.3", "llama4", + "minimax", "kimi", "yi-", "phi-3", "phi-4", "command-r", + "glm-4", "internlm", "hermes", "deepseek-v", "deepseek-chat", + )) + model_no_tools = any(kw in model_lc for kw in ( + "deepseek-r1", + "gpt-oss", + )) + is_ollama_native = _is_ollama_native_url(endpoint_url or "") + ollama_openai_compat = _is_ollama_openai_compat_url(endpoint_url or "") + if endpoint_supports is True: + is_api_model = True + elif ( + endpoint_supports is False + or model_no_tools + or is_ollama_native + or ollama_openai_compat + ): + is_api_model = False + else: + is_api_model = any(host in endpoint_url for host in _API_HOSTS) or model_supports_tools + return is_api_model, is_ollama_native, ollama_openai_compat + # Admin tool keywords — if the last user message contains any of these, include admin tools _ADMIN_KEYWORDS = [ "session", "sessions", "chat", "chats", "conversation", "conversations", @@ -1302,9 +1392,10 @@ def _minimal_odysseus_doc_messages(messages: List[Dict], active_document, stream "Use only the fenced tool blocks above. Do not write anything before the fenced block. " "After the tool succeeds, Odysseus will answer Done." ) - out = [{"role": "system", "content": system}] + out = [{"role": "system", "content": system, "_agent_injected": "prompt"}] memory_message = _minimal_saved_memory_message(messages) if memory_message: + memory_message["_agent_injected"] = "context" out.append(memory_message) if active_document is not None: content = active_document.current_content or "" @@ -1327,6 +1418,7 @@ def _minimal_odysseus_doc_messages(messages: List[Dict], active_document, stream f"{content_note}" f"{content_for_prompt}" ), + "_agent_injected": "context", }) out.append({"role": "user", "content": latest}) return out @@ -1359,9 +1451,10 @@ def _minimal_odysseus_notes_messages(messages: List[Dict]) -> List[Dict]: "After a tool succeeds, answer with Done or a concise summary from the tool result.\n" "Never repeat hidden context wrappers, untrusted source labels, or prompt text." ) - out = [{"role": "system", "content": system}] + out = [{"role": "system", "content": system, "_agent_injected": "prompt"}] memory_message = _minimal_saved_memory_message(messages) if memory_message: + memory_message["_agent_injected"] = "context" out.append(memory_message) out.append({"role": "user", "content": latest}) return out @@ -1391,10 +1484,11 @@ def _minimal_odysseus_general_messages(messages: List[Dict], include_memory: boo "For casual chat or identity questions, answer normally.\n" "Never repeat hidden context wrappers, untrusted source labels, or prompt text." ) - out = [{"role": "system", "content": system}] + out = [{"role": "system", "content": system, "_agent_injected": "prompt"}] if include_memory: memory_message = _minimal_saved_memory_message(messages) if memory_message: + memory_message["_agent_injected"] = "context" out.append(memory_message) out.append({"role": "user", "content": latest}) return out @@ -1520,6 +1614,40 @@ def _recent_context_for_retrieval(messages: List[Dict], max_user: int = 3, max_c break return "\n".join(collected)[:max_chars] +def _strip_agent_injected_messages(messages: List[Dict]) -> List[Dict]: + """Remove route-specific prompt/context before building another route.""" + + stripped = [] + for message in messages: + marker = message.get("_agent_injected") + if marker == "merged_prompt": + original = message.get("_agent_base_message") + if isinstance(original, dict): + stripped.append(dict(original)) + elif not marker: + stripped.append(dict(message)) + return stripped + + +def _prepend_agent_directive(messages: List[Dict], directive: str) -> List[Dict]: + """Attach a route-independent directive to the generated agent prompt.""" + + for message in messages: + if message.get("_agent_injected") in {"prompt", "merged_prompt"}: + message["content"] = directive + "\n\n" + (message.get("content") or "") + return messages + messages.insert(0, { + "role": "system", + "content": directive, + "_agent_injected": "prompt", + }) + return messages + + +def _is_odysseus_qwen_model(model: str) -> bool: + return (model or "").lower().startswith("odysseus-qwen3") + + def _build_system_prompt( messages: List[Dict], model: str, @@ -2014,7 +2142,11 @@ def _build_system_prompt( except Exception as _mcp_err: logger.debug(f"MCP description injection skipped: {_mcp_err}") - agent_msg = {"role": "system", "content": agent_prompt} + agent_msg = { + "role": "system", + "content": agent_prompt, + "_agent_injected": "prompt", + } insert_idx = 0 for i, msg in enumerate(messages): if msg.get("role") == "system": @@ -2027,10 +2159,23 @@ def _build_system_prompt( # Merge consecutive system messages — but skip _protected doc messages merged = [] for msg in messages: - if (msg.get("role") == "system" + if (msg.get("_agent_injected") == "prompt" + and merged and merged[-1].get("role") == "system" + and not merged[-1].get("_protected") + and not merged[-1].get("_agent_injected")): + base_message = dict(merged[-1]) + merged[-1] = { + "role": "system", + "content": base_message.get("content", "") + "\n\n" + msg["content"], + "_agent_injected": "merged_prompt", + "_agent_base_message": base_message, + } + elif (msg.get("role") == "system" and not msg.get("_protected") + and not msg.get("_agent_injected") and merged and merged[-1].get("role") == "system" - and not merged[-1].get("_protected")): + and not merged[-1].get("_protected") + and not merged[-1].get("_agent_injected")): merged[-1] = { "role": "system", "content": merged[-1]["content"] + "\n\n" + msg["content"], @@ -2047,6 +2192,17 @@ def _build_system_prompt( if merged[i].get("role") == "user": last_user_idx = i break + for injected in ( + _doc_message, + _email_message, + _email_style_message, + _integ_message, + _mcp_desc_message, + _skills_message, + _datetime_message, + ): + if injected: + injected["_agent_injected"] = "context" if _doc_message: merged.insert(last_user_idx, _doc_message) last_user_idx += 1 # the document message is now at last_user_idx @@ -2317,6 +2473,9 @@ def _compute_final_metrics( tool_events: list, round_texts: list, model: str = "", + round_models: Optional[list] = None, + round_endpoint_ids: Optional[list] = None, + round_endpoint_labels: Optional[list] = None, last_round_input_tokens: int = 0, prep_timings: Optional[Dict[str, float]] = None, backend_gen_tps: float = 0, @@ -2372,10 +2531,61 @@ def _compute_final_metrics( } if tool_events: metrics["tool_events"] = tool_events + if round_texts: metrics["round_texts"] = round_texts + metrics["round_models"] = list(round_models or []) + metrics["round_endpoint_ids"] = list(round_endpoint_ids or []) + metrics["round_endpoint_labels"] = list(round_endpoint_labels or []) return metrics +def _usage_bucket( + *, + round_num: int, + model: str, + endpoint_id, + endpoint_label, + endpoint_cost_tracked, + input_tokens: int, + output_tokens: int, + usage_source: str, +) -> dict: + """Build non-secret usage attribution for one concrete Agent round.""" + + bucket = { + "round": round_num, + "model": model, + "endpoint_id": endpoint_id, + "endpoint_label": endpoint_label, + "input_tokens": max(int(input_tokens or 0), 0), + "output_tokens": max(int(output_tokens or 0), 0), + "usage_source": "real" if usage_source == "real" else "estimated", + } + # Persist the owner-resolved route classification so saved usage remains + # stable even if the session later selects a different endpoint. + if isinstance(endpoint_cost_tracked, bool): + bucket["endpoint_cost_tracked"] = endpoint_cost_tracked + return bucket + + +def _usage_bucket_summary(usage_buckets: list) -> dict: + """Return aggregate token fields without losing per-route attribution.""" + + if not usage_buckets: + return {} + input_tokens = sum(bucket.get("input_tokens", 0) or 0 for bucket in usage_buckets) + output_tokens = sum(bucket.get("output_tokens", 0) or 0 for bucket in usage_buckets) + sources = {bucket.get("usage_source") for bucket in usage_buckets} + usage_source = next(iter(sources)) if len(sources) == 1 else "mixed" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "usage_source": usage_source, + "usage_buckets": [dict(bucket) for bucket in usage_buckets], + } + + # ── Completion verifier ── # Tools whose effects produce a checkable artifact. A turn that used one of # these is "effectful" and worth an independent completion check; pure @@ -2556,6 +2766,9 @@ async def stream_agent_loop( owner: Optional[str] = None, relevant_tools: Optional[Set[str]] = None, fallbacks: Optional[List[tuple]] = None, + route_descriptors: Optional[List[dict]] = None, + fallback_statuses: Optional[Set[int]] = None, + fallback_on_empty: bool = True, plan_mode: bool = False, approved_plan: Optional[str] = None, tool_policy: Optional[ToolPolicy] = None, @@ -2564,6 +2777,8 @@ async def stream_agent_loop( uploaded_files: Optional[List[Dict]] = None, workload: str = "foreground", _is_teacher_run: bool = False, + history_session=None, + defer_context_shaping: bool = False, ) -> AsyncGenerator[str, None]: """Streaming agent loop generator. @@ -2579,6 +2794,15 @@ async def stream_agent_loop( mcp_mgr = get_mcp_manager() prep_timings: Dict[str, float] = {} disabled_tools = set(disabled_tools or []) + route_descriptors = list(route_descriptors or []) + while len(route_descriptors) < 1 + len(fallbacks or []): + route_descriptors.append({}) + requested_route = route_descriptors[0] if route_descriptors else {} + requested_endpoint_id = requested_route.get("endpoint_id") + requested_endpoint_label = requested_route.get("endpoint_label") or "Selected route" + requested_endpoint_cost_tracked = requested_route.get("endpoint_cost_tracked") + if not isinstance(requested_endpoint_cost_tracked, bool): + requested_endpoint_cost_tracked = None if tool_policy: disabled_tools.update(tool_policy.all_disabled_names()) if tool_policy.disable_mcp: @@ -2606,7 +2830,7 @@ async def stream_agent_loop( _t0 = time.time() _needs_admin = _detect_admin_intent(messages) _last_user = _extract_last_user_message(messages) - _ody_qwen_finetune_model = (model or "").lower().startswith("odysseus-qwen3") + _ody_qwen_finetune_model = _is_odysseus_qwen_model(model) _ody_memory_identity_turn = _looks_like_memory_identity_turn(_last_user) _intent = _classify_agent_request(messages, _last_user) _low_signal_turn = bool(_intent.get("low_signal")) @@ -2664,8 +2888,79 @@ async def stream_agent_loop( direct_response = "" direct_start = time.time() direct_actual_model = model + direct_actual_endpoint_id = requested_endpoint_id + direct_actual_endpoint_label = requested_endpoint_label + direct_actual_endpoint_cost_tracked = requested_endpoint_cost_tracked + direct_actual_messages = direct_messages + direct_candidate_messages = {0: direct_messages} + direct_reasoning = "" real_input_tokens = 0 real_output_tokens = 0 + direct_has_real_usage = False + + def _direct_candidate_request(_index, _url, candidate_model, _headers): + candidate_messages = ( + _minimal_odysseus_general_messages(messages, include_memory=True) + if _is_odysseus_qwen_model(candidate_model) + else [{"role": "user", "content": _last_user}] + ) + direct_candidate_messages[_index] = candidate_messages + return {"messages": candidate_messages} + + def _direct_terminal_event(terminal_status, failure_message): + """Build truthful partial-history metadata for direct-path failure.""" + if not (direct_response.strip() or direct_reasoning.strip()): + return None + direct_usage = _usage_bucket( + round_num=1, + model=direct_actual_model, + endpoint_id=direct_actual_endpoint_id, + endpoint_label=direct_actual_endpoint_label, + endpoint_cost_tracked=direct_actual_endpoint_cost_tracked, + input_tokens=( + real_input_tokens + if direct_has_real_usage + else estimate_tokens(direct_actual_messages) + ), + output_tokens=( + real_output_tokens + if direct_has_real_usage + else max(len(direct_response + direct_reasoning) // 4, 0) + ), + usage_source="real" if direct_has_real_usage else "estimated", + ) + failure_note = f"[Agent stopped: {failure_message}]" + terminal_round = ( + f"{direct_response.strip()}\n\n{failure_note}" + if direct_response.strip() + else failure_note + ) + terminal_metadata = { + "failed": True, + "failure": { + "status": terminal_status, + "message": failure_message, + }, + "model": direct_actual_model, + "requested_model": model, + "endpoint_id": direct_actual_endpoint_id, + "endpoint_label": direct_actual_endpoint_label, + "requested_endpoint_id": requested_endpoint_id, + "requested_endpoint_label": requested_endpoint_label, + "round_texts": [terminal_round], + "round_models": [direct_actual_model], + "round_endpoint_ids": [direct_actual_endpoint_id], + "round_endpoint_labels": [direct_actual_endpoint_label], + **_usage_bucket_summary([direct_usage]), + } + if direct_reasoning.strip(): + terminal_metadata["thinking"] = direct_reasoning.strip() + if isinstance(direct_actual_endpoint_cost_tracked, bool): + terminal_metadata["endpoint_cost_tracked"] = ( + direct_actual_endpoint_cost_tracked + ) + return f'data: {json.dumps({"type": "agent_terminal", "data": terminal_metadata})}\n\n' + try: async for chunk in stream_llm_with_fallback( [(endpoint_url, model, headers)] + list(fallbacks or []), @@ -2677,6 +2972,10 @@ async def stream_agent_loop( timeout=int(get_setting("agent_stream_timeout_seconds", 300) or 300), session_id=session_id, workload=workload, + fallback_statuses=fallback_statuses, + fallback_on_empty=fallback_on_empty, + candidate_request_factory=_direct_candidate_request, + candidate_route_descriptors=route_descriptors, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: @@ -2687,49 +2986,143 @@ async def stream_agent_loop( if data.get("type") == "usage": usage = data.get("data", {}) or {} direct_actual_model = usage.get("model") or direct_actual_model - real_input_tokens += usage.get("input_tokens", 0) or 0 - real_output_tokens += usage.get("output_tokens", 0) or 0 + normalized_usage = _normalize_usage_counts( + usage.get("input_tokens", 0), + usage.get("output_tokens", 0), + ) + if normalized_usage is None: + logger.warning("[agent] ignoring malformed direct usage event") + continue + real_input_tokens += normalized_usage["input_tokens"] + real_output_tokens += normalized_usage["output_tokens"] + direct_has_real_usage = True continue if data.get("type") == "model_actual": direct_actual_model = data.get("model") or direct_actual_model data["requested_model"] = model + data["requested_endpoint_id"] = requested_endpoint_id + data["requested_endpoint_label"] = requested_endpoint_label + data["endpoint_id"] = direct_actual_endpoint_id + data["endpoint_label"] = direct_actual_endpoint_label yield f"data: {json.dumps(data)}\n\n" continue if data.get("type") == "fallback": direct_actual_model = data.get("answered_by") or direct_actual_model + direct_actual_endpoint_id = data.get("answered_by_endpoint_id") + direct_actual_endpoint_label = ( + data.get("answered_by_endpoint_label") or direct_actual_endpoint_label + ) + if isinstance(data.get("answered_by_endpoint_cost_tracked"), bool): + direct_actual_endpoint_cost_tracked = data.get( + "answered_by_endpoint_cost_tracked" + ) + candidate_index = data.get("candidate_index") + if isinstance(candidate_index, int): + direct_actual_messages = direct_candidate_messages.get( + candidate_index, + direct_actual_messages, + ) yield chunk continue if "delta" in data: - if not data.get("thinking"): + if data.get("thinking"): + direct_reasoning += data.get("delta", "") + else: direct_response += data.get("delta", "") yield chunk continue yield chunk + elif chunk.startswith("event: error"): + # A provider/request error is terminal here too. Do not + # replace it with the casual-response fallback or emit + # success metrics/[DONE]. + terminal_status = None + try: + error_line = next( + line[6:] + for line in chunk.splitlines() + if line.startswith("data: ") + ) + terminal_status = _normalize_http_status( + json.loads(error_line).get("status") + ) + except (StopIteration, json.JSONDecodeError): + terminal_status = None + failure_message = ( + f"Model request failed (HTTP {terminal_status})" + if terminal_status is not None + else "Model request failed" + ) + terminal_event = _direct_terminal_event( + terminal_status, + failure_message, + ) + if terminal_event: + yield terminal_event + yield chunk + return elif chunk.startswith("event: "): yield chunk except Exception as _direct_err: logger.warning("[agent] direct low-signal path failed: %s", _direct_err) - fallback = "Hey." - direct_response += fallback - yield f"data: {json.dumps({'delta': fallback})}\n\n" + failure_message = "Model request failed" + terminal_event = _direct_terminal_event(None, failure_message) + if terminal_event: + yield terminal_event + yield ( + "event: error\n" + f"data: {json.dumps({'error': failure_message, 'status': 500, 'fallback_eligible': False})}\n\n" + ) + return if not direct_response.strip(): - fallback = "Hey." - direct_response = fallback - yield f"data: {json.dumps({'delta': fallback})}\n\n" + failure_message = "Model returned an empty response" + terminal_event = _direct_terminal_event(None, failure_message) + if terminal_event: + yield terminal_event + yield ( + "event: error\n" + f"data: {json.dumps({'error': failure_message, 'status': 502, 'fallback_eligible': False})}\n\n" + ) + return duration = time.time() - direct_start + direct_usage = _usage_bucket( + round_num=1, + model=direct_actual_model, + endpoint_id=direct_actual_endpoint_id, + endpoint_label=direct_actual_endpoint_label, + endpoint_cost_tracked=direct_actual_endpoint_cost_tracked, + input_tokens=( + real_input_tokens + if direct_has_real_usage + else estimate_tokens(direct_actual_messages) + ), + output_tokens=( + real_output_tokens + if direct_has_real_usage + else max(len(direct_response) // 4, 1) + ), + usage_source="real" if direct_has_real_usage else "estimated", + ) metrics = { "model": direct_actual_model, "requested_model": model, - "input_tokens": real_input_tokens or estimate_tokens(direct_messages), + "endpoint_id": direct_actual_endpoint_id, + "endpoint_label": direct_actual_endpoint_label, + "requested_endpoint_id": requested_endpoint_id, + "requested_endpoint_label": requested_endpoint_label, + "input_tokens": real_input_tokens or estimate_tokens(direct_actual_messages), "output_tokens": real_output_tokens or max(len(direct_response) // 4, 1), "total_time": round(duration, 2), "response_time": round(duration, 2), "agent_rounds": 0, "tool_calls": 0, "direct_low_signal": True, + **_usage_bucket_summary([direct_usage]), } + if isinstance(direct_actual_endpoint_cost_tracked, bool): + metrics["endpoint_cost_tracked"] = direct_actual_endpoint_cost_tracked yield f"data: {json.dumps({'type': 'metrics', 'data': metrics})}\n\n" yield "data: [DONE]\n\n" return @@ -2935,36 +3328,58 @@ async def stream_agent_loop( logger.debug(f"[tool-rag] skill-aware tool include skipped: {_e}") _intent_domains = set(_intent.get("domains") or set()) - _ody_doc_finetune_mode = ( - _ody_qwen_finetune_model - and ( - "documents" in _intent_domains - or _active_document_relevant - or _prompt_active_document is not None + _base_relevant_tools = None if _relevant_tools is None else set(_relevant_tools) + _runtime_skill_tools: Set[str] = set() + + def _route_finetune_modes(candidate_model: str): + is_ody = _is_odysseus_qwen_model(candidate_model) + doc_mode = ( + is_ody + and not _runtime_skill_tools + and ( + "documents" in _intent_domains + or _active_document_relevant + or _prompt_active_document is not None + ) + and "files" not in _intent_domains + and not guide_only ) - and "files" not in _intent_domains - and not guide_only - ) - _ody_notes_finetune_mode = ( - _ody_qwen_finetune_model - and not _ody_doc_finetune_mode - and ("notes_calendar_tasks" in _intent_domains or _looks_like_notes_turn(_last_user)) - and _looks_like_notes_turn(_last_user) - and "files" not in _intent_domains - and not guide_only - ) - _ody_doc_stream_create_mode = _ody_doc_finetune_mode and _prompt_active_document is None + notes_mode = ( + is_ody + and not _runtime_skill_tools + and not doc_mode + and ("notes_calendar_tasks" in _intent_domains or _looks_like_notes_turn(_last_user)) + and _looks_like_notes_turn(_last_user) + and "files" not in _intent_domains + and not guide_only + ) + return is_ody, doc_mode, notes_mode, doc_mode and _prompt_active_document is None + + def _route_relevant_tools(candidate_model: str): + route_tools = None if _base_relevant_tools is None else set(_base_relevant_tools) + _is_ody, doc_mode, notes_mode, _stream_create = _route_finetune_modes(candidate_model) + if doc_mode and route_tools is not None: + if _prompt_active_document is not None: + route_tools = { + "edit_document", "update_document", "suggest_document", + "ask_user", "update_plan", + } + else: + route_tools = {"create_document", "ask_user", "update_plan"} + elif notes_mode and route_tools is not None: + route_tools = {"manage_notes", "ask_user", "update_plan"} + return route_tools + + ( + _ody_qwen_finetune_model, + _ody_doc_finetune_mode, + _ody_notes_finetune_mode, + _ody_doc_stream_create_mode, + ) = _route_finetune_modes(model) + _relevant_tools = _route_relevant_tools(model) if _ody_doc_finetune_mode and _relevant_tools is not None: - if _prompt_active_document is not None: - _relevant_tools = { - "edit_document", "update_document", "suggest_document", - "ask_user", "update_plan", - } - else: - _relevant_tools = {"create_document", "ask_user", "update_plan"} logger.info("[agent-intent] odysseus doc finetune tool clamp=%s", sorted(_relevant_tools)) elif _ody_notes_finetune_mode and _relevant_tools is not None: - _relevant_tools = {"manage_notes", "ask_user", "update_plan"} logger.info("[agent-intent] odysseus notes finetune tool clamp=%s", sorted(_relevant_tools)) if ( @@ -2986,6 +3401,8 @@ async def stream_agent_loop( "run_shell", "write_file", } + if _base_relevant_tools is not None: + _base_relevant_tools.difference_update(_doc_irrelevant_file_tools) _removed_doc_file_tools = sorted(_relevant_tools & _doc_irrelevant_file_tools) if _removed_doc_file_tools: _relevant_tools.difference_update(_doc_irrelevant_file_tools) @@ -3000,202 +3417,186 @@ async def stream_agent_loop( prep_timings["tool_selection"] = time.time() - _t1 _t2 = time.time() - # Hosted-API match by URL, OR the model name looks like a recent model - # known to follow OpenAI-style function calling (DeepSeek, GPT*, Claude, - # Gemini, Qwen3+, Mixtral, Llama 3.1+). Caught the DeepSeek-via-local- - # vLLM case where endpoint_url doesn't include a vendor host. - _model_lc = (model or "").lower() - # Step 1: per-endpoint override (set at registration time from the - # serve command — `--enable-auto-tool-choice` flips it on. UI can - # also toggle per endpoint). NULL = unknown; for local Ollama /v1 we - # default to fenced tools, otherwise fall through to keyword + host checks. - _endpoint_supports: Optional[bool] = None - try: - from core.database import SessionLocal as _SL, ModelEndpoint as _ME - _db = _SL() - try: - _ep = None - for _key in _endpoint_lookup_keys(endpoint_url): - _ep = _db.query(_ME).filter(_ME.base_url == _key).first() - if _ep is not None: - break - if _ep is not None: - _endpoint_supports = _ep.supports_tools - finally: - _db.close() - except Exception as _e: - logger.debug(f"endpoint supports_tools lookup failed: {_e}") - _model_supports_tools = any(kw in _model_lc for kw in ( - "gpt-4", "gpt-5", "gpt-o", "claude", "gemini", "gemma", - "qwen3", "qwen2.5", "mixtral", "mistral", "llama-3.1", "llama-3.2", - "llama-3.3", "llama-4", "llama3.1", "llama3.2", "llama3.3", "llama4", - # Local-served models that follow OpenAI-style function calling - # via vLLM's `--enable-auto-tool-choice`. Belt-and-suspenders - # with the per-endpoint flag above. - "minimax", "kimi", "yi-", "phi-3", "phi-4", "command-r", - "glm-4", "internlm", "hermes", - # deepseek-v2/v3/chat support tools via the cloud API; deepseek-r1 - # (reasoning model) does not — handled by the blocklist below. - "deepseek-v", "deepseek-chat", - )) - # Models known to reject tool schemas at the Ollama/local level even when - # the endpoint URL would otherwise enable native function calling. - # The per-endpoint supports_tools flag (True/False) always takes priority - # and can override this list for users who know their setup. - _model_no_tools = any(kw in _model_lc for kw in ( - "deepseek-r1", - # Open-weight GPT-OSS models are commonly served through llama.cpp / - # llama-cpp-python. Their names contain "gpt-o", but they do not use - # OpenAI's native tool-call channel unless the endpoint opts in. - "gpt-oss", - )) - # Native Ollama endpoints (/api/chat) handle tool schemas differently from - # the OpenAI-compat path. Models like gemma4, qwen3.5, ministral respond to - # tool schemas by emitting a single native tool_call token then stopping, - # rather than writing a fenced block — the agent loop sees 1 token and no - # recognised tool, so the round terminates immediately (issue #1567). - # Unless the endpoint is explicitly marked supports_tools=True by the user - # (via the endpoint settings toggle), treat Ollama-native as text-only so - # the fenced-block path is used instead of native function calling. - _is_ollama_native = _is_ollama_native_url(endpoint_url or "") - _ollama_openai_compat = _is_ollama_openai_compat_url(endpoint_url or "") - if _endpoint_supports is True: - _is_api_model = True - elif ( - _endpoint_supports is False - or _model_no_tools - or _is_ollama_native - or _ollama_openai_compat - ): - _is_api_model = False - else: - _is_api_model = any(h in endpoint_url for h in _API_HOSTS) or _model_supports_tools - _compact_agent_prompt = _is_api_model or _is_ollama_native or _ollama_openai_compat - messages, mcp_schemas = _build_system_prompt( - messages, model, _prompt_active_document, mcp_mgr, disabled_tools, - needs_admin=_needs_admin, relevant_tools=_relevant_tools, - mcp_disabled_map=_mcp_disabled_map, - compact=_compact_agent_prompt, - owner=owner, - suppress_local_context=guide_only, - suppress_skills=_low_signal_turn, - active_email=active_email, - ) - if _ody_doc_finetune_mode and not plan_mode and not approved_plan and not guide_only: - messages = _minimal_odysseus_doc_messages( - messages, - _prompt_active_document, - stream_create=_ody_doc_stream_create_mode, - ) - mcp_schemas = [] - logger.info( - "[agent-intent] odysseus doc minimal prompt active active_doc=%s stream_create=%s messages=%s", - bool(_prompt_active_document), - _ody_doc_stream_create_mode, - len(messages), - ) - elif _ody_notes_finetune_mode and not plan_mode and not approved_plan and not guide_only: - messages = _minimal_odysseus_notes_messages(messages) - mcp_schemas = [] - logger.info( - "[agent-intent] odysseus notes minimal prompt active messages=%s", - len(messages), - ) - elif _ody_qwen_finetune_model and not plan_mode and not approved_plan and not guide_only: - messages = _minimal_odysseus_general_messages( - messages, - include_memory=True, - ) - mcp_schemas = [] - logger.info( - "[agent-intent] odysseus general minimal prompt active include_memory=%s messages=%s", - _ody_memory_identity_turn, - len(messages), - ) - if plan_mode and not guide_only: - # Steer the model to investigate-then-propose. Hard tool gating handles - # every write path except shell; this directive is what keeps the - # intentionally-allowed bash/python read-only, so it must DOMINATE. Put - # it at the very TOP of the system prompt (the base prompt is large and - # action-oriented — appending buried it, and small models ignored it). - if messages and messages[0].get("role") == "system": - messages[0]["content"] = PLAN_MODE_DIRECTIVE + "\n\n" + (messages[0].get("content") or "") - else: - messages.insert(0, {"role": "system", "content": PLAN_MODE_DIRECTIVE}) - elif approved_plan and approved_plan.strip() and not guide_only: - # EXECUTING an approved plan. Pin the checklist as a top-of-context - # system note so a long plan on a weak model survives history - # truncation — the agent can always re-read the plan instead of losing - # the thread. (The first system message is kept by the context trimmer.) - _plan_note = build_active_plan_note(approved_plan) - if messages and messages[0].get("role") == "system": - messages[0]["content"] = _plan_note + "\n\n" + (messages[0].get("content") or "") - else: - messages.insert(0, {"role": "system", "content": _plan_note}) - logger.info("[plan] pinned approved plan (%d chars) for execution turn", len(approved_plan)) - if guide_only: - if messages and messages[0].get("role") == "system": - messages[0]["content"] = GUIDE_ONLY_DIRECTIVE + "\n\n" + (messages[0].get("content") or "") - else: - messages.insert(0, {"role": "system", "content": GUIDE_ONLY_DIRECTIVE}) - prep_timings["prompt_build"] = time.time() - _t2 + _route_context_lengths = {} - _t3 = time.time() - try: - from src.context_compactor import trim_for_context - from src.context_budget import compute_input_token_budget, DEFAULT_HARD_MAX, DEFAULT_BUDGET, budget_is_explicit as _budget_is_explicit - from src.model_context import budget_context_for_model + def _trim_route_request_messages(candidate_url, candidate_model, route_messages): + """Apply the candidate route's own context budget to its request.""" - soft_budget = int(get_setting("agent_input_token_budget", DEFAULT_BUDGET) or 0) - if soft_budget > 0: - before_trim_tokens = estimate_tokens(messages) + def _without_protection(items): + # Route markers remain internal for later prompt rebuilding; + # protection metadata is only needed during trimming. + return [{k: v for k, v in message.items() if k != "_protected"} for message in items] + + try: + from src.context_compactor import trim_for_context + from src.context_budget import ( + compute_input_token_budget, + DEFAULT_BUDGET, + DEFAULT_HARD_MAX, + budget_is_explicit as _budget_is_explicit, + ) + from src.model_context import budget_context_for_model + + candidate_context = budget_context_for_model( + candidate_url, + candidate_model, + fallback=context_length, + ) + _route_context_lengths[(candidate_url, candidate_model)] = candidate_context + soft_budget = int(get_setting("agent_input_token_budget", DEFAULT_BUDGET) or 0) + if soft_budget <= 0: + return _without_protection(route_messages) + before_trim_tokens = estimate_tokens(route_messages) reserve_tokens = min(max(max_tokens or 1024, 512), 2048) - # Ceiling for the auto-derived budget (no effect on an explicit budget; - # see #1230). Falls back to DEFAULT_HARD_MAX on missing/malformed values - # so misconfig can't zero the budget. try: - hard_max = int(get_setting("agent_input_token_hard_max", DEFAULT_HARD_MAX) or DEFAULT_HARD_MAX) + hard_max = int( + get_setting("agent_input_token_hard_max", DEFAULT_HARD_MAX) + or DEFAULT_HARD_MAX + ) except (TypeError, ValueError): hard_max = DEFAULT_HARD_MAX if hard_max <= 0: hard_max = DEFAULT_HARD_MAX - # Default value = auto sentinel (scale to the window); any other value = - # explicit cap. Value-based, not presence-based, because the save path - # materializes defaults so a persisted default must still read as auto (#4121). budget_is_explicit = _budget_is_explicit(soft_budget) - # Scale only off a window we actually discovered, bound to the value it - # proves (else 0) — not the passed-in context_length, which can be stale - # or unset for some callers (#4122 review). - ctx_for_budget = budget_context_for_model(endpoint_url, model, fallback=context_length) effective_budget = compute_input_token_budget( soft_budget, - ctx_for_budget, + candidate_context, budget_is_explicit, hard_max=hard_max, ) trimmed_messages = trim_for_context( - messages, + route_messages, effective_budget, reserve_tokens=reserve_tokens, ) after_trim_tokens = estimate_tokens(trimmed_messages) if after_trim_tokens < before_trim_tokens: logger.info( - "[agent] soft-trimmed context: %s -> %s tokens (budget=%s, reserve=%s)", + "[agent] soft-trimmed route model=%s context: %s -> %s tokens " + "(budget=%s, reserve=%s)", + candidate_model, before_trim_tokens, after_trim_tokens, effective_budget, reserve_tokens, ) - messages = trimmed_messages - except Exception as e: - logger.warning("[agent] Soft context trim skipped: %s", e) - prep_timings["context_trim"] = time.time() - _t3 + return _without_protection(trimmed_messages) + except Exception as e: + logger.warning( + "[agent] Soft context trim skipped for route model=%s: %s", + candidate_model, + e, + ) + return _without_protection(route_messages) + + async def _build_route_request_state(candidate_url, candidate_model, candidate_headers, source_messages): + compaction_state: Dict = {} + compacted_source = list(source_messages) + was_compacted = False + if defer_context_shaping or fallbacks: + compacted_source, _candidate_context, was_compacted = await maybe_compact( + None, + candidate_url, + candidate_model, + compacted_source, + candidate_headers, + owner=owner, + persist=False, + compaction_state=compaction_state, + ) + is_ody, doc_mode, notes_mode, stream_create_mode = _route_finetune_modes(candidate_model) + route_tools = _route_relevant_tools(candidate_model) + is_api, is_native_ollama, is_ollama_compat = _agent_route_tool_mode( + candidate_url, + candidate_model, + owner, + headers=candidate_headers, + ) + route_messages, route_mcp_schemas = _build_system_prompt( + _strip_agent_injected_messages(compacted_source), + candidate_model, + _prompt_active_document, + mcp_mgr, + disabled_tools, + needs_admin=_needs_admin, + relevant_tools=route_tools, + mcp_disabled_map=_mcp_disabled_map, + compact=is_api or is_native_ollama or is_ollama_compat, + owner=owner, + suppress_local_context=guide_only, + suppress_skills=_low_signal_turn, + active_email=active_email, + ) + if doc_mode and not plan_mode and not approved_plan and not guide_only: + route_messages = _minimal_odysseus_doc_messages( + route_messages, + _prompt_active_document, + stream_create=stream_create_mode, + ) + route_mcp_schemas = [] + elif notes_mode and not plan_mode and not approved_plan and not guide_only: + route_messages = _minimal_odysseus_notes_messages(route_messages) + route_mcp_schemas = [] + elif ( + is_ody + and not _runtime_skill_tools + and not plan_mode + and not approved_plan + and not guide_only + ): + route_messages = _minimal_odysseus_general_messages(route_messages, include_memory=True) + route_mcp_schemas = [] + if plan_mode and not guide_only: + _prepend_agent_directive(route_messages, PLAN_MODE_DIRECTIVE) + elif approved_plan and approved_plan.strip() and not guide_only: + _prepend_agent_directive(route_messages, build_active_plan_note(approved_plan)) + if guide_only: + _prepend_agent_directive(route_messages, GUIDE_ONLY_DIRECTIVE) + return { + "messages": route_messages, + "mcp_schemas": route_mcp_schemas, + "relevant_tools": route_tools, + "is_api_model": is_api, + "is_ollama_native": is_native_ollama, + "ollama_openai_compat": is_ollama_compat, + "ody_qwen_finetune_model": is_ody, + "ody_doc_finetune_mode": doc_mode, + "ody_notes_finetune_mode": notes_mode, + "ody_doc_stream_create_mode": stream_create_mode, + "compaction_state": compaction_state, + "was_compacted": was_compacted, + } - # Strip internal metadata keys before sending to the LLM API - messages = [{k: v for k, v in msg.items() if k != "_protected"} for msg in messages] + _initial_route_source_messages = messages + _route_state = await _build_route_request_state( + endpoint_url, + model, + headers, + _initial_route_source_messages, + ) + messages = _route_state["messages"] + mcp_schemas = _route_state["mcp_schemas"] + _relevant_tools = _route_state["relevant_tools"] + _is_api_model = _route_state["is_api_model"] + _is_ollama_native = _route_state["is_ollama_native"] + _ollama_openai_compat = _route_state["ollama_openai_compat"] + if approved_plan and approved_plan.strip() and not guide_only: + logger.info("[plan] pinned approved plan (%d chars) for execution turn", len(approved_plan)) + prep_timings["prompt_build"] = time.time() - _t2 + + _t3 = time.time() + _initial_route_request_messages = _trim_route_request_messages( + endpoint_url, + model, + messages, + ) + _initial_route_context_length = _route_context_lengths.get( + (endpoint_url, model), + context_length, + ) + prep_timings["context_trim"] = time.time() - _t3 - agent_prompt_tokens = estimate_tokens(messages) + agent_prompt_tokens = estimate_tokens(_initial_route_request_messages) logger.info( "[agent-timing] prep_done model=%s prompt_tokens=%s context_length=%s prep=%s", model, @@ -3211,6 +3612,9 @@ async def stream_agent_loop( first_token_received = False tool_events = [] # Persist tool executions for history reload round_texts = [] # Cleaned text per round for history reload + round_models = [] # Actual model for each corresponding round + round_endpoint_ids = [] + round_endpoint_labels = [] # Completion-verifier state (mechanism 3a). _effectful_used flips on when # a tool that produces a checkable artifact runs; the verifier only fires # on such turns and at most _VERIFIER_MAX_ROUNDS times. @@ -3225,8 +3629,16 @@ async def stream_agent_loop( backend_prefill_tps = 0 # backend-reported prefill speed requested_model = model actual_model = model + actual_endpoint_id = requested_endpoint_id + actual_endpoint_label = requested_endpoint_label + actual_endpoint_cost_tracked = requested_endpoint_cost_tracked + usage_buckets = [] total_tool_calls = 0 # for budget enforcement _ody_notes_tool_completed = False + _pinned_fallback_candidate = None + _pinned_fallback_route = None + _last_route_request_messages = _initial_route_request_messages + _last_route_context_length = _initial_route_context_length # Loop-breaker state. Small models (e.g. deepseek-v4-flash) can get # stuck firing the same tool call over and over with no text — burns @@ -3276,6 +3688,44 @@ async def stream_agent_loop( # so the user can resume instead of the turn silently stalling. _exhausted_rounds = False + def _tool_schemas_for_route(route_state): + route_mcp_schemas = route_state["mcp_schemas"] + route_relevant_tools = route_state["relevant_tools"] + if _force_answer: + return [] + if route_state["is_api_model"]: + if route_relevant_tools: + schema_names = set(route_relevant_tools) + if _needs_admin: + schema_names |= _ADMIN_TOOLS + base_schemas = [ + schema for schema in FUNCTION_TOOL_SCHEMAS + if schema.get("function", {}).get("name") in schema_names + ] + mcp_filtered = [ + schema for schema in route_mcp_schemas + if schema.get("function", {}).get("name") in route_relevant_tools + ] + schemas = base_schemas + mcp_filtered + else: + base_schemas = FUNCTION_TOOL_SCHEMAS if _needs_admin else [ + schema for schema in FUNCTION_TOOL_SCHEMAS + if schema.get("function", {}).get("name") not in _ADMIN_SCHEMA_NAMES + ] + schemas = base_schemas + route_mcp_schemas + if route_state["ody_qwen_finetune_model"]: + schemas = [] + if disabled_tools: + schemas = [ + schema for schema in schemas + if schema.get("function", {}).get("name") not in disabled_tools + and schema.get("name") not in disabled_tools + ] + return schemas + + wants_mcp = any(keyword in _last_user.lower() for keyword in _MCP_KEYWORDS) + return route_mcp_schemas if wants_mcp and route_mcp_schemas else [] + for round_num in range(1, max_rounds + 1): round_response = "" round_reasoning = "" # reasoning_content deltas (DeepSeek-thinking, vLLM --reasoning-parser) @@ -3290,62 +3740,105 @@ async def stream_agent_loop( # detect a SUBSEQUENT block in the same round. _doc_scan_from = 0 - # Merge native tool schemas with MCP tool schemas, filtering out - # Only send function schemas for API models (OpenAI, Anthropic, etc.). - # Local models use fenced code blocks or — schemas add overhead. - if _force_answer: - # Loop-breaker decided the model has enough info but keeps - # calling tools. Send NO tools this round so it's forced to - # write the answer instead of flailing further. - all_tool_schemas = [] - elif _is_api_model: - # Filter schemas by RAG-selected tools (if available) - if _relevant_tools: - # _build_base_prompt unions _ADMIN_TOOLS into the prompt - # sections when admin intent fires — the schema list must - # offer the same names, or the model reads prose describing - # tools it cannot call and substitutes the nearest schema - # it does have (e.g. manage_memory for manage_skills). - _schema_names = set(_relevant_tools) - if _needs_admin: - _schema_names |= _ADMIN_TOOLS - base_schemas = [ - s for s in FUNCTION_TOOL_SCHEMAS - if s.get("function", {}).get("name") in _schema_names - ] - _mcp_filtered = [ - s for s in mcp_schemas - if s.get("function", {}).get("name") in _relevant_tools - ] - all_tool_schemas = base_schemas + _mcp_filtered - else: - base_schemas = FUNCTION_TOOL_SCHEMAS if _needs_admin else [ - s for s in FUNCTION_TOOL_SCHEMAS - if s.get("function", {}).get("name") not in _ADMIN_SCHEMA_NAMES - ] - all_tool_schemas = base_schemas + mcp_schemas - if _ody_qwen_finetune_model: - all_tool_schemas = [] - if disabled_tools: - all_tool_schemas = [ - t for t in all_tool_schemas - if t.get("function", {}).get("name") not in disabled_tools - and t.get("name") not in disabled_tools - ] - else: - # Local: only MCP schemas when message suggests MCP tool usage - _last_content = _last_user.lower() - _wants_mcp = any(kw in _last_content for kw in _MCP_KEYWORDS) - all_tool_schemas = mcp_schemas if (_wants_mcp and mcp_schemas) else [] + _active_route_state = { + "messages": messages, + "mcp_schemas": mcp_schemas, + "relevant_tools": _relevant_tools, + "is_api_model": _is_api_model, + "is_ollama_native": _is_ollama_native, + "ollama_openai_compat": _ollama_openai_compat, + "ody_qwen_finetune_model": _ody_qwen_finetune_model, + "ody_doc_finetune_mode": _ody_doc_finetune_mode, + "ody_notes_finetune_mode": _ody_notes_finetune_mode, + "ody_doc_stream_create_mode": _ody_doc_stream_create_mode, + "compaction_state": ( + _route_state.get("compaction_state", {}) if round_num == 1 else {} + ), + } + if round_num == 1: + _active_route_state["request_messages"] = _initial_route_request_messages + all_tool_schemas = _tool_schemas_for_route(_active_route_state) agent_stream_timeout = int(get_setting("agent_stream_timeout_seconds", 300) or 300) _tool_names_sent = [t.get("function", {}).get("name") for t in (all_tool_schemas or []) if t.get("function")] logger.info(f"[agent-debug] round={round_num} model={model} _is_api_model={_is_api_model} tools_sent={len(_tool_names_sent)} tool_names={_tool_names_sent[:15]} relevant_tools={sorted(_relevant_tools)[:15] if _relevant_tools else 'ALL'}") - # Primary target + any configured fallback models. stream_llm_with_fallback - # only switches on a pre-content failure, so streamed output is never - # duplicated; the dead-host cooldown keeps repeat primary attempts cheap. - _candidates = [(endpoint_url, model, headers)] + list(fallbacks or []) + # Once a fallback produces substantive output, keep that exact route + # pinned for every later tool round instead of retrying the primary. + if _pinned_fallback_candidate: + _raw_candidates = [_pinned_fallback_candidate] + _raw_route_descriptors = [_pinned_fallback_route or {}] + else: + _raw_candidates = [(endpoint_url, model, headers)] + list(fallbacks or []) + _raw_route_descriptors = route_descriptors + _candidates = dedupe_model_candidates(_raw_candidates) + _candidate_route_descriptors = [] + for candidate in _candidates: + source_index = next( + ( + index + for index, source in enumerate(_raw_candidates) + if source == candidate + ), + 0, + ) + _candidate_route_descriptors.append( + _raw_route_descriptors[source_index] + if source_index < len(_raw_route_descriptors) + else {} + ) + _candidate_request_states = {0: _active_route_state} + + async def _candidate_request(index, candidate_url, candidate_model, candidate_headers): + nonlocal _last_route_request_messages, _last_route_context_length + if index == 0: + state = _active_route_state + else: + candidate_source_messages = ( + _initial_route_source_messages if round_num == 1 else messages + ) + state = await _build_route_request_state( + candidate_url, + candidate_model, + candidate_headers, + candidate_source_messages, + ) + request_messages = state.get("request_messages") + if request_messages is None: + request_messages = _trim_route_request_messages( + candidate_url, + candidate_model, + state["messages"], + ) + state["request_messages"] = request_messages + _last_route_request_messages = request_messages + state["context_length"] = _route_context_lengths.get( + (candidate_url, candidate_model), + context_length, + ) + _last_route_context_length = state["context_length"] + candidate_tools = _tool_schemas_for_route(state) + state["tools"] = candidate_tools + _candidate_request_states[index] = state + return { + "messages": request_messages, + "kwargs": { + "tools": candidate_tools or None, + "tool_choice_none": state["ody_doc_finetune_mode"], + }, + } + + def _apply_candidate_compaction(index: int) -> bool: + state = _candidate_request_states.get(index) or {} + if history_session is not None: + return apply_compaction_state( + history_session, + state.get("compaction_state"), + ) + return apply_compaction_state_for_session( + session_id, + state.get("compaction_state"), + ) # stream_llm enforces a per-read INACTIVITY timeout (httpx read=timeout), # which kills a wedged/silent endpoint. This wall-clock deadline is the # complementary cap for the rare stream that trickles bytes forever and @@ -3354,6 +3847,49 @@ async def stream_agent_loop( _round_start = time.time() _round_first_event_logged = False _round_first_token_logged = False + _round_actual_model = model + _round_actual_endpoint_id = actual_endpoint_id + _round_actual_endpoint_label = actual_endpoint_label + _round_real_input_tokens = 0 + _round_real_output_tokens = 0 + _round_has_real_usage = False + _round_usage_finalized = False + candidate_index = 0 + + def _finalize_round_usage(*, include_empty: bool = True): + nonlocal _round_usage_finalized + if _round_usage_finalized: + return + _round_usage_finalized = True + if ( + not include_empty + and not _round_has_real_usage + and not round_response + and not round_reasoning + and not native_tool_calls + ): + return + if _round_has_real_usage: + round_input_tokens = _round_real_input_tokens + round_output_tokens = _round_real_output_tokens + usage_source = "real" + else: + round_input_tokens = estimate_tokens(_last_route_request_messages) + round_output_tokens = max( + len(round_response + round_reasoning) // 4, + 0, + ) + usage_source = "estimated" + usage_buckets.append(_usage_bucket( + round_num=round_num, + model=_round_actual_model, + endpoint_id=_round_actual_endpoint_id, + endpoint_label=_round_actual_endpoint_label, + endpoint_cost_tracked=actual_endpoint_cost_tracked, + input_tokens=round_input_tokens, + output_tokens=round_output_tokens, + usage_source=usage_source, + )) logger.info( "[agent-timing] round_start round=%s model=%s endpoint=%s prompt_tokens=%s tools=%s native_tools=%s timeout=%s", round_num, @@ -3375,6 +3911,10 @@ async def stream_agent_loop( timeout=agent_stream_timeout, session_id=session_id, workload=workload, + fallback_statuses=fallback_statuses, + fallback_on_empty=fallback_on_empty, + candidate_request_factory=_candidate_request, + candidate_route_descriptors=_candidate_route_descriptors, ): if not _round_first_event_logged: _round_first_event_logged = True @@ -3400,8 +3940,73 @@ async def stream_agent_loop( time.time() - _round_start, chunk[:500], ) + terminal_status = None + try: + error_line = next( + line[6:] + for line in chunk.splitlines() + if line.startswith("data: ") + ) + error_data = json.loads(error_line) + terminal_status = _normalize_http_status( + error_data.get("status") + ) + except Exception: + pass + terminal_error = { + "message": ( + f"Model request failed (HTTP {terminal_status})" + if terminal_status is not None + else "Model request failed" + ), + "status": terminal_status, + } + if full_response.strip() or round_reasoning.strip() or tool_events or round_texts: + _finalize_round_usage(include_empty=False) + partial_round = strip_tool_blocks( + round_response, + skip_fenced=( + _is_api_model + and not native_tool_calls + and not guide_only + ), + ).strip() + if _ody_qwen_finetune_model: + partial_round = _strip_doc_model_artifacts(partial_round).strip() + failure_note = f"[Agent stopped: {terminal_error['message']}]" + terminal_round = ( + f"{partial_round}\n\n{failure_note}" + if partial_round + else failure_note + ) + terminal_metadata = { + "failed": True, + "failure": terminal_error, + "model": actual_model, + "requested_model": requested_model, + "endpoint_id": actual_endpoint_id, + "endpoint_label": actual_endpoint_label, + "requested_endpoint_id": requested_endpoint_id, + "requested_endpoint_label": requested_endpoint_label, + "tool_events": tool_events, + "round_texts": [*round_texts, terminal_round], + "round_models": [*round_models, _round_actual_model], + "round_endpoint_ids": [*round_endpoint_ids, _round_actual_endpoint_id], + "round_endpoint_labels": [*round_endpoint_labels, _round_actual_endpoint_label], + **_usage_bucket_summary(usage_buckets), + } + if round_reasoning.strip(): + terminal_metadata["thinking"] = round_reasoning.strip() + if isinstance(actual_endpoint_cost_tracked, bool): + terminal_metadata["endpoint_cost_tracked"] = ( + actual_endpoint_cost_tracked + ) + yield f'data: {json.dumps({"type": "agent_terminal", "data": terminal_metadata})}\n\n' yield chunk - continue + # A terminal provider/request failure is not a completed Agent + # round. Stop before empty-response synthesis, metrics, + # teacher escalation, post-processing, or a success [DONE]. + return if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: data = json.loads(chunk[6:]) @@ -3446,16 +4051,33 @@ async def stream_agent_loop( _doc_last_len = len(decoded) yield f'data: {json.dumps({"type": "doc_stream_delta", "content": decoded})}\n\n' elif data.get("type") == "tool_calls": + if _apply_candidate_compaction(candidate_index): + yield f'data: {json.dumps({"type": "compacted", "context_length": _last_route_context_length})}\n\n' native_tool_calls = data.get("calls", []) logger.info(f"Agent round {round_num}: received {len(native_tool_calls)} native tool call(s)") elif data.get("type") == "usage": u = data.get("data", {}) actual_model = u.get("model") or actual_model - round_input = u.get("input_tokens", 0) + _round_actual_model = u.get("model") or _round_actual_model + normalized_usage = _normalize_usage_counts( + u.get("input_tokens", 0), + u.get("output_tokens", 0), + ) + if normalized_usage is None: + logger.warning( + "[agent] ignoring malformed usage event in round %s", + round_num, + ) + continue + round_input = normalized_usage["input_tokens"] + round_output = normalized_usage["output_tokens"] real_input_tokens += round_input - real_output_tokens += u.get("output_tokens", 0) + real_output_tokens += round_output + _round_real_input_tokens += round_input + _round_real_output_tokens += round_output last_round_input_tokens = round_input has_real_usage = True + _round_has_real_usage = True # Backend-reported TRUE generation speed (llama.cpp # timings.predicted_per_second) — pure decode, excludes # prefill/network. Preferred over tokens/wall-clock, which @@ -3468,14 +4090,83 @@ async def stream_agent_loop( # The selected model failed and another answered; surface # the notice so a misconfigured provider isn't masked. actual_model = data.get("answered_by") or actual_model + actual_endpoint_id = data.get("answered_by_endpoint_id") + actual_endpoint_label = ( + data.get("answered_by_endpoint_label") or actual_endpoint_label + ) + if isinstance(data.get("answered_by_endpoint_cost_tracked"), bool): + actual_endpoint_cost_tracked = data.get( + "answered_by_endpoint_cost_tracked" + ) + candidate_index = data.get("candidate_index") + if ( + _pinned_fallback_candidate is None + and isinstance(candidate_index, int) + and 0 < candidate_index < len(_candidates) + ): + _pinned_fallback_candidate = _candidates[candidate_index] + _pinned_fallback_route = ( + _candidate_route_descriptors[candidate_index] + if candidate_index < len(_candidate_route_descriptors) + else {} + ) + endpoint_url, model, headers = _pinned_fallback_candidate + answering_state = _candidate_request_states.get(candidate_index) + if answering_state is None: + answering_state = await _build_route_request_state( + endpoint_url, + model, + headers, + messages, + ) + answering_state["request_messages"] = _trim_route_request_messages( + endpoint_url, + model, + answering_state["messages"], + ) + answering_state["context_length"] = _route_context_lengths.get( + (endpoint_url, model), + context_length, + ) + messages = answering_state["messages"] + mcp_schemas = answering_state["mcp_schemas"] + _relevant_tools = answering_state["relevant_tools"] + _is_api_model = answering_state["is_api_model"] + _is_ollama_native = answering_state["is_ollama_native"] + _ollama_openai_compat = answering_state["ollama_openai_compat"] + _ody_qwen_finetune_model = answering_state["ody_qwen_finetune_model"] + _ody_doc_finetune_mode = answering_state["ody_doc_finetune_mode"] + _ody_notes_finetune_mode = answering_state["ody_notes_finetune_mode"] + _ody_doc_stream_create_mode = answering_state["ody_doc_stream_create_mode"] + data["pinned_for_run"] = True + if _apply_candidate_compaction(candidate_index): + yield f'data: {json.dumps({"type": "compacted", "context_length": _last_route_context_length})}\n\n' + _round_actual_model = data.get("answered_by") or model + _round_actual_endpoint_id = actual_endpoint_id + _round_actual_endpoint_label = actual_endpoint_label + data["round"] = round_num logger.warning(f"[agent] round {round_num} fell back: " f"{data.get('selected_model')} -> {data.get('answered_by')}") - yield chunk + yield f"data: {json.dumps(data)}\n\n" elif data.get("type") == "model_actual": + if _apply_candidate_compaction( + candidate_index if isinstance(candidate_index, int) else 0 + ): + yield f'data: {json.dumps({"type": "compacted", "context_length": _last_route_context_length})}\n\n' actual_model = data.get("model") or actual_model + _round_actual_model = data.get("model") or _round_actual_model data["requested_model"] = requested_model + data["requested_endpoint_id"] = requested_endpoint_id + data["requested_endpoint_label"] = requested_endpoint_label + data["endpoint_id"] = _round_actual_endpoint_id + data["endpoint_label"] = _round_actual_endpoint_label + data["round"] = round_num yield f"data: {json.dumps(data)}\n\n" elif "delta" in data: + if _apply_candidate_compaction( + candidate_index if isinstance(candidate_index, int) else 0 + ): + yield f'data: {json.dumps({"type": "compacted", "context_length": _last_route_context_length})}\n\n' if not first_token_received: time_to_first_token = time.time() - total_start first_token_received = True @@ -3585,6 +4276,7 @@ async def stream_agent_loop( _round_first_event_logged, _round_first_token_logged, ) + _finalize_round_usage() _normalized_doc_round = ( _normalize_stream_document_fences( round_response, @@ -3709,17 +4401,30 @@ async def stream_agent_loop( url=endpoint_url, model=model, messages=_synth_messages, headers=headers, temperature=0.3, max_tokens=max_tokens, timeout=60, ) - _synth = _strip_think_blocks(strip_tool_blocks(_raw or "")).strip() + _raw_text = _raw or "" + _synth = _strip_think_blocks(strip_tool_blocks(_raw_text)).strip() + usage_buckets.append(_usage_bucket( + round_num=round_num, + model=model, + endpoint_id=_round_actual_endpoint_id, + endpoint_label=_round_actual_endpoint_label, + endpoint_cost_tracked=actual_endpoint_cost_tracked, + input_tokens=estimate_tokens(_synth_messages), + output_tokens=max(len(_raw_text) // 4, 0), + usage_source="estimated", + )) except Exception as _e: logger.warning(f"[agent] grace synthesis failed: {_e}") if _synth: yield f'data: {json.dumps({"delta": _synth})}\n\n' + round_response += _synth full_response += _synth else: _fb = ("I gathered some search results but couldn't pull a clean " "answer together. Want me to try a more specific question, " "or summarize what I did find?") yield f'data: {json.dumps({"delta": _fb})}\n\n' + round_response += _fb full_response += _fb # ── Fallback: auto-create document if model dumped large code in chat ── @@ -3762,6 +4467,9 @@ async def stream_agent_loop( # on reload (#3222 follow-up). cleaned_round = strip_tool_blocks(round_response, skip_fenced=(_is_api_model and not used_native and not guide_only)).strip() round_texts.append(cleaned_round) + round_models.append(_round_actual_model) + round_endpoint_ids.append(_round_actual_endpoint_id) + round_endpoint_labels.append(_round_actual_endpoint_label) if _ody_qwen_finetune_model and not tool_blocks and cleaned_round: yield f'data: {json.dumps({"delta": cleaned_round})}\n\n' @@ -4110,6 +4818,9 @@ async def _run_tool(): } if _new: _relevant_tools.update(_new) + _runtime_skill_tools.update(_new) + if _base_relevant_tools is not None: + _base_relevant_tools.update(_new) logger.info( "[tool-rag] skill '%s' unlocked tools for next round: %s", _ms_name, sorted(_new), @@ -4362,6 +5073,9 @@ async def _run_tool(): # Save for history persistence tool_event = { "round": round_num, + "model": _round_actual_model, + "endpoint_id": _round_actual_endpoint_id, + "endpoint_label": _round_actual_endpoint_label, "tool": block.tool_type, "command": cmd_display, "output": output_text, @@ -4496,15 +5210,40 @@ async def _run_tool(): # --- Final metrics --- total_duration = time.time() - total_start metrics = _compute_final_metrics( - messages, full_response, total_duration, time_to_first_token, - context_length, real_input_tokens, real_output_tokens, + _last_route_request_messages, full_response, total_duration, time_to_first_token, + _last_route_context_length, real_input_tokens, real_output_tokens, has_real_usage, tool_events, round_texts, model=actual_model, + round_models=round_models, + round_endpoint_ids=round_endpoint_ids, + round_endpoint_labels=round_endpoint_labels, last_round_input_tokens=last_round_input_tokens, prep_timings=prep_timings, backend_gen_tps=backend_gen_tps, backend_prefill_tps=backend_prefill_tps, ) metrics["requested_model"] = requested_model + metrics["endpoint_id"] = actual_endpoint_id + metrics["endpoint_label"] = actual_endpoint_label + if isinstance(actual_endpoint_cost_tracked, bool): + metrics["endpoint_cost_tracked"] = actual_endpoint_cost_tracked + usage_summary = _usage_bucket_summary(usage_buckets) + if usage_summary: + metrics.update(usage_summary) + if not backend_gen_tps and total_duration > 0: + metrics["tokens_per_second"] = round( + usage_summary["output_tokens"] / total_duration, + 2, + ) + if _last_route_context_length: + metrics["context_percent"] = min( + round( + (usage_buckets[-1]["input_tokens"] / _last_route_context_length) * 100, + 1, + ), + 100.0, + ) + metrics["requested_endpoint_id"] = requested_endpoint_id + metrics["requested_endpoint_label"] = requested_endpoint_label yield f"data: {json.dumps({'type': 'metrics', 'data': metrics})}\n\n" # Teacher-escalation: inline takeover visible in the chat stream. diff --git a/src/agent_runs.py b/src/agent_runs.py index 3431347c72..20a86df73b 100644 --- a/src/agent_runs.py +++ b/src/agent_runs.py @@ -17,13 +17,14 @@ import asyncio import json import logging +import uuid from typing import AsyncGenerator, Dict, Optional logger = logging.getLogger(__name__) class _Run: - __slots__ = ("buffer", "subscribers", "status", "task", "evict_task") + __slots__ = ("buffer", "subscribers", "status", "task", "evict_task", "run_id") def __init__(self) -> None: self.buffer: list = [] # ordered SSE event strings (replay log) @@ -31,6 +32,9 @@ def __init__(self) -> None: self.status: str = "running" # running | done | error | stopped self.task: Optional[asyncio.Task] = None self.evict_task: Optional[asyncio.Task] = None + # Stable across every subscription/replay of this exact detached run. + # The browser uses it to make local cost accounting replay-idempotent. + self.run_id: str = uuid.uuid4().hex _RUNS: Dict[str, _Run] = {} @@ -53,13 +57,15 @@ def _publish(run: _Run, ev: str) -> None: pass -def _schedule_evict(session_id: str) -> None: +def _schedule_evict(session_id: str, expected_run: Optional[_Run] = None) -> None: """(Re)arm a grace-period eviction for a terminal run with no subscribers. Identity-checked so a run that gets replaced/reused is never evicted by a stale timer.""" run = _RUNS.get(session_id) if run is None: return + if expected_run is not None and run is not expected_run: + return if run.evict_task and not run.evict_task.done(): run.evict_task.cancel() @@ -85,25 +91,42 @@ def get_status(session_id: str) -> Optional[str]: return r.status if r else None -async def _drain(session_id: str, agen: AsyncGenerator[str, None], +def get_run_id(session_id: str) -> Optional[str]: + """Return the opaque identity of the current detached run, if present.""" + r = _RUNS.get(session_id) + return r.run_id if r else None + + +def get_active_run(session_id: str) -> Optional[_Run]: + """Return the exact active run currently registered for a session.""" + r = _RUNS.get(session_id) + return r if r and r.status == "running" else None + + +async def _drain(session_id: str, run: _Run, agen: AsyncGenerator[str, None], prev_task: Optional[asyncio.Task] = None) -> None: """Pull every event from the wrapped generator into the run buffer, fanning each out to live subscribers. Runs to completion regardless of subscribers.""" - run = _RUNS.get(session_id) - if run is None: - return + subscribers_woken = False + + def _wake_subscribers() -> None: + nonlocal subscribers_woken + if subscribers_woken: + return + subscribers_woken = True + for q in list(run.subscribers): + try: + q.put_nowait((None, None)) + except Exception: + pass + # If this run replaced an in-flight one (rapid double-send), wait for that # one to fully finish first. Its CancelledError handler calls aclose(), which # persists its partial response — letting it complete before we start writing # keeps the two runs' session saves sequential instead of interleaved. - if prev_task is not None and not prev_task.done(): - try: - await asyncio.wait({prev_task}) - except asyncio.CancelledError: - raise # our own cancellation — propagate - except Exception: - pass try: + if prev_task is not None and not prev_task.done(): + await asyncio.wait({prev_task}) async for ev in agen: _publish(run, ev) if run.status == "running": @@ -116,6 +139,16 @@ async def _drain(session_id: str, agen: AsyncGenerator[str, None], await agen.aclose() except Exception: pass + # A rapid third replacement can cancel this task while it is still + # waiting for its predecessor. Close this run's subscribers promptly, + # but keep the task alive until the predecessor finishes so the next + # run still observes the transitive session-save ordering barrier. + _wake_subscribers() + if prev_task is not None and not prev_task.done(): + try: + await asyncio.shield(prev_task) + except (asyncio.CancelledError, Exception): + pass except Exception as e: logger.error("[agent-run] %s failed: %s", session_id, e, exc_info=True) run.status = "error" @@ -127,15 +160,11 @@ async def _drain(session_id: str, agen: AsyncGenerator[str, None], _publish(run, "data: [DONE]\n\n") finally: # Wake every subscriber with the end sentinel so their SSE closes. - for q in list(run.subscribers): - try: - q.put_nowait((None, None)) - except Exception: - pass + _wake_subscribers() # Run is terminal — arm the grace timer so it (and its buffer) is # eventually freed even if nobody ever reconnects. subscribe() cancels # this on connect and re-arms on disconnect. - _schedule_evict(session_id) + _schedule_evict(session_id, run) def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run: @@ -151,14 +180,23 @@ def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run: prev.evict_task.cancel() run = _Run() _RUNS[session_id] = run - run.task = asyncio.create_task(_drain(session_id, agen, prev_task)) + run.task = asyncio.create_task(_drain(session_id, run, agen, prev_task)) return run -async def subscribe(session_id: str) -> AsyncGenerator[str, None]: +async def subscribe( + session_id: str, + expected_run: Optional[_Run] = None, +) -> AsyncGenerator[str, None]: """Replay the run's buffer from the start, then stream live until it ends. - Safe to call repeatedly (reconnect) and from multiple clients at once.""" - run = _RUNS.get(session_id) + Safe to call repeatedly (reconnect) and from multiple clients at once. + + ``expected_run`` binds a lazy StreamingResponse body to the same run whose + identity was put in its response headers. Without that binding, a rapid + replacement between response construction and body iteration could replay + the replacement run under the prior run's identity. + """ + run = expected_run or _RUNS.get(session_id) if run is None: return q: asyncio.Queue = asyncio.Queue() @@ -201,12 +239,19 @@ async def subscribe(session_id: str) -> AsyncGenerator[str, None]: # Last subscriber gone on a finished run — (re)arm eviction so the # buffer doesn't linger indefinitely. if not run.subscribers and run.status != "running": - _schedule_evict(session_id) + _schedule_evict(session_id, run) + +def stop(session_id: str, expected_run_id: Optional[str] = None) -> bool: + """Cancel the matching in-flight run (which saves its partial output). -def stop(session_id: str) -> bool: - """Cancel an in-flight run (the wrapped generator saves its partial).""" + A stale browser may issue Stop after another tab has replaced the session's + run. Once the caller knows its opaque run identity, fail closed rather than + cancelling that newer run. + """ run = _RUNS.get(session_id) + if not expected_run_id or run is None or run.run_id != expected_run_id: + return False if run and run.task and not run.task.done(): run.task.cancel() return True diff --git a/src/agent_tools/admin_tools.py b/src/agent_tools/admin_tools.py index 2cd6dc1a8f..227b068981 100644 --- a/src/agent_tools/admin_tools.py +++ b/src/agent_tools/admin_tools.py @@ -510,7 +510,12 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: # set/get/list/delete operate on the REAL app settings (the same store # the Settings panel writes), so changing a model / voice / search # engine / reminder channel from chat actually takes effect. - from src.settings import load_settings, save_settings, DEFAULT_SETTINGS + from src.settings import ( + DEFAULT_SETTINGS, + RETIRED_SETTING_KEYS, + load_settings, + save_settings, + ) # Secrets/credentials the agent must NOT write: kept read-only (masked) # so API keys never flow through chat. User sets these in the panel. @@ -562,6 +567,9 @@ def _resolve(k): return k2 return _ALIASES_SET.get(k2, (k or "").strip()) + def _is_managed_key(key): + return key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS + _ENUMS = { "image_quality": ["low", "medium", "high"], "reminder_channel": ["browser", "email", "ntfy", "webhook"], @@ -624,14 +632,18 @@ def _mask(k, v): if action == "list": s = load_settings() - shown = {k: _mask(k, v) for k, v in s.items() if k in DEFAULT_SETTINGS and not isinstance(v, dict)} + shown = { + k: _mask(k, v) + for k, v in s.items() + if _is_managed_key(k) and not isinstance(v, dict) + } return {"response": f"{len(shown)} settings (use get/set with a key)", "settings": shown, "exit_code": 0} elif action == "get": key = _resolve(args.get("key", "")) if not key: return {"error": "key is required", "exit_code": 1} - if key not in DEFAULT_SETTINGS: + if not _is_managed_key(key): return {"error": f"Unknown setting '{args.get('key')}'. Use action='list' to see them.", "exit_code": 1} val = load_settings().get(key, DEFAULT_SETTINGS.get(key)) return {"response": f"{key} = {_mask(key, val)}", "value": _mask(key, val), "exit_code": 0} @@ -642,11 +654,11 @@ def _mask(k, v): if not raw: return {"error": "key is required", "exit_code": 1} key = _resolve(raw) - if key not in DEFAULT_SETTINGS: + if not _is_managed_key(key): return {"error": f"Unknown setting '{raw}'. Use action='list' to see available settings.", "exit_code": 1} if _is_secret(key): return {"response": f"'{key}' is a credential/secret. For security I can't set it from chat. Open Settings and set it there.", "exit_code": 0} - # Structured settings (dicts/lists like keybinds, default_model_fallbacks) + # Structured settings (dicts/lists like keybinds or vision fallbacks) # have no safe scalar coercion; _coerce would pass a bare string # straight through and clobber the structure. Refuse them here; they're # edited in their dedicated panels. (reset/delete still restore the @@ -675,7 +687,7 @@ def _mask(k, v): elif action == "delete" or action == "reset": key = _resolve(args.get("key", "")) - if key not in DEFAULT_SETTINGS: + if not _is_managed_key(key): return {"error": f"Unknown setting '{args.get('key')}'.", "exit_code": 1} if _is_secret(key): return {"response": f"'{key}' is a credential. Reset it in the panel.", "exit_code": 0} diff --git a/src/context_compactor.py b/src/context_compactor.py index 3a4f6c072d..8310541000 100644 --- a/src/context_compactor.py +++ b/src/context_compactor.py @@ -273,7 +273,9 @@ def _is_research_primer(m): if essential_system: sys_text = essential_system[0].get("content", "") if len(sys_text) > 2000: - essential_system[0] = {"role": "system", "content": sys_text[:2000] + "\n[System prompt truncated for context limits]"} + truncated_system = dict(essential_system[0]) + truncated_system["content"] = sys_text[:2000] + "\n[System prompt truncated for context limits]" + essential_system[0] = truncated_system trimmed = essential_system + convo_msgs if estimate_tokens(trimmed) <= budget: return _sanitize_tool_messages(essential_system + protected_msgs + convo_msgs) @@ -316,6 +318,9 @@ async def maybe_compact( messages: List[Dict], headers: Optional[Dict] = None, owner: Optional[str] = None, + *, + persist: bool = True, + compaction_state: Optional[Dict[str, Any]] = None, ) -> tuple: """Check context usage and compact if above threshold. @@ -406,7 +411,17 @@ async def maybe_compact( # offset — session.history INCLUDES the system messages, but # split_point is indexed against convo_msgs which does NOT. Without # this, the slice drops the leading system message(s). - _update_session_history(session, split_point, summary, system_msg_count=len(system_msgs)) + if compaction_state is not None: + compaction_state.update({ + "split_point": split_point, + "summary": summary, + "system_msg_count": len(system_msgs), + "applied": False, + }) + if persist: + _update_session_history(session, split_point, summary, system_msg_count=len(system_msgs)) + if compaction_state is not None: + compaction_state["applied"] = True new_used = estimate_tokens(compacted) logger.info( @@ -417,6 +432,51 @@ async def maybe_compact( return compacted, context_length, True +def apply_compaction_state(session, compaction_state: Optional[Dict[str, Any]]) -> bool: + """Persist a route-specific compaction after that route commits output. + + Candidate prompts may be compacted speculatively while an explicit + foreground fallback chain is being tried. Persisting at construction time + would let an unavailable route rewrite history before another route answers, + so callers hold this small plan and apply only the winning route's plan. + """ + + state = compaction_state if isinstance(compaction_state, dict) else None + if not state or state.get("applied"): + return False + summary = state.get("summary") + split_point = state.get("split_point") + system_msg_count = state.get("system_msg_count", 0) + if not isinstance(summary, str) or not isinstance(split_point, int): + return False + _update_session_history( + session, + split_point, + summary, + system_msg_count=system_msg_count if isinstance(system_msg_count, int) else 0, + ) + state["applied"] = True + return True + + +def apply_compaction_state_for_session( + session_id: Optional[str], + compaction_state: Optional[Dict[str, Any]], +) -> bool: + """Resolve an in-memory session and apply a deferred compaction plan.""" + + if not session_id: + return False + try: + from core.models import get_session_manager_instance + + manager = get_session_manager_instance() + session = manager.get_session(session_id) if manager else None + except Exception: + session = None + return apply_compaction_state(session, compaction_state) if session else False + + def _update_session_history(session, split_point: int, summary: str, system_msg_count: int = 0): """Update the in-memory session history after compaction. diff --git a/src/endpoint_resolver.py b/src/endpoint_resolver.py index 1bb8fc3afe..406abd5f95 100644 --- a/src/endpoint_resolver.py +++ b/src/endpoint_resolver.py @@ -5,6 +5,7 @@ """ import json +import ipaddress import logging import socket import subprocess @@ -27,6 +28,50 @@ ) +def endpoint_cost_tracked(url: str, endpoint_kind: Optional[str] = None) -> bool: + """Return whether token cost should be tracked for a concrete route. + + This is intentionally a non-secret route classification. It mirrors the + frontend's local/subscription exclusions without exposing endpoint URLs to + message metadata. + """ + + try: + parsed = urlparse(url or "") + host = (parsed.hostname or "").lower().rstrip(".") + path = (parsed.path or "").rstrip("/") + except Exception: + return False + if not host: + return False + if host == "chatgpt.com" and ( + path == "/backend-api/codex" or path.startswith("/backend-api/codex/") + ): + return False + kind = str(endpoint_kind or "auto").strip().lower() + if kind == "local": + return False + if kind in {"api", "proxy"}: + return True + if host in {"localhost", "0.0.0.0", "host.docker.internal"} or host.endswith(".local"): + return False + if "." not in host: + return False + try: + ip = ipaddress.ip_address(host) + local_networks = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("100.64.0.0/10"), + ) + if ip.is_loopback or any(ip in network for network in local_networks): + return False + except ValueError: + pass + return True + + def _first_chat_model(models) -> Optional[str]: """First model that isn't an embedding/tts/etc.; falls back to models[0].""" for m in (models or []): @@ -396,10 +441,14 @@ def _stg(key: str) -> str: db.close() -def resolve_endpoint_by_id( - ep_id: str, model: Optional[str] = None, owner: Optional[str] = None -) -> Optional[Tuple[str, str, Dict]]: - """Resolve a specific endpoint id (+ optional model) to (chat_url, model, headers). +def _resolve_endpoint_by_id_with_descriptor( + ep_id: str, + model: Optional[str] = None, + owner: Optional[str] = None, + *, + require_exact_model: bool = False, +) -> Optional[Tuple[Tuple[str, str, Dict], dict]]: + """Resolve a concrete endpoint/model plus its non-secret descriptor. Returns None if the endpoint doesn't exist or is disabled. Used to turn a configured fallback entry ({endpoint_id, model}) into a dispatch target. @@ -426,15 +475,34 @@ def resolve_endpoint_by_id( chat_url = build_chat_url(base) headers = build_headers(api_key, base) m = (model or "").strip() - # Drop a model the user disabled on the endpoint, then pick the first - # enabled chat model rather than a hidden one. - if m and m in _endpoint_hidden_models(ep): - m = "" - if not m: - m = _first_chat_model(_endpoint_enabled_models(ep)) or "" + enabled_models = _endpoint_enabled_models(ep) + if require_exact_model: + # Explicit foreground fallback entries are concrete choices. A + # hidden or known-missing model must disable the entry instead of + # silently substituting another model from the endpoint. + if not m or m in _endpoint_hidden_models(ep): + return None + if enabled_models and m not in enabled_models: + return None + else: + # Legacy Utility/Vision chains retain their model-repair behavior. + if m and m in _endpoint_hidden_models(ep): + m = "" + if not m: + m = _first_chat_model(enabled_models) or "" if not m: return None - return chat_url, m, headers + return ( + (chat_url, m, headers), + { + "endpoint_id": ep.id, + "endpoint_label": getattr(ep, "name", None) or ep.id, + "endpoint_cost_tracked": endpoint_cost_tracked( + chat_url, + getattr(ep, "endpoint_kind", None), + ), + }, + ) except Exception as e: logger.debug(f"Could not resolve endpoint {ep_id}: {e}") return None @@ -442,11 +510,72 @@ def resolve_endpoint_by_id( db.close() -def resolve_chat_fallback_candidates(owner: Optional[str] = None) -> list: - """Compatibility shim for the retired default-chat fallback chain.""" +def resolve_endpoint_by_id( + ep_id: str, + model: Optional[str] = None, + owner: Optional[str] = None, + *, + require_exact_model: bool = False, +) -> Optional[Tuple[str, str, Dict]]: + """Resolve a specific endpoint id (+ optional model) to its runtime route.""" + + resolved = _resolve_endpoint_by_id_with_descriptor( + ep_id, + model, + owner=owner, + require_exact_model=require_exact_model, + ) + return resolved[0] if resolved else None - del owner - return [] + +def resolve_route_descriptor( + endpoint_url: str, + model: str, + headers: Optional[Dict] = None, + owner: Optional[str] = None, +) -> dict: + """Return the visible endpoint identity for an already-resolved route. + + Headers are compared only inside the process so two endpoints using the + same provider URL/model but different credentials remain distinguishable. + No credential material is returned or logged. + """ + + if not endpoint_url or not model: + return { + "endpoint_id": None, + "endpoint_label": "Selected route", + "endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url), + } + db = SessionLocal() + try: + q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + if owner: + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + expected = (endpoint_url.rstrip("/"), model, headers or {}) + for ep in q.all(): + resolved = _resolve_endpoint_by_id_with_descriptor( + ep.id, + model, + owner=owner, + require_exact_model=True, + ) + if not resolved: + continue + candidate, descriptor = resolved + actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {}) + if actual == expected: + return descriptor + except Exception as e: + logger.debug("Could not identify selected endpoint route: %s", e) + finally: + db.close() + return { + "endpoint_id": None, + "endpoint_label": "Selected route", + "endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url), + } def resolve_utility_fallback_candidates(owner: Optional[str] = None) -> list: @@ -460,17 +589,62 @@ def resolve_vision_fallback_candidates(owner: Optional[str] = None) -> list: def _resolve_fallback_candidates(setting_key: str, owner: Optional[str] = None) -> list: - out = [] try: from src.settings import get_user_setting, load_settings settings = load_settings() chain = get_user_setting(setting_key, owner or "", settings.get(setting_key) or []) or [] except Exception: - return out - for entry in chain: + return [] + return resolve_fallback_entries(chain, owner=owner) + + +def resolve_fallback_entries( + entries, + owner: Optional[str] = None, + *, + require_exact_model: bool = False, +) -> list: + """Resolve ordered endpoint/model entries within the caller's owner scope.""" + + out = [] + for entry in entries or []: if not isinstance(entry, dict): continue - resolved = resolve_endpoint_by_id(entry.get("endpoint_id", ""), entry.get("model", ""), owner=owner) - if resolved: + resolved = resolve_endpoint_by_id( + entry.get("endpoint_id", ""), + entry.get("model", ""), + owner=owner, + require_exact_model=require_exact_model, + ) + if resolved and resolved not in out: out.append(resolved) return out + + +def resolve_fallback_entries_with_descriptors( + entries, + owner: Optional[str] = None, + *, + require_exact_model: bool = False, +) -> list: + """Resolve ordered entries while retaining safe endpoint provenance.""" + + out = [] + seen = [] + for entry in entries or []: + if not isinstance(entry, dict): + continue + resolved = _resolve_endpoint_by_id_with_descriptor( + entry.get("endpoint_id", ""), + entry.get("model", ""), + owner=owner, + require_exact_model=require_exact_model, + ) + if not resolved: + continue + candidate, descriptor = resolved + if any(candidate == prior for prior in seen): + continue + seen.append(candidate) + out.append((candidate, descriptor)) + return out diff --git a/src/foreground_model_routing.py b/src/foreground_model_routing.py index 241ccf26b5..2f437e0f54 100644 --- a/src/foreground_model_routing.py +++ b/src/foreground_model_routing.py @@ -1,22 +1,139 @@ -"""Foreground Chat and Agent model-routing policy. +"""Explicit foreground Chat and Agent model-routing policy.""" -The selected session model is strict by default. Historical -``default_model_fallbacks`` values remain stored for compatibility, but this -policy intentionally does not read or migrate them. -""" +from dataclasses import dataclass +from typing import Any, Collection, Dict, FrozenSet, Optional, Tuple -from typing import Any, Dict, Optional +from src.endpoint_resolver import ( + endpoint_cost_tracked, + resolve_fallback_entries, + resolve_fallback_entries_with_descriptors, + resolve_route_descriptor, +) +_DEFAULT_FALLBACK_ENTRY_RESOLVER = resolve_fallback_entries -def resolve_foreground_fallback_candidates(owner: Optional[str] = None) -> list: - """Return fallback candidates for a foreground Chat or Agent request. - Foreground routing is strict, so no alternate endpoint/model is eligible. - ``owner`` is accepted to keep this policy boundary owner-aware. +FOREGROUND_FALLBACK_ENABLED_KEY = "foreground_fallback_enabled" +FOREGROUND_FALLBACK_LIST_KEY = "foreground_model_fallbacks" +FOREGROUND_AVAILABILITY_STATUSES: FrozenSet[int] = frozenset({ + 408, 425, 429, 500, 502, 503, 504, 507, 508, 529, +}) +MAX_FOREGROUND_FALLBACKS = 10 + + +@dataclass(frozen=True) +class ForegroundModelPolicy: + """Resolved per-user foreground fallback policy.""" + + enabled: bool = False + fallback_candidates: Tuple[tuple, ...] = () + fallback_descriptors: Tuple[dict, ...] = () + eligible_statuses: FrozenSet[int] = FOREGROUND_AVAILABILITY_STATUSES + fallback_on_empty: bool = False + + +def _load_policy_preferences(owner: Optional[str]) -> dict: + """Load only preferences that explicitly belong to ``owner``. + + The generic preferences loader intentionally treats a legacy flat store as + the single-user preferences object. That compatibility must not cross an + authentication transition: once a named owner is present, foreground + fallback consent exists only in an actual ``_users[owner]`` dictionary. + """ + + from routes import prefs_routes + + if owner is None: + prefs = prefs_routes._load_for_user(None) + return dict(prefs) if isinstance(prefs, dict) else {} + + raw = prefs_routes._load() + users = raw.get("_users") if isinstance(raw, dict) else None + if not isinstance(users, dict): + return {} + prefs = users.get(owner) + return dict(prefs) if isinstance(prefs, dict) else {} + + +def resolve_foreground_model_policy( + owner: Optional[str] = None, + allowed_models: Optional[Collection[str]] = None, +) -> ForegroundModelPolicy: + """Resolve an explicit owner-scoped policy, failing closed to strict mode. + + The policy is stored in user preferences even when authentication is + disabled. Historical ``default_model_fallbacks`` values are deliberately + unrelated and are never read or migrated. """ - del owner - return [] + try: + prefs = _load_policy_preferences(owner) + except Exception: + return ForegroundModelPolicy() + + if prefs.get(FOREGROUND_FALLBACK_ENABLED_KEY) is not True: + return ForegroundModelPolicy() + + entries = prefs.get(FOREGROUND_FALLBACK_LIST_KEY) + if not isinstance(entries, list) or not entries: + return ForegroundModelPolicy() + entries = entries[:MAX_FOREGROUND_FALLBACKS] + if allowed_models is not None: + allowed = frozenset(allowed_models) + entries = [ + entry for entry in entries + if ( + isinstance(entry, dict) + and isinstance(entry.get("model"), str) + and entry.get("model") in allowed + ) + ] + if not entries: + return ForegroundModelPolicy() + + if resolve_fallback_entries is not _DEFAULT_FALLBACK_ENTRY_RESOLVER: + # Preserve the long-standing resolver seam used by downstream tests and + # integrations. Production uses the descriptor-aware resolver below. + compatibility_candidates = resolve_fallback_entries( + entries, + owner=owner, + require_exact_model=True, + ) + resolved_routes = [ + ( + candidate, + { + "endpoint_id": entries[index].get("endpoint_id"), + "endpoint_label": entries[index].get("endpoint_id") or "Fallback route", + "endpoint_cost_tracked": endpoint_cost_tracked(candidate[0]), + }, + ) + for index, candidate in enumerate(compatibility_candidates) + if index < len(entries) + ] + else: + resolved_routes = resolve_fallback_entries_with_descriptors( + entries, + owner=owner, + require_exact_model=True, + ) + candidates = [candidate for candidate, _descriptor in resolved_routes] + if not candidates: + return ForegroundModelPolicy() + + return ForegroundModelPolicy( + enabled=True, + fallback_candidates=tuple(candidates), + fallback_descriptors=tuple( + dict(descriptor) for _candidate, descriptor in resolved_routes + ), + ) + + +def resolve_foreground_fallback_candidates(owner: Optional[str] = None) -> list: + """Return only candidates explicitly enabled by the current user.""" + + return list(resolve_foreground_model_policy(owner).fallback_candidates) def build_foreground_model_candidates( @@ -24,8 +141,39 @@ def build_foreground_model_candidates( model: str, headers: Optional[Dict[str, Any]] = None, owner: Optional[str] = None, + policy: Optional[ForegroundModelPolicy] = None, ) -> list: """Build the ordered candidate list for a foreground request.""" + policy = policy or resolve_foreground_model_policy(owner) + primary = (endpoint_url, model, headers or {}) + candidates = [primary] + for candidate in policy.fallback_candidates: + if candidate not in candidates: + candidates.append(candidate) + return candidates + + +def build_foreground_route_descriptors( + endpoint_url: str, + model: str, + headers: Optional[Dict[str, Any]] = None, + owner: Optional[str] = None, + policy: Optional[ForegroundModelPolicy] = None, +) -> list: + """Build safe route metadata parallel to foreground candidates.""" + + policy = policy or resolve_foreground_model_policy(owner) + selected = resolve_route_descriptor(endpoint_url, model, headers or {}, owner=owner) primary = (endpoint_url, model, headers or {}) - return [primary] + resolve_foreground_fallback_candidates(owner=owner) + candidates = [primary] + descriptors = [selected] + for candidate, descriptor in zip( + policy.fallback_candidates, + policy.fallback_descriptors, + ): + if candidate in candidates: + continue + candidates.append(candidate) + descriptors.append(dict(descriptor)) + return descriptors diff --git a/src/llm_core.py b/src/llm_core.py index 760ef19f7a..ab7b178f84 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -8,6 +8,7 @@ import threading import re import os +import math from contextlib import asynccontextmanager from fastapi import HTTPException from typing import Optional, Dict, List, Tuple @@ -21,6 +22,53 @@ _LOCAL_MODEL_CURRENT: Dict[str, object] = {} +def _normalize_usage_counts(input_value=0, output_value=0): + """Return safe integer token counts, or ``None`` for malformed usage.""" + + def _count(value): + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if isinstance(value, int): + count = value + else: + if not math.isfinite(value) or not value.is_integer(): + return None + count = int(value) + if count < 0 or count > (2**63 - 1): + return None + return count + + input_tokens = _count(input_value) + output_tokens = _count(output_value) + if input_tokens is None or output_tokens is None: + return None + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + + +def _normalize_http_status(value) -> Optional[int]: + """Accept only genuine three-digit integral HTTP status values.""" + + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + status = value + elif isinstance(value, float): + if not math.isfinite(value) or not value.is_integer(): + return None + status = int(value) + elif isinstance(value, str): + text = value.strip() + if not re.fullmatch(r"\d{3}", text): + return None + status = int(text) + else: + return None + return status if 100 <= status <= 599 else None + + def _local_model_gate_enabled() -> bool: return os.getenv("ODYSSEUS_LOCAL_MODEL_GATE", "true").lower() not in {"0", "false", "no", "off"} @@ -108,6 +156,12 @@ class LLMConfig: CONNECT_TIMEOUT = float(os.getenv('LLM_CONNECT_TIMEOUT', '10') or '10') +class _FallbackIneligibleHTTPException(HTTPException): + """HTTP-shaped provider failure that must never advance a route chain.""" + + fallback_eligible = False + + def _call_timeout(read_timeout) -> httpx.Timeout: """Per-request timeout for non-streaming LLM calls (connect from config).""" return httpx.Timeout(connect=LLMConfig.CONNECT_TIMEOUT, read=float(read_timeout), write=10.0, pool=5.0) @@ -119,9 +173,28 @@ def _stream_timeout(read_timeout) -> httpx.Timeout: # Cache for LLM responses -def _get_cache_key(url: str, model: str, messages: List[Dict], - temperature: float, max_tokens: int) -> str: - """Generate cache key for LLM requests.""" +def _cache_header_identity(headers) -> str: + """Return a non-secret identity for credential-distinct request routes.""" + + if isinstance(headers, str): + try: + headers = json.loads(headers) + except (TypeError, ValueError, json.JSONDecodeError): + headers = {"_raw": headers} + if not isinstance(headers, dict): + headers = {} + canonical = [ + (str(key).strip().lower(), str(value)) + for key, value in headers.items() + ] + canonical.sort() + encoded = json.dumps(canonical, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(encoded.encode()).hexdigest() + + +def _get_cache_key(url: str, model: str, messages: List[Dict], + temperature: float, max_tokens: int, headers=None) -> str: + """Generate a cache key partitioned by endpoint and credential identity.""" hashable_messages = [] for msg in messages: sorted_items = tuple(sorted(msg.items())) @@ -132,11 +205,16 @@ def _get_cache_key(url: str, model: str, messages: List[Dict], 'model': model, 'messages': hashable_messages, 'temp': temperature, - 'max_tokens': max_tokens + 'max_tokens': max_tokens, + # Never put credentials in a cache key or loggable cache payload. The + # digest only prevents responses from one configured account/route + # being returned under another route with the same URL and model. + 'header_identity': _cache_header_identity(headers), }, sort_keys=True) return hashlib.sha256(content.encode()).hexdigest() _response_cache = {} +_response_model_cache = {} # Dead-host cooldown: maps host (scheme://host:port) -> unix ts when cooldown expires. # When a connect to a host fails, we mark it dead for DEAD_HOST_COOLDOWN seconds so @@ -340,7 +418,7 @@ def check(self, text: str) -> Optional[str]: f"Stopped generation: {self.model} started repeating tokens " f"({reason}). Try a different model or lower temperature." ) - return f'event: error\ndata: {json.dumps({"status": 502, "text": message, "error": message})}\n\n' + return f'event: error\ndata: {json.dumps({"status": 502, "text": message, "error": message, "fallback_eligible": False})}\n\n' def _model_activity_key(url: str, model: str) -> str: @@ -349,6 +427,29 @@ def _model_activity_key(url: str, model: str) -> str: def _same_model_identity(left: str, right: str) -> bool: return (left or "").strip().lower() == (right or "").strip().lower() +def _reported_model_name(value) -> str: + """Return a provider model identifier only when it is usable metadata.""" + return value.strip() if isinstance(value, str) and value.strip() else "" + + +def _model_actual_event(requested_model: str, reported_model) -> Optional[str]: + """Build a provenance event when a provider resolves a different model.""" + actual_model = _reported_model_name(reported_model) + if not actual_model or _same_model_identity(actual_model, requested_model): + return None + return f'data: {json.dumps({"type": "model_actual", "requested_model": requested_model, "model": actual_model})}\n\n' + + +def _annotate_usage_model(usage: dict, requested_model: str, actual_model: str) -> dict: + """Attach provider model provenance to a normalized usage payload.""" + actual_model = _reported_model_name(actual_model) + if actual_model: + usage["model"] = actual_model + if not _same_model_identity(actual_model, requested_model): + usage["requested_model"] = requested_model + return usage + + def note_model_activity(url: str, model: str): """Record that a real upstream request used this endpoint/model.""" if not url or not model: @@ -419,7 +520,19 @@ def _get_cached_response(cache_key: str) -> Optional[str]: """Get cached response if it exists.""" return _response_cache.get(cache_key) -def _set_cached_response(cache_key: str, response: str) -> None: + +def _get_cached_response_model(cache_key: str) -> Optional[str]: + """Return provider-reported model metadata paired with a cached reply.""" + model = _response_model_cache.get(cache_key) + return model if isinstance(model, str) and model.strip() else None + + +def _set_cached_response( + cache_key: str, + response: str, + *, + actual_model: Optional[str] = None, +) -> None: """Store response in cache.""" if len(_response_cache) > 128: keys_to_remove = list(_response_cache.keys())[:64] @@ -428,7 +541,12 @@ def _set_cached_response(cache_key: str, response: str) -> None: # threadpool) may have already evicted the same snapshotted key, # and del would raise KeyError mid-eviction (issue #659). _response_cache.pop(key, None) + _response_model_cache.pop(key, None) _response_cache[cache_key] = response + if isinstance(actual_model, str) and actual_model.strip(): + _response_model_cache[cache_key] = actual_model.strip() + else: + _response_model_cache.pop(cache_key, None) # ── Anthropic native API adapter ── @@ -1760,7 +1878,7 @@ def normalize_model_id( return None def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE, - max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None, + max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None, timeout: int = LLMConfig.DEFAULT_TIMEOUT, prompt_type: Optional[str] = None) -> str: """Synchronous LLM call with optional prompt type enhancement.""" h = _provider_headers(_detect_provider(url)) @@ -1791,7 +1909,9 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL messages_copy = non_sys provider = _detect_provider(url) - cache_key = _get_cache_key(url, model, messages_copy, temperature, max_tokens) + cache_key = _get_cache_key( + url, model, messages_copy, temperature, max_tokens, headers=headers, + ) cached_response = _get_cached_response(cache_key) if cached_response: logger.debug(f"Returning cached response for key: {cache_key}") @@ -1856,27 +1976,70 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL raise HTTPException(502, f"Unexpected schema from {target_url}: {str(data)[:400]}") -def _dedupe_candidates(candidates): +def _candidate_is_configured(candidate) -> bool: + return bool( + isinstance(candidate, (tuple, list)) + and len(candidate) == 3 + and isinstance(candidate[0], str) + and candidate[0].strip() + and isinstance(candidate[1], str) + and candidate[1].strip() + ) + + +def _safe_route_descriptor(value) -> dict: + value = value if isinstance(value, dict) else {} + endpoint_id = value.get("endpoint_id") + endpoint_label = value.get("endpoint_label") + endpoint_cost_tracked = value.get("endpoint_cost_tracked") + return { + "endpoint_id": endpoint_id if isinstance(endpoint_id, str) and endpoint_id else None, + "endpoint_label": ( + endpoint_label + if isinstance(endpoint_label, str) and endpoint_label.strip() + else "Selected route" + ), + "endpoint_cost_tracked": ( + endpoint_cost_tracked + if isinstance(endpoint_cost_tracked, bool) + else None + ), + } + + +def _dedupe_model_candidates_with_descriptors(candidates, descriptors=None): + """Dedupe routes and their parallel non-secret descriptors together.""" + + seen = [] + out = [] + out_descriptors = [] + descriptors = list(descriptors or []) + for index, candidate in enumerate(candidates or []): + if not _candidate_is_configured(candidate): + continue + route = (candidate[0], candidate[1], candidate[2] or {}) + if any(route == prior for prior in seen): + continue + seen.append(route) + out.append(candidate) + raw_descriptor = descriptors[index] if index < len(descriptors) else {} + out_descriptors.append(_safe_route_descriptor(raw_descriptor)) + return out, out_descriptors + + +def dedupe_model_candidates(candidates): """Filter malformed entries and drop a later repeat of an already-seen - ``(url, model)`` route, preserving order (first occurrence wins). + ``(url, model, headers)`` route, preserving order (first occurrence wins). The chain is the primary target followed by any caller-authorized fallbacks. A fallback that repeats the session's current model would otherwise make the chain re-attempt the very route that just failed: a wasted round-trip plus a spurious ``fallback`` notice for a switch that did - not happen. Headers are not part of the key; the first tuple (with its - headers) is the one kept. + not happen. Credentials are part of route identity: two configured + endpoints may intentionally use the same provider URL/model with different + keys, and rate limiting on one must not discard the other candidate. """ - seen = set() - out = [] - for c in candidates or []: - if not c or not c[0] or not c[1]: - continue - key = (c[0], c[1]) - if key in seen: - continue - seen.add(key) - out.append(c) + out, _descriptors = _dedupe_model_candidates_with_descriptors(candidates) return out @@ -1888,7 +2051,7 @@ def llm_call_with_fallback(candidates, messages, **kwargs) -> str: the next candidate. The dead-host cooldown inside `llm_call` makes repeat attempts at an offline primary effectively free. """ - cands = _dedupe_candidates(candidates) + cands = dedupe_model_candidates(candidates) if not cands: raise HTTPException(503, "No model endpoint configured") last_err = None @@ -1905,7 +2068,7 @@ def llm_call_with_fallback(candidates, messages, **kwargs) -> str: async def llm_call_async_with_fallback(candidates, messages, **kwargs) -> str: """Async variant of `llm_call_with_fallback` — same semantics.""" - cands = _dedupe_candidates(candidates) + cands = dedupe_model_candidates(candidates) if not cands: raise HTTPException(503, "No model endpoint configured") last_err = None @@ -1920,6 +2083,93 @@ async def llm_call_async_with_fallback(candidates, messages, **kwargs) -> str: raise last_err if last_err else HTTPException(503, "All fallback candidates failed") +def _nonstream_error_status(error: Exception) -> Optional[int]: + """Normalize a non-stream provider failure for explicit fallback policy.""" + + status = getattr(error, "status_code", None) + if not isinstance(status, bool) and status is not None: + return _normalize_http_status(status) + if isinstance(error, (httpx.ConnectError, httpx.ConnectTimeout)): + return 503 + if isinstance(error, httpx.ReadTimeout): + return 504 + return None + + +async def llm_call_async_with_route_fallback( + candidates, + messages, + *, + fallback_statuses, + **kwargs, +): + """Call an ordered non-stream route chain and return route provenance. + + Unlike the legacy utility helper, this advances only for an explicitly + eligible status. A successful empty response still commits the current + candidate; empty output is not availability evidence. The third return + value is the provider-reported model when available, otherwise the exact + configured candidate model. + """ + + raw_candidates = list(candidates or []) + if not raw_candidates or not _candidate_is_configured(raw_candidates[0]): + raise _FallbackIneligibleHTTPException(400, "Selected model endpoint is not configured") + candidate_request_factory = kwargs.pop("candidate_request_factory", None) + cands = dedupe_model_candidates(raw_candidates) + if not cands: + raise HTTPException(503, "No model endpoint configured") + eligible_statuses = frozenset(fallback_statuses or ()) + for index, candidate in enumerate(cands): + url, model, headers = candidate + try: + candidate_messages = messages + candidate_kwargs = kwargs + if candidate_request_factory is not None: + request = candidate_request_factory(index, url, model, headers) or {} + if hasattr(request, "__await__"): + request = await request + candidate_messages = request.get("messages", messages) + candidate_kwargs = {**kwargs, **(request.get("kwargs") or {})} + candidate_kwargs = { + **candidate_kwargs, + "availability_only_transport": True, + } + response = await llm_call_async( + url, + model, + candidate_messages, + headers=headers, + return_model_metadata=True, + **candidate_kwargs, + ) + actual_model = model + if ( + isinstance(response, tuple) + and len(response) == 2 + and isinstance(response[0], str) + ): + response, reported_model = response + if isinstance(reported_model, str) and reported_model.strip(): + actual_model = reported_model.strip() + return response, candidate, actual_model + except Exception as error: + if getattr(error, "fallback_eligible", None) is False: + raise + status = _nonstream_error_status(error) + if index >= len(cands) - 1 or status not in eligible_statuses: + raise + tag = "primary" if index == 0 else "candidate" + logger.warning( + "[fallback] %s %s failed with eligible status %s; trying next", + tag, + model, + status, + ) + + raise HTTPException(503, "All fallback candidates failed") + + async def llm_call_async( url: str, model: str, @@ -1932,7 +2182,9 @@ async def llm_call_async( prompt_type: Optional[str] = None, session_id: Optional[str] = None, workload: str = "foreground", -) -> str: + availability_only_transport: bool = False, + return_model_metadata: bool = False, +) -> str | tuple[str, str]: """Asynchronous LLM call using httpx with connection pooling, timeout, retry logic, and performance logging.""" provider = _detect_provider(url) messages_copy = _sanitize_llm_messages(messages) @@ -1950,10 +2202,14 @@ async def llm_call_async( else: messages_copy = non_sys - cache_key = _get_cache_key(url, model, messages_copy, temperature, max_tokens) + cache_key = _get_cache_key( + url, model, messages_copy, temperature, max_tokens, headers=headers, + ) cached_response = _get_cached_response(cache_key) if cached_response: logger.debug(f"Returning cached response for key: {cache_key}") + if return_model_metadata: + return cached_response, (_get_cached_response_model(cache_key) or model) return cached_response if provider == "chatgpt-subscription": @@ -1961,6 +2217,7 @@ async def llm_call_async( # that want a plain string (auto-title, memory extraction, etc.). # Reuse stream_llm's validated Codex SSE path and collect deltas. parts: List[str] = [] + actual_model = model async for chunk in stream_llm( url, model, @@ -1983,8 +2240,16 @@ async def llm_call_async( continue if raw == "[DONE]": response = "".join(parts) - _set_cached_response(cache_key, response) - return response + _set_cached_response( + cache_key, + response, + actual_model=actual_model, + ) + return ( + (response, actual_model) + if return_model_metadata + else response + ) try: data = json.loads(raw) except json.JSONDecodeError: @@ -1992,13 +2257,22 @@ async def llm_call_async( if event_is_error or data.get("error") or (data.get("status") and data.get("text")): status = int(data.get("status") or 502) text = data.get("text") or data.get("error") or "ChatGPT Subscription request failed" - raise HTTPException(status, text) + error_type = ( + _FallbackIneligibleHTTPException + if data.get("fallback_eligible") is False + else HTTPException + ) + raise error_type(status, text) + if data.get("type") == "model_actual": + reported_model = data.get("model") + if isinstance(reported_model, str) and reported_model.strip(): + actual_model = reported_model.strip() delta = data.get("delta") if isinstance(delta, str): parts.append(delta) response = "".join(parts) - _set_cached_response(cache_key, response) - return response + _set_cached_response(cache_key, response, actual_model=actual_model) + return (response, actual_model) if return_model_metadata else response if provider == "anthropic": target_url = _normalize_anthropic_url(url) @@ -2064,7 +2338,21 @@ async def llm_call_async( logger.info(f"LLM async call to {target_url} succeeded in {duration:.2f}s (attempt {attempt})") _clear_host_dead(target_url) data = r.json() + if isinstance(data, dict) and data.get("error"): + provider_error = data["error"] + status = _provider_stream_error_status(provider_error, default=400) + if isinstance(provider_error, dict): + detail = provider_error.get("message") or provider_error.get("type") or str(provider_error) + else: + detail = str(provider_error) + raise HTTPException(status, detail or "Upstream request failed") try: + reported_model = data.get("model") if isinstance(data, dict) else None + actual_model = ( + reported_model.strip() + if isinstance(reported_model, str) and reported_model.strip() + else model + ) if provider == "anthropic": response = _parse_anthropic_response(data) elif provider == "ollama": @@ -2072,10 +2360,23 @@ async def llm_call_async( else: msg = data["choices"][0]["message"] response = msg.get("content") or msg.get("reasoning_content") or "" - _set_cached_response(cache_key, response) - return response + _set_cached_response( + cache_key, + response, + actual_model=actual_model, + ) + return ( + (response, actual_model) + if return_model_metadata + else response + ) + except HTTPException: + raise except Exception: - raise HTTPException(502, f"Unexpected schema from {target_url}: {str(data)[:400]}") + raise _FallbackIneligibleHTTPException( + 502, + f"Unexpected schema from {target_url}: {str(data)[:400]}", + ) except (httpx.ConnectError, httpx.ConnectTimeout) as e: _cooled = _mark_host_dead(target_url) duration = time.time() - start @@ -2084,12 +2385,55 @@ async def llm_call_async( if _cooled or attempt >= max_retries: raise HTTPException(503, f"Cannot reach {_host_key(target_url)}: {e}") await asyncio.sleep(LLMConfig.RETRY_DELAY) - except (httpx.RequestError, httpx.HTTPStatusError) as e: + except httpx.ReadTimeout as e: duration = time.time() - start - logger.warning(f"LLM async call attempt {attempt} failed after {duration:.2f}s: {e}") + logger.warning(f"LLM async read timed out after {duration:.2f}s: {e}") + if attempt >= max_retries: + raise HTTPException(504, f"POST {target_url} timed out after {max_retries} attempts") + await asyncio.sleep(LLMConfig.RETRY_DELAY) + except (httpx.WriteTimeout, httpx.PoolTimeout) as e: + duration = time.time() - start + logger.warning(f"LLM async upstream timeout after {duration:.2f}s: {e}") + if availability_only_transport: + raise _FallbackIneligibleHTTPException( + 504, + f"POST {target_url} failed during request delivery", + ) + if attempt >= max_retries: + raise HTTPException(504, f"POST {target_url} timed out after {max_retries} attempts") + await asyncio.sleep(LLMConfig.RETRY_DELAY) + except httpx.ProtocolError as e: + duration = time.time() - start + logger.warning(f"LLM async protocol failure after {duration:.2f}s: {e}") + if availability_only_transport: + raise _FallbackIneligibleHTTPException( + 502, + f"POST {target_url} failed with a protocol error", + ) + if attempt >= max_retries: + raise HTTPException(502, f"POST {target_url} failed after {max_retries} attempts: {e}") + await asyncio.sleep(LLMConfig.RETRY_DELAY) + except httpx.NetworkError as e: + duration = time.time() - start + logger.warning(f"LLM async network failure after {duration:.2f}s: {e}") + if availability_only_transport: + raise _FallbackIneligibleHTTPException( + 502, + f"POST {target_url} failed with a network error", + ) if attempt >= max_retries: raise HTTPException(502, f"POST {target_url} failed after {max_retries} attempts: {e}") await asyncio.sleep(LLMConfig.RETRY_DELAY) + except httpx.HTTPStatusError as e: + status = e.response.status_code if e.response is not None else 502 + raise HTTPException(status, str(e)) + except httpx.RequestError as e: + duration = time.time() - start + logger.warning(f"LLM async request configuration failed after {duration:.2f}s: {e}") + raise _FallbackIneligibleHTTPException( + 502, + f"POST {target_url} could not be configured: {e}", + ) def _stream_target_url(url: str) -> str: provider = _detect_provider(url) @@ -2227,6 +2571,8 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat event_name = "" input_tokens = 0 output_tokens = 0 + _responses_actual_model = "" + _responses_model_announced = False try: client = _get_http_client() async with client.stream('POST', target_url, json=payload, headers=h, timeout=stream_timeout) as r: @@ -2252,6 +2598,22 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat except json.JSONDecodeError: continue evt = data.get("type") or event_name + response_data = data.get("response") or {} + reported_model = ( + response_data.get("model") + if isinstance(response_data, dict) + else None + ) + reported_model = _reported_model_name( + reported_model or data.get("model") + ) + if reported_model: + _responses_actual_model = reported_model + if not _responses_model_announced: + model_event = _model_actual_event(model, reported_model) + if model_event: + _responses_model_announced = True + yield model_event if evt == "response.output_text.delta": delta = data.get("delta") or "" if delta: @@ -2262,16 +2624,48 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat yield f'data: {json.dumps({"delta": delta})}\n\n' elif evt == "response.completed": usage = (data.get("response") or {}).get("usage") or data.get("usage") or {} - input_tokens = usage.get("input_tokens") or usage.get("prompt_tokens") or input_tokens - output_tokens = usage.get("output_tokens") or usage.get("completion_tokens") or output_tokens - if input_tokens or output_tokens: - yield f'data: {json.dumps({"type": "usage", "data": {"input_tokens": input_tokens, "output_tokens": output_tokens}})}\n\n' + if isinstance(usage, dict): + raw_input = ( + usage.get("input_tokens") + if "input_tokens" in usage + else usage.get("prompt_tokens", input_tokens) + ) + raw_output = ( + usage.get("output_tokens") + if "output_tokens" in usage + else usage.get("completion_tokens", output_tokens) + ) + normalized_usage = _normalize_usage_counts( + raw_input, + raw_output, + ) + if normalized_usage and ( + "input_tokens" in usage + or "prompt_tokens" in usage + or "output_tokens" in usage + or "completion_tokens" in usage + ): + _annotate_usage_model( + normalized_usage, + model, + _responses_actual_model, + ) + yield f'data: {json.dumps({"type": "usage", "data": normalized_usage})}\n\n' yield "data: [DONE]\n\n" return elif evt in ("response.failed", "error"): err = data.get("error") or (data.get("response") or {}).get("error") or {} + if evt == "error" and not err: + # Responses API ``error`` events carry code/message + # at the top level, unlike ``response.failed``. + err = { + key: data[key] + for key in ("type", "code", "message", "status", "status_code", "http_status") + if key in data + } text = err.get("message") if isinstance(err, dict) else str(err or "ChatGPT Subscription request failed") - yield f'event: error\ndata: {json.dumps({"status": 502, "text": text})}\n\n' + status = _provider_stream_error_status(err, default=400) + yield f'event: error\ndata: {json.dumps({"status": status, "text": text})}\n\n' return yield "data: [DONE]\n\n" except (httpx.ConnectError, httpx.ConnectTimeout) as e: @@ -2281,17 +2675,23 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat yield f'event: error\ndata: {json.dumps({"error": f"Cannot reach {_host_key(target_url)}", "status": 503})}\n\n' except httpx.ReadTimeout: yield f'event: error\ndata: {json.dumps({"error": "Read timeout", "status": 504})}\n\n' + except (httpx.WriteTimeout, httpx.PoolTimeout): + yield f'event: error\ndata: {json.dumps({"error": "Upstream timeout", "status": 504, "fallback_eligible": False})}\n\n' + except httpx.ProtocolError: + yield f'event: error\ndata: {json.dumps({"error": "Upstream protocol error", "status": 502, "fallback_eligible": False})}\n\n' except httpx.NetworkError: - yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n' except Exception as e: logger.error(f"ChatGPT Subscription stream error: {e}") - yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n' return # ── Native Ollama streaming ── if provider == "ollama": _ollama_tool_calls: List[Dict] = [] _harmony_router = _HarmonyStreamRouter() + _ollama_actual_model = "" + _ollama_model_announced = False try: client = _get_http_client() async with client.stream('POST', target_url, json=payload, headers=h, timeout=stream_timeout) as r: @@ -2308,6 +2708,20 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat j = json.loads(line) except json.JSONDecodeError: continue + if j.get("error"): + err = j.get("error") + status = _provider_stream_error_status(err, default=400) + text = err.get("message") if isinstance(err, dict) else str(err) + yield f'event: error\ndata: {json.dumps({"error": text or "Ollama request failed", "status": status})}\n\n' + return + reported_model = _reported_model_name(j.get("model")) + if reported_model: + _ollama_actual_model = reported_model + if not _ollama_model_announced: + model_event = _model_actual_event(model, reported_model) + if model_event: + _ollama_model_announced = True + yield model_event message = j.get("message") or {} thinking = message.get("thinking") or "" if thinking: @@ -2330,7 +2744,17 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat if _ollama_tool_calls: yield f'data: {json.dumps({"type": "tool_calls", "calls": _ollama_tool_calls})}\n\n' if j.get("prompt_eval_count") is not None or j.get("eval_count") is not None: - yield f'data: {json.dumps({"type": "usage", "data": {"input_tokens": j.get("prompt_eval_count", 0), "output_tokens": j.get("eval_count", 0)}})}\n\n' + normalized_usage = _normalize_usage_counts( + j.get("prompt_eval_count", 0), + j.get("eval_count", 0), + ) + if normalized_usage: + _annotate_usage_model( + normalized_usage, + model, + _ollama_actual_model, + ) + yield f'data: {json.dumps({"type": "usage", "data": normalized_usage})}\n\n' yield "data: [DONE]\n\n" return for part, is_thinking in _harmony_router.flush(): @@ -2343,17 +2767,24 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat yield f'event: error\ndata: {json.dumps({"error": f"Cannot reach {_host_key(target_url)}", "status": 503})}\n\n' except httpx.ReadTimeout: yield f'event: error\ndata: {json.dumps({"error": "Read timeout", "status": 504})}\n\n' + except (httpx.WriteTimeout, httpx.PoolTimeout): + yield f'event: error\ndata: {json.dumps({"error": "Upstream timeout", "status": 504, "fallback_eligible": False})}\n\n' + except httpx.ProtocolError: + yield f'event: error\ndata: {json.dumps({"error": "Upstream protocol error", "status": 502, "fallback_eligible": False})}\n\n' except httpx.NetworkError: - yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n' except Exception as e: logger.error(f"Ollama stream error: {e}") - yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n' return # ── Anthropic streaming ── if provider == "anthropic": _anth_input_tokens = 0 _anth_output_tokens = 0 + _anth_usage_seen = False + _anth_actual_model = "" + _anth_model_announced = False # Track tool_use blocks: {index: {id, name, arguments_json}} _anth_tool_blocks: Dict[int, Dict] = {} _anth_block_idx = -1 @@ -2407,7 +2838,28 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat if partial and _anth_tool_blocks[idx].get("name") in ("create_document", "update_document", "edit_document"): yield f'data: {json.dumps({"type": "tool_call_delta", "index": idx, "name": _anth_tool_blocks[idx]["name"], "arg_delta": partial})}\n\n' elif evt == "message_start": - _u = j.get("message", {}).get("usage", {}) + message_data = j.get("message") or {} + reported_model = _reported_model_name( + message_data.get("model") + if isinstance(message_data, dict) + else None + ) + if reported_model: + _anth_actual_model = reported_model + if not _anth_model_announced: + model_event = _model_actual_event(model, reported_model) + if model_event: + _anth_model_announced = True + yield model_event + _u = ( + message_data.get("usage") + if isinstance(message_data, dict) + else {} + ) or {} + if not isinstance(_u, dict): + _u = {} + if "input_tokens" in _u: + _anth_usage_seen = True _anth_input_tokens = _u.get("input_tokens", 0) # Surface prompt-cache effectiveness: cache_read > 0 means the # stable system+tools prefix was served from cache this round. @@ -2419,7 +2871,12 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat _c_read, _c_write, _anth_input_tokens, ) elif evt == "message_delta": - _anth_output_tokens = j.get("usage", {}).get("output_tokens", 0) + _u = j.get("usage") or {} + if not isinstance(_u, dict): + _u = {} + if "output_tokens" in _u: + _anth_usage_seen = True + _anth_output_tokens = _u.get("output_tokens", 0) elif evt == "message_stop": # Emit accumulated tool calls in OpenAI-compatible format if _anth_tool_blocks: @@ -2432,13 +2889,24 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat "arguments": tb["arguments"], }) yield f'data: {json.dumps({"type": "tool_calls", "calls": calls})}\n\n' - if _anth_input_tokens or _anth_output_tokens: - yield f'data: {json.dumps({"type": "usage", "data": {"input_tokens": _anth_input_tokens, "output_tokens": _anth_output_tokens}})}\n\n' + normalized_usage = _normalize_usage_counts( + _anth_input_tokens, + _anth_output_tokens, + ) + if normalized_usage and _anth_usage_seen: + _annotate_usage_model( + normalized_usage, + model, + _anth_actual_model, + ) + yield f'data: {json.dumps({"type": "usage", "data": normalized_usage})}\n\n' yield "data: [DONE]\n\n" return elif evt == "error": - err_msg = j.get("error", {}).get("message", "Unknown error") - yield f'event: error\ndata: {json.dumps({"error": err_msg, "status": 400})}\n\n' + err = j.get("error") or {} + err_msg = err.get("message", "Unknown error") if isinstance(err, dict) else str(err) + status = _provider_stream_error_status(err, default=400) + yield f'event: error\ndata: {json.dumps({"error": err_msg, "status": status})}\n\n' return except json.JSONDecodeError: continue @@ -2450,11 +2918,15 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat yield f'event: error\ndata: {json.dumps({"error": f"Cannot reach {_host_key(target_url)}", "status": 503})}\n\n' except httpx.ReadTimeout: yield f'event: error\ndata: {json.dumps({"error": "Read timeout", "status": 504})}\n\n' + except (httpx.WriteTimeout, httpx.PoolTimeout): + yield f'event: error\ndata: {json.dumps({"error": "Upstream timeout", "status": 504, "fallback_eligible": False})}\n\n' + except httpx.ProtocolError: + yield f'event: error\ndata: {json.dumps({"error": "Upstream protocol error", "status": 502, "fallback_eligible": False})}\n\n' except httpx.NetworkError: - yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n' except Exception as e: logger.error(f"Anthropic stream error: {e}") - yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n' return # ── OpenAI-compatible streaming ── @@ -2528,6 +3000,12 @@ def _format_routed_content(parts: List[Tuple[str, bool]]) -> List[str]: if data.strip(): if data.startswith("{"): j = json.loads(data) + if j.get("error"): + err = j.get("error") + status = _provider_stream_error_status(err, default=400) + text = err.get("message") if isinstance(err, dict) else str(err) + yield f'event: error\ndata: {json.dumps({"error": text or "Upstream request failed", "status": status})}\n\n' + return chunk_model = j.get("model") if isinstance(chunk_model, str) and chunk_model.strip(): _actual_model = chunk_model.strip() @@ -2553,9 +3031,21 @@ def _format_routed_content(parts: List[Tuple[str, bool]]) -> List[str]: or _delta0.get("thinking") or _delta0.get("tool_calls") ) - if "usage" in j and not _delta_has_output: - u = j["usage"] or {} - _usage_data = {"input_tokens": u.get("prompt_tokens", 0), "output_tokens": u.get("completion_tokens", 0)} + u = j.get("usage") + _has_genuine_usage = ( + isinstance(u, dict) + and ( + "prompt_tokens" in u + or "completion_tokens" in u + ) + ) + if _has_genuine_usage and not _delta_has_output: + _usage_data = _normalize_usage_counts( + u.get("prompt_tokens", 0), + u.get("completion_tokens", 0), + ) + if _usage_data is None: + continue # llama.cpp puts a `timings` block alongside `usage` with the # TRUE generation speed (predicted_per_second) — pure decode, # excluding prefill/network. Pass it through so the UI shows the @@ -2739,11 +3229,15 @@ def _format_routed_content(parts: List[Tuple[str, bool]]) -> List[str]: yield f'event: error\ndata: {json.dumps({"error": f"Cannot reach {_host_key(target_url)}", "status": 503})}\n\n' except httpx.ReadTimeout: yield f'event: error\ndata: {json.dumps({"error": "Read timeout", "status": 504})}\n\n' + except (httpx.WriteTimeout, httpx.PoolTimeout): + yield f'event: error\ndata: {json.dumps({"error": "Upstream timeout", "status": 504, "fallback_eligible": False})}\n\n' + except httpx.ProtocolError: + yield f'event: error\ndata: {json.dumps({"error": "Upstream protocol error", "status": 502, "fallback_eligible": False})}\n\n' except httpx.NetworkError: - yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n' except Exception as e: logger.error(f"Stream error: {e}") - yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502})}\n\n' + yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n' def _summarize_stream_error(err_chunk: Optional[str]) -> str: @@ -2764,12 +3258,119 @@ def _summarize_stream_error(err_chunk: Optional[str]) -> str: return "primary model failed" +def _stream_error_status(err_chunk: Optional[str]) -> Optional[int]: + """Return the integer status from an SSE error chunk when present.""" + + if not err_chunk: + return None + try: + for line in err_chunk.split("\n"): + if not line.startswith("data: "): + continue + status = json.loads(line[6:]).get("status") + return _normalize_http_status(status) + except (TypeError, ValueError, json.JSONDecodeError): + return None + return None + + +def _stream_error_fallback_override(err_chunk: Optional[str]) -> Optional[bool]: + """Return an adapter's explicit eligibility decision when present.""" + + if not err_chunk: + return None + try: + for line in err_chunk.split("\n"): + if not line.startswith("data: "): + continue + value = json.loads(line[6:]).get("fallback_eligible") + return value if isinstance(value, bool) else None + except (TypeError, ValueError, json.JSONDecodeError): + return None + return None + + +def _request_factory_error_chunk(error: Exception, status: Optional[int]) -> str: + """Convert route-request preparation failures into a safe SSE error.""" + + wire_status = status if status is not None else 500 + payload = { + "error": f"Model request preparation failed (HTTP {wire_status})", + "status": wire_status, + } + override = getattr(error, "fallback_eligible", None) + if isinstance(override, bool): + payload["fallback_eligible"] = override + elif status is None: + # An unclassified internal/configuration failure must never become an + # availability fallback merely because its safe wire status is 500. + payload["fallback_eligible"] = False + return f'event: error\ndata: {json.dumps(payload)}\n\n' + + +def _provider_stream_error_status(error, *, default: int = 400) -> int: + """Classify structured provider stream errors without making them eligible by default. + + Some streaming APIs report an HTTP 200 handshake and put the real failure + in a later event. Unknown application errors are request failures, not + availability evidence; only explicit transient/server markers become 5xx + or rate-limit statuses. + """ + + if isinstance(error, dict): + # A structured numeric status is authoritative. Text heuristics are + # only a fallback for providers that omit it. + saw_explicit_status = False + for key in ("status", "status_code", "http_status", "code"): + if key not in error: + continue + value = error.get(key) + if value is None: + continue + # Google-style errors use a symbolic ``status`` together with a + # numeric HTTP ``code``. A symbolic ``code`` remains part of the + # marker heuristics below; it is not itself an explicit status. + if key != "code": + saw_explicit_status = True + status = _normalize_http_status(value) + if status is not None: + return status + if saw_explicit_status: + return default + marker = " ".join(str(error.get(key) or "") for key in ("type", "code", "message")).lower() + else: + marker = str(error or "").lower() + + if "insufficient_quota" in marker or "billing" in marker: + return 402 + if any(token in marker for token in ("authentication", "unauthorized", "invalid api key", "invalid_api_key")): + return 401 + if any(token in marker for token in ("permission", "forbidden")): + return 403 + if any(token in marker for token in ("not_found", "not found", "unknown model")): + return 404 + if any(token in marker for token in ("invalid_request", "invalid request", "unsupported", "malformed", "bad request")): + return 400 + if any(token in marker for token in ("rate_limit", "rate limit", "too many requests")): + return 429 + if any(token in marker for token in ("overloaded", "over capacity")): + return 529 + if any(token in marker for token in ("timeout", "timed out")): + return 504 + if any(token in marker for token in ("api_error", "server_error", "server error", "internal error", "temporarily unavailable")): + return 500 + + return default + + async def stream_llm_with_fallback(candidates, messages, **kwargs): """Wrap stream_llm with an ordered fallback chain. `candidates` is a list of (url, model, headers). Each is tried in order, - but only retried on a *pre-content* failure — an ``event: error`` or an - empty completion before any assistant text / completed tool call is yielded. + but only retried on an eligible *pre-content* failure. Callers can restrict + errors with ``fallback_statuses`` and disable empty-completion switching + with ``fallback_on_empty=False``. Omitting both preserves the generic + fallback behavior for non-foreground call sites. Metadata is held until substantive output commits the candidate. Once a candidate has emitted real output we never switch (that would duplicate streamed tokens); a later error from that candidate passes @@ -2778,91 +3379,207 @@ async def stream_llm_with_fallback(candidates, messages, **kwargs): Yields the same SSE chunk protocol as stream_llm. """ - cands = _dedupe_candidates(candidates) - if not cands: - yield f'event: error\ndata: {json.dumps({"error": "No model endpoint configured", "status": 503})}\n\n' + fallback_statuses = kwargs.pop("fallback_statuses", None) + fallback_on_empty = bool(kwargs.pop("fallback_on_empty", True)) + candidate_request_factory = kwargs.pop("candidate_request_factory", None) + candidate_route_descriptors = kwargs.pop("candidate_route_descriptors", None) + eligible_statuses = None if fallback_statuses is None else frozenset(fallback_statuses) + + raw_candidates = list(candidates or []) + if not raw_candidates or not _candidate_is_configured(raw_candidates[0]): + yield f'event: error\ndata: {json.dumps({"error": "Selected model endpoint is not configured", "status": 400, "fallback_eligible": False})}\n\n' return + cands, route_descriptors = _dedupe_model_candidates_with_descriptors( + raw_candidates, + candidate_route_descriptors, + ) primary_model = cands[0][1] + primary_route = route_descriptors[0] last_error = None + failures = [] for i, (url, model, headers) in enumerate(cands): is_last = (i == len(cands) - 1) emitted = False retried = False pending_metadata = [] - async for chunk in stream_llm(url, model, messages, headers=headers, **kwargs): - if chunk.startswith("event: error"): - if not emitted and not is_last: - # Pre-content failure with fallbacks left — swallow and - # move to the next candidate. - last_error = chunk - retried = True - if i == 0: - logger.warning(f"[fallback] primary {model} failed before output; trying fallback") - else: - logger.warning(f"[fallback] candidate {model} failed; trying next") - break - if not emitted: - # A last-candidate error is already the clearest terminal - # result; do not append an empty-completion error as well. + candidate_messages = messages + candidate_kwargs = kwargs + if candidate_request_factory is not None: + try: + request = candidate_request_factory(i, url, model, headers) or {} + if hasattr(request, "__await__"): + request = await request + candidate_messages = request.get("messages", messages) + candidate_kwargs = {**kwargs, **(request.get("kwargs") or {})} + except Exception as error: + status = _nonstream_error_status(error) + eligibility_override = getattr(error, "fallback_eligible", None) + eligible = ( + True + if eligible_statuses is None + else ( + eligibility_override + if isinstance(eligibility_override, bool) + else status in eligible_statuses + ) + ) + error_chunk = _request_factory_error_chunk(error, status) + if not is_last and eligible: + last_error = error_chunk + failures.append({ + "candidate_index": i, + "model": model, + "status": status, + "reason": _summarize_stream_error(error_chunk), + }) + tag = "primary" if i == 0 else "candidate" + logger.warning( + "[fallback] %s %s request preparation failed with " + "eligible status %s; trying next", + tag, + model, + status, + ) + continue + yield error_chunk + return + candidate_stream = stream_llm( + url, + model, + candidate_messages, + headers=headers, + **candidate_kwargs, + ) + try: + async for chunk in candidate_stream: + if chunk.startswith("event: error"): + status = _stream_error_status(chunk) + eligibility_override = _stream_error_fallback_override(chunk) + eligible = ( + True + if eligible_statuses is None + else ( + eligibility_override + if eligibility_override is not None + else status in eligible_statuses + ) + ) + if not emitted and not is_last and eligible: + # Pre-content failure with fallbacks left — swallow and + # move to the next candidate. + last_error = chunk + failures.append({ + "candidate_index": i, + "model": model, + "status": status, + "reason": _summarize_stream_error(chunk), + }) + retried = True + if i == 0: + logger.warning(f"[fallback] primary {model} failed before output; trying fallback") + else: + logger.warning(f"[fallback] candidate {model} failed; trying next") + break + if not emitted: + # A last-candidate error is already the clearest terminal + # result; do not append an empty-completion error as well. + yield chunk + return yield chunk - return - yield chunk - continue + continue - event_data = {} - is_done = chunk.startswith("data: [DONE]") - if chunk.startswith("data: ") and not is_done: - try: - event_data = json.loads(chunk[6:]) - except Exception: - pass - - delta = event_data.get("delta") - event_type = event_data.get("type") - substantive = ( - isinstance(delta, str) and bool(delta) - ) or ( - event_type == "tool_call_delta" - ) or ( - event_type == "tool_calls" - and bool(event_data.get("calls")) - ) + event_data = {} + is_done = chunk.startswith("data: [DONE]") + if chunk.startswith("data: ") and not is_done: + try: + event_data = json.loads(chunk[6:]) + except Exception: + pass + + delta = event_data.get("delta") + event_type = event_data.get("type") + substantive = ( + isinstance(delta, str) and bool(delta.strip()) + ) or ( + event_type == "tool_calls" + and bool(event_data.get("calls")) + ) - if substantive and not emitted: - # First real output from a NON-primary candidate: tell the client - # the selected model failed and another answered. Without this the - # fallback is invisible — a misconfigured provider looks like it - # works because the reply is shown under the originally selected - # model's name (e.g. a Bedrock/Claude endpoint that 400s every - # request but appears fine because another model silently answered). - if i > 0: - yield ('data: ' + json.dumps({ - "type": "fallback", - "selected_model": primary_model, - "answered_by": model, - "reason": _summarize_stream_error(last_error), - }) + '\n\n') - # Metadata must not commit a candidate. Once real output arrives, - # flush it after any fallback notice and before the output itself. - for metadata_chunk in pending_metadata: - yield metadata_chunk - pending_metadata.clear() - emitted = True - - if substantive or emitted: - yield chunk - elif not is_done: - pending_metadata.append(chunk) + if substantive and not emitted: + # First real output from a NON-primary candidate: tell the client + # the selected model failed and another answered. Without this the + # fallback is invisible — a misconfigured provider looks like it + # works because the reply is shown under the originally selected + # model's name (e.g. a Bedrock/Claude endpoint that 400s every + # request but appears fine because another model silently answered). + if i > 0: + primary_reason = ( + failures[0]["reason"] + if failures + else _summarize_stream_error(last_error) + ) + yield ('data: ' + json.dumps({ + "type": "fallback", + "selected_model": primary_model, + "answered_by": model, + "selected_endpoint_id": primary_route.get("endpoint_id"), + "selected_endpoint_label": primary_route.get("endpoint_label"), + "selected_endpoint_cost_tracked": primary_route.get("endpoint_cost_tracked"), + "answered_by_endpoint_id": route_descriptors[i].get("endpoint_id"), + "answered_by_endpoint_label": route_descriptors[i].get("endpoint_label"), + "answered_by_endpoint_cost_tracked": route_descriptors[i].get("endpoint_cost_tracked"), + "candidate_index": i, + "reason": primary_reason, + "failures": [ + { + "candidate_index": failure["candidate_index"], + "model": failure["model"], + "status": failure["status"], + } + for failure in failures + ], + }) + '\n\n') + # Metadata must not commit a candidate. Once real output arrives, + # flush it after any fallback notice and before the output itself. + for metadata_chunk in pending_metadata: + yield metadata_chunk + pending_metadata.clear() + emitted = True + + if substantive or emitted: + yield chunk + elif not is_done: + pending_metadata.append(chunk) + finally: + close_candidate = getattr(candidate_stream, "aclose", None) + if callable(close_candidate): + try: + await close_candidate() + except Exception as close_error: + logger.warning( + "[fallback] failed to close candidate %s stream: %s", + model, + type(close_error).__name__, + ) if emitted: return if retried: continue - if not is_last: + if not is_last and fallback_on_empty: last_error = f'event: error\ndata: {json.dumps({"error": f"Model {model} returned no substantive output", "status": 502})}\n\n' + failures.append({ + "candidate_index": i, + "model": model, + "status": 502, + "reason": _summarize_stream_error(last_error), + }) tag = "primary" if i == 0 else "candidate" logger.warning(f"[fallback] {tag} {model} returned no substantive output; trying next") continue + if not is_last: + yield f'event: error\ndata: {json.dumps({"error": f"Model {model} returned no substantive output", "status": 502})}\n\n' + return yield f'event: error\ndata: {json.dumps({"error": "All model candidates returned no substantive output", "status": 502})}\n\n' return diff --git a/src/settings.py b/src/settings.py index 1151de7ca4..47d38308cf 100644 --- a/src/settings.py +++ b/src/settings.py @@ -14,6 +14,13 @@ logger = logging.getLogger(__name__) +# Keys retained in the raw settings store for compatibility and rollback, but +# deliberately unavailable through generic settings APIs or agent tools. They +# must stay in ``DEFAULT_SETTINGS`` so old files continue to load without data +# loss; callers that present or mutate settings should use this set as a +# tombstone boundary. +RETIRED_SETTING_KEYS = frozenset({"default_model_fallbacks"}) + # Tiny TTL cache for settings/features. get_setting() is called on hot paths # (every chat, every preprocess); without this it re-parses the JSON each call. # Picks up edits within _CACHE_TTL seconds, which is fine for human-edited config. @@ -199,6 +206,17 @@ def _invalidate_caches(): }, } + +def without_retired_settings(settings: dict) -> dict: + """Return a shallow copy suitable for generic settings interfaces.""" + if not isinstance(settings, dict): + return {} + return { + key: value + for key, value in settings.items() + if key not in RETIRED_SETTING_KEYS + } + DEFAULT_FEATURES = { "web_search": True, "web_fetch": True, @@ -271,7 +289,7 @@ def is_setting_overridden(key: str) -> bool: # Default chat endpoint / model — without per-user resolution every new # account inherited whatever the most-recent admin picked, which then # got injected into the chat composer on first open. - "default_endpoint_id", "default_model", "default_model_fallbacks", + "default_endpoint_id", "default_model", "utility_endpoint_id", "utility_model", "utility_model_fallbacks", "research_endpoint_id", "research_model", } diff --git a/src/task_endpoint.py b/src/task_endpoint.py index ae57a81f7b..28897f2a66 100644 --- a/src/task_endpoint.py +++ b/src/task_endpoint.py @@ -1,7 +1,6 @@ """Shared resolver for background-task AI endpoints.""" from src.endpoint_resolver import ( - resolve_chat_fallback_candidates, resolve_endpoint, resolve_utility_fallback_candidates, ) @@ -32,7 +31,6 @@ def resolve_task_candidates( 2. Utility endpoint/model 3. Default endpoint/model 4. Utility fallback chain - 5. Retired default-fallback compatibility hook (currently empty) """ candidates = [] @@ -49,9 +47,6 @@ def _append(url, model, headers): _append(*resolve_endpoint("default", owner=owner)) for url, model, headers in resolve_utility_fallback_candidates(owner=owner): _append(url, model, headers) - for url, model, headers in resolve_chat_fallback_candidates(owner=owner): - _append(url, model, headers) - return candidates diff --git a/static/index.html b/static/index.html index 5f1312f403..d63efe9d7c 100644 --- a/static/index.html +++ b/static/index.html @@ -1437,13 +1437,6 @@

diff --git a/static/js/chat.js b/static/js/chat.js index 06c7a1dc7a..bbb1a765e5 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -23,6 +23,12 @@ import slashCommands, { initSlashCommands, isCommand, handleSlashCommand, handle import createResearchSynapse from './researchSynapse.js'; import { createStreamRenderer } from './streamingRenderer.js'; import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composerArrowUpRecall.js'; +import { + applyModelMetricsState, + applyModelRouteEventState, + inheritModelRouteState, +} from './chatModelProvenance.js'; +import { createTerminalStreamError, isRecoverableStreamError } from './chatStreamErrors.js'; const RESEARCH_TIMEOUT_MS = 360000; const DEFAULT_TIMEOUT_MS = 120000; @@ -113,13 +119,27 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const tsSpan = roleEl.querySelector('.role-timestamp'); const req = requestedModel || actualModel || ''; const actual = actualModel || requestedModel || ''; - let label = _modelRouteLabel(req, actual); + let label = _modelRouteLabel( + req, + actual, + opts.requestedEndpointLabel, + opts.actualEndpointLabel, + opts.requestedEndpointId, + opts.actualEndpointId, + ); if (opts.suffix) label += ' (' + opts.suffix + ')'; if (opts.characterName) label = opts.characterName; roleEl.textContent = label + ' '; _applyModelColor(roleEl, actual || req); - if (req && actual && !_sameModelName(req, actual)) { - roleEl.title = req + ' -> ' + actual + (opts.reason ? ': ' + opts.reason : ''); + const endpointChanged = Boolean( + opts.requestedEndpointId + && opts.actualEndpointId + && opts.requestedEndpointId !== opts.actualEndpointId + ); + if (req && actual && (!_sameModelName(req, actual) || endpointChanged)) { + roleEl.title = req + ' -> ' + actual + + (endpointChanged ? ' (' + opts.requestedEndpointLabel + ' -> ' + opts.actualEndpointLabel + ')' : '') + + (opts.reason ? ': ' + opts.reason : ''); } else if (!opts.reason) { roleEl.removeAttribute('title'); } @@ -265,6 +285,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Background streaming support const _backgroundStreams = new Map(); // sessionId -> { status, accumulated, sourcesHtml, abortCtrl, query, metrics } const _resumingStreams = new Set(); // sessionId -> a resumeStream() reader is live (re-attach lock) + const _terminalSavedStreams = new Set(); // sessionId -> canonical terminal event seen by active reader + const _streamRunIds = new Map(); // sessionId -> opaque identity of the exact detached run let _streamSessionId = null; // Session ID for the currently active reader loop let _lastReaderActivity = 0; // Timestamp of last reader.read() success — used to detect frozen streams let _webLockRelease = null; // Function to release the Web Lock held during streaming @@ -277,6 +299,25 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer _resumingStreams.has(sessionId); } + /** Stable cost identity for one logical metrics segment within a run. */ + function _metricsCostRecordId(runId, event) { + if (!runId) return ''; + return `${runId}:${event && event.teacher ? 'teacher' : 'primary'}`; + } + + /** Stop only the exact detached run whose identity this browser observed. */ + function _stopExactRun(sessionId) { + if (!sessionId) return false; + const runId = _streamRunIds.get(sessionId); + if (!runId) return false; + fetch(`/api/chat/stop/${encodeURIComponent(sessionId)}`, { + method: 'POST', + credentials: 'same-origin', + headers: { 'X-Odysseus-Run-Id': runId }, + }).catch(() => {}); + return true; + } + // Sources box builder and toggleSources are now in chatRenderer.js var _buildSourcesBox = chatRenderer.buildSourcesBox; @@ -880,6 +921,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Capture session ID for background stream detection const streamSessionId = sessionModule.getCurrentSessionId(); _streamSessionId = streamSessionId; + _terminalSavedStreams.delete(streamSessionId); + _streamRunIds.delete(streamSessionId); const streamQuery = msg; _lastReaderActivity = Date.now(); @@ -899,6 +942,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer let _thinkOpen = false; let holder = null; let finalMeta = null; + let _canonicalTerminalSaved = false; let spinner = null; let timedOut = false; let processingProbeTimer = null; @@ -1251,14 +1295,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer if (!abortCtrl.signal.aborted) { timedOut = true; abortCtrl._reason = 'timeout'; - try { - if (streamSessionId) { - fetch(`/api/chat/stop/${encodeURIComponent(streamSessionId)}`, { - method: 'POST', - credentials: 'same-origin', - }).catch(() => {}); - } - } catch (_) {} + try { _stopExactRun(streamSessionId); } catch (_) {} abortCtrl.abort(); } }, timeoutMs); @@ -1396,6 +1433,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer enableResearchBtn(); return; } + const streamRunId = res.headers.get('X-Odysseus-Run-Id') || ''; + if (streamRunId) _streamRunIds.set(streamSessionId, streamRunId); // Mark the chat log busy while streaming so screen readers wait for the // settled response instead of announcing every token. Cleared in finally. @@ -1459,9 +1498,17 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const newRole = document.createElement('div'); newRole.className = 'role'; const metaS = sessionModule.getSessions().find(s => s.id === streamSessionId); - const requested = holder?._requestedModel || metaS?.model || modelName; - const actual = holder?._actualModel || requested; - newRole.textContent = _modelRouteLabel(requested, actual) || ''; + inheritModelRouteState(holder, roundHolder, newWrap, metaS?.model || modelName); + const requested = newWrap._requestedModel; + const actual = newWrap._actualModel; + newRole.textContent = _modelRouteLabel( + requested, + actual, + newWrap._requestedEndpointLabel, + newWrap._actualEndpointLabel, + newWrap._requestedEndpointId, + newWrap._actualEndpointId, + ) || ''; _applyModelColor(newRole, actual); newWrap.appendChild(newRole); const newBody = document.createElement('div'); @@ -1691,6 +1738,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer let _nextIsError = false; let _streamSawDone = false; + let _streamTerminalError = null; let _firstVisibleOutputSeen = false; const markFirstVisibleOutput = () => { if (_firstVisibleOutputSeen) return; @@ -1816,10 +1864,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Handle SSE error events (e.g. HTTP 404 from provider) if (_nextIsError || json.status >= 400) { _nextIsError = false; - const errMsg = json.text || json.error?.message || `Error ${json.status || 'unknown'}`; - console.error('Stream error:', errMsg); + _streamTerminalError = createTerminalStreamError(json); + console.error('Stream error:', _streamTerminalError.message); if (spinner && spinner.element) spinner.destroy(); - typewriterInto(roundHolder.querySelector('.body'), errMsg); break; } if (json.delta || json.type === 'agent_prep' || json.type === 'tool_start' || json.type === 'tool_output' || json.type === 'tool_progress' || json.type === 'agent_step' || json.type === 'loop_breaker_triggered' || json.type === 'intent_nudge_exhausted' || json.type === 'doc_stream_open' || json.type === 'doc_stream_delta' || json.type === 'research_progress') { @@ -2258,18 +2305,6 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer 6000 ); continue; - } else if (json.type === 'model_fallback') { - // Model went offline — switched to fallback - var _fbData = json.data || {}; - uiModule.showToast( - `Model ${_fbData.old_model || '?'} offline — switched to ${_fbData.new_model || '?'}`, - 5000 - ); - // Update the model picker to reflect the new model - if (sessionModule && sessionModule.updateModelPicker) { - sessionModule.updateModelPicker(); - } - continue; } else if (json.type === 'model_info') { // Update role label with model name as soon as we know it if (!_isBg && holder) { @@ -2277,6 +2312,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer if (roleEl) { holder._requestedModel = json.requested_model || json.model || holder._requestedModel; holder._actualModel = json.model || holder._actualModel || holder._requestedModel; + holder._requestedEndpointId = json.requested_endpoint_id || json.endpoint_id || holder._requestedEndpointId || null; + holder._requestedEndpointLabel = json.requested_endpoint_label || json.endpoint_label || holder._requestedEndpointLabel || 'Selected route'; + holder._actualEndpointId = json.endpoint_id || holder._actualEndpointId || holder._requestedEndpointId; + holder._actualEndpointLabel = json.endpoint_label || holder._actualEndpointLabel || holder._requestedEndpointLabel; if (json.suffix) holder._roleSuffix = json.suffix; // Prepend character name if sent by server or set locally var _charName = json.character_name || (presetsModule.getCharacterName ? presetsModule.getCharacterName() : ''); @@ -2284,6 +2323,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer _setRoleModelLabel(roleEl, holder._requestedModel, holder._actualModel, { suffix: holder._roleSuffix, characterName: holder._characterName, + requestedEndpointId: holder._requestedEndpointId, + requestedEndpointLabel: holder._requestedEndpointLabel, + actualEndpointId: holder._actualEndpointId, + actualEndpointLabel: holder._actualEndpointLabel, }); } } @@ -2294,9 +2337,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer if (!_isBg) { var _selM = _shortModel(json.selected_model || ''); var _ansM = _shortModel(json.answered_by || ''); - uiModule.showToast('⚠ ' + _selM + ' failed — answered by ' + _ansM, 6000); - if (holder) { - var _rEl = holder.querySelector('.role'); + uiModule.showToast('Fallback: ' + _selM + ' failed — answered by ' + _ansM, 6000); + var _fallbackHolder = applyModelRouteEventState(json, holder, roundHolder, modelName); + if (_fallbackHolder) { + var _rEl = _fallbackHolder.querySelector('.role'); if (_rEl) { var _tsS = _rEl.querySelector('.role-timestamp'); _rEl.textContent = _ansM + ' (fallback) '; @@ -2304,13 +2348,14 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer (json.reason ? ': ' + json.reason : '') + ' — answered by ' + (json.answered_by || ''); _applyModelColor(_rEl, json.answered_by); if (_tsS) _rEl.appendChild(_tsS); - holder._requestedModel = json.selected_model || holder._requestedModel || modelName; - const _hasResolvedActual = holder._actualModel && !_sameModelName(holder._actualModel, holder._requestedModel); - holder._actualModel = _hasResolvedActual ? holder._actualModel : (json.answered_by || holder._actualModel || holder._requestedModel); - _setRoleModelLabel(_rEl, holder._requestedModel, holder._actualModel, { - suffix: holder._roleSuffix, - characterName: holder._characterName, + _setRoleModelLabel(_rEl, _fallbackHolder._requestedModel, _fallbackHolder._actualModel, { + suffix: _fallbackHolder._roleSuffix, + characterName: _fallbackHolder._characterName, reason: json.reason, + requestedEndpointId: _fallbackHolder._requestedEndpointId, + requestedEndpointLabel: _fallbackHolder._requestedEndpointLabel, + actualEndpointId: _fallbackHolder._actualEndpointId, + actualEndpointLabel: _fallbackHolder._actualEndpointLabel, }); } } @@ -2354,12 +2399,15 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer try { note.scrollIntoView({ block: 'end', behavior: 'smooth' }); } catch (_) { uiModule.scrollHistory && uiModule.scrollHistory(); } } } else if (json.type === 'model_actual') { - if (!_isBg && holder) { - holder._requestedModel = json.requested_model || holder._requestedModel || modelName; - holder._actualModel = json.model || holder._actualModel || holder._requestedModel; - _setRoleModelLabel(holder.querySelector('.role'), holder._requestedModel, holder._actualModel, { - suffix: holder._roleSuffix, - characterName: holder._characterName, + if (!_isBg) { + var _modelHolder = applyModelRouteEventState(json, holder, roundHolder, modelName); + if (_modelHolder) _setRoleModelLabel(_modelHolder.querySelector('.role'), _modelHolder._requestedModel, _modelHolder._actualModel, { + suffix: _modelHolder._roleSuffix, + characterName: _modelHolder._characterName, + requestedEndpointId: _modelHolder._requestedEndpointId, + requestedEndpointLabel: _modelHolder._requestedEndpointLabel, + actualEndpointId: _modelHolder._actualEndpointId, + actualEndpointLabel: _modelHolder._actualEndpointLabel, }); } } else if (json.type === 'attachments') { @@ -2445,15 +2493,60 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const detail = before && after && before > after ? ` (${after}/${before} messages sent)` : ''; uiModule.showToast(`Context trimmed for this model${detail}`); } + } else if (json.type === 'agent_terminal' || json.type === 'chat_terminal') { + // The backend persisted canonical partial output, sanitized + // failure metadata, and actual-route provenance before this + // event. The terminal catch below reloads that exact record. + _canonicalTerminalSaved = true; + _terminalSavedStreams.add(streamSessionId); + const priorMetrics = metrics; + metrics = json.data || metrics; + if (metrics && streamRunId) { + metrics._costRecordId = _metricsCostRecordId(streamRunId, json); + } + // Direct Chat may have emitted provider usage before its + // terminal event. Carry that already-recorded state onto the + // canonical terminal metadata instead of billing it twice. + if (priorMetrics && priorMetrics._costRecorded && metrics) { + metrics._costRecorded = true; + } + if (_isBg) { + var bgTerminal = _backgroundStreams.get(streamSessionId); + if (bgTerminal) { + if ( + bgTerminal.metrics + && bgTerminal.metrics._costRecorded + && metrics + ) { + metrics._costRecorded = true; + } + bgTerminal.metrics = metrics; + bgTerminal.status = 'completed'; + if (metrics) { + chatRenderer.recordSessionMetricsCost(metrics, streamSessionId); + } + } + continue; + } + if (holder && metrics) { + applyModelMetricsState(metrics, holder, roundHolder, modelName); + const terminalMetricsTarget = _metricsTargetForTurn(); + if (terminalMetricsTarget) displayMetrics(terminalMetricsTarget, metrics); + } } else if (json.type === 'metrics') { metrics = json.data; + if (metrics && streamRunId) { + metrics._costRecordId = _metricsCostRecordId(streamRunId, json); + } if (!_isBg && holder && metrics) { - holder._requestedModel = metrics.requested_model || holder._requestedModel || modelName; - holder._actualModel = metrics.model || holder._actualModel || holder._requestedModel; + applyModelMetricsState(metrics, holder, roundHolder, modelName); } if (_isBg) { var bgM = _backgroundStreams.get(streamSessionId); - if (bgM) bgM.metrics = json.data; + if (bgM) { + bgM.metrics = json.data; + chatRenderer.recordSessionMetricsCost(bgM.metrics, streamSessionId); + } continue; } if (metrics) { @@ -2821,9 +2914,17 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const newRole = document.createElement('div'); newRole.className = 'role'; const metaS = sessionModule.getSessions().find(s => s.id === streamSessionId); - const _roundRequested = holder?._requestedModel || metaS?.model; - const _roundActual = holder?._actualModel || _roundRequested; - newRole.textContent = _modelRouteLabel(_roundRequested, _roundActual) || ''; + inheritModelRouteState(holder, roundHolder, newWrap, metaS?.model || modelName); + const _roundRequested = newWrap._requestedModel; + const _roundActual = newWrap._actualModel; + newRole.textContent = _modelRouteLabel( + _roundRequested, + _roundActual, + newWrap._requestedEndpointLabel, + newWrap._actualEndpointLabel, + newWrap._requestedEndpointId, + newWrap._actualEndpointId, + ) || ''; _applyModelColor(newRole, _roundActual); newWrap.appendChild(newRole); const newBody = document.createElement('div'); @@ -2929,6 +3030,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer } } + if (_streamTerminalError) { + throw _streamTerminalError; + } if (!_streamSawDone) { throw new Error('Stream closed before completion'); } @@ -2947,15 +3051,25 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const _isBgFinal = (sessionModule.getCurrentSessionId() !== streamSessionId) || _backgroundStreams.has(streamSessionId); if (!_isBgFinal) { finalMeta = sessionModule.getSessions().find(s => s.id === sessionModule.getCurrentSessionId()); - const _finalActualModel = metrics?.model || holder._actualModel || finalMeta?.model; - const _finalRequestedModel = metrics?.requested_model || holder._requestedModel || finalMeta?.model || _finalActualModel; + const _finalModelHolder = applyModelMetricsState( + metrics, + holder, + roundHolder, + finalMeta?.model || modelName, + ) || holder; + const _finalActualModel = _finalModelHolder._actualModel || finalMeta?.model; + const _finalRequestedModel = _finalModelHolder._requestedModel || finalMeta?.model || _finalActualModel; // Prepend character name if set var _charNameFinal = presetsModule.getCharacterName ? presetsModule.getCharacterName() : ''; - const roleEl = holder.querySelector('.role'); + const roleEl = _finalModelHolder.querySelector('.role'); if (roleEl) { _setRoleModelLabel(roleEl, _finalRequestedModel, _finalActualModel, { - suffix: holder._roleSuffix, - characterName: _charNameFinal || holder._characterName, + suffix: _finalModelHolder._roleSuffix, + characterName: _charNameFinal || _finalModelHolder._characterName, + requestedEndpointId: _finalModelHolder._requestedEndpointId, + requestedEndpointLabel: _finalModelHolder._requestedEndpointLabel, + actualEndpointId: _finalModelHolder._actualEndpointId, + actualEndpointLabel: _finalModelHolder._actualEndpointLabel, }); } holder.dataset.raw = accumulated; @@ -3217,7 +3331,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Error happened while backgrounded — update map, don't touch DOM console.error('Background stream error:', err); var bgErr = _backgroundStreams.get(streamSessionId); - if (bgErr && bgErr.status === 'completed') { + if (bgErr && ( + bgErr.status === 'completed' || _terminalSavedStreams.has(streamSessionId) + )) { + bgErr.status = 'completed'; // [DONE] was already processed — this error is benign (e.g. reader.read() after close) // Don't override the completed status; just ensure the completed dot stays if (sessionModule && sessionModule.clearStreaming) { @@ -3377,7 +3494,30 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // cap. Only auto-recover from connection-class failures; deterministic // errors (unsupported tools, 4xx/5xx, parse failures) surface right away // instead of burning the nudge budget on a guaranteed-to-fail retry. - if (!(_isRecoverableStreamErr(err) && _tryAutoRecover(holder, accumulated, streamSessionId))) { + if (!(isRecoverableStreamError(err) && _tryAutoRecover(holder, accumulated, streamSessionId))) { + if (err.terminalStreamError) { + if (_canonicalTerminalSaved || accumulated.trim()) { + // Let this stream's finally block clear foreground state before + // reselecting; otherwise selectSession would detach the already + // terminal reader and leave a stale background-stream marker. + setTimeout(async () => { + if (sessionModule.getCurrentSessionId() === streamSessionId) { + await sessionModule.selectSession(streamSessionId, { showLoading: false }); + } else { + await sessionModule.loadSessions(); + } + }, 0); + } else { + const terminalBody = roundHolder && roundHolder.querySelector('.body'); + if (terminalBody) { + const terminalNote = document.createElement('div'); + terminalNote.style.cssText = 'color: var(--color-error); font-style: italic; padding: 4px 0;'; + terminalNote.textContent = `[Error: ${err.message}]`; + terminalBody.appendChild(terminalNote); + } + } + return; + } const errorHolder = document.querySelector('.msg-ai:last-of-type .body'); if (errorHolder) { let errMsg = `Error: ${err.message}`; @@ -3391,6 +3531,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer } } } finally { + if (_streamSessionId === streamSessionId) _streamSessionId = null; clearResponseTimeout(); clearProcessingProbe(); clearFirstTokenWaitTimers(); @@ -3406,6 +3547,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Only reset UI state if still on the stream's session and was never backgrounded const _isBgFinally = (sessionModule.getCurrentSessionId() !== streamSessionId) || _backgroundStreams.has(streamSessionId); + _terminalSavedStreams.delete(streamSessionId); if (!_isBgFinally) { // Reset button to idle state @@ -3511,29 +3653,20 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const _sid = _streamSessionId || (window.sessionModule && window.sessionModule.getCurrentSessionId && window.sessionModule.getCurrentSessionId()); if (_sid) { - fetch(`/api/chat/stop/${encodeURIComponent(_sid)}`, { method: 'POST', credentials: 'same-origin' }).catch(() => {}); + _stopExactRun(_sid); } } catch (_) {} } } // ── Stall watchdog ────────────────────────────────────────────── - // Auto-recover a turn whose stream died (connection drop) or went silent: - // preserve the partial, then re-submit a completion handshake by reusing the - // existing continue/resume path. Returns false at the cap so the caller can - // surface the failure instead of nudging forever. + // Auto-recover a turn whose browser stream died by reconnecting to the exact + // detached server run. Returns false at the cap so the caller can surface + // the failure instead of retrying forever. // Only auto-recover from connection-class failures (the genuine "silently // died" case). Deterministic errors — unsupported tools, HTTP 4xx/5xx, JSON // parse failures — will fail identically on retry, so surfacing them // immediately is both more honest and avoids wasting the nudge budget. - function _isRecoverableStreamErr(err) { - if (!err) return false; - if (err.name === 'TypeError') return true; // fetch/reader network failure - const m = (err.message || '').toLowerCase(); - if (/\btool\b|unsupported|json|parse|\b4\d\d\b|\b5\d\d\b/.test(m)) return false; - return /network|fetch|connection|reset|closed|aborted|stream|tim(?:e|ed)\s?out|econn|eof/.test(m); - } - function _tryAutoRecover(holder, accumulated, sessionId) { if (_autoNudges >= _AUTO_NUDGE_CAP) return false; _autoNudges++; @@ -3544,28 +3677,18 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer markdownModule.processWithThinking(markdownModule.squashOutsideCode(accumulated)); } catch (_) {} } - _pendingContinue = holder || null; // merge the continuation into the same bubble - _hideUserBubble = true; // no user bubble for the handshake - _autoContinuePending = true; // don't reset the counter on this submit - const _abandon = () => { // clear the pending flags so they can't - _pendingContinue = null; // leak into whatever chat is now open - _hideUserBubble = false; - _autoContinuePending = false; - }; - // Defer so the stream's finally resets state first — otherwise the send - // button is still in "stop" mode and clicking it would toggle, not send. - setTimeout(() => { + // The server run is detached and keeps its exact pinned model/tool state. + // Reconnect to that run instead of submitting a new user turn, which would + // cancel it, retry the selected model, and risk duplicating side effects. + setTimeout(async () => { // The stream that died may not be the chat the user is now looking at — - // never inject the recovery handshake into the wrong conversation. - if (sessionId && sessionModule.getCurrentSessionId() !== sessionId) { _abandon(); return; } - const msgInput = uiModule.el('message'); - const sb = document.querySelector('.send-btn'); - if (!msgInput || !sb) { _abandon(); return; } - const tail = (accumulated || '').slice(-400); - msgInput.value = tail - ? `The stream dropped before you finished. It ended with:\n\n${tail}\n\nIf the task is fully complete, reply with just: DONE. Otherwise continue exactly where you left off and finish it — do not repeat what you already wrote.` - : `The stream dropped before you produced anything. If the task is already done, reply with just: DONE. Otherwise complete it now.`; - sb.click(); + // never attach the recovery reader to the wrong conversation. + if (sessionId && sessionModule.getCurrentSessionId() !== sessionId) return; + const resumed = await resumeStream(sessionId, holder || null); + if (!resumed && holder && holder.isConnected) { + const body = holder.querySelector('.body'); + if (body) typewriterInto(body, 'Connection lost. The existing run could not be resumed.'); + } }, 200); return true; } @@ -3719,9 +3842,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer abortCurrentRequest(); return; } - // Store background stream state + const terminalSaved = _terminalSavedStreams.has(sessionId); + // Store background stream state. A canonical terminal event can precede + // its SSE error event; preserve completion if the user switches sessions + // during that gap instead of creating a fresh running/error marker. _backgroundStreams.set(sessionId, { - status: 'running', + status: terminalSaved ? 'completed' : 'running', accumulated: currentAccumulated, sourcesHtml: '', findingsData: null, @@ -3730,8 +3856,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer metrics: null, }); // Mark session with pulsing dot in sidebar - if (sessionModule && sessionModule.markStreaming) { + if (!terminalSaved && sessionModule && sessionModule.markStreaming) { sessionModule.markStreaming(sessionId); + } else if (terminalSaved && sessionModule && sessionModule.clearStreaming) { + sessionModule.clearStreaming(sessionId); } // Clear local state WITHOUT aborting the fetch currentAbort = null; @@ -3758,7 +3886,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer * reloaded from the DB so its full render stays faithful. Returns true if it * attached, false to let the caller fall back to spinner+poll. */ - export async function resumeStream(sessionId) { + export async function resumeStream(sessionId, replaceHolder = null) { if (!sessionId) return false; if (hasActiveStream(sessionId)) return false; @@ -3769,9 +3897,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer return false; } if (!res.ok || !res.body) return false; + const resumeRunId = res.headers.get('X-Odysseus-Run-Id') || ''; + if (resumeRunId) _streamRunIds.set(sessionId, resumeRunId); const box = document.getElementById('chat-history'); if (!box) return false; + if (replaceHolder && replaceHolder.parentNode) replaceHolder.remove(); // Block duplicate re-attach attempts while this reader is live. A dedicated // set (not _backgroundStreams) so checkBackgroundStream doesn't mistake this @@ -3786,6 +3917,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer holder.innerHTML = '
' + uiModule.esc(roleLabel) + ' ' + roleTs + '
' + '
'; + holder._requestedModel = meta && meta.model; + holder._actualModel = holder._requestedModel; _applyModelColor(holder.querySelector('.role'), meta && meta.model); const contentDiv = holder.querySelector('.stream-content'); box.appendChild(holder); @@ -3803,6 +3936,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer let gotDelta = false; let leftSession = false; let metricsData = null; + let replayError = null; + let canonicalTerminalSeen = false; // "Rich" responses (tool calls, sources, doc streaming, multi-round) need the // full canonical render, which is rebuilt from the saved DB record on reload. // Plain text replies can be finalized in place without a reload. @@ -3839,6 +3974,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const parts = buffer.split('\n\n'); buffer = parts.pop(); for (const part of parts) { + const eventIsError = part.split('\n').some(l => l.trim() === 'event: error'); + if (eventIsError) rich = true; const line = part.split('\n').find(l => l.startsWith('data: ')); if (!line) continue; const payload = line.slice(6); @@ -3848,7 +3985,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer } let json; try { json = JSON.parse(payload); } catch (_) { continue; } - if (json.delta) { + if (eventIsError) { + replayError = createTerminalStreamError(json); + } else if (json.delta) { roundText += json.delta; if (!docFenceOpened && (roundText.includes('```create_document\n') || roundText.includes('```document\n') || roundText.includes('```documen\n'))) { docFenceOpened = true; @@ -3864,6 +4003,64 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer if (documentModule) documentModule.streamDocDelta(json.content || json.delta || ''); } else if (json.type === 'metrics') { metricsData = json.data || metricsData; + if (metricsData && resumeRunId) { + metricsData._costRecordId = _metricsCostRecordId(resumeRunId, json); + } + if (metricsData) { + chatRenderer.recordSessionMetricsCost(metricsData, sessionId); + } + } else if (json.type === 'fallback') { + // Replay can attach after the selected route has already failed. + // Reflect the fallback immediately, then reload the canonical + // multi-round record when the detached run completes. + rich = true; + const fallbackHolder = applyModelRouteEventState(json, holder, null, meta && meta.model); + if (fallbackHolder) { + _setRoleModelLabel( + fallbackHolder.querySelector('.role'), + fallbackHolder._requestedModel, + fallbackHolder._actualModel, + { + reason: json.reason, + requestedEndpointId: fallbackHolder._requestedEndpointId, + requestedEndpointLabel: fallbackHolder._requestedEndpointLabel, + actualEndpointId: fallbackHolder._actualEndpointId, + actualEndpointLabel: fallbackHolder._actualEndpointLabel, + }, + ); + } + uiModule.showToast( + 'Fallback: ' + _shortModel(json.selected_model || '') + ' failed — answered by ' + + _shortModel(json.answered_by || ''), + 6000, + ); + } else if (json.type === 'model_actual') { + rich = true; + const modelHolder = applyModelRouteEventState(json, holder, null, meta && meta.model); + if (modelHolder) { + _setRoleModelLabel( + modelHolder.querySelector('.role'), + modelHolder._requestedModel, + modelHolder._actualModel, + { + requestedEndpointId: modelHolder._requestedEndpointId, + requestedEndpointLabel: modelHolder._requestedEndpointLabel, + actualEndpointId: modelHolder._actualEndpointId, + actualEndpointLabel: modelHolder._actualEndpointLabel, + }, + ); + } + } else if (json.type === 'agent_terminal' || json.type === 'chat_terminal') { + // The server has already persisted canonical partial content plus + // a sanitized failure note and actual route provenance. Do not + // finalize replayed deltas as a successful local-only answer. + rich = true; + canonicalTerminalSeen = true; + metricsData = json.data || metricsData; + if (metricsData && resumeRunId) { + metricsData._costRecordId = _metricsCostRecordId(resumeRunId, json); + } + if (metricsData) displayMetrics(holder, metricsData); } else if (json.type === 'tool_start' || json.type === 'tool_output' || json.type === 'tool_progress' || json.type === 'agent_step' || json.type === 'web_sources' || json.type === 'rag_sources' || @@ -3874,7 +4071,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer } } } catch (e) { - // Network drop or parse failure: fall through to the reload below. + // Network drop or parse failure: fall through to the canonical reload. + rich = true; } cleanup(); @@ -3884,6 +4082,18 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const onThisSession = sessionModule.getCurrentSessionId && sessionModule.getCurrentSessionId() === sessionId; + // A failure before substantive output has no persisted assistant record to + // recover through a canonical reload. Keep its sanitized provider/request + // error visible in the replay holder instead of deleting the only evidence. + if (onThisSession && replayError && !canonicalTerminalSeen) { + const errorDiv = document.createElement('div'); + errorDiv.style.cssText = 'color: var(--color-error); font-style: italic; padding: 4px 0;'; + errorDiv.textContent = `[Error: ${replayError.message}]`; + contentDiv.appendChild(errorDiv); + uiModule.scrollHistory(); + return true; + } + // Plain text reply: finalize in place. Replace the live bubble with a // canonical single message (markdown + footer actions + metrics) using the // same renderer history does. No history refetch, no end-of-stream flicker. @@ -3900,6 +4110,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // reload from the DB for the full canonical render. if (holder._docWritingThread && holder._docWritingThread.parentNode) holder._docWritingThread.remove(); if (holder.parentNode) holder.remove(); + if (metricsData) { + chatRenderer.recordSessionMetricsCost(metricsData, sessionId); + } if (onThisSession) sessionModule.selectSession(sessionId); else sessionModule.loadSessions(); return true; diff --git a/static/js/chatModelProvenance.js b/static/js/chatModelProvenance.js new file mode 100644 index 0000000000..2274537cdf --- /dev/null +++ b/static/js/chatModelProvenance.js @@ -0,0 +1,104 @@ +/** Select and update the response holder for a route-provenance event. */ +export function applyModelRouteEventState(event, holder, roundHolder, defaultModel = '') { + const target = event && event.round && roundHolder ? roundHolder : holder; + if (!target) return null; + + target._requestedModel = ( + event.requested_model + || event.selected_model + || target._requestedModel + || defaultModel + ); + target._actualModel = ( + event.model + || event.answered_by + || target._actualModel + || target._requestedModel + ); + const hasEndpointRoute = Boolean( + event.requested_endpoint_id + || event.selected_endpoint_id + || event.endpoint_id + || event.answered_by_endpoint_id + || event.requested_endpoint_label + || event.selected_endpoint_label + || event.endpoint_label + || event.answered_by_endpoint_label + || target._requestedEndpointLabel + ); + if (hasEndpointRoute) { + target._requestedEndpointId = ( + event.requested_endpoint_id + || event.selected_endpoint_id + || target._requestedEndpointId + || null + ); + target._requestedEndpointLabel = ( + event.requested_endpoint_label + || event.selected_endpoint_label + || target._requestedEndpointLabel + || 'Selected route' + ); + target._actualEndpointId = ( + event.endpoint_id + || event.answered_by_endpoint_id + || target._actualEndpointId + || target._requestedEndpointId + || null + ); + target._actualEndpointLabel = ( + event.endpoint_label + || event.answered_by_endpoint_label + || target._actualEndpointLabel + || target._requestedEndpointLabel + ); + } + return target; +} + +/** Copy the active route into the bubble created for the next Agent round. */ +export function inheritModelRouteState(holder, roundHolder, target, defaultModel = '') { + if (!target) return null; + const source = roundHolder || holder; + target._requestedModel = source?._requestedModel || defaultModel; + target._actualModel = source?._actualModel || target._requestedModel; + if (source?._requestedEndpointLabel || source?._actualEndpointLabel) { + target._requestedEndpointId = source?._requestedEndpointId || null; + target._requestedEndpointLabel = source?._requestedEndpointLabel || 'Selected route'; + target._actualEndpointId = source?._actualEndpointId || target._requestedEndpointId; + target._actualEndpointLabel = source?._actualEndpointLabel || target._requestedEndpointLabel; + } + return target; +} + +/** Apply final/metrics provenance to the active round, not the first bubble. */ +export function applyModelMetricsState(metrics, holder, roundHolder, defaultModel = '') { + const target = roundHolder || holder; + if (!target || !metrics) return target || null; + const roundModels = Array.isArray(metrics.round_models) ? metrics.round_models : []; + const roundModel = roundHolder && roundModels.length + ? roundModels[roundModels.length - 1] + : null; + target._requestedModel = metrics.requested_model || target._requestedModel || defaultModel; + target._actualModel = roundModel || metrics.model || target._actualModel || target._requestedModel; + const roundEndpointIds = Array.isArray(metrics.round_endpoint_ids) ? metrics.round_endpoint_ids : []; + const roundEndpointLabels = Array.isArray(metrics.round_endpoint_labels) ? metrics.round_endpoint_labels : []; + if ( + metrics.requested_endpoint_label + || metrics.endpoint_label + || roundEndpointLabels.length + || target._requestedEndpointLabel + ) { + target._requestedEndpointId = metrics.requested_endpoint_id || target._requestedEndpointId || null; + target._requestedEndpointLabel = metrics.requested_endpoint_label || target._requestedEndpointLabel || 'Selected route'; + const hasRoundEndpointId = Boolean(roundHolder && roundEndpointIds.length); + const hasRoundEndpointLabel = Boolean(roundHolder && roundEndpointLabels.length); + target._actualEndpointId = hasRoundEndpointId + ? roundEndpointIds[roundEndpointIds.length - 1] + : (metrics.endpoint_id || target._actualEndpointId || target._requestedEndpointId); + target._actualEndpointLabel = hasRoundEndpointLabel + ? roundEndpointLabels[roundEndpointLabels.length - 1] + : (metrics.endpoint_label || target._actualEndpointLabel || target._requestedEndpointLabel); + } + return target; +} diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index 82e4c2b5b5..7356fe2a23 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -611,10 +611,36 @@ export function sameModelName(left, right) { || shortModel(a).toLowerCase() === shortModel(b).toLowerCase(); } -export function modelRouteLabel(requestedModel, actualModel) { +function shortEndpointLabel(label) { + const value = modelValue(label); + if (!value) return ''; + return value.length > 18 ? value.slice(0, 17) + '…' : value; +} + +export function modelRouteLabel( + requestedModel, + actualModel, + requestedEndpointLabel = '', + actualEndpointLabel = '', + requestedEndpointId = '', + actualEndpointId = '', +) { const requested = modelValue(requestedModel); const actual = modelValue(actualModel) || requested; - if (!requested || sameModelName(requested, actual)) return shortModel(actual || requested); + const requestedRoute = modelValue(requestedEndpointId || requestedEndpointLabel); + const actualRoute = modelValue(actualEndpointId || actualEndpointLabel); + const routeChanged = Boolean( + actualRoute + && requestedRoute + && actualRoute !== requestedRoute + ); + if (!requested || sameModelName(requested, actual)) { + const model = shortModel(actual || requested); + if (!routeChanged) return model; + const from = shortEndpointLabel(requestedEndpointLabel || 'Selected route'); + const to = shortEndpointLabel(actualEndpointLabel || actualEndpointId); + return model + ' (' + from + ' -> ' + to + ')'; + } return shortModel(requested) + ' -> ' + shortModel(actual); } @@ -625,10 +651,24 @@ export function replyModelPair(modelName, metadata) { if (actualFromMeta || requestedFromMeta) { const actual = actualFromMeta || requestedFromMeta || modelValue(modelName); const requested = requestedFromMeta || actual; - return { requestedModel: requested, actualModel: actual }; + return { + requestedModel: requested, + actualModel: actual, + requestedEndpointId: meta.requested_endpoint_id || null, + requestedEndpointLabel: meta.requested_endpoint_label || 'Selected route', + actualEndpointId: meta.endpoint_id || null, + actualEndpointLabel: meta.endpoint_label || meta.requested_endpoint_label || 'Selected route', + }; } const fallback = modelValue(modelName); - return { requestedModel: fallback, actualModel: fallback }; + return { + requestedModel: fallback, + actualModel: fallback, + requestedEndpointId: null, + requestedEndpointLabel: 'Selected route', + actualEndpointId: null, + actualEndpointLabel: 'Selected route', + }; } /** @@ -820,12 +860,50 @@ export function isCostTrackedEndpoint(url) { } /** Cost for the current turn, returning null for non-billable endpoints. */ -function _billableCost(model, inputTokens, outputTokens) { - const url = _currentEndpointUrl(); - if (!isCostTrackedEndpoint(url)) return null; +function _billableCost(model, inputTokens, outputTokens, endpointCostTracked, selectedEndpointUrl) { + // Foreground fallback can answer on a different endpoint than the session's + // selected route. Prefer the backend's non-secret actual-route + // classification; retain the selected-endpoint check for older history. + if (endpointCostTracked === false) return null; + const selectedUrl = selectedEndpointUrl === undefined + ? _currentEndpointUrl() + : selectedEndpointUrl; + if (endpointCostTracked !== true && !isCostTrackedEndpoint(selectedUrl)) { + return null; + } return getModelCost(model, inputTokens, outputTokens); } +/** Sum cost using the route/model that produced each Agent round. */ +function _metricsBillableCost(metrics, model, inputTokens, outputTokens, selectedEndpointUrl) { + const buckets = Array.isArray(metrics.usage_buckets) ? metrics.usage_buckets : []; + if (!buckets.length) { + return _billableCost( + model, + inputTokens, + outputTokens, + metrics.endpoint_cost_tracked, + selectedEndpointUrl, + ); + } + let total = 0; + let hasPricedUsage = false; + for (const bucket of buckets) { + if (!bucket || typeof bucket !== 'object') continue; + const bucketCost = _billableCost( + bucket.model || model, + Number(bucket.input_tokens) || 0, + Number(bucket.output_tokens) || 0, + bucket.endpoint_cost_tracked, + selectedEndpointUrl, + ); + if (bucketCost === null) continue; + total += bucketCost; + hasPricedUsage = true; + } + return hasPricedUsage ? total : null; +} + export function getImageCost(model, quality, size) { if (!model) return null; const m = model.toLowerCase(); @@ -840,6 +918,8 @@ export function getImageCost(model, quality, size) { /* ── Session cost helpers ─────────────────────────────────────────── */ const _COST_KEY = 'ody-session-cost'; +const _COST_RUNS_KEY = 'ody-session-cost-runs'; +const _MAX_COST_RUNS_PER_SESSION = 256; /** Return the accumulated cost for the current (or given) session. */ export function getSessionCost(sessionId) { @@ -847,7 +927,14 @@ export function getSessionCost(sessionId) { if (!sid) return 0; try { const costs = JSON.parse(localStorage.getItem(_COST_KEY) || '{}'); - return costs[sid] || 0; + const runCosts = JSON.parse(localStorage.getItem(_COST_RUNS_KEY) || '{}'); + const recordedRuns = runCosts[sid] && typeof runCosts[sid] === 'object' + ? Object.values(runCosts[sid]) + : []; + return (costs[sid] || 0) + recordedRuns.reduce( + (total, value) => total + (Number(value) || 0), + 0, + ); } catch (_e) { return 0; } } @@ -859,6 +946,9 @@ export function resetSessionCost(sessionId) { const costs = JSON.parse(localStorage.getItem(_COST_KEY) || '{}'); delete costs[sid]; localStorage.setItem(_COST_KEY, JSON.stringify(costs)); + const runCosts = JSON.parse(localStorage.getItem(_COST_RUNS_KEY) || '{}'); + delete runCosts[sid]; + localStorage.setItem(_COST_RUNS_KEY, JSON.stringify(runCosts)); } catch (_e) { /* ignore */ } updateSessionCostUI(); } @@ -867,21 +957,8 @@ export function resetSessionCost(sessionId) { export function updateSessionCostUI() { const el = document.getElementById('session-cost-display'); if (!el) return; - // Non-billable endpoint? Hide the badge and clear stale cost that a previous - // cloud-rate calculation may have left in localStorage for this session. - const _url = _currentEndpointUrl(); - if (!isCostTrackedEndpoint(_url)) { - const sid = window.sessionModule && window.sessionModule.getCurrentSessionId(); - if (sid && getSessionCost(sid) > 0) { - try { - const costs = JSON.parse(localStorage.getItem(_COST_KEY) || '{}'); - delete costs[sid]; - localStorage.setItem(_COST_KEY, JSON.stringify(costs)); - } catch (_e) { /* ignore */ } - } - el.style.display = 'none'; - return; - } + // The ledger records billable work already performed in this session. A + // selected local endpoint does not erase cost from a paid fallback route. const cost = getSessionCost(); if (cost > 0) { el.textContent = '$' + (cost < 0.01 ? cost.toFixed(4) : cost < 1 ? cost.toFixed(3) : cost.toFixed(2)); @@ -891,6 +968,61 @@ export function updateSessionCostUI() { } } +/** Record one metrics payload in a session ledger at most once. */ +export function recordSessionMetricsCost(metrics, sessionId, selectedEndpointUrl) { + if (!metrics || typeof metrics !== 'object') return null; + const cost = _metricsBillableCost( + metrics, + metrics.model || 'Unknown', + metrics.input_tokens || 0, + metrics.output_tokens || 0, + selectedEndpointUrl, + ); + if (metrics._fromHistory) return cost; + const sid = sessionId || ( + window.sessionModule && window.sessionModule.getCurrentSessionId() + ); + if (!sid || cost === null) return cost; + const runId = typeof metrics._costRecordId === 'string' + ? metrics._costRecordId.trim() + : ''; + if (metrics._costRecorded && !runId) return cost; + metrics._costRecorded = true; + if (runId) { + try { + const runCosts = JSON.parse(localStorage.getItem(_COST_RUNS_KEY) || '{}'); + const sessionRuns = runCosts[sid] && typeof runCosts[sid] === 'object' + ? runCosts[sid] + : {}; + // Assigning by detached-run identity is replay-idempotent even when a + // refresh produces a fresh metrics object or two tabs race to write it. + sessionRuns[runId] = cost; + const entries = Object.entries(sessionRuns); + if (entries.length > _MAX_COST_RUNS_PER_SESSION) { + const overflow = entries.slice(0, entries.length - _MAX_COST_RUNS_PER_SESSION); + const costs = JSON.parse(localStorage.getItem(_COST_KEY) || '{}'); + costs[sid] = (costs[sid] || 0) + overflow.reduce( + (total, entry) => total + (Number(entry[1]) || 0), + 0, + ); + overflow.forEach(([oldRunId]) => delete sessionRuns[oldRunId]); + localStorage.setItem(_COST_KEY, JSON.stringify(costs)); + } + runCosts[sid] = sessionRuns; + localStorage.setItem(_COST_RUNS_KEY, JSON.stringify(runCosts)); + } catch (_e) { /* ignore */ } + } else { + try { + const costs = JSON.parse(localStorage.getItem(_COST_KEY) || '{}'); + costs[sid] = (costs[sid] || 0) + cost; + localStorage.setItem(_COST_KEY, JSON.stringify(costs)); + } catch (_e) { /* ignore */ } + } + const currentSid = window.sessionModule && window.sessionModule.getCurrentSessionId(); + if (currentSid === sid) updateSessionCostUI(); + return cost; +} + /** Create a timestamp span for role labels. * Pass an ISO string / Date / epoch-ms to render the message's own time * (used when replaying history). Falls back to "now" when no value is given. */ @@ -1827,23 +1959,19 @@ export function displayMetrics(messageElement, metrics) { const isReal = metrics.usage_source === 'real'; const ctxPct = metrics.context_percent; const model = metrics.model || 'Unknown'; - const cost = _billableCost(model, inputTokens, outputTokens); + const cost = _metricsBillableCost( + metrics, + model, + inputTokens, + outputTokens, + ); // Nothing useful to show — bail out (only if ALL metrics are missing) if (!responseTime && !inputTokens && !outputTokens && tps == null && !ctxPct) return; - // Accumulate session cost (only on fresh metrics, not history reload) - if (!metrics._fromHistory) { - const _sid = window.sessionModule && window.sessionModule.getCurrentSessionId(); - if (_sid && cost !== null) { - try { - const _costs = JSON.parse(localStorage.getItem(_COST_KEY) || '{}'); - _costs[_sid] = (_costs[_sid] || 0) + cost; - localStorage.setItem(_COST_KEY, JSON.stringify(_costs)); - } catch (_e) { /* ignore */ } - updateSessionCostUI(); - } - } + // Rendering can occur when metrics arrive and again after [DONE]. The + // ledger mutation is idempotent for that shared payload. + recordSessionMetricsCost(metrics); // Keep token counts in the Message Stats popup; the footer should stay slim. const costStr0 = cost !== null ? `$${cost < 0.01 ? cost.toFixed(4) : cost.toFixed(3)}` : null; @@ -2260,9 +2388,19 @@ export function addMessage(role, content, modelName, metadata) { const textRaw = Array.isArray(content) ? markdownModule.renderContent(content) : content; // --- Agent multi-bubble reconstruction from saved metadata --- - if (role === 'assistant' && metadata && metadata.tool_events && metadata.tool_events.length > 0) { + if ( + role === 'assistant' + && metadata + && ( + (Array.isArray(metadata.tool_events) && metadata.tool_events.length > 0) + || (Array.isArray(metadata.round_texts) && metadata.round_texts.length > 1) + ) + ) { const roundTexts = metadata.round_texts || []; - const toolEvents = metadata.tool_events; + const roundModels = metadata.round_models || []; + const roundEndpointIds = metadata.round_endpoint_ids || []; + const roundEndpointLabels = metadata.round_endpoint_labels || []; + const toolEvents = metadata.tool_events || []; let pendingAskUser = null; let lastWrap = null; let firstMsgAi = null; @@ -2275,7 +2413,8 @@ export function addMessage(role, content, modelName, metadata) { toolsByRound[r].push(ev); } - const maxRound = Math.max(...Object.keys(toolsByRound).map(Number), roundTexts.length); + const toolRounds = Object.keys(toolsByRound).map(Number); + const maxRound = Math.max(toolRounds.length ? Math.max(...toolRounds) : 0, roundTexts.length); for (let r = 0; r < maxRound; r++) { const roundNum = r + 1; @@ -2287,10 +2426,31 @@ export function addMessage(role, content, modelName, metadata) { const roleEl = document.createElement('div'); roleEl.className = 'role'; const pair = replyModelPair(modelName, metadata); - const contModel = pair.actualModel || pair.requestedModel; - roleEl.textContent = modelRouteLabel(pair.requestedModel, contModel); - if (pair.requestedModel && contModel && !sameModelName(pair.requestedModel, contModel)) { - roleEl.title = pair.requestedModel + ' -> ' + contModel; + const contModel = roundModels[r] || pair.actualModel || pair.requestedModel; + const contEndpointId = r < roundEndpointIds.length + ? roundEndpointIds[r] + : pair.actualEndpointId; + const contEndpointLabel = r < roundEndpointLabels.length + ? roundEndpointLabels[r] + : pair.actualEndpointLabel; + roleEl.textContent = modelRouteLabel( + pair.requestedModel, + contModel, + pair.requestedEndpointLabel, + contEndpointLabel, + pair.requestedEndpointId, + contEndpointId, + ); + if ( + pair.requestedModel + && contModel + && ( + !sameModelName(pair.requestedModel, contModel) + || (pair.requestedEndpointId && contEndpointId && pair.requestedEndpointId !== contEndpointId) + ) + ) { + roleEl.title = pair.requestedModel + ' -> ' + contModel + + ' (' + pair.requestedEndpointLabel + ' -> ' + contEndpointLabel + ')'; } applyModelColor(roleEl, contModel); if (r === 0) roleEl.appendChild(roleTimestamp(metadata?.timestamp)); @@ -2445,7 +2605,14 @@ export function addMessage(role, content, modelName, metadata) { const isCompacted = metadata?.compacted; const replyModels = replyModelPair(modelName, metadata); const resolvedModel = replyModels.actualModel || replyModels.requestedModel; - var _roleText = role === 'user' ? 'You' : (isSlash || isCompacted) ? 'Odysseus' : modelRouteLabel(replyModels.requestedModel, resolvedModel); + var _roleText = role === 'user' ? 'You' : (isSlash || isCompacted) ? 'Odysseus' : modelRouteLabel( + replyModels.requestedModel, + resolvedModel, + replyModels.requestedEndpointLabel, + replyModels.actualEndpointLabel, + replyModels.requestedEndpointId, + replyModels.actualEndpointId, + ); if (role === 'assistant' && (metadata?.research || metadata?.research_clarification)) { _roleText += ' (Research)'; } @@ -2456,8 +2623,14 @@ export function addMessage(role, content, modelName, metadata) { } r.textContent = _roleText; if (role !== 'user') { - if (!isSlash && !isCompacted && replyModels.requestedModel && resolvedModel && !sameModelName(replyModels.requestedModel, resolvedModel)) { - r.title = replyModels.requestedModel + ' -> ' + resolvedModel; + const endpointChanged = Boolean( + replyModels.requestedEndpointId + && replyModels.actualEndpointId + && replyModels.requestedEndpointId !== replyModels.actualEndpointId + ); + if (!isSlash && !isCompacted && replyModels.requestedModel && resolvedModel && (!sameModelName(replyModels.requestedModel, resolvedModel) || endpointChanged)) { + r.title = replyModels.requestedModel + ' -> ' + resolvedModel + + ' (' + replyModels.requestedEndpointLabel + ' -> ' + replyModels.actualEndpointLabel + ')'; } if (!isSlash && !isCompacted) applyModelColor(r, resolvedModel); r.appendChild(roleTimestamp(metadata?.timestamp)); @@ -2741,6 +2914,7 @@ const chatRenderer = { getSessionCost, resetSessionCost, updateSessionCostUI, + recordSessionMetricsCost, roleTimestamp, stripToolBlocks, copyMessageText, diff --git a/static/js/chatStreamErrors.js b/static/js/chatStreamErrors.js new file mode 100644 index 0000000000..250cb290d6 --- /dev/null +++ b/static/js/chatStreamErrors.js @@ -0,0 +1,23 @@ +/** Build a terminal stream error while preserving provider-supplied text. */ +export function createTerminalStreamError(payload = {}) { + const rawError = payload.error; + const message = ( + payload.text + || (typeof rawError === 'string' ? rawError : rawError?.message) + || `Error ${payload.status || 'unknown'}` + ); + const error = new Error(message); + error.name = 'TerminalStreamError'; + error.terminalStreamError = true; + error.status = payload.status; + return error; +} + +/** Only connection-class stream failures are safe to resubmit automatically. */ +export function isRecoverableStreamError(error) { + if (!error || error.terminalStreamError || error.name === 'TerminalStreamError') return false; + if (error.name === 'TypeError') return true; + const message = (error.message || '').toLowerCase(); + if (/\btool\b|unsupported|json|parse|\b4\d\d\b|\b5\d\d\b/.test(message)) return false; + return /network|fetch|connection|reset|closed|aborted|stream|tim(?:e|ed)\s?out|econn|eof/.test(message); +} diff --git a/static/js/settings.js b/static/js/settings.js index df0a492fbb..c9d68debed 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -445,14 +445,7 @@ async function initDefaultChat() { var epSel = el('set-defaultEpSelect'); var modelSel = el('set-defaultModelSelect'); var msg = el('set-defaultChatMsg'); - var fbContainer = el('set-defaultFallbacks'); - var addFbBtn = el('set-defaultAddFallback'); var _endpoints = []; - var _fallbacks = []; // Hidden legacy DOM hook; stored values are not loaded or saved. - - function enabledEndpoints() { - return _endpoints.filter(function(e) { return e.is_enabled; }); - } // Fill any