From cc41bd0cbe12c3b5267068b17bb087181601b529 Mon Sep 17 00:00:00 2001 From: Greg Land Date: Sun, 12 Jul 2026 09:40:50 -0400 Subject: [PATCH 1/2] feat: warn when configured model belongs to a different provider isValidModelId only checks the shape of the model string, so a saved OPENWIKI_MODEL_ID left over from a previous provider (e.g. an Anthropic model while the provider is now OpenAI) passed validation and failed later with an opaque provider-side 400/404. Add isModelIdForOtherProvider / getProvidersForKnownModelId to constants.ts, which use exact matching against each provider's known model options (so namespaced overlaps like OpenRouter's "anthropic/claude-..." don't false-match, and custom/gateway model IDs are never flagged). resolveModelId now emits a non-fatal, actionable warning on mismatch; the run still proceeds since a custom endpoint may legitimately serve the model. Add unit tests for both helpers, including exact-match, shared openai/openai-chatgpt models, and custom-id cases. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/agent/index.ts | 30 +++++++++++++++++++ src/constants.ts | 48 ++++++++++++++++++++++++++++++ test/constants.test.ts | 66 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) diff --git a/src/agent/index.ts b/src/agent/index.ts index 1c9e37ca..f0713921 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -41,6 +41,8 @@ import { getProviderApiKeyEnvKey, getProviderBaseUrlEnvKey, getProviderLabel, + getProvidersForKnownModelId, + isModelIdForOtherProvider, isValidModelId, normalizeModelId, OPENAI_COMPATIBLE_BASE_URL_ENV_KEY, @@ -414,9 +416,37 @@ function resolveModelId( ); } + warnOnProviderModelMismatch(options, provider, modelId); + return modelId; } +// Non-fatal: if the configured model is a known model of a different provider +// (e.g. an Anthropic model left in OPENWIKI_MODEL_ID while the provider is now +// OpenAI), surface an actionable warning instead of letting the request fail +// later with an opaque provider-side 400/404. The run still proceeds, since a +// custom endpoint or gateway may legitimately serve the model. +function warnOnProviderModelMismatch( + options: OpenWikiRunOptions, + provider: OpenWikiProvider, + modelId: string, +): void { + if (!isModelIdForOtherProvider(modelId, provider)) { + return; + } + + const otherProviders = getProvidersForKnownModelId(modelId, provider) + .map((otherProvider) => getProviderLabel(otherProvider)) + .join(", "); + const message = + `Warning: model "${modelId}" is not a known ${getProviderLabel(provider)} model ` + + `(it belongs to ${otherProviders}). The request may fail. ` + + `Set ${OPENWIKI_MODEL_ID_ENV_KEY} to a ${getProviderLabel(provider)} model, or switch providers.`; + + emitDebug(options, `model.mismatch provider=${provider} model=${modelId}`); + options.onEvent?.({ type: "text", text: message }); +} + function createModel( provider: OpenWikiProvider, modelId: string, diff --git a/src/constants.ts b/src/constants.ts index 1c1eb2bb..613ae6b7 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -298,6 +298,54 @@ export function getDefaultModelId(provider: OpenWikiProvider): string { return getProviderModelOptions(provider)[0]?.id ?? DEFAULT_MODEL_ID; } +// Returns the list of built-in providers whose known model options include the +// given model ID by exact match, excluding the provider passed in. Used to warn +// when a saved model plainly belongs to a different provider (e.g. an Anthropic +// model left over while the provider is now OpenAI). Exact matching avoids false +// positives from namespaced overlaps such as OpenRouter's "anthropic/claude-...". +// Returns an empty array for custom/unknown model IDs, so gateway and +// OpenAI-compatible model names are never flagged. +export function getProvidersForKnownModelId( + modelId: string, + excludeProvider: OpenWikiProvider, +): OpenWikiProvider[] { + const normalized = normalizeModelId(modelId); + const providers: OpenWikiProvider[] = []; + + for (const provider of Object.keys(PROVIDER_CONFIGS) as OpenWikiProvider[]) { + if (provider === excludeProvider) { + continue; + } + if ( + getProviderModelOptions(provider).some( + (option) => option.id === normalized, + ) + ) { + providers.push(provider); + } + } + + return providers; +} + +// True when the model ID is a known model of some other provider and is NOT a +// known model of the configured provider — a clear provider/model mismatch. +export function isModelIdForOtherProvider( + modelId: string, + provider: OpenWikiProvider, +): boolean { + const normalized = normalizeModelId(modelId); + const isKnownForProvider = getProviderModelOptions(provider).some( + (option) => option.id === normalized, + ); + + if (isKnownForProvider) { + return false; + } + + return getProvidersForKnownModelId(normalized, provider).length > 0; +} + export function normalizeProvider( value: string | null | undefined, ): OpenWikiProvider | null { diff --git a/test/constants.test.ts b/test/constants.test.ts index bbcae399..37546bd3 100644 --- a/test/constants.test.ts +++ b/test/constants.test.ts @@ -5,6 +5,8 @@ import { DEFAULT_PROVIDER, getDefaultModelId, getProviderModelOptions, + getProvidersForKnownModelId, + isModelIdForOtherProvider, isValidBaseUrl, isValidModelId, isValidProvider, @@ -200,3 +202,67 @@ describe("getDefaultModelId", () => { }, ); }); + +describe("getProvidersForKnownModelId", () => { + test("finds the provider(s) whose known models include the id", () => { + expect(getProvidersForKnownModelId("claude-opus-4-8", "openai")).toEqual([ + "anthropic", + ]); + }); + + test("excludes the provider passed in", () => { + expect(getProvidersForKnownModelId("claude-opus-4-8", "anthropic")).toEqual( + [], + ); + }); + + test("uses exact matching, so namespaced overlaps do not false-match", () => { + // Anthropic's bare "claude-opus-4-8" must not match OpenRouter's + // namespaced "anthropic/claude-opus-4-8" or vice versa. + expect( + getProvidersForKnownModelId("anthropic/claude-opus-4-8", "openai"), + ).toEqual(["openrouter"]); + expect( + getProvidersForKnownModelId("anthropic/claude-opus-4-8", "openrouter"), + ).toEqual([]); + }); + + test("returns empty for custom / unknown model ids", () => { + expect( + getProvidersForKnownModelId("my-gateway-model", "openai-compatible"), + ).toEqual([]); + }); +}); + +describe("isModelIdForOtherProvider", () => { + test("flags a model that clearly belongs to a different provider", () => { + expect(isModelIdForOtherProvider("claude-opus-4-8", "openai")).toBe(true); + }); + + test("does not flag a model that is valid for the configured provider", () => { + expect(isModelIdForOtherProvider("claude-opus-4-8", "anthropic")).toBe( + false, + ); + }); + + test("does not flag shared OpenAI models across openai / openai-chatgpt", () => { + const [firstOpenAiModel] = getProviderModelOptions("openai"); + if (firstOpenAiModel) { + expect( + isModelIdForOtherProvider(firstOpenAiModel.id, "openai-chatgpt"), + ).toBe(false); + } + }); + + test("does not flag custom / unknown model ids", () => { + expect( + isModelIdForOtherProvider("my-gateway-model", "openai-compatible"), + ).toBe(false); + }); + + test("trims whitespace before comparing", () => { + expect(isModelIdForOtherProvider(" claude-opus-4-8 ", "openai")).toBe( + true, + ); + }); +}); From 03c984996467f756defe70017c240d5855089323 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Sun, 19 Jul 2026 20:39:27 -0700 Subject: [PATCH 2/2] emit provider/model mismatch warning to stderr so it survives run failures --- src/agent/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/agent/index.ts b/src/agent/index.ts index 39c68be8..9def7776 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -621,6 +621,9 @@ function warnOnProviderModelMismatch( emitDebug(options, `model.mismatch provider=${provider} model=${modelId}`); options.onEvent?.({ type: "text", text: message }); + // Also emit to stderr so the warning survives on failure, where the TUI + // re-renders the log away and --print discards buffered streamed text. + process.stderr.write(`${message}\n`); } export function createModel(