diff --git a/src/agent/index.ts b/src/agent/index.ts index e38ce91c..9def7776 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -65,6 +65,8 @@ import { getProviderModelOptions, getProviderRegionEnvKey, getProviderSecretKeyEnvKey, + getProvidersForKnownModelId, + isModelIdForOtherProvider, DEFAULT_VERTEX_LOCATION, GOOGLE_CLOUD_PROJECT_ENV_KEY, isValidModelId, @@ -590,9 +592,40 @@ export function resolveModelId( ); } + warnOnProviderModelMismatch(options, provider, modelId); + return modelId; } +// Non-fatal: if the configured model is a known model of a different provider +// (e.g. an Anthropic model left in OPENWIKI_MODEL_ID while the provider is now +// OpenAI), surface an actionable warning instead of letting the request fail +// later with an opaque provider-side 400/404. The run still proceeds, since a +// custom endpoint or gateway may legitimately serve the model. +function warnOnProviderModelMismatch( + options: OpenWikiRunOptions, + provider: OpenWikiProvider, + modelId: string, +): void { + if (!isModelIdForOtherProvider(modelId, provider)) { + return; + } + + const otherProviders = getProvidersForKnownModelId(modelId, provider) + .map((otherProvider) => getProviderLabel(otherProvider)) + .join(", "); + const message = + `Warning: model "${modelId}" is not a known ${getProviderLabel(provider)} model ` + + `(it belongs to ${otherProviders}). The request may fail. ` + + `Set ${OPENWIKI_MODEL_ID_ENV_KEY} to a ${getProviderLabel(provider)} model, or switch providers.`; + + emitDebug(options, `model.mismatch provider=${provider} model=${modelId}`); + options.onEvent?.({ type: "text", text: message }); + // Also emit to stderr so the warning survives on failure, where the TUI + // re-renders the log away and --print discards buffered streamed text. + process.stderr.write(`${message}\n`); +} + export function createModel( provider: OpenWikiProvider, modelId: string, diff --git a/src/constants.ts b/src/constants.ts index ca74925f..b4372959 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -557,6 +557,54 @@ export function getDefaultModelId(provider: OpenWikiProvider): string { return getProviderModelOptions(provider)[0]?.id ?? DEFAULT_MODEL_ID; } +// Returns the list of built-in providers whose known model options include the +// given model ID by exact match, excluding the provider passed in. Used to warn +// when a saved model plainly belongs to a different provider (e.g. an Anthropic +// model left over while the provider is now OpenAI). Exact matching avoids false +// positives from namespaced overlaps such as OpenRouter's "anthropic/claude-...". +// Returns an empty array for custom/unknown model IDs, so gateway and +// OpenAI-compatible model names are never flagged. +export function getProvidersForKnownModelId( + modelId: string, + excludeProvider: OpenWikiProvider, +): OpenWikiProvider[] { + const normalized = normalizeModelId(modelId); + const providers: OpenWikiProvider[] = []; + + for (const provider of Object.keys(PROVIDER_CONFIGS) as OpenWikiProvider[]) { + if (provider === excludeProvider) { + continue; + } + if ( + getProviderModelOptions(provider).some( + (option) => option.id === normalized, + ) + ) { + providers.push(provider); + } + } + + return providers; +} + +// True when the model ID is a known model of some other provider and is NOT a +// known model of the configured provider — a clear provider/model mismatch. +export function isModelIdForOtherProvider( + modelId: string, + provider: OpenWikiProvider, +): boolean { + const normalized = normalizeModelId(modelId); + const isKnownForProvider = getProviderModelOptions(provider).some( + (option) => option.id === normalized, + ); + + if (isKnownForProvider) { + return false; + } + + return getProvidersForKnownModelId(normalized, provider).length > 0; +} + export function normalizeProvider( value: string | null | undefined, ): OpenWikiProvider | null { diff --git a/test/constants.test.ts b/test/constants.test.ts index 40db95d5..9d585942 100644 --- a/test/constants.test.ts +++ b/test/constants.test.ts @@ -11,6 +11,8 @@ import { getProviderModelOptions, getProviderRegionEnvKey, getProviderSecretKeyEnvKey, + getProvidersForKnownModelId, + isModelIdForOtherProvider, isValidBaseUrl, isValidProviderBaseUrl, isValidModelId, @@ -416,3 +418,71 @@ describe("getDefaultModelId", () => { }, ); }); + +describe("getProvidersForKnownModelId", () => { + test("finds the provider(s) whose known models include the id", () => { + // claude-opus-4-8 is a known model of both anthropic and the + // gemini-enterprise gateway, which also serves Claude models. + expect(getProvidersForKnownModelId("claude-opus-4-8", "openai")).toEqual([ + "anthropic", + "gemini-enterprise", + ]); + }); + + test("excludes the provider passed in", () => { + // Excluding anthropic still leaves gemini-enterprise, which also lists it. + expect(getProvidersForKnownModelId("claude-opus-4-8", "anthropic")).toEqual( + ["gemini-enterprise"], + ); + }); + + test("uses exact matching, so namespaced overlaps do not false-match", () => { + // Anthropic's bare "claude-opus-4-8" must not match OpenRouter's + // namespaced "anthropic/claude-opus-4-8" or vice versa. + expect( + getProvidersForKnownModelId("anthropic/claude-opus-4-8", "openai"), + ).toEqual(["openrouter"]); + expect( + getProvidersForKnownModelId("anthropic/claude-opus-4-8", "openrouter"), + ).toEqual([]); + }); + + test("returns empty for custom / unknown model ids", () => { + expect( + getProvidersForKnownModelId("my-gateway-model", "openai-compatible"), + ).toEqual([]); + }); +}); + +describe("isModelIdForOtherProvider", () => { + test("flags a model that clearly belongs to a different provider", () => { + expect(isModelIdForOtherProvider("claude-opus-4-8", "openai")).toBe(true); + }); + + test("does not flag a model that is valid for the configured provider", () => { + expect(isModelIdForOtherProvider("claude-opus-4-8", "anthropic")).toBe( + false, + ); + }); + + test("does not flag shared OpenAI models across openai / openai-chatgpt", () => { + const [firstOpenAiModel] = getProviderModelOptions("openai"); + if (firstOpenAiModel) { + expect( + isModelIdForOtherProvider(firstOpenAiModel.id, "openai-chatgpt"), + ).toBe(false); + } + }); + + test("does not flag custom / unknown model ids", () => { + expect( + isModelIdForOtherProvider("my-gateway-model", "openai-compatible"), + ).toBe(false); + }); + + test("trims whitespace before comparing", () => { + expect(isModelIdForOtherProvider(" claude-opus-4-8 ", "openai")).toBe( + true, + ); + }); +});