diff --git a/src/genai/instrumentations/langchain/tracer.ts b/src/genai/instrumentations/langchain/tracer.ts index ee9f29e..5314aa1 100644 --- a/src/genai/instrumentations/langchain/tracer.ts +++ b/src/genai/instrumentations/langchain/tracer.ts @@ -188,6 +188,7 @@ export class LangChainTracer extends BaseTracer { } } Utils.setModelAttribute(run, span); + Utils.setChoiceCountAttribute(run, span); Utils.setResponseIdAttribute(run, span); Utils.setProviderNameAttribute(run, span); Utils.setSessionIdAttribute(run, span); diff --git a/src/genai/instrumentations/langchain/utils.ts b/src/genai/instrumentations/langchain/utils.ts index 11f0caa..d3ec0e6 100644 --- a/src/genai/instrumentations/langchain/utils.ts +++ b/src/genai/instrumentations/langchain/utils.ts @@ -11,6 +11,7 @@ import { ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_CHOICE_COUNT, ATTR_GEN_AI_REQUEST_MODEL, ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_RESPONSE_MODEL, @@ -461,12 +462,24 @@ export function getRequestModel(run: Run): string | undefined { // served the request). export function getResponseModel(run: Run): string | undefined { const llmOutput = run.outputs?.llmOutput as Record | undefined; + const v1Metadata = run.outputs?.generations?.[0]?.[0]?.message?.response_metadata as + | Record + | undefined; + const v0Metadata = run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata as + | Record + | undefined; + return [ - // v1: response_metadata directly on message - run.outputs?.generations?.[0]?.[0]?.message?.response_metadata?.model_name, - // v0: response_metadata nested under kwargs - run.outputs?.generations?.[0]?.[0]?.message?.kwargs?.response_metadata?.model_name, - // LLMResult.llmOutput.model_name (common for Chat models) + // v1: response_metadata directly on message. Prefer the canonical OpenAI + // Responses-API field (`model`) and fall back to the `model_name` alias + // LangChain keeps "for backwards compat with chat completion calls" (see + // langchain-ai/langchainjs libs/providers/langchain-openai/src/converters/responses.ts). + v1Metadata?.model, + v1Metadata?.model_name, + // v0: response_metadata nested under kwargs. + v0Metadata?.model, + v0Metadata?.model_name, + // LLMResult.llmOutput.* (common for Chat Completions API). llmOutput?.model_name, llmOutput?.model, ] @@ -528,6 +541,31 @@ export function setModelAttribute(run: Run, span: Span) { } } +// Choice count - Helper to extract the requested number of candidate completions +// (`n` in OpenAI / LangChain `invocation_params`). Per OTel GenAI semconv, this +// attribute is conditionally required when available in the request and not +// equal to 1, so we omit it for the common single-completion case to avoid +// emitting redundant data. +export function getChoiceCount(run: Run): number | undefined { + const invocationParams = run.extra?.invocation_params as Record | undefined; + const raw = invocationParams?.n; + if (typeof raw === "number" && Number.isFinite(raw) && Number.isInteger(raw) && raw >= 1) { + return raw; + } + if (typeof raw === "string") { + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed >= 1) return parsed; + } + return undefined; +} + +export function setChoiceCountAttribute(run: Run, span: Span) { + const n = getChoiceCount(run); + if (n !== undefined && n !== 1) { + span.setAttribute(ATTR_GEN_AI_REQUEST_CHOICE_COUNT, n); + } +} + // Response identifier - Helper to extract the unique response id returned by // the underlying provider (e.g. OpenAI chat completion id). LangChain.js // typically surfaces this as the AIMessage id (top-level for v1, nested diff --git a/src/genai/semconv.ts b/src/genai/semconv.ts index 308c128..e48925c 100644 --- a/src/genai/semconv.ts +++ b/src/genai/semconv.ts @@ -26,6 +26,7 @@ export const ATTR_ERROR_MESSAGE = "error.message" as const; // GenAI core export const ATTR_GEN_AI_OPERATION_NAME = "gen_ai.operation.name" as const; export const ATTR_GEN_AI_REQUEST_MODEL = "gen_ai.request.model" as const; +export const ATTR_GEN_AI_REQUEST_CHOICE_COUNT = "gen_ai.request.choice.count" as const; export const ATTR_GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" as const; export const ATTR_GEN_AI_RESPONSE_ID = "gen_ai.response.id" as const; export const ATTR_GEN_AI_PROVIDER_NAME = "gen_ai.provider.name" as const; diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts index 25799a3..1e028ab 100644 --- a/test/internal/unit/genai/langchain/tracer.test.ts +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -16,6 +16,12 @@ import { ATTR_ERROR_MESSAGE, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_CHOICE_COUNT, + ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_RESPONSE_ID, + ATTR_GEN_AI_RESPONSE_MODEL, + ATTR_GEN_AI_USAGE_INPUT_TOKENS, + ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, } from "../../../../../src/genai/index.js"; function makeRun(overrides: Partial = {}): Run { @@ -318,4 +324,94 @@ describe("LangChainTracer", () => { assert.strictEqual(startSpanCalls.length, 0, "should not create span when MAX_RUNS exceeded"); }); }); + + // End-to-end shape test for #128 scenario 3 / #150. Mirrors the Run object + // LangChain's `langchain-openai/converters/responses.ts` produces for a single + // ChatOpenAI invocation with `useResponsesApi: true`, and asserts the full + // bundle of GenAI semconv attributes lands on the span. + describe("Responses API (useResponsesApi: true) end-to-end", () => { + it("populates request/response model, response id, and token attrs on a RAPI run", async () => { + const tracer = createMockTracer(); + const lct = new LangChainTracer(tracer); + + const runId = "rapi-run-1"; + const startedRun = makeRun({ + id: runId, + run_type: "llm", + name: "ChatOpenAI", + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { + metadata: { ls_model_name: "deployment-o4-mini", ls_provider: "openai" }, + invocation_params: { model: "deployment-o4-mini", n: 3 }, + }, + inputs: { + messages: [[{ role: "user", content: "hello" }]], + }, + }); + await lct.onRunCreate(startedRun); + + const completedRun = { + ...startedRun, + outputs: { + generations: [ + [ + { + message: { + // Shape produced by libs/providers/langchain-openai/src/converters/responses.ts. + // We pin distinct sentinels on the fields the test is supposed + // to ignore so the assertions actually prove that `model` + // (canonical) and `response_metadata.id` (provider-supplied) + // win over the `model_name` alias and `message.id`. + response_metadata: { + model_provider: "openai", + model: "o4-mini-2025-04-16", + model_name: "model_name-alias-should-be-ignored", + id: "resp_rapi_abc", + created_at: 1_700_000_000, + }, + usage_metadata: { + input_tokens: 4, + output_tokens: 7, + }, + id: "message-id-should-be-ignored", + }, + }, + ], + ], + }, + } as unknown as Run; + await (lct as unknown as { _endTrace(run: Run): Promise })._endTrace(completedRun); + + const span = tracer.lastSpan!; + const calls = (span.setAttribute as ReturnType).mock.calls; + const got = (key: string) => calls.find((c: unknown[]) => c[0] === key)?.[1]; + + assert.strictEqual(got(ATTR_GEN_AI_OPERATION_NAME), "chat", "operation name"); + assert.strictEqual( + got(ATTR_GEN_AI_REQUEST_MODEL), + "deployment-o4-mini", + "request model should be the deployment alias", + ); + assert.strictEqual( + got(ATTR_GEN_AI_REQUEST_CHOICE_COUNT), + 3, + "request choice count should come from invocation_params.n when >1", + ); + assert.strictEqual( + got(ATTR_GEN_AI_RESPONSE_MODEL), + "o4-mini-2025-04-16", + "response model should come from response_metadata.model (not the model_name alias)", + ); + assert.strictEqual( + got(ATTR_GEN_AI_RESPONSE_ID), + "resp_rapi_abc", + "response id should come from response_metadata.id (not message.id)", + ); + assert.strictEqual(got(ATTR_GEN_AI_USAGE_INPUT_TOKENS), 4, "input tokens"); + assert.strictEqual(got(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS), 7, "output tokens"); + assert.strictEqual(span.statusObj?.code, SpanStatusCode.OK); + }); + }); }); diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts index 822dd32..15d526f 100644 --- a/test/internal/unit/genai/langchain/utils.test.ts +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -12,6 +12,7 @@ import { setInputMessagesAttribute, setOutputMessagesAttribute, setModelAttribute, + setChoiceCountAttribute, setProviderNameAttribute, setResponseIdAttribute, setSessionIdAttribute, @@ -25,6 +26,7 @@ import { ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_PROVIDER_NAME, + ATTR_GEN_AI_REQUEST_CHOICE_COUNT, ATTR_GEN_AI_REQUEST_MODEL, ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_RESPONSE_MODEL, @@ -544,6 +546,224 @@ describe("setModelAttribute", () => { ), ); }); + + // Responses API (useResponsesApi: true) — LangChain's openai provider + // populates response_metadata.model (canonical) and, for backwards compat + // with chat completion calls, also response_metadata.model_name. We must + // honor both shapes so non-OpenAI RAPI providers (e.g. @langchain/perplexity) + // and any future major where the model_name alias is dropped keep working. + it("RAPI v1: extracts response model from response_metadata.model when only `model` is set", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "o4-mini-2025-04-16", + model_provider: "openai", + id: "resp_abc", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "deployment-o4-mini", + ), + "request model should come from invocation_params.model for non-Azure RAPI clients", + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + "response model should be sourced from response_metadata.model (RAPI canonical field)", + ); + }); + + it("RAPI v1: prefers response_metadata.model over response_metadata.model_name when both are present", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "o4-mini-2025-04-16", + // LangChain duplicates `model` into `model_name` "for + // backwards compat with chat completion calls". We pin a + // distinct sentinel here so the assertion proves we read + // the canonical `model` field first rather than coupling + // to the `model_name` alias. + model_name: "model_name-alias-should-be-ignored", + model_provider: "openai", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + "response model should come from response_metadata.model (canonical)", + ); + assert.ok( + !calls.some( + (c: unknown[]) => + c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "model_name-alias-should-be-ignored", + ), + "response model must not fall back to the model_name alias when model is set", + ); + }); + + it("RAPI v0: extracts response model from kwargs.response_metadata.model", () => { + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "openai", "ChatOpenAI"], + }, + extra: { invocation_params: { model: "deployment-o4-mini" } }, + outputs: { + generations: [ + [ + { + message: { + kwargs: { + response_metadata: { + model: "o4-mini-2025-04-16", + model_provider: "openai", + }, + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "deployment-o4-mini", + ), + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "o4-mini-2025-04-16", + ), + ); + }); + + it("AzureChatOpenAI + RAPI: response_metadata.model still drives the workaround request model", () => { + // Combines the AzureChatOpenAI ls_model_name=gpt-3.5-turbo regression (see + // langchain-ai/langchainjs#10874) with the RAPI response shape. The + // response-side model must populate gen_ai.request.model (via the Azure + // workaround) AND gen_ai.response.model, even when LangChain only sets + // `model` (no `model_name` alias). + const span = makeSpan(); + const run = makeRun({ + serialized: { + id: ["langchain", "chat_models", "azure_openai", "AzureChatOpenAI"], + }, + extra: { metadata: { ls_model_name: "gpt-3.5-turbo" } }, + outputs: { + generations: [ + [ + { + message: { + response_metadata: { + model: "gpt-4o-mini-2024-07-18", + model_provider: "openai", + }, + }, + }, + ], + ], + }, + }); + setModelAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_MODEL && c[1] === "gpt-4o-mini-2024-07-18", + ), + "AzureChatOpenAI request model should use response_metadata.model when model_name is absent", + ); + assert.ok( + calls.some( + (c: unknown[]) => c[0] === ATTR_GEN_AI_RESPONSE_MODEL && c[1] === "gpt-4o-mini-2024-07-18", + ), + ); + }); +}); + +describe("setChoiceCountAttribute", () => { + it("sets gen_ai.request.choice.count from invocation_params.n when > 1", () => { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { n: 3 } } }); + setChoiceCountAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_CHOICE_COUNT && c[1] === 3), + ); + }); + + it("parses numeric strings", () => { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { n: "5" } } }); + setChoiceCountAttribute(run, span); + const calls = (span.setAttribute as ReturnType).mock.calls; + assert.ok( + calls.some((c: unknown[]) => c[0] === ATTR_GEN_AI_REQUEST_CHOICE_COUNT && c[1] === 5), + ); + }); + + it("does not emit when n === 1 (semconv: conditionally required only when != 1)", () => { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { n: 1 } } }); + setChoiceCountAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); + + it("does not emit when n is missing", () => { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { model: "gpt-4o" } } }); + setChoiceCountAttribute(run, span); + assert.strictEqual((span.setAttribute as ReturnType).mock.calls.length, 0); + }); + + it("ignores non-positive / non-integer / non-numeric values", () => { + for (const n of [0, -1, 1.5, "abc", null, true, [], {}]) { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { n } as Record } }); + setChoiceCountAttribute(run, span); + assert.strictEqual( + (span.setAttribute as ReturnType).mock.calls.length, + 0, + `should ignore invalid n=${JSON.stringify(n)}`, + ); + } + }); }); describe("setResponseIdAttribute", () => {