feat(observability): OpenTelemetry instrumentation (spans + metrics + traceparent)

packages/moss-agent/package.json

@@ -172,6 +172,14 @@
     "docs": "typedoc --out docs-api src/index.ts src/core/index.ts src/knowledge/index.ts --tsconfig tsconfig.build.json"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.220.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.220.0",
+    "@opentelemetry/resources": "^2.9.0",
+    "@opentelemetry/sdk-metrics": "^2.9.0",
+    "@opentelemetry/sdk-node": "^0.220.0",
+    "@opentelemetry/sdk-trace-base": "^2.9.0",
+    "@opentelemetry/semantic-conventions": "^1.43.0",
     "@rdk-moss/core": "^0.6.0",
     "ink": "^7.0.5",
     "marked": "^5.1.0",

packages/moss-agent/src/cli-main.ts

@@ -67,6 +67,7 @@ import { AgentMesh, createMeshTools, isMeshVerboseEnabled } from './mesh/agent-m
 import { MeshEventBus } from './mesh/index.js';
 import { LanDiscovery } from './mesh/lan-discovery.js';
 import { setTracer } from './observability/tracing.js';
+import { initObservability, shutdownObservability } from './observability/index.js';
 import { resolveLLMUsageLogPath } from './observability/llm-usage.js';
 import { redactSensitiveData } from './observability/redact.js';
 import { resolveCliDetailMode } from './cli/output.js';
@@ -621,6 +622,11 @@ async function main() {
   });
 
   const cliLlmProvider = parsedArgs.mock ? createMockLLMProvider() : createCliProvider(providerConfig);
+
+  // Initialize observability (OTel tracing + metrics + local file trace) based on env.
+  // No-op when MOSS_OTEL_ENABLED is unset and MOSS_OTEL_URL absent.
+  initObservability({ workspaceDir: workspace });
+
   const agent = new MossAgent({
     llmProvider: cliLlmProvider, sessionStore, model,
     workspaceDir: workspace,
@@ -1041,9 +1047,13 @@ async function main() {
   } finally {
     await agent.close();
     await closeMcpConnections(mcpConnections);
+    await shutdownObservability();
   }
 }
 
+// Flush any in-flight traces/metrics on unexpected exit paths.
+process.on('beforeExit', () => { void shutdownObservability(); });
+
 main().catch((err) => {
   // Config file errors already carry a clean, actionable one-liner — show it
   // alone instead of a raw Node stack. A malformed/hand-edited config.json

packages/moss-agent/src/core/agent/moss-agent.ts

@@ -37,7 +37,8 @@ import {
 } from '@rdk-moss/core/contracts/async-task';
 import { compactHistoryIfNeeded, type SummarizeFn } from '../../context/compaction.js';
 import { createRemoteCompactProviderFromEnv } from '../../context/remote-compaction.js';
-import { setTraceRedactor } from '../../observability/tracing.js';
+import { setTraceRedactor, startSpan, sessionAttributes } from '../../observability/tracing.js';
+import { mossMetrics } from '../../observability/index.js';
 import { logLLMUsage } from '../../observability/llm-usage.js';
 import {
   PlatformExtensionRegistry,
@@ -1897,28 +1898,53 @@ export class MossAgent {
     userMessage: string,
     options?: ChatOptions
   ): AsyncGenerator<MossAgentEvent> {
+    const model = String(this.config.model);
+    const sessionStart = Date.now();
+    // Session root span covers every CLI entry (oneshot / piped / TUI all
+    // funnel through streamChat). turn → llm/tool child spans nest under it
+    // via the active context.
+    const sessionSpan = startSpan('moss.session', sessionAttributes(sessionKey, model, sessionKey));
+    let sessionResult: { toolCalls?: unknown[]; stopReason?: string } | undefined;
     const run = await this.createAgentLoopRun(sessionKey, userMessage, options);
     this.noteRunStarted(sessionKey, run.params.runId);
-    const miniStream = runAgentLoop(run.params);
 
     let done: Extract<MossAgentEvent, { type: 'done' }> | undefined;
+    // Run the agent loop inside the session span's context so child spans
+    // (moss.agent.turn, moss.llm.request, moss.tool.invoke) nest under
+    // moss.session — share its traceId with the session span as parent.
+    // runInSpanContextGen keeps the span's context active across the
+    // generator's yields/awaits (AsyncLocalStorage propagation). The mini
+    // stream is created INSIDE the generator so runAgentLoop's background
+    // async task inherits the session context at startup.
     try {
-      for await (const event of this.adaptMiniStreamEvents(miniStream, run)) {
+      for await (const event of sessionSpan.runInSpanContextGen(
+        async function* (self: MossAgent) {
+          const miniStream = runAgentLoop(run.params);
+          for await (const ev of self.adaptMiniStreamEvents(miniStream, run)) {
+            yield ev;
+          }
+          const miniResult = await miniStream.result();
+          done = run.adapter.getDoneEvent(miniResult);
+          sessionResult = done?.result;
+        }.call(this, this),
+      )) {
         yield event;
       }
-
-      const miniResult = await miniStream.result();
-      done = run.adapter.getDoneEvent(miniResult);
-      this.noteRunFinished(sessionKey, run.params.runId);
-
-
-
-
-
-
-
-
     } finally {
+      // Session metrics on every exit path (success or failure).
+      // end_turn → ok; explicit error stopReason → error; otherwise incomplete.
+      const outcome = sessionResult
+        ? (sessionResult.stopReason === 'end_turn'
+            ? 'ok'
+            : sessionResult.stopReason === 'error' ? 'error' : 'incomplete')
+        : 'error';
+      mossMetrics.sessionCount.add(1, { outcome });
+      mossMetrics.sessionDuration.record(Date.now() - sessionStart, { outcome });
+      mossMetrics.sessionToolCount.record(
+        Array.isArray(sessionResult?.toolCalls) ? sessionResult!.toolCalls!.length : 0,
+        { outcome },
+      );
+      sessionSpan.end(outcome !== 'error');
       this.noteRunFinished(sessionKey, run.params.runId);
       this.retireSteeringMessages(sessionKey, run.params.runId);
       clearPendingStructuredValidation(sessionKey, run.params.runId);
@@ -1927,7 +1953,7 @@ export class MossAgent {
       }
     }
 
-    yield done;
+    yield done!;
   }
 
   async *streamChat(

packages/moss-agent/src/core/loop/agent-loop-llm-call.ts

@@ -20,6 +20,7 @@ import type { AgentLoopMutableState } from './agent-loop-state.js';
 import type { CompactHookRegistry } from './compact-hooks.js';
 import { isContextOverflowError, describeError } from '../../provider/errors.js';
 import { withSpan, turnAttributes } from '../../observability/tracing.js';
+import { mossMetrics } from '../../observability/index.js';
 import { redactSensitiveData } from '../../observability/redact.js';
 import { totalPromptTokens } from '../llm/usage.js';
 import { runAgentLoopLlmTurn } from './agent-loop-stream-helpers.js';
@@ -138,7 +139,7 @@ export async function executeLlmTurn(params: ExecuteLlmTurnParams): Promise<Exec
 
   try {
     const llmTurn = await withSpan(
-      'agent.llm_turn',
+      'moss.llm.request',
       turnAttributes(runId, state.turns, String(modelDef.id)),
       async (span) => {
         span.addEvent('prompt_window', {
@@ -199,6 +200,12 @@ export async function executeLlmTurn(params: ExecuteLlmTurnParams): Promise<Exec
         durationMs: Date.now() - llmTurnStartedAt,
         success: true,
       });
+      // Metrics (noop when metrics disabled)
+      const _llmModel = String(modelDef.id);
+      const _llmDuration = Date.now() - llmTurnStartedAt;
+      mossMetrics.llmDuration.record(_llmDuration, { model: _llmModel });
+      mossMetrics.llmTokens.add(llmTurn.usage.inputTokens, { direction: 'input', model: _llmModel });
+      mossMetrics.llmTokens.add(llmTurn.usage.outputTokens, { direction: 'output', model: _llmModel });
     }
 
     return {
@@ -220,6 +227,8 @@ export async function executeLlmTurn(params: ExecuteLlmTurnParams): Promise<Exec
       success: false,
       error: String(redactSensitiveData(describeError(llmError))),
     });
+    // Metrics: record failed LLM call
+    mossMetrics.llmDuration.record(Date.now() - llmTurnStartedAt, { model: String(modelDef.id), status: 'error' });
     const errorText = describeError(llmError);
     if (
       isContextOverflowError(errorText) &&

packages/moss-agent/src/core/loop/agent-loop.ts

@@ -34,6 +34,7 @@ import { createInitialLoopState, resetIterationState } from './agent-loop-state.
 import type { SteeringContext } from './steering.js';
 import { prepareTurnContext, shouldIncludeThinkingInBudget } from './agent-loop-context-prep.js';
 import { executeLlmTurn } from './agent-loop-llm-call.js';
+import { startSpan, turnAttributes } from '../../observability/tracing.js';
 import { processLlmResponse } from './agent-loop-response.js';
 import {
   buildBackgroundCompletionSystemText,
@@ -1069,6 +1070,11 @@ export function runAgentLoop(
 
 
           let turnToolCalls: { id: string; name: string; input: Record<string, unknown> }[] = [];
+          // Open a turn span covering this iteration's LLM + tool work.
+          // runInSpanContext() activates it around executeLlmTurn/processLlmResponse
+          // so child spans (moss.llm.request, moss.tool.invoke) nest under it.
+          // The finally ends the span on every exit: continue, break, throw.
+          const turnSpan = startSpan('moss.agent.turn', turnAttributes(runId, state.turns, String(modelDef.id)));
           try {
 
             const ctxResult = await prepareTurnContext({
@@ -1112,7 +1118,7 @@ export function runAgentLoop(
             }
 
 
-            const llmResult = await executeLlmTurn({
+            const llmResult = await turnSpan.runInSpanContext(() => executeLlmTurn({
               state,
               modelDef,
               piContext: ctxResult.piContext,
@@ -1142,7 +1148,7 @@ export function runAgentLoop(
               suppressVisibleDeltas: Boolean(
                 params.guardAssistantOutput || params.shouldBufferAssistantOutput?.()
               ),
-            });
+            }));
 
             if (llmResult.control === 'retry') {
               state.turns--;
@@ -1153,7 +1159,7 @@ export function runAgentLoop(
             turnToolCalls = llmResult.toolCalls;
 
 
-            const responseResult = await processLlmResponse({
+            const responseResult = await turnSpan.runInSpanContext(() => processLlmResponse({
               state,
               runId,
               assistantContent: llmResult.assistantContent,
@@ -1191,7 +1197,7 @@ export function runAgentLoop(
               push: (e) => stream.push(e),
               buildCorrectionMessage,
               pendingToolAborts,
-            });
+            }));
 
 
             state.consecutiveTurnErrors = 0;
@@ -1315,9 +1321,12 @@ export function runAgentLoop(
                 remainingBuffer: turnAssistantBuffer.length,
                 sessionKey,
               });
-
-
+
+
+
             }
+            // Ends the turn span on every exit path: continue, break, throw.
+            turnSpan.end(state.consecutiveTurnErrors === 0);
           }
         }
 

packages/moss-agent/src/core/tools/execute-tool-call.ts

@@ -29,6 +29,8 @@ import { describeError, isTimeoutError, isTransientError } from '../../provider/
 import { getRootLogger } from '../../logger.js';
 import { runPreToolHookChain, validateToolInputObject } from './tool-pipeline.js';
 import { MossError, ErrorCode, errorMessage, isMossError } from '../../errors.js';
+import { withSpan, toolAttributes } from '../../observability/tracing.js';
+import { mossMetrics } from '../../observability/index.js';
 
 const logger = getRootLogger();
 
@@ -288,6 +290,42 @@ export type ExecuteToolCallOutcome =
 export async function executeOneToolCall(
   call: { id: string; name: string; input: Record<string, unknown> },
   deps: ExecuteToolCallDeps
+): Promise<ExecuteToolCallOutcome> {
+  const startMs = Date.now();
+  return withSpan(
+    'moss.tool.invoke',
+    toolAttributes(deps.sessionKey, call.name, call.id),
+    async (span) => {
+      try {
+        const outcome = await executeOneToolCallInner(call, deps);
+        // Only a completed-and-failed execution is a real tool error.
+        // denied (user refusal) / pre-blocked / hook-blocked / unknown-tool are
+        // not execution errors — classify as 'blocked' so the tool error-rate
+        // metric doesn't over-count them.
+        const isExecError = outcome.kind === 'completed' && Boolean(outcome.isError);
+        const metricStatus: string =
+          outcome.kind === 'completed' ? (isExecError ? 'error' : 'ok') : 'blocked';
+        const durationMs = outcome.kind === 'completed' && typeof outcome.durationMs === 'number'
+          ? outcome.durationMs
+          : Date.now() - startMs;
+        span.setAttribute('is_error', isExecError);
+        span.setAttribute('outcome_kind', outcome.kind);
+        if (outcome.kind === 'completed' && outcome.outcome) span.setAttribute('outcome', outcome.outcome);
+        mossMetrics.toolInvocations.add(1, { tool: call.name, status: metricStatus });
+        mossMetrics.toolDuration.record(durationMs, { tool: call.name });
+        return outcome;
+      } catch (err) {
+        mossMetrics.toolInvocations.add(1, { tool: call.name, status: 'error' });
+        mossMetrics.toolDuration.record(Date.now() - startMs, { tool: call.name });
+        throw err;
+      }
+    },
+  );
+}
+
+async function executeOneToolCallInner(
+  call: { id: string; name: string; input: Record<string, unknown> },
+  deps: ExecuteToolCallDeps
 ): Promise<ExecuteToolCallOutcome> {
   try {
 

packages/moss-agent/src/observability/console-trace.ts

@@ -0,0 +1,68 @@
+/**
+ * ConsoleSpanProcessor — emits a human-readable span lifecycle to stderr.
+ *
+ * For local debugging: MOSS_TRACE=console attaches this processor so you can
+ * watch the trace tree stream by as the agent runs (start/end per span,
+ * indented by parent depth, with duration + status). Zero network, zero disk.
+ *
+ * Uses SDK span lifecycle: onStart opens the line, onEnd completes it with
+ * duration + status. Because spans end out of order (children end before
+ * parents), we print on END (when duration/status are known) with a depth
+ * derived from the span's nesting via a parent stack — approximated by
+ * tracking open spans and their parent ids.
+ */
+import type { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
+import type { Context } from '@opentelemetry/api';
+import { SpanStatusCode } from '@opentelemetry/api';
+
+interface OpenSpan {
+  name: string;
+  spanId: string;
+  parentSpanId?: string;
+  depth: number;
+}
+
+export class ConsoleSpanProcessor implements SpanProcessor {
+  private open = new Map<string, OpenSpan>();
+
+  onStart(span: ReadableSpan, _parentContext: Context): void {
+    const spanId = span.spanContext().spanId;
+    const parentSpanId = span.parentSpanContext?.spanId;
+    // Depth: parent's depth + 1, or 0 for a root.
+    const parentDepth = parentSpanId ? this.open.get(parentSpanId)?.depth ?? 0 : -1;
+    const depth = parentDepth + 1;
+    this.open.set(spanId, { name: span.name, spanId, parentSpanId, depth });
+    const indent = '  '.repeat(depth);
+    process.stderr.write(`[trace] ${indent}▶ ${span.name}\n`);
+  }
+
+  onEnd(span: ReadableSpan): void {
+    const spanId = span.spanContext().spanId;
+    const info = this.open.get(spanId);
+    const depth = info?.depth ?? 0;
+    const indent = '  '.repeat(depth);
+    const durMs = Number(span.duration[0]) * 1000 + Math.trunc(Number(span.duration[1]) / 1_000_000);
+    const status = span.status.code === SpanStatusCode.ERROR ? 'ERROR' : 'ok';
+    const msg = span.status.message ? ` (${span.status.message.slice(0, 100)})` : '';
+    const attrs = this.formatAttrs(span);
+    process.stderr.write(`[trace] ${indent}◀ ${span.name} (${durMs}ms, ${status})${attrs}${msg}\n`);
+    this.open.delete(spanId);
+  }
+
+  private formatAttrs(span: ReadableSpan): string {
+    const entries = Object.entries(span.attributes ?? {});
+    if (entries.length === 0) return '';
+    const shown = entries
+      .filter(([k]) => !['runId', 'sessionKey'].includes(k))
+      .slice(0, 4)
+      .map(([k, v]) => `${k}=${typeof v === 'string' ? v : String(v)}`)
+      .join(' ');
+    return shown ? ` {${shown}}` : '';
+  }
+
+  async forceFlush(): Promise<void> {}
+
+  async shutdown(): Promise<void> {
+    this.open.clear();
+  }
+}