feat(telemetry): add OpenTelemetry tracing, metrics, logs and cross-service e2e test

.env.example

@@ -212,8 +212,24 @@ ENCRYPTION_KEY=your-encryption-key-must-be-at-least-32-characters-long
 ENCRYPTION_SALT=your-salt-16-chars
 
 # OpenTelemetry Configuration
-# Leave OTEL_EXPORTER_OTLP_ENDPOINT empty for local-only telemetry
+#
+# Omni does not bundle or require an OpenTelemetry Collector or backend.
+# Point OTEL_EXPORTER_OTLP_ENDPOINT at any compatible OTLP/HTTP collector or
+# backend (e.g. http://my-collector:4318).  When left empty, instrumentation
+# is active but signals are not exported and are dropped.
+#
+# Example with an external collector:
+#   OTEL_EXPORTER_OTLP_ENDPOINT=http://my-otel-collector:4318
 OTEL_EXPORTER_OTLP_ENDPOINT=
+# Set OTEL_DEPLOYMENT_ID to one shared non-empty value (e.g. deploy-YYYY-MM-DD)
+# so all services report under the same deployment identity.
+# When empty, Rust and Python export an empty deployment.id while Node uses
+# "unknown". When the variable is absent outside Compose, Rust and Python
+# generate a ULID. Set it explicitly for consistent cross-service identity.
 OTEL_DEPLOYMENT_ID=
 OTEL_DEPLOYMENT_ENVIRONMENT=production
 SERVICE_VERSION=0.1.0
+
+# Metric export interval in milliseconds (default: 60000). Applied when an OTLP
+# endpoint is configured.
+# OTEL_METRIC_EXPORT_INTERVAL=60000

.gitignore

@@ -9,3 +9,5 @@ venv/
 .venv/
 __pycache__/
 omni-sa-key.json
+.pi/
+.pi-subagents/

Cargo.toml

@@ -62,11 +62,12 @@ futures-util = "0.3"
 
 # OpenTelemetry dependencies
 opentelemetry = "0.32"
-opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] }
-opentelemetry-otlp = { version = "0.32", features = ["http-proto", "reqwest-client", "trace", "metrics"] }
+opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio", "metrics", "logs", "experimental_trace_batch_span_processor_with_async_runtime", "experimental_logs_batch_log_processor_with_async_runtime"] }
+opentelemetry-otlp = { version = "0.32", features = ["http-proto", "http-json", "reqwest-client", "trace", "metrics", "logs"] }
 opentelemetry-semantic-conventions = "0.32"
 tracing-opentelemetry = "0.33"
 opentelemetry-http = "0.32"
+opentelemetry-appender-tracing = "0.32"
 
 # Test dependencies
 testcontainers = { version = "0.21", features = ["watchdog"] }

README.md

@@ -47,6 +47,11 @@ The agent runtime can execute code in a sandboxed container on an isolated Docke
 
 See the full [architecture documentation](https://docs.getomni.co/architecture) for more details.
 
+OpenTelemetry traces, metrics, and logs are exported from the five core
+services (web, AI, searcher, indexer, connector-manager).
+See [docs/observability.md](docs/observability.md) for signal flow,
+metric inventory, and endpoint configuration.
+
 ## Deployment
 
 Omni can be deployed entirely on your own infra. See our deployment guides:

docker/docker-compose.yml

@@ -17,6 +17,7 @@ x-otel-config: &otel-config
   OTEL_DEPLOYMENT_ID: ${OTEL_DEPLOYMENT_ID}
   OTEL_DEPLOYMENT_ENVIRONMENT: ${OTEL_DEPLOYMENT_ENVIRONMENT:-development}
   SERVICE_VERSION: ${SERVICE_VERSION:-0.1.0}
+  OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-60000}
 
 x-storage-config: &storage-config
   STORAGE_BACKEND: ${STORAGE_BACKEND}

services/ai/db/embedding_queue.py

@@ -3,7 +3,7 @@
 import logging
 from enum import StrEnum
 from typing import Optional, List
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from asyncpg import Pool
 

services/ai/email_service/sender.py

@@ -94,7 +94,10 @@ async def send(
                 data = resp.json()
                 return SendResult(success=True, message_id=data.get("id"))
 
-            logger.error("Resend error: status=%d body=%s", resp.status_code, resp.text)
+            logger.error(
+                "Resend error: status=%d body=%s", resp.status_code, resp.text,
+                extra={"otel_skip": True},
+            )
             return SendResult(success=False, error="Failed to send email via Resend")
         except Exception as e:
             logger.error("Resend send error: %s", e)