[WIP] feat: add MLflow tracing integration via OpenTelemetry bridge

.env.example

@@ -284,3 +284,26 @@ OTEL_TRACES_SAMPLER=always_on
 
 # Sampler argument (e.g., 0.1 for 10% sampling with traceidratio)
 OTEL_TRACES_SAMPLER_ARG=1.0
+
+# -----------------------------------------------------------------------------
+# MLflow Tracing Configuration (via OpenTelemetry bridge)
+# -----------------------------------------------------------------------------
+# Enable MLflow tracing (works independently of OTEL_ENABLED; requires pip install 'lightspeed-agent[mlflow]')
+# MLFLOW_ENABLED=false
+
+# MLflow tracking server URI
+# MLFLOW_TRACKING_URI=http://localhost:5000
+
+# MLflow experiment name
+# MLFLOW_EXPERIMENT_NAME=lightspeed-agent
+
+# MLflow experiment ID (sent as x-mlflow-experiment-id header; overrides name if set).
+# The experiment must already exist on the MLflow server — use MLFLOW_EXPERIMENT_NAME
+# instead for auto-creation on first trace.
+# MLFLOW_EXPERIMENT_ID=
+
+# Enable logging of prompts and responses in MLflow traces
+# MLFLOW_LOG_PROMPTS=false
+
+# Comma-separated key=value tags for MLflow runs (e.g., env=prod,team=ai)
+# MLFLOW_RUN_TAGS=

Makefile

@@ -1,7 +1,7 @@
 # Red Hat Lightspeed Agent for Google Cloud - Makefile
 # Common development and deployment commands
 
-.PHONY: help build build-agent build-marketplace run stop logs logs-mcp clean test test-shell lint dev check-env lock lock-agent lock-handler lock-dev lock-check audit
+.PHONY: help build build-agent build-marketplace run stop logs logs-mcp clean test test-shell lint dev check-env lock lock-agent lock-handler lock-dev lock-mlflow lock-check audit
 
 # Default target
 help:
@@ -18,6 +18,7 @@ help:
 	@echo "  make lock-agent   - Regenerate agent lock file only"
 	@echo "  make lock-handler - Regenerate marketplace handler lock file only"
 	@echo "  make lock-dev     - Regenerate dev lock file only"
+	@echo "  make lock-mlflow  - Regenerate mlflow lock file only"
 	@echo "  make lock-check   - Verify lock files are in sync (used by CI)"
 	@echo "  make audit        - Scan dependencies for known vulnerabilities (pip-audit)"
 	@echo ""
@@ -71,7 +72,7 @@ lint:
 # =============================================================================
 
 lock:
-	$(MAKE) lock-agent lock-handler lock-dev
+	$(MAKE) lock-agent lock-handler lock-dev lock-mlflow
 
 lock-agent:
 	@echo "Regenerating agent lock file..."
@@ -91,6 +92,12 @@ lock-dev:
 		--extra dev --output-file=requirements-dev.txt pyproject.toml
 	@echo "✓ requirements-dev.txt updated"
 
+lock-mlflow:
+	@echo "Regenerating mlflow lock file..."
+	source .venv/bin/activate && uv pip compile --generate-hashes --python-version=3.12 --python-platform=linux \
+		--extra mlflow --output-file=requirements-mlflow.txt pyproject.toml
+	@echo "✓ requirements-mlflow.txt updated"
+
 lock-check:
 	@echo "Checking if lock files are in sync with pyproject.toml..."
 	@cp requirements-agent.txt /tmp/requirements-agent-check.txt
@@ -111,6 +118,12 @@ lock-check:
 	@diff -u <(tail -n +3 requirements-dev.txt) <(tail -n +3 /tmp/requirements-dev-check.txt) || \
 		(echo "ERROR: requirements-dev.txt is out of sync. Run 'make lock' to update." && rm -f /tmp/requirements-dev-check.txt && exit 1)
 	@rm -f /tmp/requirements-dev-check.txt
+	@cp requirements-mlflow.txt /tmp/requirements-mlflow-check.txt
+	@uv pip compile --generate-hashes --python-version=3.12 --python-platform=linux \
+		--extra mlflow --output-file=/tmp/requirements-mlflow-check.txt pyproject.toml
+	@diff -u <(tail -n +3 requirements-mlflow.txt) <(tail -n +3 /tmp/requirements-mlflow-check.txt) || \
+		(echo "ERROR: requirements-mlflow.txt is out of sync. Run 'make lock' to update." && rm -f /tmp/requirements-mlflow-check.txt && exit 1)
+	@rm -f /tmp/requirements-mlflow-check.txt
 	@echo "✓ Lock files are in sync"
 
 audit:

deploy/cloudrun/README.md

@@ -2616,6 +2616,64 @@ gcloud logging read 'resource.type="cloud_run_revision" AND resource.labels.serv
 
 No additional configuration is required — audit logging is automatically active when `LOG_FORMAT=json`.
 
+## MLflow Tracing (LLM Observability)
+
+The agent supports sending LLM traces to an external MLflow Tracking Server via
+an OpenTelemetry bridge. Google ADK natively generates OTel traces for agent
+runs, LLM calls, and tool invocations — MLflow accepts them via its OTLP
+endpoint at `/v1/traces`.
+
+On Cloud Run, MLflow is **not deployed as a new service** — the agent connects
+to an existing MLflow instance on Red Hat infrastructure (or any external MLflow
+server). No additional Cloud SQL databases or GCS buckets are needed.
+
+### Configuration
+
+Set the following environment variables on the agent service (already present in
+`service.yaml` with defaults):
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MLFLOW_ENABLED` | `false` | Enable MLflow tracing |
+| `MLFLOW_TRACKING_URI` | (empty) | External MLflow server URL (e.g., `https://mlflow.example.redhat.com`) |
+| `MLFLOW_EXPERIMENT_NAME` | `lightspeed-agent` | MLflow experiment name |
+| `MLFLOW_LOG_PROMPTS` | `false` | Log prompts/responses (**security-sensitive**) |
+
+### Enabling MLflow
+
+```bash
+gcloud run services update ${SERVICE_NAME:-lightspeed-agent} \
+  --region=$GOOGLE_CLOUD_LOCATION \
+  --project=$GOOGLE_CLOUD_PROJECT \
+  --update-env-vars="\
+MLFLOW_ENABLED=true,\
+MLFLOW_TRACKING_URI=https://mlflow.example.redhat.com"
+```
+
+**Networking:** If the MLflow instance is on a public HTTPS endpoint, no
+additional configuration is needed — Cloud Run has outbound internet access by
+default. If it's behind a VPN or private network, configure a VPC connector +
+Cloud VPN or Cloud Interconnect (the existing VPC connector for Redis can be
+reused).
+
+**Authentication:** If the MLflow instance requires auth, set credentials via
+Secret Manager and configure the `OTEL_EXPORTER_OTLP_HEADERS` env var with
+the appropriate auth headers.
+
+### Dependencies
+
+The agent needs `opentelemetry-exporter-otlp-proto-http` for OTLP span export to
+MLflow. This package is already included in the default container image (pulled in
+by `opentelemetry-exporter-otlp` in the `[agent]` extra) — no additional
+installation is needed.
+
+For minimal (non-agent) installs, the `[mlflow]` optional dependency group
+provides only the OTLP HTTP exporter:
+
+```bash
+pip install 'lightspeed-agent[mlflow]'
+```
+
 ## Monitoring
 
 ### Built-in Cloud Run Metrics

deploy/cloudrun/service.yaml

@@ -187,6 +187,19 @@ spec:
               value: "8002"
             - name: FORWARDED_ALLOW_IPS
               value: "*"
+            # MLflow Configuration
+            - name: MLFLOW_ENABLED
+              value: "${MLFLOW_ENABLED:-false}"
+            - name: MLFLOW_TRACKING_URI
+              value: "${MLFLOW_TRACKING_URI:-}"
+            - name: MLFLOW_EXPERIMENT_NAME
+              value: "${MLFLOW_EXPERIMENT_NAME:-lightspeed-agent}"
+            - name: MLFLOW_EXPERIMENT_ID
+              value: "${MLFLOW_EXPERIMENT_ID:-}"
+            - name: MLFLOW_LOG_PROMPTS
+              value: "${MLFLOW_LOG_PROMPTS:-false}"
+            - name: MLFLOW_RUN_TAGS
+              value: "${MLFLOW_RUN_TAGS:-}"
           volumeMounts:
             - name: redis-ca-cert
               mountPath: /secrets/redis-ca-cert

deploy/openshift/README.md

@@ -77,6 +77,8 @@ a web interface to:
 | **postgresql** | PostgreSQL 16 for session persistence | Both (only when `sessionBackend: database`) |
 | **marketplace-postgresql** | PostgreSQL 16 for marketplace/entitlement data | Standalone only |
 | **redis** | Redis 7 for distributed rate limiting | Both modes |
+| **mlflow** | MLflow Tracking Server for LLM observability | Both (only when `mlflow.enabled: true`) |
+| **mlflow-postgresql** | PostgreSQL 16 for MLflow metadata | Both (only when `mlflow.enabled` and `mlflow.postgresql.mode: dedicated`) |
 | **marketplace-handler** | DCR and marketplace event handler | Standalone only |
 | **standalone-ui** | Web UI for DCR + A2A testing | Standalone only |
 
@@ -1077,6 +1079,62 @@ they are distinguishable in PromQL queries.
 > `monitoring.serviceMonitor.*`, `monitoring.grafanaDashboard.*`). Only
 > `otel.serviceName` must be set per agent since it must be unique.
 
+### MLflow Tracking (LLM Observability)
+
+MLflow provides LLM-specific tracing — token counts, latency, tool call traces,
+and optional prompt/response logging. It works via an OpenTelemetry bridge:
+Google ADK natively generates OTel traces, and MLflow accepts them via its OTLP
+endpoint at `/v1/traces`. The agent adds a second OTel span processor alongside
+the existing exporters. Requires MLflow >= 3.6.0 on the server side and
+`opentelemetry-exporter-otlp-proto-http` on the agent side (already included in
+the default container image via the `[agent]` dependency group).
+
+| Value | Description | Default |
+|---|---|---|
+| `mlflow.enabled` | Deploy MLflow Tracking Server and enable agent tracing | `false` |
+| `mlflow.trackingUri` | MLflow server URI (on OCP the ConfigMap auto-resolves to the ClusterIP Service) | `http://localhost:5000` |
+| `mlflow.experimentName` | MLflow experiment name | `lightspeed-agent` |
+| `mlflow.experimentId` | MLflow experiment ID (sent as `x-mlflow-experiment-id` OTLP header). Must already exist on the server — use `experimentName` for auto-creation. | `""` |
+| `mlflow.logPrompts` | Log LLM prompts/responses (**security-sensitive** — may contain PII) | `false` |
+| `mlflow.runTags` | Extra run tags as `key=value` pairs | `""` |
+| `mlflow.image.repository` | MLflow container image | `ghcr.io/mlflow/mlflow` |
+| `mlflow.image.tag` | Image tag | `v3.14.0` |
+| `mlflow.port` | MLflow server port | `5000` |
+| `mlflow.route.enabled` | Create an OpenShift Route for the MLflow UI | `false` |
+| `mlflow.auth.type` | Auth type: `none`, `oidc`, `proxy` | `none` |
+| `mlflow.postgresql.mode` | `dedicated` (new PostgreSQL) or `shared` (reuse marketplace PostgreSQL) | `dedicated` |
+| `mlflow.postgresql.database` | Database name | `mlflow_tracking` |
+| `mlflow.storage.artifactSize` | PVC size for MLflow artifacts | `10Gi` |
+| `secrets.mlflowDbPassword` | MLflow dedicated PostgreSQL password | `""` |
+
+**Enabling MLflow:**
+
+```yaml
+# In my-values.yaml:
+mlflow:
+  enabled: true
+
+# In secrets.yaml (when using dedicated PostgreSQL):
+secrets:
+  mlflowDbPassword: "a-strong-mlflow-password"
+```
+
+After install, the agent automatically sends OTel traces to the MLflow server.
+Access the MLflow UI via `oc port-forward`:
+
+```bash
+oc port-forward svc/lightspeed-agent-mlflow 5000:5000 -n lightspeed-agent
+# Open http://localhost:5000 in your browser
+```
+
+Or enable an OpenShift Route for persistent access:
+
+```yaml
+mlflow:
+  route:
+    enabled: true
+```
+
 ## Authentication
 
 The agent authenticates requests via Red Hat SSO token introspection:
@@ -1194,6 +1252,10 @@ oc logs deployment/lightspeed-agent-postgresql -n lightspeed-agent
 oc logs deployment/lightspeed-agent-marketplace-postgresql -n lightspeed-agent
 # Redis
 oc logs deployment/lightspeed-agent-redis -n lightspeed-agent
+# MLflow (only when mlflow.enabled=true)
+oc logs deployment/lightspeed-agent-mlflow -n lightspeed-agent
+# MLflow PostgreSQL (only when mlflow.enabled and mlflow.postgresql.mode=dedicated)
+oc logs deployment/lightspeed-agent-mlflow-postgresql -n lightspeed-agent
 ```
 
 ### Common issues

deploy/openshift/templates/_helpers.tpl

@@ -116,6 +116,36 @@ UI service name
 {{- include "lightspeed-agent.fullname" . }}-ui
 {{- end }}
 
+{{/*
+MLflow service name
+*/}}
+{{- define "lightspeed-agent.mlflowServiceName" -}}
+{{- include "lightspeed-agent.fullname" . }}-mlflow
+{{- end }}
+
+{{/*
+Selector labels for MLflow
+*/}}
+{{- define "lightspeed-agent.mlflowSelectorLabels" -}}
+app.kubernetes.io/name: {{ include "lightspeed-agent.fullname" . }}-mlflow
+app.kubernetes.io/component: mlflow
+{{- end }}
+
+{{/*
+MLflow PostgreSQL service name
+*/}}
+{{- define "lightspeed-agent.mlflowPostgresqlServiceName" -}}
+{{- include "lightspeed-agent.fullname" . }}-mlflow-postgresql
+{{- end }}
+
+{{/*
+Selector labels for MLflow PostgreSQL
+*/}}
+{{- define "lightspeed-agent.mlflowPostgresqlSelectorLabels" -}}
+app.kubernetes.io/name: {{ include "lightspeed-agent.fullname" . }}-mlflow-postgresql
+app.kubernetes.io/component: mlflow-database
+{{- end }}
+
 {{/*
 Whether the handler should be deployed on OCP.
 Only deployed in standalone mode — in hybrid the handler stays on GCP.

deploy/openshift/templates/configmap.yaml

@@ -131,3 +131,17 @@ data:
   OTEL_METRICS_ENABLED: {{ .Values.otel.metricsEnabled | quote }}
   OTEL_METRICS_PROMETHEUS_PORT: {{ .Values.otel.metricsPrometheusPort | quote }}
   OTEL_METRICS_COLLECTION_INTERVAL: {{ .Values.otel.metricsCollectionInterval | quote }}
+
+  # MLflow
+  MLFLOW_ENABLED: {{ .Values.mlflow.enabled | quote }}
+  {{- if .Values.mlflow.enabled }}
+  MLFLOW_TRACKING_URI: "http://{{ include "lightspeed-agent.mlflowServiceName" . }}:{{ .Values.mlflow.port }}"
+  MLFLOW_EXPERIMENT_NAME: {{ .Values.mlflow.experimentName | quote }}
+  {{- if .Values.mlflow.experimentId }}
+  MLFLOW_EXPERIMENT_ID: {{ .Values.mlflow.experimentId | quote }}
+  {{- end }}
+  MLFLOW_LOG_PROMPTS: {{ .Values.mlflow.logPrompts | quote }}
+  {{- if .Values.mlflow.runTags }}
+  MLFLOW_RUN_TAGS: {{ .Values.mlflow.runTags | quote }}
+  {{- end }}
+  {{- end }}

deploy/openshift/templates/mlflow-deployment.yaml

@@ -0,0 +1,93 @@
+{{- if .Values.mlflow.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "lightspeed-agent.mlflowServiceName" . }}
+  labels:
+    {{- include "lightspeed-agent.labels" . | nindent 4 }}
+    app.kubernetes.io/component: mlflow
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "lightspeed-agent.mlflowSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "lightspeed-agent.mlflowSelectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+      containers:
+        - name: mlflow
+          image: {{ printf "%s:%s" .Values.mlflow.image.repository .Values.mlflow.image.tag }}
+          imagePullPolicy: {{ .Values.mlflow.image.pullPolicy }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          args:
+            - "mlflow"
+            - "server"
+            - "--host"
+            - "0.0.0.0"
+            - "--port"
+            - {{ .Values.mlflow.port | quote }}
+            - "--artifacts-destination"
+            - "/mlflow/artifacts"
+            - "--backend-store-uri"
+            {{- if eq .Values.mlflow.postgresql.mode "dedicated" }}
+            - "postgresql://$(MLFLOW_DB_USER):$(MLFLOW_DB_PASSWORD)@$(MLFLOW_DB_HOST):5432/$(MLFLOW_DB_NAME)"
+            {{- else }}
+            {{- fail "mlflow.postgresql.mode must be 'dedicated' — MLflow requires a SQL backend for OpenTelemetry OTLP trace ingestion (file-based backends silently drop traces)" }}
+            {{- end }}
+          ports:
+            - containerPort: {{ .Values.mlflow.port }}
+              protocol: TCP
+          env:
+            {{- if eq .Values.mlflow.postgresql.mode "dedicated" }}
+            - name: MLFLOW_DB_USER
+              value: {{ .Values.mlflow.postgresql.user | quote }}
+            - name: MLFLOW_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "lightspeed-agent.fullname" . }}-secrets
+                  key: MLFLOW_DB_PASSWORD
+            - name: MLFLOW_DB_HOST
+              value: {{ include "lightspeed-agent.mlflowPostgresqlServiceName" . }}
+            - name: MLFLOW_DB_NAME
+              value: {{ .Values.mlflow.postgresql.database | quote }}
+            {{- end }}
+          volumeMounts:
+            - name: mlflow-artifacts
+              mountPath: /mlflow/artifacts
+            - name: tmp
+              mountPath: /tmp
+          resources:
+            {{- toYaml .Values.mlflow.resources | nindent 12 }}
+          readinessProbe:
+            httpGet:
+              path: /api/2.0/mlflow/experiments/search?max_results=1
+              port: {{ .Values.mlflow.port }}
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.mlflow.port }}
+            initialDelaySeconds: 30
+            periodSeconds: 10
+      volumes:
+        - name: mlflow-artifacts
+          persistentVolumeClaim:
+            claimName: {{ include "lightspeed-agent.fullname" . }}-mlflow-artifacts
+        - name: tmp
+          emptyDir: {}
+{{- end }}