SamGalanakis · SamGalanakis · Nov 11, 2025 · Nov 11, 2025
diff --git a/README.md b/README.md
@@ -27,6 +27,18 @@ It works by monitoring your devices, having a LLM process all the information an
 
 [See setup guide](docs/SETUP.md)
 
+### Monitoring
+
+To run the monitoring stack, which includes Prometheus for metrics and Loki for logging, use the following command:
+
+```bash
+docker compose -f docker-compose.prod.yml -f docker-compose.prod.local.yml -f docker-compose.prod.monitoring.yml up --build
+```
+
+- **Grafana:** [http://localhost:3000](http://localhost:3000) (admin/admin)
+- **Prometheus:** [http://localhost:9090](http://localhost:9090)
+- **Loki:** Logs are viewable within Grafana.
+
 
 ## Architecture
 ![architecture](docs/media/architecture.png)

diff --git a/docker-compose.prod.monitoring.yml b/docker-compose.prod.monitoring.yml
@@ -30,6 +30,33 @@ services:
         condition: service_started
     restart: unless-stopped
 
+  loki:
+    image: grafana/loki:latest
+    networks:
+      - web
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yml
+    volumes:
+      - ./ops/loki-config.yml:/etc/loki/local-config.yml:ro
+      - loki_data:/loki
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    networks:
+      - web
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./ops/promtail-config.yml:/etc/promtail/config.yml:ro
+      - ./ops/promtail-data:/data
+    command: -config.file=/etc/promtail/config.yml
+    depends_on:
+      loki:
+        condition: service_started
+    restart: unless-stopped
+
   grafana:
     image: grafana/grafana:latest
     networks:
@@ -46,8 +73,11 @@ services:
     depends_on:
       prometheus:
         condition: service_started
+      loki:
+        condition: service_started
     restart: unless-stopped
 
 volumes:
   grafana_data:
   prometheus_data:
+  loki_data:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -32,8 +32,8 @@ services:
     healthcheck:
       test: ["CMD", "curl", "-fsS", "http://localhost:8000/healthz"]
       interval: 10s
-      timeout: 2s
-      retries: 10
+      timeout: 180s
+      retries: 3
     depends_on:
       valkey:
         condition: service_healthy

diff --git a/ops/grafana/provisioning/dashboards/loki_13186_rev1.json b/ops/grafana/provisioning/dashboards/loki_13186_rev1.json
@@ -0,0 +1,139 @@
+{
+  "title": "Loki Dashboard",
+  "uid": "liz0yRCZz",
+  "timezone": "",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": true,
+  "description": "Loki dashboard with quick search and timeline.",
+  "style": "dark",
+  "tags": [],
+  "refresh": "30s",
+  "annotations": {
+    "list": [
+      {
+        "type": "dashboard",
+        "name": "Annotations & Alerts",
+        "enable": true,
+        "hide": true,
+        "builtIn": 1,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "datasource": "-- Grafana --"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 6,
+      "title": "Timeline",
+      "type": "graph",
+      "datasource": { "type": "loki", "uid": "loki" },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
+      "lines": true,
+      "bars": false,
+      "points": false,
+      "stack": false,
+      "fill": 1,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "legend": { "show": true },
+      "xaxis": { "mode": "time", "show": true },
+      "yaxes": [
+        { "format": "short", "logBase": 1, "show": true },
+        { "format": "short", "logBase": 1, "show": false }
+      ],
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (container) (count_over_time({project=\"$project\", service=\"$service\", container=~\"$container\"} |= \"$search\" [$__interval]))",
+          "legendFormat": "{{container}}",
+          "datasource": { "type": "loki", "uid": "loki" }
+        }
+      ],
+      "options": { "dataLinks": [] }
+    },
+    {
+      "id": 2,
+      "title": "Logs",
+      "type": "logs",
+      "datasource": { "type": "loki", "uid": "loki" },
+      "gridPos": { "h": 25, "w": 24, "x": 0, "y": 6 },
+      "maxDataPoints": 5000,
+      "options": {
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "{project=\"$project\", service=\"$service\", container=~\"$container\"} |= \"$search\"",
+          "datasource": { "type": "loki", "uid": "loki" }
+        }
+      ]
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "name": "project",
+        "label": "Project",
+        "type": "query",
+        "hide": 0,
+        "multi": false,
+        "includeAll": false,
+        "refresh": 1,
+        "sort": 0,
+        "skipUrlSync": false,
+        "datasource": { "type": "loki", "uid": "loki" },
+        "query": "label_values({project=~\".+\"}, project)",
+        "current": {}
+      },
+      {
+        "name": "service",
+        "label": "Service",
+        "type": "query",
+        "hide": 0,
+        "multi": false,
+        "includeAll": false,
+        "allValue": ".*",
+        "refresh": 1,
+        "sort": 1,
+        "skipUrlSync": false,
+        "datasource": { "type": "loki", "uid": "loki" },
+        "query": "label_values({project=\"$project\"}, service)",
+        "current": {}
+      },
+      {
+        "name": "container",
+        "label": "Container",
+        "type": "query",
+        "hide": 0,
+        "multi": false,
+        "includeAll": true,
+        "allValue": ".*",
+        "refresh": 1,
+        "sort": 2,
+        "skipUrlSync": false,
+        "datasource": { "type": "loki", "uid": "loki" },
+        "query": "label_values({project=\"$project\", service=\"$service\"}, container)",
+        "current": {}
+      },
+      {
+        "name": "search",
+        "label": "Search",
+        "type": "textbox",
+        "hide": 0,
+        "skipUrlSync": false,
+        "query": "",
+        "current": { "selected": false, "text": "", "value": "" },
+        "options": [ { "selected": true, "text": "", "value": "" } ]
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "timepicker": {
+    "refresh_intervals": ["10s","30s","1m","5m","15m","30m","1h","2h","1d"]
+  }
+}
diff --git a/ops/grafana/provisioning/datasources/prometheus.yml b/ops/grafana/provisioning/datasources/prometheus.yml
@@ -8,3 +8,11 @@ datasources:
     url: http://prometheus:9090
     isDefault: true
     editable: true
+
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    isDefault: false
+    editable: true
diff --git a/ops/loki-config.yml b/ops/loki-config.yml
@@ -0,0 +1,54 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  filesystem:
+    directory: /loki/chunks
+  tsdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+
+compactor:
+  working_directory: /loki/compactor
+  # optional but useful knobs:
+  # compaction_interval: 5m
+  # retention_enabled: true
+  # retention_delete_delay: 2h
+  # delete_request_store: filesystem  # only if you plan to use delete API
+
+limits_config:
+  ingestion_rate_mb: 16
+  ingestion_burst_size_mb: 32
+  max_streams_per_user: 0
+  max_global_streams_per_user: 0
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+ruler:
+  enable_api: true
+  alertmanager_url: http://alertmanager:9093
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-tmp
diff --git a/ops/promtail-config.yml b/ops/promtail-config.yml
@@ -0,0 +1,38 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /data/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+
+    relabel_configs:
+      # keep only compose project "clepsy"
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
+        regex: clepsy
+        action: keep
+
+      # nice labels
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: container
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
+        target_label: service
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
+        target_label: project
+
+      # point promtail at the Docker JSON log file
+      - source_labels: ['__meta_docker_container_id']
+        target_label: __path__
+        replacement: /var/lib/docker/containers/$1/*-json.log
+
+    pipeline_stages:
+      - docker: {}  # parse Docker JSON logs
diff --git a/ops/promtail-data/positions.yaml b/ops/promtail-data/positions.yaml
@@ -0,0 +1,9 @@
+positions:
+  cursor-3ff1b418dd37ae09ca7444caf917a24b1f4e224bcf6f920c56a2368b763deaa2: "1762901689"
+  cursor-57e3f73f251e5bce8b015c2ef9d95e3500ccc5c875d315e16a4e354bf828e5a9: "1762901689"
+  cursor-5aa9ada74b53a48769dcdad3e134246dd3b95bd5b8e667efcbde63d34e08bc77: "1762901694"
+  cursor-79e2f983fb75a8f3f58f7130a15c3f9c9c6ee75abcd11d073434f5322478d035: "1762901699"
+  cursor-a7e2f68d8d5f96463f4d2e529b26f7f832725005ffab8fc88be5aa30cc3d8fcf: "1762901695"
+  cursor-c6ceaa0045a6c5ebd1b6164892df420adaf3b12a18363c40516f9373cf5a1c10: "1762901689"
+  cursor-dbbf766a03bbbc03f5ca6f1505b775e9f9c14243408c9cfa1243fdfa8d65b062: "1762901691"
+  cursor-f80169f08c85ba3de526916630a60657b456960e0cf767b8cb2ef9a41eba70d9: "1762901694"
diff --git a/src/clepsy/infra/streams.py b/src/clepsy/infra/streams.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from datetime import datetime, timezone
 from typing import Iterable
 
@@ -40,7 +38,7 @@ def xadd_source_event(
         # This happens if the specified ID would be <= the top item (e.g., backfill older timestamps).
         # Fallback to server-assigned ID to avoid dropping the event; the payload still carries the true event ts.
         logger.warning(
-            "XADD with event-time ID %s failed (%r); falling back to server-assigned ID (*)",
+            "XADD with event-time ID {} failed ({}); falling back to server-assigned ID (*)",
             msg_id_base,
             e,
         )