From 3aa4c0342dd1871099bd7b757ed50aa50f10cdf0 Mon Sep 17 00:00:00 2001 From: samouil galanakis Date: Tue, 11 Nov 2025 23:55:29 +0100 Subject: [PATCH] loki logs --- README.md | 12 ++ docker-compose.prod.monitoring.yml | 30 ++++ docker-compose.prod.yml | 4 +- .../dashboards/loki_13186_rev1.json | 139 ++++++++++++++++++ .../provisioning/datasources/prometheus.yml | 8 + ops/loki-config.yml | 54 +++++++ ops/promtail-config.yml | 38 +++++ ops/promtail-data/positions.yaml | 9 ++ src/clepsy/infra/streams.py | 4 +- 9 files changed, 293 insertions(+), 5 deletions(-) create mode 100755 ops/grafana/provisioning/dashboards/loki_13186_rev1.json create mode 100755 ops/loki-config.yml create mode 100755 ops/promtail-config.yml create mode 100755 ops/promtail-data/positions.yaml diff --git a/README.md b/README.md index 5e5ba28..e4de56d 100755 --- a/README.md +++ b/README.md @@ -27,6 +27,18 @@ It works by monitoring your devices, having a LLM process all the information an [See setup guide](docs/SETUP.md) +### Monitoring + +To run the monitoring stack, which includes Prometheus for metrics and Loki for logging, use the following command: + +```bash +docker compose -f docker-compose.prod.yml -f docker-compose.prod.local.yml -f docker-compose.prod.monitoring.yml up --build +``` + +- **Grafana:** [http://localhost:3000](http://localhost:3000) (admin/admin) +- **Prometheus:** [http://localhost:9090](http://localhost:9090) +- **Loki:** Logs are viewable within Grafana. + ## Architecture ![architecture](docs/media/architecture.png) diff --git a/docker-compose.prod.monitoring.yml b/docker-compose.prod.monitoring.yml index b256ba6..e424318 100755 --- a/docker-compose.prod.monitoring.yml +++ b/docker-compose.prod.monitoring.yml @@ -30,6 +30,33 @@ services: condition: service_started restart: unless-stopped + loki: + image: grafana/loki:latest + networks: + - web + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yml + volumes: + - ./ops/loki-config.yml:/etc/loki/local-config.yml:ro + - loki_data:/loki + restart: unless-stopped + + promtail: + image: grafana/promtail:latest + networks: + - web + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + - ./ops/promtail-config.yml:/etc/promtail/config.yml:ro + - ./ops/promtail-data:/data + command: -config.file=/etc/promtail/config.yml + depends_on: + loki: + condition: service_started + restart: unless-stopped + grafana: image: grafana/grafana:latest networks: @@ -46,8 +73,11 @@ services: depends_on: prometheus: condition: service_started + loki: + condition: service_started restart: unless-stopped volumes: grafana_data: prometheus_data: + loki_data: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e95f221..42ef3c2 100755 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -32,8 +32,8 @@ services: healthcheck: test: ["CMD", "curl", "-fsS", "http://localhost:8000/healthz"] interval: 10s - timeout: 2s - retries: 10 + timeout: 180s + retries: 3 depends_on: valkey: condition: service_healthy diff --git a/ops/grafana/provisioning/dashboards/loki_13186_rev1.json b/ops/grafana/provisioning/dashboards/loki_13186_rev1.json new file mode 100755 index 0000000..6155041 --- /dev/null +++ b/ops/grafana/provisioning/dashboards/loki_13186_rev1.json @@ -0,0 +1,139 @@ +{ + "title": "Loki Dashboard", + "uid": "liz0yRCZz", + "timezone": "", + "schemaVersion": 39, + "version": 1, + "editable": true, + "description": "Loki dashboard with quick search and timeline.", + "style": "dark", + "tags": [], + "refresh": "30s", + "annotations": { + "list": [ + { + "type": "dashboard", + "name": "Annotations & Alerts", + "enable": true, + "hide": true, + "builtIn": 1, + "iconColor": "rgba(0, 211, 255, 1)", + "datasource": "-- Grafana --" + } + ] + }, + "panels": [ + { + "id": 6, + "title": "Timeline", + "type": "graph", + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "lines": true, + "bars": false, + "points": false, + "stack": false, + "fill": 1, + "linewidth": 1, + "nullPointMode": "null", + "legend": { "show": true }, + "xaxis": { "mode": "time", "show": true }, + "yaxes": [ + { "format": "short", "logBase": 1, "show": true }, + { "format": "short", "logBase": 1, "show": false } + ], + "targets": [ + { + "refId": "A", + "expr": "sum by (container) (count_over_time({project=\"$project\", service=\"$service\", container=~\"$container\"} |= \"$search\" [$__interval]))", + "legendFormat": "{{container}}", + "datasource": { "type": "loki", "uid": "loki" } + } + ], + "options": { "dataLinks": [] } + }, + { + "id": 2, + "title": "Logs", + "type": "logs", + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 25, "w": 24, "x": 0, "y": 6 }, + "maxDataPoints": 5000, + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "refId": "A", + "expr": "{project=\"$project\", service=\"$service\", container=~\"$container\"} |= \"$search\"", + "datasource": { "type": "loki", "uid": "loki" } + } + ] + } + ], + "templating": { + "list": [ + { + "name": "project", + "label": "Project", + "type": "query", + "hide": 0, + "multi": false, + "includeAll": false, + "refresh": 1, + "sort": 0, + "skipUrlSync": false, + "datasource": { "type": "loki", "uid": "loki" }, + "query": "label_values({project=~\".+\"}, project)", + "current": {} + }, + { + "name": "service", + "label": "Service", + "type": "query", + "hide": 0, + "multi": false, + "includeAll": false, + "allValue": ".*", + "refresh": 1, + "sort": 1, + "skipUrlSync": false, + "datasource": { "type": "loki", "uid": "loki" }, + "query": "label_values({project=\"$project\"}, service)", + "current": {} + }, + { + "name": "container", + "label": "Container", + "type": "query", + "hide": 0, + "multi": false, + "includeAll": true, + "allValue": ".*", + "refresh": 1, + "sort": 2, + "skipUrlSync": false, + "datasource": { "type": "loki", "uid": "loki" }, + "query": "label_values({project=\"$project\", service=\"$service\"}, container)", + "current": {} + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "hide": 0, + "skipUrlSync": false, + "query": "", + "current": { "selected": false, "text": "", "value": "" }, + "options": [ { "selected": true, "text": "", "value": "" } ] + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": { + "refresh_intervals": ["10s","30s","1m","5m","15m","30m","1h","2h","1d"] + } +} diff --git a/ops/grafana/provisioning/datasources/prometheus.yml b/ops/grafana/provisioning/datasources/prometheus.yml index 67a5699..bbf0818 100755 --- a/ops/grafana/provisioning/datasources/prometheus.yml +++ b/ops/grafana/provisioning/datasources/prometheus.yml @@ -8,3 +8,11 @@ datasources: url: http://prometheus:9090 isDefault: true editable: true + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: false + editable: true diff --git a/ops/loki-config.yml b/ops/loki-config.yml new file mode 100755 index 0000000..949f2dd --- /dev/null +++ b/ops/loki-config.yml @@ -0,0 +1,54 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + +compactor: + working_directory: /loki/compactor + # optional but useful knobs: + # compaction_interval: 5m + # retention_enabled: true + # retention_delete_delay: 2h + # delete_request_store: filesystem # only if you plan to use delete API + +limits_config: + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + max_streams_per_user: 0 + max_global_streams_per_user: 0 + reject_old_samples: true + reject_old_samples_max_age: 168h + +ruler: + enable_api: true + alertmanager_url: http://alertmanager:9093 + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-tmp diff --git a/ops/promtail-config.yml b/ops/promtail-config.yml new file mode 100755 index 0000000..4953ead --- /dev/null +++ b/ops/promtail-config.yml @@ -0,0 +1,38 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /data/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + + relabel_configs: + # keep only compose project "clepsy" + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + regex: clepsy + action: keep + + # nice labels + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: service + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + target_label: project + + # point promtail at the Docker JSON log file + - source_labels: ['__meta_docker_container_id'] + target_label: __path__ + replacement: /var/lib/docker/containers/$1/*-json.log + + pipeline_stages: + - docker: {} # parse Docker JSON logs diff --git a/ops/promtail-data/positions.yaml b/ops/promtail-data/positions.yaml new file mode 100755 index 0000000..4168815 --- /dev/null +++ b/ops/promtail-data/positions.yaml @@ -0,0 +1,9 @@ +positions: + cursor-3ff1b418dd37ae09ca7444caf917a24b1f4e224bcf6f920c56a2368b763deaa2: "1762901689" + cursor-57e3f73f251e5bce8b015c2ef9d95e3500ccc5c875d315e16a4e354bf828e5a9: "1762901689" + cursor-5aa9ada74b53a48769dcdad3e134246dd3b95bd5b8e667efcbde63d34e08bc77: "1762901694" + cursor-79e2f983fb75a8f3f58f7130a15c3f9c9c6ee75abcd11d073434f5322478d035: "1762901699" + cursor-a7e2f68d8d5f96463f4d2e529b26f7f832725005ffab8fc88be5aa30cc3d8fcf: "1762901695" + cursor-c6ceaa0045a6c5ebd1b6164892df420adaf3b12a18363c40516f9373cf5a1c10: "1762901689" + cursor-dbbf766a03bbbc03f5ca6f1505b775e9f9c14243408c9cfa1243fdfa8d65b062: "1762901691" + cursor-f80169f08c85ba3de526916630a60657b456960e0cf767b8cb2ef9a41eba70d9: "1762901694" diff --git a/src/clepsy/infra/streams.py b/src/clepsy/infra/streams.py index d6284a0..f94ae1e 100755 --- a/src/clepsy/infra/streams.py +++ b/src/clepsy/infra/streams.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from datetime import datetime, timezone from typing import Iterable @@ -40,7 +38,7 @@ def xadd_source_event( # This happens if the specified ID would be <= the top item (e.g., backfill older timestamps). # Fallback to server-assigned ID to avoid dropping the event; the payload still carries the true event ts. logger.warning( - "XADD with event-time ID %s failed (%r); falling back to server-assigned ID (*)", + "XADD with event-time ID {} failed ({}); falling back to server-assigned ID (*)", msg_id_base, e, )