diff --git a/packages/oyasai-alloy.nix b/packages/oyasai-alloy.nix new file mode 100644 index 000000000..f55105d6f --- /dev/null +++ b/packages/oyasai-alloy.nix @@ -0,0 +1,60 @@ +{ + lib, + grafana-alloy, + oyasaiDockerTools, + stdenv, + writeTextFile, +}: + +let + alloyConfig = writeTextFile { + name = "config.alloy"; + text = '' + // Discover all Docker containers via the Docker socket. + discovery.docker "all" { + host = "unix:///var/run/docker.sock" + } + + // Relabel: expose container name and log stream as Loki labels. + discovery.relabel "containers" { + targets = discovery.docker.all.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } + } + + // Tail logs from Docker containers and forward to Loki. + loki.source.docker "all" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.containers.output + forward_to = [loki.write.default.receiver] + } + + loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } + } + ''; + }; +in +lib.optionalAttrs stdenv.hostPlatform.isLinux { + docker = oyasaiDockerTools.buildLayeredImage { + name = "oyasai-alloy"; + config = { + Cmd = [ + "${grafana-alloy}/bin/alloy" + "run" + "${alloyConfig}" + ]; + }; + }; +} diff --git a/packages/oyasai-cdktf/src/stacks/platform-services.ts b/packages/oyasai-cdktf/src/stacks/platform-services.ts index e99900761..496d1ebaf 100644 --- a/packages/oyasai-cdktf/src/stacks/platform-services.ts +++ b/packages/oyasai-cdktf/src/stacks/platform-services.ts @@ -50,12 +50,17 @@ export class PlatformServices extends OyasaiPlatformTerraformStack { const imageIds = JSON.parse(process.env.OYASAI_IMAGE_ID as string); const images = { // keep-sorted start + alloy: imageIds["oyasai-alloy"], + grafana: imageIds["oyasai-grafana"], + loki: imageIds["oyasai-loki"], mariadb: imageIds.mariadb, + mcMonitorExporter: imageIds["oyasai-mc-monitor-exporter"], minecraftAxiom: imageIds["oyasai-minecraft-axiom"], minecraftBackup: imageIds["mc-backup"], minecraftLobby: imageIds["oyasai-minecraft-lobby"], minecraftMain: imageIds["oyasai-minecraft-main"], mysqlBackup: imageIds["mysql-backup"], + prometheus: imageIds["oyasai-prometheus"], velocity: imageIds["oyasai-velocity"], // keep-sorted end } as const; @@ -63,10 +68,12 @@ export class PlatformServices extends OyasaiPlatformTerraformStack { const baseHostPath = join("/opt/platform", this.environment); const hostPaths = { // keep-sorted start + loki: join(baseHostPath, "loki"), mariadb: join(baseHostPath, "mariadb"), minecraftAxiom: join(baseHostPath, "minecraft-axiom"), minecraftLobby: join(baseHostPath, "minecraft-lobby"), minecraftMain: join(baseHostPath, "minecraft-main"), + prometheus: join(baseHostPath, "prometheus"), velocity: join(baseHostPath, "velocity"), // keep-sorted end } as const; @@ -290,5 +297,65 @@ export class PlatformServices extends OyasaiPlatformTerraformStack { }), }); } + + new Container(this, this.t("mc-monitor-exporter-container"), { + image: images.mcMonitorExporter, + name: "mc-monitor-exporter", + restart: "unless-stopped", + networksAdvanced: [network], + env: envs({ + EXPORT_SERVERS: [ + `${minecraftMainContainer.name}:25565`, + `${minecraftLobbyContainer.name}:25565`, + ].join(","), + }), + }); + + new Container(this, this.t("loki-container"), { + image: images.loki, + name: "loki", + restart: "unless-stopped", + networksAdvanced: [network], + volumes: [ + { + containerPath: "/data", + hostPath: hostPaths.loki, + }, + ], + }); + + new Container(this, this.t("alloy-container"), { + image: images.alloy, + name: "alloy", + restart: "unless-stopped", + networksAdvanced: [network], + volumes: [ + { + containerPath: "/var/run/docker.sock", + hostPath: "/var/run/docker.sock", + }, + ], + }); + + new Container(this, this.t("prometheus-container"), { + image: images.prometheus, + name: "prometheus", + restart: "unless-stopped", + networksAdvanced: [network], + volumes: [ + { + containerPath: "/data", + hostPath: hostPaths.prometheus, + }, + ], + }); + + new Container(this, this.t("grafana-container"), { + image: images.grafana, + name: "grafana", + restart: "unless-stopped", + networksAdvanced: [network], + ports: ports({ tcp: [3000] }), + }); } } diff --git a/packages/oyasai-grafana/oyasai-grafana-dashboard.json b/packages/oyasai-grafana/oyasai-grafana-dashboard.json new file mode 100644 index 000000000..252a0478c --- /dev/null +++ b/packages/oyasai-grafana/oyasai-grafana-dashboard.json @@ -0,0 +1,462 @@ +{ + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Offline" + }, + "1": { + "color": "green", + "index": 0, + "text": "Online" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "mc_status_healthy{host=\"oyasai-minecraft-main\"}", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Main Server", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Offline" + }, + "1": { + "color": "green", + "index": 0, + "text": "Online" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "mc_status_healthy{host=\"oyasai-minecraft-lobby\"}", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Lobby Server", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "mc_status_players_online{host=\"oyasai-minecraft-main\"}", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Main Players Online", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "mc_status_players_online{host=\"oyasai-minecraft-lobby\"}", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Lobby Players Online", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "mc_status_players_online{host=\"oyasai-minecraft-main\"}", + "legendFormat": "Main", + "refId": "A" + }, + { + "expr": "mc_status_players_online{host=\"oyasai-minecraft-lobby\"}", + "legendFormat": "Lobby", + "refId": "B" + } + ], + "title": "Players Online", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "mc_status_response_time_seconds{host=\"oyasai-minecraft-main\"}", + "legendFormat": "Main", + "refId": "A" + }, + { + "expr": "mc_status_response_time_seconds{host=\"oyasai-minecraft-lobby\"}", + "legendFormat": "Lobby", + "refId": "B" + } + ], + "title": "Server Response Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 7, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "expr": "{container=~\"oyasai-minecraft.*\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Minecraft Logs", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["minecraft"], + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Oyasai Minecraft", + "uid": "oyasai-minecraft", + "version": 1 +} diff --git a/packages/oyasai-grafana/package.nix b/packages/oyasai-grafana/package.nix new file mode 100644 index 000000000..4d0319bc1 --- /dev/null +++ b/packages/oyasai-grafana/package.nix @@ -0,0 +1,91 @@ +{ + lib, + grafana, + oyasaiDockerTools, + stdenv, + formats, + runCommandLocal, +}: + +let + datasourceYml = (formats.yaml { }).generate "prometheus.yaml" { + apiVersion = 1; + datasources = [ + { + name = "Prometheus"; + uid = "prometheus"; + type = "prometheus"; + url = "http://prometheus:9090"; + isDefault = true; + access = "proxy"; + } + ]; + }; + + lokiYml = (formats.yaml { }).generate "loki.yaml" { + apiVersion = 1; + datasources = [ + { + name = "Loki"; + uid = "loki"; + type = "loki"; + url = "http://loki:3100"; + access = "proxy"; + } + ]; + }; + + dashboardsDir = runCommandLocal "grafana-dashboards" { } '' + mkdir -p $out + cp ${./oyasai-grafana-dashboard.json} $out/minecraft.json + ''; + + dashboardsProvisionerYml = (formats.yaml { }).generate "dashboards.yaml" { + apiVersion = 1; + providers = [ + { + name = "oyasai"; + type = "file"; + disableDeletion = true; + allowUiUpdates = false; + options.path = "${dashboardsDir}"; + } + ]; + }; + + # Bake data sources and dashboards into the image via provisioning directory. + # Grafana reads this on startup and auto-configures them. + provisioningDir = runCommandLocal "grafana-provisioning" { } '' + mkdir -p $out/datasources $out/dashboards + cp ${datasourceYml} $out/datasources/prometheus.yaml + cp ${lokiYml} $out/datasources/loki.yaml + cp ${dashboardsProvisionerYml} $out/dashboards/oyasai.yaml + ''; +in +lib.optionalAttrs stdenv.hostPlatform.isLinux { + docker = oyasaiDockerTools.buildLayeredImage { + name = "oyasai-grafana"; + config = { + Cmd = [ + "${grafana}/bin/grafana" + "server" + "--homepath=${grafana}/share/grafana" + ]; + Env = [ + "GF_PATHS_HOME=${grafana}/share/grafana" + "GF_PATHS_PROVISIONING=${provisioningDir}" + "GF_PATHS_DATA=/data" + "GF_PATHS_LOGS=/data/log" + "GF_PATHS_PLUGINS=/data/plugins" + "GF_SERVER_HTTP_PORT=3000" + "GF_AUTH_ANONYMOUS_ENABLED=true" + "GF_AUTH_ANONYMOUS_ORG_ROLE=Admin" + "GF_AUTH_DISABLE_LOGIN_FORM=true" + "GF_USERS_ALLOW_SIGN_UP=false" + ]; + ExposedPorts = { + "3000/tcp" = { }; + }; + }; + }; +} diff --git a/packages/oyasai-loki.nix b/packages/oyasai-loki.nix new file mode 100644 index 000000000..b545b952d --- /dev/null +++ b/packages/oyasai-loki.nix @@ -0,0 +1,53 @@ +{ + lib, + grafana-loki, + oyasaiDockerTools, + stdenv, + formats, +}: + +let + lokiYml = (formats.yaml { }).generate "loki.yml" { + auth_enabled = false; + + server.http_listen_port = 3100; + + common = { + instance_addr = "127.0.0.1"; + path_prefix = "/data"; + storage.filesystem = { + chunks_directory = "/data/chunks"; + rules_directory = "/data/rules"; + }; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + + schema_config.configs = [ + { + from = "2020-10-24"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; +in +lib.optionalAttrs stdenv.hostPlatform.isLinux { + docker = oyasaiDockerTools.buildLayeredImage { + name = "oyasai-loki"; + config = { + Cmd = [ + "${grafana-loki}/bin/loki" + "-config.file=${lokiYml}" + ]; + ExposedPorts = { + "3100/tcp" = { }; + }; + }; + }; +} diff --git a/packages/oyasai-mc-monitor-exporter.nix b/packages/oyasai-mc-monitor-exporter.nix new file mode 100644 index 000000000..585dfbb8c --- /dev/null +++ b/packages/oyasai-mc-monitor-exporter.nix @@ -0,0 +1,29 @@ +{ + lib, + mc-monitor, + oyasaiDockerTools, + stdenv, + writeShellApplication, +}: + +let + result = writeShellApplication { + name = "oyasai-mc-monitor-exporter"; + runtimeInputs = [ mc-monitor ]; + text = '' + exec mc-monitor export-for-prometheus "$@" + ''; + passthru = lib.optionalAttrs stdenv.hostPlatform.isLinux { + docker = oyasaiDockerTools.buildLayeredImage { + name = "oyasai-mc-monitor-exporter"; + config = { + Cmd = [ (lib.getExe result) ]; + ExposedPorts = { + "8080/tcp" = { }; + }; + }; + }; + }; + }; +in +result diff --git a/packages/oyasai-prometheus.nix b/packages/oyasai-prometheus.nix new file mode 100644 index 000000000..c3fe8382c --- /dev/null +++ b/packages/oyasai-prometheus.nix @@ -0,0 +1,39 @@ +{ + lib, + prometheus, + oyasaiDockerTools, + stdenv, + formats, +}: + +let + prometheusYml = (formats.yaml { }).generate "prometheus.yml" { + global = { + scrape_interval = "15s"; + evaluation_interval = "15s"; + }; + scrape_configs = [ + { + job_name = "minecraft"; + static_configs = [ { targets = [ "mc-monitor-exporter:8080" ]; } ]; + } + ]; + }; +in +lib.optionalAttrs stdenv.hostPlatform.isLinux { + docker = oyasaiDockerTools.buildLayeredImage { + name = "oyasai-prometheus"; + config = { + Cmd = [ + "${prometheus}/bin/prometheus" + "--config.file=${prometheusYml}" + "--storage.tsdb.path=/data" + "--web.console.libraries=${prometheus}/share/prometheus/console_libraries" + "--web.console.templates=${prometheus}/share/prometheus/consoles" + ]; + ExposedPorts = { + "9090/tcp" = { }; + }; + }; + }; +}