Complete Phase 7 telemetry, analytics, and usage

.env.prod.example

@@ -38,6 +38,7 @@ DNS_BIND_V4=0.0.0.0
 # -----------------------------------------------------------------------------
 # [required for telemetry profile] Unique high-entropy ClickHouse password.
 CLICKHOUSE_PASSWORD=replace-with-a-unique-high-entropy-clickhouse-password
+CLICKHOUSE_URL=http://clickhouse:8123
 
 # -----------------------------------------------------------------------------
 # Container images — owned by Compose on hosts running the corresponding profile

Makefile

@@ -2,7 +2,7 @@ COMPOSE_DEV := docker compose -f compose.dev.yml
 COMPOSE_PROD := docker compose --env-file .env.prod -f compose.prod.yml
 COMPOSE_PROD_EXAMPLE := docker compose --env-file .env.prod.example -f compose.prod.yml
 
-.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check
+.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check
 
 dev-assets:
 	docker build --target frontend-assets-export --output type=local,dest=./core/public/build ./core
@@ -40,11 +40,15 @@ dev-e2e:
 	python3 tests/e2e/phase4_mtls.py
 	python3 tests/e2e/phase5_tls.py
 	python3 tests/e2e/phase6_security.py
+	python3 tests/e2e/phase7_analytics.py
 	python3 tests/e2e/phase4_runtime.py
 
 dev-scale-e2e:
 	python3 tests/e2e/phase2_scale.py
 
+dev-phase7-e2e:
+	python3 tests/e2e/phase7_analytics.py
+
 dev-logs:
 	$(COMPOSE_DEV) logs -f --tail=200
 

compose.dev.yml

@@ -24,6 +24,10 @@ x-core-env: &core-env
   ACME_INITIAL_JITTER_SECONDS: "1"
   EDGE_IDENTITY_CA_CERTIFICATE: /run/dev-pki/edge-identity-ca.crt
   EDGE_IDENTITY_CA_PRIVATE_KEY: /run/dev-pki/edge-identity-ca.key
+  CLICKHOUSE_URL: http://clickhouse:8123
+  CLICKHOUSE_DATABASE: cdnf
+  CLICKHOUSE_USER: cdnf
+  CLICKHOUSE_PASSWORD: cdnf-dev-only
 
 x-core: &core
   build:
@@ -46,7 +50,7 @@ x-core: &core
     vendor-init: { condition: service_completed_successfully }
     control-db: { condition: service_healthy }
     redis: { condition: service_healthy }
-  networks: [control]
+  networks: [control, telemetry]
   restart: unless-stopped
 
 services:
@@ -241,17 +245,22 @@ services:
 
   vector:
     image: timberio/vector:0.55.0-alpine
+    environment:
+      CLICKHOUSE_ENDPOINT: http://clickhouse:8123
+      CLICKHOUSE_USER: cdnf
+      CLICKHOUSE_PASSWORD: cdnf-dev-only
     volumes:
       - ./docker/vector/vector.yaml:/etc/vector/vector.yaml:ro
       - vector-data:/vector-data-dir
-    networks: [telemetry]
+    networks: [telemetry, edge, dns]
     restart: unless-stopped
 
   prometheus:
     image: prom/prometheus:v3.12.0
     ports: ["9090:9090"]
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro
       - prometheus:/prometheus
     networks: [telemetry]
 

compose.prod.yml

@@ -26,14 +26,18 @@ x-core-env: &core-env
   EDGE_IDENTITY_CA_CERTIFICATE: /run/secrets/edge-identity-ca.crt
   EDGE_IDENTITY_CA_PRIVATE_KEY: /run/secrets/edge-identity-ca.key
   EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE: ${EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE:-}
+  CLICKHOUSE_URL: ${CLICKHOUSE_URL:?CLICKHOUSE_URL is required}
+  CLICKHOUSE_DATABASE: cdnf
+  CLICKHOUSE_USER: cdnf
+  CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required}
 
 x-core: &core
   image: ghcr.io/vaheed/cdnfoundry-core:${CDNF_RELEASE:?CDNF_RELEASE must be an immutable commit SHA}
   environment: *core-env
   depends_on:
     control-db: { condition: service_healthy }
     redis: { condition: service_healthy }
-  networks: [control]
+  networks: [control, telemetry]
   restart: unless-stopped
   read_only: true
   tmpfs: [/tmp]
@@ -206,17 +210,22 @@ services:
   vector:
     image: timberio/vector:0.55.0-alpine
     profiles: [telemetry, edge]
+    environment:
+      CLICKHOUSE_ENDPOINT: ${CLICKHOUSE_URL:?CLICKHOUSE_URL is required}
+      CLICKHOUSE_USER: cdnf
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required}
     volumes:
       - ./docker/vector/vector.yaml:/etc/vector/vector.yaml:ro
       - vector-data:/vector-data-dir
-    networks: [telemetry]
+    networks: [telemetry, edge, dns-private]
     restart: unless-stopped
 
   prometheus:
     image: prom/prometheus:v3.12.0
     profiles: [telemetry]
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro
       - prometheus:/prometheus
     networks: [telemetry]
     restart: unless-stopped

core/app/Console/Commands/FinalizeUsage.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\BuildUsageRollups;
+use Carbon\CarbonImmutable;
+use Illuminate\Console\Command;
+
+class FinalizeUsage extends Command
+{
+    protected $signature = 'usage:finalize';
+
+    protected $description = 'Dispatch an idempotent rebuild for the most recently finalized usage hour';
+
+    public function handle(): int
+    {
+        $to = CarbonImmutable::now('UTC')->subHour()->startOfHour();
+        BuildUsageRollups::dispatch($to->subHour()->toIso8601String(), $to->toIso8601String());
+        $this->info("Queued usage interval ending {$to->toIso8601String()}.");
+
+        return self::SUCCESS;
+    }
+}

core/app/Exceptions/AnalyticsUnavailableException.php

@@ -0,0 +1,7 @@
+<?php
+
+namespace App\Exceptions;
+
+use RuntimeException;
+
+class AnalyticsUnavailableException extends RuntimeException {}

core/app/Filament/Admin/Pages/Telemetry.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Filament\Admin\Pages;
+
+use App\Models\UsageRollup;
+use App\Support\AnalyticsStore;
+use Carbon\CarbonImmutable;
+use Filament\Pages\Page;
+use Illuminate\Support\Facades\Http;
+use Throwable;
+
+class Telemetry extends Page
+{
+    protected static string|\BackedEnum|null $navigationIcon = 'heroicon-o-presentation-chart-line';
+
+    protected static ?string $navigationLabel = 'Telemetry and usage';
+
+    protected static string|\UnitEnum|null $navigationGroup = 'Observe';
+
+    protected static ?int $navigationSort = 10;
+
+    protected string $view = 'filament.admin.pages.telemetry';
+
+    public function getStateProperty(): array
+    {
+        $store = app(AnalyticsStore::class);
+        $to = CarbonImmutable::now('UTC');
+        $range = ['from' => $to->subDay(), 'to' => $to, 'raw' => false];
+        $rawRange = ['from' => $to->subHour(), 'to' => $to, 'raw' => true];
+        $state = [
+            'available' => false,
+            'meta' => ['from' => $range['from']->toIso8601String(), 'to' => $range['to']->toIso8601String(), 'partial' => true],
+            'summary' => [],
+            'traffic' => [],
+            'dns' => [],
+            'logs' => ['errors' => [], 'security' => [], 'edges' => []],
+            'buffer' => $this->bufferStatus(),
+            'usage' => $this->recentUsage(),
+        ];
+        try {
+            $state['meta'] = $store->metadata($range);
+            $state['summary'] = $store->summary(null, $range);
+            $state['traffic'] = $store->aggregate(null, $range, 'traffic');
+            $state['dns'] = $store->aggregate(null, $range, 'dns');
+            foreach (array_keys($state['logs']) as $stream) {
+                $state['logs'][$stream] = array_slice($store->logs(null, $rawRange, $stream, null)['items'], 0, 10);
+            }
+            $state['available'] = true;
+        } catch (Throwable) {
+            // PostgreSQL usage and Vector health remain independently useful.
+        }
+
+        return $state;
+    }
+
+    private function recentUsage(): array
+    {
+        return UsageRollup::query()->with('domain:id,name')->latest('interval_start')->limit(20)->get()
+            ->map(fn (UsageRollup $row): array => [
+                'domain' => $row->domain?->name ?? "Domain #{$row->domain_id}",
+                'interval' => $row->interval_start->toIso8601String(),
+                'requests' => $row->requests,
+                'bytes' => $row->bytes_in + $row->bytes_out,
+                'dns_queries' => $row->dns_queries,
+                'status' => $row->status,
+            ])->all();
+    }
+
+    private function bufferStatus(): array
+    {
+        try {
+            $metrics = Http::connectTimeout(1)->timeout(2)->get('http://vector:9598/metrics')->throw()->body();
+            preg_match_all('/^(vector_buffer_[a-z_]+|vector_component_(?:discarded_events_total|errors_total))[^ ]* ([0-9.e+-]+)$/m', $metrics, $matches, PREG_SET_ORDER);
+
+            return ['available' => true, 'metrics' => collect($matches)->mapWithKeys(fn (array $match): array => [$match[1] => $match[2]])->all()];
+        } catch (Throwable) {
+            return ['available' => false, 'metrics' => []];
+        }
+    }
+}

core/app/Filament/Domain/Pages/Analytics.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Filament\Domain\Pages;
+
+use App\Models\Domain;
+use App\Models\UsageRollup;
+use App\Support\AnalyticsStore;
+use Carbon\CarbonImmutable;
+use Filament\Pages\Page;
+use Illuminate\Database\Eloquent\Builder;
+use Throwable;
+
+class Analytics extends Page
+{
+    protected static string|\BackedEnum|null $navigationIcon = 'heroicon-o-chart-bar-square';
+
+    protected static ?string $navigationLabel = 'Analytics and logs';
+
+    protected static string|\UnitEnum|null $navigationGroup = 'Observe';
+
+    protected static ?int $navigationSort = 10;
+
+    protected string $view = 'filament.domain.pages.analytics';
+
+    public function getStateProperty(): array
+    {
+        $domain = $this->selectedDomain();
+        if ($domain === null) {
+            return ['domain' => null, 'available' => true, 'views' => [], 'meta' => [], 'logs' => [], 'usage' => []];
+        }
+        $store = app(AnalyticsStore::class);
+        $to = CarbonImmutable::now('UTC');
+        $range = ['from' => $to->subDay(), 'to' => $to, 'raw' => false];
+        $rawRange = ['from' => $to->subHour(), 'to' => $to, 'raw' => true];
+        try {
+            $summary = $store->summary($domain, $range);
+            $views = [
+                'Request and bandwidth timeseries' => $store->aggregate($domain, $range, 'timeseries'),
+                'Status codes' => $store->aggregate($domain, $range, 'status-codes'),
+                'Cache ratio' => $store->aggregate($domain, $range, 'cache'),
+                'Countries and continents' => $store->aggregate($domain, $range, 'countries'),
+                'Hostnames' => $store->aggregate($domain, $range, 'hostnames'),
+                'Top URLs (last hour)' => $store->topUrls($domain, $rawRange),
+                'Origin health and latency' => $store->aggregate($domain, $range, 'origin'),
+                'Edge distribution' => $store->aggregate($domain, $range, 'edges'),
+                'DNS activity' => $store->aggregate($domain, $range, 'dns'),
+            ];
+            $logs = [];
+            foreach (['requests', 'dns', 'errors', 'security'] as $stream) {
+                $logs[$stream] = array_slice($store->logs($domain, $rawRange, $stream, null)['items'], 0, 10);
+            }
+
+            return ['domain' => $domain, 'available' => true, 'meta' => $store->metadata($range), 'summary' => $summary, 'views' => $views, 'logs' => $logs, 'usage' => $this->recentUsage($domain)];
+        } catch (Throwable) {
+            return ['domain' => $domain, 'available' => false, 'summary' => [], 'views' => [], 'logs' => [], 'usage' => $this->recentUsage($domain), 'meta' => ['from' => $range['from']->toIso8601String(), 'to' => $range['to']->toIso8601String(), 'partial' => true]];
+        }
+    }
+
+    private function recentUsage(Domain $domain): array
+    {
+        return UsageRollup::query()->whereBelongsTo($domain)->latest('interval_start')->limit(20)->get()
+            ->map(fn (UsageRollup $row): array => [
+                'interval' => $row->interval_start->toIso8601String(),
+                'requests' => $row->requests,
+                'bytes' => $row->bytes_in + $row->bytes_out,
+                'dns_queries' => $row->dns_queries,
+                'status' => $row->status,
+            ])->all();
+    }
+
+    public function getDomainsProperty()
+    {
+        return $this->domainQuery()->orderBy('name')->get(['domains.id', 'domains.name', 'domains.display_name']);
+    }
+
+    private function selectedDomain(): ?Domain
+    {
+        $requested = request()->integer('domain');
+
+        return $this->domainQuery()->when($requested > 0, fn (Builder $query): Builder => $query->whereKey($requested))->orderBy('domains.id')->first();
+    }
+
+    private function domainQuery(): Builder
+    {
+        return Domain::query()->whereHas('users', fn (Builder $query): Builder => $query->where('users.id', auth()->id()));
+    }
+}

core/app/Http/Controllers/Admin/AnalyticsController.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace App\Http\Controllers\Admin;
+
+use App\Http\Controllers\Controller;
+use App\Support\AnalyticsStore;
+use Illuminate\Http\JsonResponse;
+use Illuminate\Http\Request;
+
+class AnalyticsController extends Controller
+{
+    public function summary(Request $request, AnalyticsStore $store): JsonResponse
+    {
+        $range = $store->range($request);
+
+        return response()->json(['data' => $store->summary(null, $range), 'meta' => $store->metadata($range)]);
+    }
+
+    public function view(Request $request, string $view, AnalyticsStore $store): JsonResponse
+    {
+        $range = $store->range($request);
+
+        return response()->json(['data' => $store->aggregate(null, $range, $view), 'meta' => $store->metadata($range)]);
+    }
+}

core/app/Http/Controllers/Admin/LogController.php

@@ -0,0 +1,19 @@
+<?php
+
+namespace App\Http\Controllers\Admin;
+
+use App\Http\Controllers\Controller;
+use App\Support\AnalyticsStore;
+use Illuminate\Http\JsonResponse;
+use Illuminate\Http\Request;
+
+class LogController extends Controller
+{
+    public function index(Request $request, string $stream, AnalyticsStore $store): JsonResponse
+    {
+        $range = $store->range($request, true);
+        $result = $store->logs(null, $range, $stream, $request->query('cursor'));
+
+        return response()->json(['data' => $result['items'], 'meta' => [...$store->metadata($range), 'next_cursor' => $result['next_cursor']]]);
+    }
+}