diff --git a/.env.prod.example b/.env.prod.example index e666d5e..9e72b22 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -38,6 +38,7 @@ DNS_BIND_V4=0.0.0.0 # ----------------------------------------------------------------------------- # [required for telemetry profile] Unique high-entropy ClickHouse password. CLICKHOUSE_PASSWORD=replace-with-a-unique-high-entropy-clickhouse-password +CLICKHOUSE_URL=http://clickhouse:8123 # ----------------------------------------------------------------------------- # Container images — owned by Compose on hosts running the corresponding profile diff --git a/Makefile b/Makefile index c87239b..57f8b55 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ COMPOSE_DEV := docker compose -f compose.dev.yml COMPOSE_PROD := docker compose --env-file .env.prod -f compose.prod.yml COMPOSE_PROD_EXAMPLE := docker compose --env-file .env.prod.example -f compose.prod.yml -.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check +.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check dev-assets: docker build --target frontend-assets-export --output type=local,dest=./core/public/build ./core @@ -40,11 +40,15 @@ dev-e2e: python3 tests/e2e/phase4_mtls.py python3 tests/e2e/phase5_tls.py python3 tests/e2e/phase6_security.py + python3 tests/e2e/phase7_analytics.py python3 tests/e2e/phase4_runtime.py dev-scale-e2e: python3 tests/e2e/phase2_scale.py +dev-phase7-e2e: + python3 tests/e2e/phase7_analytics.py + dev-logs: $(COMPOSE_DEV) logs -f --tail=200 diff --git a/compose.dev.yml b/compose.dev.yml index a3b7d32..60ebf19 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -24,6 +24,10 @@ x-core-env: &core-env ACME_INITIAL_JITTER_SECONDS: "1" EDGE_IDENTITY_CA_CERTIFICATE: /run/dev-pki/edge-identity-ca.crt EDGE_IDENTITY_CA_PRIVATE_KEY: /run/dev-pki/edge-identity-ca.key + CLICKHOUSE_URL: http://clickhouse:8123 + CLICKHOUSE_DATABASE: cdnf + CLICKHOUSE_USER: cdnf + CLICKHOUSE_PASSWORD: cdnf-dev-only x-core: &core build: @@ -46,7 +50,7 @@ x-core: &core vendor-init: { condition: service_completed_successfully } control-db: { condition: service_healthy } redis: { condition: service_healthy } - networks: [control] + networks: [control, telemetry] restart: unless-stopped services: @@ -241,10 +245,14 @@ services: vector: image: timberio/vector:0.55.0-alpine + environment: + CLICKHOUSE_ENDPOINT: http://clickhouse:8123 + CLICKHOUSE_USER: cdnf + CLICKHOUSE_PASSWORD: cdnf-dev-only volumes: - ./docker/vector/vector.yaml:/etc/vector/vector.yaml:ro - vector-data:/vector-data-dir - networks: [telemetry] + networks: [telemetry, edge, dns] restart: unless-stopped prometheus: @@ -252,6 +260,7 @@ services: ports: ["9090:9090"] volumes: - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro - prometheus:/prometheus networks: [telemetry] diff --git a/compose.prod.yml b/compose.prod.yml index 5cb15b0..215c477 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -26,6 +26,10 @@ x-core-env: &core-env EDGE_IDENTITY_CA_CERTIFICATE: /run/secrets/edge-identity-ca.crt EDGE_IDENTITY_CA_PRIVATE_KEY: /run/secrets/edge-identity-ca.key EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE: ${EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE:-} + CLICKHOUSE_URL: ${CLICKHOUSE_URL:?CLICKHOUSE_URL is required} + CLICKHOUSE_DATABASE: cdnf + CLICKHOUSE_USER: cdnf + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required} x-core: &core image: ghcr.io/vaheed/cdnfoundry-core:${CDNF_RELEASE:?CDNF_RELEASE must be an immutable commit SHA} @@ -33,7 +37,7 @@ x-core: &core depends_on: control-db: { condition: service_healthy } redis: { condition: service_healthy } - networks: [control] + networks: [control, telemetry] restart: unless-stopped read_only: true tmpfs: [/tmp] @@ -206,10 +210,14 @@ services: vector: image: timberio/vector:0.55.0-alpine profiles: [telemetry, edge] + environment: + CLICKHOUSE_ENDPOINT: ${CLICKHOUSE_URL:?CLICKHOUSE_URL is required} + CLICKHOUSE_USER: cdnf + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required} volumes: - ./docker/vector/vector.yaml:/etc/vector/vector.yaml:ro - vector-data:/vector-data-dir - networks: [telemetry] + networks: [telemetry, edge, dns-private] restart: unless-stopped prometheus: @@ -217,6 +225,7 @@ services: profiles: [telemetry] volumes: - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro - prometheus:/prometheus networks: [telemetry] restart: unless-stopped diff --git a/core/app/Console/Commands/FinalizeUsage.php b/core/app/Console/Commands/FinalizeUsage.php new file mode 100644 index 0000000..486cd59 --- /dev/null +++ b/core/app/Console/Commands/FinalizeUsage.php @@ -0,0 +1,23 @@ +subHour()->startOfHour(); + BuildUsageRollups::dispatch($to->subHour()->toIso8601String(), $to->toIso8601String()); + $this->info("Queued usage interval ending {$to->toIso8601String()}."); + + return self::SUCCESS; + } +} diff --git a/core/app/Exceptions/AnalyticsUnavailableException.php b/core/app/Exceptions/AnalyticsUnavailableException.php new file mode 100644 index 0000000..b47efc4 --- /dev/null +++ b/core/app/Exceptions/AnalyticsUnavailableException.php @@ -0,0 +1,7 @@ + $to->subDay(), 'to' => $to, 'raw' => false]; + $rawRange = ['from' => $to->subHour(), 'to' => $to, 'raw' => true]; + $state = [ + 'available' => false, + 'meta' => ['from' => $range['from']->toIso8601String(), 'to' => $range['to']->toIso8601String(), 'partial' => true], + 'summary' => [], + 'traffic' => [], + 'dns' => [], + 'logs' => ['errors' => [], 'security' => [], 'edges' => []], + 'buffer' => $this->bufferStatus(), + 'usage' => $this->recentUsage(), + ]; + try { + $state['meta'] = $store->metadata($range); + $state['summary'] = $store->summary(null, $range); + $state['traffic'] = $store->aggregate(null, $range, 'traffic'); + $state['dns'] = $store->aggregate(null, $range, 'dns'); + foreach (array_keys($state['logs']) as $stream) { + $state['logs'][$stream] = array_slice($store->logs(null, $rawRange, $stream, null)['items'], 0, 10); + } + $state['available'] = true; + } catch (Throwable) { + // PostgreSQL usage and Vector health remain independently useful. + } + + return $state; + } + + private function recentUsage(): array + { + return UsageRollup::query()->with('domain:id,name')->latest('interval_start')->limit(20)->get() + ->map(fn (UsageRollup $row): array => [ + 'domain' => $row->domain?->name ?? "Domain #{$row->domain_id}", + 'interval' => $row->interval_start->toIso8601String(), + 'requests' => $row->requests, + 'bytes' => $row->bytes_in + $row->bytes_out, + 'dns_queries' => $row->dns_queries, + 'status' => $row->status, + ])->all(); + } + + private function bufferStatus(): array + { + try { + $metrics = Http::connectTimeout(1)->timeout(2)->get('http://vector:9598/metrics')->throw()->body(); + preg_match_all('/^(vector_buffer_[a-z_]+|vector_component_(?:discarded_events_total|errors_total))[^ ]* ([0-9.e+-]+)$/m', $metrics, $matches, PREG_SET_ORDER); + + return ['available' => true, 'metrics' => collect($matches)->mapWithKeys(fn (array $match): array => [$match[1] => $match[2]])->all()]; + } catch (Throwable) { + return ['available' => false, 'metrics' => []]; + } + } +} diff --git a/core/app/Filament/Domain/Pages/Analytics.php b/core/app/Filament/Domain/Pages/Analytics.php new file mode 100644 index 0000000..81c598f --- /dev/null +++ b/core/app/Filament/Domain/Pages/Analytics.php @@ -0,0 +1,87 @@ +selectedDomain(); + if ($domain === null) { + return ['domain' => null, 'available' => true, 'views' => [], 'meta' => [], 'logs' => [], 'usage' => []]; + } + $store = app(AnalyticsStore::class); + $to = CarbonImmutable::now('UTC'); + $range = ['from' => $to->subDay(), 'to' => $to, 'raw' => false]; + $rawRange = ['from' => $to->subHour(), 'to' => $to, 'raw' => true]; + try { + $summary = $store->summary($domain, $range); + $views = [ + 'Request and bandwidth timeseries' => $store->aggregate($domain, $range, 'timeseries'), + 'Status codes' => $store->aggregate($domain, $range, 'status-codes'), + 'Cache ratio' => $store->aggregate($domain, $range, 'cache'), + 'Countries and continents' => $store->aggregate($domain, $range, 'countries'), + 'Hostnames' => $store->aggregate($domain, $range, 'hostnames'), + 'Top URLs (last hour)' => $store->topUrls($domain, $rawRange), + 'Origin health and latency' => $store->aggregate($domain, $range, 'origin'), + 'Edge distribution' => $store->aggregate($domain, $range, 'edges'), + 'DNS activity' => $store->aggregate($domain, $range, 'dns'), + ]; + $logs = []; + foreach (['requests', 'dns', 'errors', 'security'] as $stream) { + $logs[$stream] = array_slice($store->logs($domain, $rawRange, $stream, null)['items'], 0, 10); + } + + return ['domain' => $domain, 'available' => true, 'meta' => $store->metadata($range), 'summary' => $summary, 'views' => $views, 'logs' => $logs, 'usage' => $this->recentUsage($domain)]; + } catch (Throwable) { + return ['domain' => $domain, 'available' => false, 'summary' => [], 'views' => [], 'logs' => [], 'usage' => $this->recentUsage($domain), 'meta' => ['from' => $range['from']->toIso8601String(), 'to' => $range['to']->toIso8601String(), 'partial' => true]]; + } + } + + private function recentUsage(Domain $domain): array + { + return UsageRollup::query()->whereBelongsTo($domain)->latest('interval_start')->limit(20)->get() + ->map(fn (UsageRollup $row): array => [ + 'interval' => $row->interval_start->toIso8601String(), + 'requests' => $row->requests, + 'bytes' => $row->bytes_in + $row->bytes_out, + 'dns_queries' => $row->dns_queries, + 'status' => $row->status, + ])->all(); + } + + public function getDomainsProperty() + { + return $this->domainQuery()->orderBy('name')->get(['domains.id', 'domains.name', 'domains.display_name']); + } + + private function selectedDomain(): ?Domain + { + $requested = request()->integer('domain'); + + return $this->domainQuery()->when($requested > 0, fn (Builder $query): Builder => $query->whereKey($requested))->orderBy('domains.id')->first(); + } + + private function domainQuery(): Builder + { + return Domain::query()->whereHas('users', fn (Builder $query): Builder => $query->where('users.id', auth()->id())); + } +} diff --git a/core/app/Http/Controllers/Admin/AnalyticsController.php b/core/app/Http/Controllers/Admin/AnalyticsController.php new file mode 100644 index 0000000..bc87501 --- /dev/null +++ b/core/app/Http/Controllers/Admin/AnalyticsController.php @@ -0,0 +1,25 @@ +range($request); + + return response()->json(['data' => $store->summary(null, $range), 'meta' => $store->metadata($range)]); + } + + public function view(Request $request, string $view, AnalyticsStore $store): JsonResponse + { + $range = $store->range($request); + + return response()->json(['data' => $store->aggregate(null, $range, $view), 'meta' => $store->metadata($range)]); + } +} diff --git a/core/app/Http/Controllers/Admin/LogController.php b/core/app/Http/Controllers/Admin/LogController.php new file mode 100644 index 0000000..850f803 --- /dev/null +++ b/core/app/Http/Controllers/Admin/LogController.php @@ -0,0 +1,19 @@ +range($request, true); + $result = $store->logs(null, $range, $stream, $request->query('cursor')); + + return response()->json(['data' => $result['items'], 'meta' => [...$store->metadata($range), 'next_cursor' => $result['next_cursor']]]); + } +} diff --git a/core/app/Http/Controllers/Admin/UsageController.php b/core/app/Http/Controllers/Admin/UsageController.php new file mode 100644 index 0000000..a3391e4 --- /dev/null +++ b/core/app/Http/Controllers/Admin/UsageController.php @@ -0,0 +1,58 @@ +range($request); + + return response()->json(['data' => $usage->query(null, $from, $to)->cursorPaginate(100), 'meta' => ['from' => $from->toIso8601String(), 'to' => $to->toIso8601String(), 'units' => ['bandwidth' => 'bytes']]]); + } + + public function export(Request $request, DomainUsageController $usage): JsonResponse|StreamedResponse + { + [$from, $to] = $usage->range($request); + $format = $request->validate(['format' => ['sometimes', 'in:json,csv']])['format'] ?? 'json'; + $query = $usage->query(null, $from, $to); + if ($format === 'json') { + return response()->json(['data' => $query->limit(10000)->get(), 'meta' => ['contract_version' => 1, 'from' => $from->toIso8601String(), 'to' => $to->toIso8601String()]]); + } + + return response()->streamDownload(function () use ($query): void { + $output = fopen('php://output', 'wb'); + fputcsv($output, ['contract_version', 'domain_id', 'interval_start', 'interval_end', 'granularity', 'requests', 'bytes_in', 'bytes_out', 'cache_hits', 'dns_queries', 'status']); + $query->lazyById(500)->each(fn ($row) => fputcsv($output, [1, $row->domain_id, $row->interval_start->toIso8601String(), $row->interval_end->toIso8601String(), $row->granularity, $row->requests, $row->bytes_in, $row->bytes_out, $row->cache_hits, $row->dns_queries, $row->status])); + fclose($output); + }, 'usage-all-domains.csv', ['Content-Type' => 'text/csv; charset=UTF-8']); + } + + public function csv(Request $request, DomainUsageController $usage): StreamedResponse + { + $request->merge(['format' => 'csv']); + + return $this->export($request, $usage); + } + + public function rebuild(Request $request): JsonResponse + { + $data = $request->validate(['domain_id' => ['sometimes', 'integer', 'exists:domains,id'], 'from' => ['required', 'date'], 'to' => ['required', 'date']]); + $from = CarbonImmutable::parse($data['from'])->utc(); + $to = CarbonImmutable::parse($data['to'])->utc(); + abort_if($from >= $to || abs($to->diffInDays($from)) > 31 || ! $from->isStartOfHour() || ! $to->isStartOfHour(), 422, 'Rebuild ranges must use complete UTC hours and span at most 31 days.'); + $operation = Operation::query()->create(['actor_id' => $request->user()->id, 'type' => 'usage.rebuild', 'status' => 'pending', 'input' => ['domain_id' => $data['domain_id'] ?? null, 'from' => $from->toIso8601String(), 'to' => $to->toIso8601String()]]); + BuildUsageRollups::dispatch($from->toIso8601String(), $to->toIso8601String(), $data['domain_id'] ?? null, $operation->id)->afterCommit(); + + return response()->json(['data' => ['operation_id' => $operation->id, 'status' => 'pending']], 202); + } +} diff --git a/core/app/Http/Controllers/AnalyticsController.php b/core/app/Http/Controllers/AnalyticsController.php new file mode 100644 index 0000000..c0e1348 --- /dev/null +++ b/core/app/Http/Controllers/AnalyticsController.php @@ -0,0 +1,30 @@ +range($request); + + return response()->json(['data' => $store->summary($domain, $range), 'meta' => $store->metadata($range)]); + } + + public function view(Request $request, Domain $domain, string $view, AnalyticsStore $store): JsonResponse + { + Gate::authorize('view', $domain); + $rawAggregate = $view === 'top-urls'; + $range = $store->range($request, $rawAggregate); + $data = $rawAggregate ? $store->topUrls($domain, $range) : $store->aggregate($domain, $range, $view); + + return response()->json(['data' => $data, 'meta' => $store->metadata($range)]); + } +} diff --git a/core/app/Http/Controllers/DomainLogController.php b/core/app/Http/Controllers/DomainLogController.php new file mode 100644 index 0000000..721e09d --- /dev/null +++ b/core/app/Http/Controllers/DomainLogController.php @@ -0,0 +1,21 @@ +range($request, true); + $result = $store->logs($domain, $range, $stream, $request->query('cursor')); + + return response()->json(['data' => $result['items'], 'meta' => [...$store->metadata($range), 'next_cursor' => $result['next_cursor']]]); + } +} diff --git a/core/app/Http/Controllers/UsageController.php b/core/app/Http/Controllers/UsageController.php new file mode 100644 index 0000000..a54e774 --- /dev/null +++ b/core/app/Http/Controllers/UsageController.php @@ -0,0 +1,65 @@ +range($request); + $page = $this->query($domain, $from, $to)->cursorPaginate(100); + + return response()->json(['data' => $page, 'meta' => ['from' => $from->toIso8601String(), 'to' => $to->toIso8601String(), 'units' => ['bandwidth' => 'bytes']]]); + } + + public function export(Request $request, Domain $domain): JsonResponse|StreamedResponse + { + Gate::authorize('view', $domain); + [$from, $to] = $this->range($request); + $format = $request->validate(['format' => ['sometimes', 'in:json,csv']])['format'] ?? 'json'; + $query = $this->query($domain, $from, $to); + if ($format === 'json') { + return response()->json(['data' => $query->limit(10000)->get(), 'meta' => ['contract_version' => 1, 'from' => $from->toIso8601String(), 'to' => $to->toIso8601String(), 'units' => ['bandwidth' => 'bytes']]]); + } + + return response()->streamDownload(function () use ($query): void { + $output = fopen('php://output', 'wb'); + fputcsv($output, ['contract_version', 'domain_id', 'interval_start', 'interval_end', 'granularity', 'requests', 'bytes_in', 'bytes_out', 'cache_hits', 'dns_queries', 'status']); + $query->lazyById(500)->each(fn (UsageRollup $row) => fputcsv($output, [1, $row->domain_id, $row->interval_start->toIso8601String(), $row->interval_end->toIso8601String(), $row->granularity, $row->requests, $row->bytes_in, $row->bytes_out, $row->cache_hits, $row->dns_queries, $row->status])); + fclose($output); + }, "usage-domain-{$domain->id}.csv", ['Content-Type' => 'text/csv; charset=UTF-8']); + } + + public function csv(Request $request, Domain $domain): StreamedResponse + { + $request->merge(['format' => 'csv']); + + return $this->export($request, $domain); + } + + public function range(Request $request): array + { + $validated = $request->validate(['from' => ['sometimes', 'date'], 'to' => ['sometimes', 'date']]); + $to = isset($validated['to']) ? CarbonImmutable::parse($validated['to'])->utc() : CarbonImmutable::now('UTC'); + $from = isset($validated['from']) ? CarbonImmutable::parse($validated['from'])->utc() : $to->subDays(30); + abort_if($from >= $to || abs($to->diffInDays($from)) > 400, 422, 'Usage ranges must be positive and no longer than 400 days.'); + + return [$from, $to]; + } + + public function query(?Domain $domain, CarbonImmutable $from, CarbonImmutable $to): Builder + { + return UsageRollup::query()->when($domain !== null, fn (Builder $query): Builder => $query->where('domain_id', $domain->id)) + ->where('interval_start', '>=', $from)->where('interval_start', '<', $to)->orderBy('id'); + } +} diff --git a/core/app/Jobs/BuildUsageRollups.php b/core/app/Jobs/BuildUsageRollups.php new file mode 100644 index 0000000..97b1c1f --- /dev/null +++ b/core/app/Jobs/BuildUsageRollups.php @@ -0,0 +1,61 @@ +onQueue('bulk_maintenance'); + } + + public function handle(AnalyticsStore $store): void + { + $operation = $this->operationId ? Operation::query()->find($this->operationId) : null; + $operation?->update(['status' => 'running', 'started_at' => now()]); + try { + $from = CarbonImmutable::parse($this->from)->utc(); + $to = CarbonImmutable::parse($this->to)->utc(); + for ($interval = $from; $interval < $to; $interval = $interval->addHour()) { + $end = $interval->addHour(); + Domain::query()->when($this->domainId !== null, fn ($query) => $query->whereKey($this->domainId)) + ->orderBy('id')->select(['id', 'name'])->chunkById(250, function ($domains) use ($store, $interval, $end): void { + $scope = $domains->pluck('name', 'id')->all(); + $totals = collect($store->usageInterval($interval, $end, $scope))->groupBy('domain_id')->map(fn ($rows): array => [ + 'requests' => $rows->sum('requests'), 'bytes_in' => $rows->sum('bytes_in'), 'bytes_out' => $rows->sum('bytes_out'), + 'cache_hits' => $rows->sum('cache_hits'), 'dns_queries' => $rows->sum('dns_queries'), + ]); + foreach (array_keys($scope) as $domainId) { + $values = $totals->get((string) $domainId, $totals->get($domainId, ['requests' => 0, 'bytes_in' => 0, 'bytes_out' => 0, 'cache_hits' => 0, 'dns_queries' => 0])); + UsageRollup::query()->updateOrCreate( + ['domain_id' => $domainId, 'interval_start' => $interval, 'granularity' => 'hour'], + [...$values, 'interval_end' => $end, 'status' => 'finalized', 'source_finalized_at' => now()], + ); + } + }); + } + $operation?->update(['status' => 'succeeded', 'finished_at' => now(), 'result' => ['from' => $this->from, 'to' => $this->to]]); + } catch (Throwable $exception) { + $operation?->update(['status' => 'failed', 'finished_at' => now(), 'error' => 'analytics_unavailable: Usage source is unavailable.']); + throw $exception; + } + } +} diff --git a/core/app/Models/Domain.php b/core/app/Models/Domain.php index cfcea93..67bf698 100644 --- a/core/app/Models/Domain.php +++ b/core/app/Models/Domain.php @@ -61,6 +61,11 @@ public function securityEvents(): HasMany return $this->hasMany(SecurityEvent::class); } + public function usageRollups(): HasMany + { + return $this->hasMany(UsageRollup::class); + } + public function activeTlsCertificate(): BelongsTo { return $this->belongsTo(TlsCertificate::class, 'active_tls_certificate_id'); diff --git a/core/app/Models/UsageRollup.php b/core/app/Models/UsageRollup.php new file mode 100644 index 0000000..b98e298 --- /dev/null +++ b/core/app/Models/UsageRollup.php @@ -0,0 +1,21 @@ +belongsTo(Domain::class); + } + + protected function casts(): array + { + return ['interval_start' => 'immutable_datetime', 'interval_end' => 'immutable_datetime', 'source_finalized_at' => 'immutable_datetime']; + } +} diff --git a/core/app/Support/AnalyticsStore.php b/core/app/Support/AnalyticsStore.php new file mode 100644 index 0000000..f622c6b --- /dev/null +++ b/core/app/Support/AnalyticsStore.php @@ -0,0 +1,198 @@ +filled('to') ? CarbonImmutable::parse($request->string('to')->toString())->utc() : CarbonImmutable::now('UTC'); + $from = $request->filled('from') ? CarbonImmutable::parse($request->string('from')->toString())->utc() : ($raw ? $to->subHour() : $to->subDay()); + } catch (Throwable) { + throw ValidationException::withMessages(['from' => 'The time range must use ISO 8601 timestamps.']); + } + $maximum = $raw ? self::RAW_MAX_HOURS * 3600 : self::AGGREGATE_MAX_DAYS * 86400; + if ($from >= $to || abs($to->diffInSeconds($from)) > $maximum || $to->isAfter(CarbonImmutable::now('UTC')->addMinute())) { + throw ValidationException::withMessages(['from' => $raw ? 'Raw-log ranges must be positive and no longer than 24 hours.' : 'Analytics ranges must be positive and no longer than 90 days.']); + } + + return ['from' => $from, 'to' => $to, 'raw' => $raw]; + } + + public function summary(?Domain $domain, array $range): array + { + $scope = $domain === null ? '1' : 'domain_id = {domain_id:UInt64}'; + $parameters = $this->parameters($range, $domain); + $edge = $this->query("SELECT sum(requests) AS requests, sum(bytes_in) AS bytes_in, sum(bytes_out) AS bytes_out, sumIf(requests, cache_status = 'HIT') AS cache_hits, sum(origin_errors) AS origin_errors, sum(tls_failures) AS tls_failures, sum(security_blocks) AS security_blocks FROM cdnf.edge_hourly WHERE {$scope} AND interval_start >= {from:DateTime64} AND interval_start < {to:DateTime64}", $parameters)[0] ?? []; + $dns = $this->query("SELECT sum(queries) AS dns_queries FROM cdnf.dns_hourly WHERE {$scope} AND interval_start >= {from:DateTime64} AND interval_start < {to:DateTime64}", $parameters)[0] ?? []; + $requests = (int) ($edge['requests'] ?? 0); + + return [...$edge, ...$dns, 'cache_ratio' => $requests === 0 ? 0 : round(((int) ($edge['cache_hits'] ?? 0)) / $requests, 6)]; + } + + public function aggregate(?Domain $domain, array $range, string $view): array + { + $scope = $domain === null ? '1' : 'domain_id = {domain_id:UInt64}'; + $parameters = $this->parameters($range, $domain); + [$select, $group, $order, $table] = match ($view) { + 'timeseries', 'traffic' => ['toStartOfHour(interval_start) AS bucket, sum(requests) AS requests, sum(bytes_in) AS bytes_in, sum(bytes_out) AS bytes_out', 'bucket', 'bucket', 'edge_hourly'], + 'status-codes' => ['status, sum(requests) AS requests', 'status', 'sum(requests) DESC, status', 'edge_hourly'], + 'cache' => ['cache_status, sum(requests) AS requests', 'cache_status', 'sum(requests) DESC, cache_status', 'edge_hourly'], + 'countries' => ['country, continent, sum(requests) AS requests, sum(bytes_out) AS bytes_out', 'country, continent', 'sum(requests) DESC, country', 'edge_hourly'], + 'hostnames' => ['hostname, sum(requests) AS requests, sum(bytes_out) AS bytes_out', 'hostname', 'sum(requests) DESC, hostname', 'edge_hourly'], + 'origin' => ['sum(origin_errors) AS errors, sum(origin_latency_sum) AS latency_sum_ms, sum(origin_latency_samples) AS latency_samples, if(latency_samples = 0, 0, latency_sum_ms / latency_samples) AS average_latency_ms', '', 'errors DESC', 'edge_hourly'], + 'edges' => ['edge_id, sum(requests) AS requests, sum(bytes_out) AS bytes_out', 'edge_id', 'sum(requests) DESC, edge_id', 'edge_hourly'], + 'dns' => ['qtype, rcode, sum(queries) AS queries', 'qtype, rcode', 'sum(queries) DESC, qtype, rcode', 'dns_hourly'], + default => throw ValidationException::withMessages(['view' => 'The analytics view is invalid.']), + }; + if ($view === 'dns' && $domain !== null) { + $scope = '(domain_id = {domain_id:UInt64} OR zone = {domain_name:String} OR endsWith(zone, concat(\'.\', {domain_name:String})))'; + } + $groupSql = $group === '' ? '' : " GROUP BY {$group}"; + + return $this->query("SELECT {$select} FROM cdnf.{$table} WHERE {$scope} AND interval_start >= {from:DateTime64} AND interval_start < {to:DateTime64}{$groupSql} ORDER BY {$order} LIMIT 1000", $parameters); + } + + public function topUrls(Domain $domain, array $range): array + { + return $this->query('SELECT path, count() AS requests, sum(bytes_out) AS bytes_out FROM cdnf.edge_events WHERE domain_id = {domain_id:UInt64} AND occurred_at >= {from:DateTime64} AND occurred_at < {to:DateTime64} GROUP BY path ORDER BY requests DESC, path LIMIT 100', $this->parameters($range, $domain)); + } + + public function logs(?Domain $domain, array $range, string $stream, ?string $cursor): array + { + $decoded = $this->decodeCursor($cursor); + $parameters = [...$this->parameters($range, $domain), 'cursor_time' => $decoded['occurred_at'] ?? '9999-12-31 23:59:59.999', 'cursor_id' => $decoded['event_id'] ?? 'ffffffff-ffff-ffff-ffff-ffffffffffff']; + $scope = $domain === null ? '1' : 'domain_id = {domain_id:UInt64}'; + $cursorSql = '(occurred_at, event_id) < ({cursor_time:DateTime64}, {cursor_id:UUID})'; + if ($stream === 'dns') { + if ($domain !== null) { + $scope = '(domain_id = {domain_id:UInt64} OR zone = {domain_name:String} OR endsWith(zone, concat(\'.\', {domain_name:String})))'; + } + $rows = $this->query("SELECT occurred_at, event_id, domain_id, zone, qname, qtype, rcode, client_ip, dns_cluster, country, continent, outcome FROM cdnf.dns_events WHERE {$scope} AND occurred_at >= {from:DateTime64} AND occurred_at < {to:DateTime64} AND {$cursorSql} ORDER BY occurred_at DESC, event_id DESC LIMIT 101", $parameters); + } else { + $filter = match ($stream) { + 'errors' => "status >= 500 OR origin_error != '' OR tls_error != ''", + 'security' => "security_action = 'block'", + 'edges' => "event_type IN ('deployment', 'health')", + 'requests' => "event_type = 'request'", + default => throw ValidationException::withMessages(['stream' => 'The log stream is invalid.']), + }; + $rows = $this->query("SELECT occurred_at, event_id, domain_id, hostname, method, path, status, bytes_in, bytes_out, cache_status, origin_latency_ms, origin_error, tls_error, security_action, security_reason, edge_id, client_ip, country, continent, event_type FROM cdnf.edge_events WHERE {$scope} AND occurred_at >= {from:DateTime64} AND occurred_at < {to:DateTime64} AND {$cursorSql} AND ({$filter}) ORDER BY occurred_at DESC, event_id DESC LIMIT 101", $parameters); + } + $hasMore = count($rows) > 100; + $rows = array_slice($rows, 0, 100); + foreach ($rows as &$row) { + if (isset($row['client_ip'])) { + $row['client_ip'] = $this->maskAddress((string) $row['client_ip']); + } + } + $last = end($rows); + + return ['items' => $rows, 'next_cursor' => $hasMore && is_array($last) ? $this->encodeCursor($last) : null]; + } + + /** @param array $domains */ + public function usageInterval(CarbonImmutable $from, CarbonImmutable $to, array $domains): array + { + if ($domains === []) { + return []; + } + $ids = array_keys($domains); + $names = array_values($domains); + $parameters = [ + 'from' => $from->format('Y-m-d H:i:s.u'), 'to' => $to->format('Y-m-d H:i:s.u'), + 'domain_ids' => '['.implode(',', $ids).']', + 'domain_names' => '['.implode(',', array_map(fn (string $name): string => "'".str_replace("'", "\\'", $name)."'", $names)).']', + ]; + + return $this->query("SELECT domain_id, sum(requests) AS requests, sum(bytes_in) AS bytes_in, sum(bytes_out) AS bytes_out, sumIf(requests, cache_status = 'HIT') AS cache_hits, 0 AS dns_queries FROM cdnf.edge_hourly WHERE domain_id IN {domain_ids:Array(UInt64)} AND interval_start >= {from:DateTime64} AND interval_start < {to:DateTime64} GROUP BY domain_id UNION ALL WITH {domain_names:Array(String)} AS names, {domain_ids:Array(UInt64)} AS ids SELECT arrayElement(ids, arrayFirstIndex(name -> zone = name OR endsWith(zone, concat('.', name)), names)) AS mapped_domain_id, 0, 0, 0, 0, sum(queries) FROM cdnf.dns_hourly WHERE arrayFirstIndex(name -> zone = name OR endsWith(zone, concat('.', name)), names) > 0 AND interval_start >= {from:DateTime64} AND interval_start < {to:DateTime64} GROUP BY mapped_domain_id", $parameters); + } + + public function metadata(array $range): array + { + $delay = app(PlatformSettings::class)->integer('telemetry', 'finalization_delay_minutes'); + $finalizedUntil = CarbonImmutable::now('UTC')->subMinutes($delay); + + return ['from' => $range['from']->toIso8601String(), 'to' => $range['to']->toIso8601String(), 'timezone' => 'UTC', 'units' => ['bandwidth' => 'bytes', 'latency' => 'milliseconds'], 'finalized_until' => $finalizedUntil->toIso8601String(), 'partial' => $range['to']->isAfter($finalizedUntil), 'sampling' => 'none']; + } + + private function query(string $sql, array $parameters): array + { + $configuration = config('services.clickhouse'); + $query = ['database' => $configuration['database'], 'default_format' => 'JSONEachRow', 'max_execution_time' => $configuration['max_execution_time'], 'max_memory_usage' => $configuration['max_memory_usage'], 'max_rows_to_read' => $configuration['max_rows_to_read'], 'max_result_rows' => $configuration['max_result_rows'], 'prefer_column_name_to_alias' => 1]; + foreach ($parameters as $key => $value) { + $query["param_{$key}"] = $value; + } + try { + $response = Http::withBasicAuth($configuration['username'], $configuration['password']) + ->connectTimeout($configuration['connect_timeout'])->timeout($configuration['timeout']) + ->withQueryParameters($query)->withBody($sql.' FORMAT JSONEachRow', 'text/plain') + ->post(rtrim($configuration['url'], '/')); + if (! $response->successful()) { + throw new AnalyticsUnavailableException; + } + $body = trim($response->body()); + + return $body === '' ? [] : collect(explode("\n", $body))->map(fn (string $line): array => json_decode($line, true, flags: JSON_THROW_ON_ERROR))->all(); + } catch (AnalyticsUnavailableException $exception) { + throw $exception; + } catch (Throwable $exception) { + throw new AnalyticsUnavailableException(previous: $exception); + } + } + + private function parameters(array $range, ?Domain $domain): array + { + return array_filter(['from' => $range['from']->format('Y-m-d H:i:s.u'), 'to' => $range['to']->format('Y-m-d H:i:s.u'), 'domain_id' => $domain?->getKey(), 'domain_name' => $domain?->name], fn ($value): bool => $value !== null); + } + + private function maskAddress(string $address): string + { + if (filter_var($address, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + $bits = app(PlatformSettings::class)->integer('telemetry', 'ipv4_mask_bits'); + $packed = inet_pton($address); + } elseif (filter_var($address, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + $bits = app(PlatformSettings::class)->integer('telemetry', 'ipv6_mask_bits'); + $packed = inet_pton($address); + } else { + return 'unknown'; + } + for ($bit = $bits; $bit < strlen($packed) * 8; $bit++) { + $byte = intdiv($bit, 8); + $packed[$byte] = chr(ord($packed[$byte]) & ~(1 << (7 - ($bit % 8)))); + } + + return inet_ntop($packed)."/{$bits}"; + } + + private function decodeCursor(?string $cursor): array + { + if ($cursor === null || $cursor === '') { + return []; + } + $decoded = json_decode(base64_decode(strtr($cursor, '-_', '+/'), true) ?: '', true); + if (! is_array($decoded) || ! isset($decoded['occurred_at'], $decoded['event_id'])) { + throw ValidationException::withMessages(['cursor' => 'The log cursor is invalid.']); + } + + return $decoded; + } + + private function encodeCursor(array $row): string + { + return rtrim(strtr(base64_encode(json_encode(['occurred_at' => $row['occurred_at'], 'event_id' => $row['event_id']], JSON_THROW_ON_ERROR)), '+/', '-_'), '='); + } +} diff --git a/core/bootstrap/app.php b/core/bootstrap/app.php index f797e45..6454797 100644 --- a/core/bootstrap/app.php +++ b/core/bootstrap/app.php @@ -1,5 +1,6 @@ validateCsrfTokens(except: ['edge/v1/*']); }) ->withExceptions(function (Exceptions $exceptions): void { + $exceptions->render(function (AnalyticsUnavailableException $exception, Request $request) { + if (! $request->is('api/*')) { + return null; + } + + return response()->json(['message' => 'Analytics are temporarily unavailable.', 'code' => 'analytics_unavailable', 'details' => (object) []], 503); + }); $exceptions->shouldRenderJsonWhen( fn (Request $request) => $request->is('api/*') || $request->is('edge/*'), ); diff --git a/core/config/platform.php b/core/config/platform.php index 43d12c8..2b16843 100644 --- a/core/config/platform.php +++ b/core/config/platform.php @@ -2,6 +2,18 @@ return [ 'groups' => [ + 'telemetry' => [ + 'label' => 'Telemetry retention and privacy', + 'description' => 'Bounded raw telemetry, aggregate retention, finalization delay, and client-address masking.', + 'fields' => [ + 'raw_retention_days' => ['type' => 'integer', 'label' => 'Raw retention (days)', 'default' => 7, 'description' => 'Maximum time raw request and DNS events remain queryable.', 'rules' => ['required', 'integer', 'between:1,30']], + 'hourly_retention_days' => ['type' => 'integer', 'label' => 'Hourly retention (days)', 'default' => 400, 'description' => 'Retention for hourly operational aggregates.', 'rules' => ['required', 'integer', 'between:30,730']], + 'daily_retention_days' => ['type' => 'integer', 'label' => 'Daily retention (days)', 'default' => 1095, 'description' => 'Retention for compact daily aggregates.', 'rules' => ['required', 'integer', 'between:365,3650']], + 'finalization_delay_minutes' => ['type' => 'integer', 'label' => 'Finalization delay (minutes)', 'default' => 15, 'description' => 'Intervals newer than this delay are labelled provisional.', 'rules' => ['required', 'integer', 'between:5,1440']], + 'ipv4_mask_bits' => ['type' => 'integer', 'label' => 'IPv4 display mask', 'default' => 24, 'description' => 'Prefix retained in normal views and exports.', 'rules' => ['required', 'integer', 'between:8,32']], + 'ipv6_mask_bits' => ['type' => 'integer', 'label' => 'IPv6 display mask', 'default' => 48, 'description' => 'Prefix retained in normal views and exports.', 'rules' => ['required', 'integer', 'between:16,64']], + ], + ], 'dns_lifecycle' => [ 'label' => 'DNS lifecycle', 'description' => 'Retention windows for disabled domains and released domain names.', diff --git a/core/config/services.php b/core/config/services.php index 0ff8a02..214fb36 100644 --- a/core/config/services.php +++ b/core/config/services.php @@ -1,6 +1,18 @@ [ + 'url' => env('CLICKHOUSE_URL', 'http://clickhouse:8123'), + 'database' => env('CLICKHOUSE_DATABASE', 'cdnf'), + 'username' => env('CLICKHOUSE_USER', 'cdnf'), + 'password' => env('CLICKHOUSE_PASSWORD', ''), + 'connect_timeout' => 1.0, + 'timeout' => 4.0, + 'max_execution_time' => 3, + 'max_memory_usage' => 134217728, + 'max_rows_to_read' => 10000000, + 'max_result_rows' => 10001, + ], 'geoip' => ['database' => env('GEOIP_DATABASE', '/mmdb/GeoLite2-City.mmdb')], 'acme' => [ 'enabled' => filter_var(env('ACME_ENABLED', false), FILTER_VALIDATE_BOOL), diff --git a/core/database/migrations/2026_07_20_000000_create_usage_rollups.php b/core/database/migrations/2026_07_20_000000_create_usage_rollups.php new file mode 100644 index 0000000..89915b6 --- /dev/null +++ b/core/database/migrations/2026_07_20_000000_create_usage_rollups.php @@ -0,0 +1,46 @@ +bigIncrements('id'); + $table->foreignId('domain_id')->constrained()->cascadeOnDelete(); + $table->timestampTz('interval_start'); + $table->timestampTz('interval_end'); + $table->string('granularity', 12)->default('hour'); + $table->unsignedBigInteger('requests')->default(0); + $table->unsignedBigInteger('bytes_in')->default(0); + $table->unsignedBigInteger('bytes_out')->default(0); + $table->unsignedBigInteger('cache_hits')->default(0); + $table->unsignedBigInteger('dns_queries')->default(0); + $table->string('status', 16)->default('finalized'); + $table->timestampTz('source_finalized_at')->nullable(); + $table->timestampsTz(); + $table->unique(['domain_id', 'interval_start', 'granularity'], 'usage_rollups_interval_unique'); + $table->index(['interval_start', 'domain_id']); + }); + DB::table('system_settings')->insertOrIgnore([ + 'group' => 'telemetry', + 'values' => json_encode(['raw_retention_days' => 7, 'hourly_retention_days' => 400, 'daily_retention_days' => 1095, 'finalization_delay_minutes' => 15, 'ipv4_mask_bits' => 24, 'ipv6_mask_bits' => 48], JSON_THROW_ON_ERROR), + 'revision' => 1, 'created_at' => now(), 'updated_at' => now(), + ]); + if (DB::getDriverName() === 'pgsql') { + DB::statement("ALTER TABLE usage_rollups ADD CONSTRAINT usage_rollups_granularity_check CHECK (granularity IN ('hour', 'day'))"); + DB::statement("ALTER TABLE usage_rollups ADD CONSTRAINT usage_rollups_status_check CHECK (status IN ('provisional', 'finalized'))"); + DB::statement('ALTER TABLE usage_rollups ADD CONSTRAINT usage_rollups_interval_check CHECK (interval_end > interval_start)'); + } + } + + public function down(): void + { + DB::table('system_settings')->where('group', 'telemetry')->delete(); + Schema::dropIfExists('usage_rollups'); + } +}; diff --git a/core/resources/views/filament/admin/pages/telemetry.blade.php b/core/resources/views/filament/admin/pages/telemetry.blade.php new file mode 100644 index 0000000..49e6e74 --- /dev/null +++ b/core/resources/views/filament/admin/pages/telemetry.blade.php @@ -0,0 +1,136 @@ + + @php + $state = $this->state; + $summary = $state['summary']; + $formatBytes = function (int|float|string|null $value): string { + $bytes = max(0, (float) ($value ?? 0)); + $units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + $index = $bytes > 0 ? min((int) floor(log($bytes, 1024)), count($units) - 1) : 0; + + return number_format($bytes / (1024 ** $index), $index === 0 ? 0 : 1) . ' ' . $units[$index]; + }; + @endphp + +
+ +
+ ClickHouse {{ $state['available'] ? 'available' : 'unavailable' }} + Vector metrics {{ $state['buffer']['available'] ? 'available' : 'unavailable' }} + {{ ($state['meta']['partial'] ?? true) ? 'Partial / provisional' : 'Finalized' }} +
+ @if (!$state['available']) +
Analytics unavailable. Traffic serving is independent and remains active. Finalized PostgreSQL usage is still shown below.
+ @endif + + + @if ($state['available']) +
+ @foreach ([ + ['label' => 'Requests', 'value' => number_format((int) ($summary['requests'] ?? 0)), 'description' => 'HTTP requests in the selected range', 'tone' => 'success'], + ['label' => 'Bandwidth', 'value' => $formatBytes(((int) ($summary['bytes_in'] ?? 0)) + ((int) ($summary['bytes_out'] ?? 0))), 'description' => 'Inbound and outbound transfer', 'tone' => 'success'], + ['label' => 'Cache hit ratio', 'value' => number_format(((float) ($summary['cache_ratio'] ?? 0)) * 100, 1) . '%', 'description' => number_format((int) ($summary['cache_hits'] ?? 0)) . ' requests served from cache', 'tone' => 'success'], + ['label' => 'DNS queries', 'value' => number_format((int) ($summary['dns_queries'] ?? 0)), 'description' => 'Authoritative queries in the range', 'tone' => 'success'], + ['label' => 'Origin errors', 'value' => number_format((int) ($summary['origin_errors'] ?? 0)), 'description' => 'Origin failures requiring review', 'tone' => ((int) ($summary['origin_errors'] ?? 0)) > 0 ? 'warning' : 'success'], + ['label' => 'Security blocks', 'value' => number_format((int) ($summary['security_blocks'] ?? 0)), 'description' => 'Bounded protection decisions', 'tone' => 'warning'], + ] as $stat) +
+
{{ $stat['label'] }}
+
{{ $stat['value'] }}
+
{{ $stat['description'] }}
+
+ @endforeach +
+ +
+ +
+ + + + @forelse ($state['traffic'] as $row) + + @empty + + @endforelse + +
UTC hourRequestsTransfer
{{ $row['bucket'] ?? 'Unknown' }}{{ number_format((int) ($row['requests'] ?? 0)) }}{{ $formatBytes(((int) ($row['bytes_in'] ?? 0)) + ((int) ($row['bytes_out'] ?? 0))) }}
No traffic was recorded in this range.
+
+ + + +
+ + + + @forelse ($state['dns'] as $row) + + @empty + + @endforelse + +
TypeResponseQueries
{{ $row['qtype'] ?? 'Unknown' }}{{ $row['rcode'] ?? 'Unknown' }}{{ number_format((int) ($row['queries'] ?? 0)) }}
No DNS activity was recorded in this range.
+
+ +
+ + +
+ @foreach ($state['logs'] as $stream => $rows) +
+
{{ str($stream)->headline() }}
+
+ @forelse ($rows as $row) +
+
+
{{ $row['hostname'] ?? $row['edge_id'] ?? ('Domain #' . ($row['domain_id'] ?? 'unknown')) }}
+
{{ $row['occurred_at'] ?? 'Unknown time' }} · {{ $row['method'] ?? $row['event_type'] ?? 'event' }} {{ $row['path'] ?? '' }} · {{ $row['security_reason'] ?? $row['origin_error'] ?? $row['tls_error'] ?? ('HTTP ' . ($row['status'] ?? '—')) }}
+
+ @if (isset($row['status'])){{ $row['status'] }}@endif +
+ @empty +
No {{ $stream }} events in the last hour.
+ @endforelse +
+
+ @endforeach +
+ + @endif + +
+ +
+ @forelse ($state['buffer']['metrics'] as $metric => $value) + @php + $problem = (str_contains($metric, 'discarded') || str_contains($metric, 'errors')) && (float) $value > 0; + @endphp +
+
{{ str($metric)->after('vector_')->replace('_', ' ')->headline() }}
{{ $metric }}
+ {{ is_numeric($value) ? number_format((float) $value, 0) : $value }} +
+ @empty +
Vector delivery metrics are unavailable.
+ @endforelse +
+ + + +
+ Global usage CSV +
+
+ + + + @forelse ($state['usage'] as $row) + + @empty + + @endforelse + +
Domain / intervalRequestsTransferDNSState
{{ $row['domain'] }}
{{ $row['interval'] }}
{{ number_format($row['requests']) }}{{ $formatBytes($row['bytes']) }}{{ number_format($row['dns_queries']) }}{{ str($row['status'])->headline() }}
No finalized usage intervals are available yet.
+
+ +
+
+ diff --git a/core/resources/views/filament/domain/pages/analytics.blade.php b/core/resources/views/filament/domain/pages/analytics.blade.php new file mode 100644 index 0000000..49b272f --- /dev/null +++ b/core/resources/views/filament/domain/pages/analytics.blade.php @@ -0,0 +1,132 @@ + + @php + $state = $this->state; + $formatBytes = function (int|float|string|null $value): string { + $bytes = max(0, (float) ($value ?? 0)); + $units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + $index = $bytes > 0 ? min((int) floor(log($bytes, 1024)), count($units) - 1) : 0; + + return number_format($bytes / (1024 ** $index), $index === 0 ? 0 : 1) . ' ' . $units[$index]; + }; + $dimensions = ['bucket', 'status', 'cache_status', 'country', 'hostname', 'path', 'edge_id', 'qtype']; + $formatMetric = function (string $key, mixed $value) use ($formatBytes): string { + if (str_starts_with($key, 'bytes')) return $formatBytes($value); + if (str_contains($key, 'latency')) return number_format((float) $value, 1) . ' ms'; + if (is_numeric($value)) return number_format((float) $value, str_contains($key, 'ratio') ? 2 : 0); + + return (string) $value; + }; + @endphp + +
+ +
+ @forelse ($this->domains as $domain) + + {{ $domain->display_name ?: $domain->name }} + + @empty +
No domains are assigned to this account.
+ @endforelse +
+ + + @if ($state['domain']) + +
+ ClickHouse {{ $state['available'] ? 'available' : 'unavailable' }} + {{ ($state['meta']['partial'] ?? true) ? 'Partial / provisional' : 'Finalized' }} +
+ @if (!$state['available']) +
Analytics unavailable. DNS and edge serving continue normally; finalized PostgreSQL usage remains available below.
+ @endif + + + @if ($state['available']) + @php + $summary = $state['summary']; + @endphp +
+ @foreach ([ + ['label' => 'Requests', 'value' => number_format((int) ($summary['requests'] ?? 0)), 'description' => 'HTTP requests in the last 24 hours', 'tone' => 'success'], + ['label' => 'Bandwidth', 'value' => $formatBytes(((int) ($summary['bytes_in'] ?? 0)) + ((int) ($summary['bytes_out'] ?? 0))), 'description' => 'Inbound and outbound transfer', 'tone' => 'success'], + ['label' => 'Cache hit ratio', 'value' => number_format(((float) ($summary['cache_ratio'] ?? 0)) * 100, 1) . '%', 'description' => number_format((int) ($summary['cache_hits'] ?? 0)) . ' cached requests', 'tone' => 'success'], + ['label' => 'DNS queries', 'value' => number_format((int) ($summary['dns_queries'] ?? 0)), 'description' => 'Authoritative queries', 'tone' => 'success'], + ['label' => 'Origin errors', 'value' => number_format((int) ($summary['origin_errors'] ?? 0)), 'description' => 'Origin failures', 'tone' => ((int) ($summary['origin_errors'] ?? 0)) > 0 ? 'warning' : 'success'], + ['label' => 'Security blocks', 'value' => number_format((int) ($summary['security_blocks'] ?? 0)), 'description' => 'Protection decisions', 'tone' => 'warning'], + ] as $stat) +
+
{{ $stat['label'] }}
+
{{ $stat['value'] }}
+
{{ $stat['description'] }}
+
+ @endforeach +
+ +
+ @foreach ($state['views'] as $label => $rows) + +
+ @forelse ($rows as $row) + @php + $dimension = collect($dimensions)->first(fn (string $key): bool => array_key_exists($key, $row)); + $title = $dimension ? ($row[$dimension] ?? 'Unknown') : 'Summary'; + if ($dimension === 'country') $title = ($row['country'] ?? 'ZZ') . ' · ' . ($row['continent'] ?? 'Unknown'); + if ($dimension === 'qtype') $title = ($row['qtype'] ?? 'Unknown') . ' · ' . ($row['rcode'] ?? 'Unknown'); + $metrics = collect($row)->except(array_filter([$dimension, 'continent', 'rcode']))->map(fn ($value, $key) => str($key)->replace('_', ' ')->headline() . ': ' . $formatMetric($key, $value))->implode(' · '); + @endphp +
+
{{ $title }}
{{ $metrics ?: 'No activity' }}
+
+ @empty +
No data was recorded for this view.
+ @endforelse +
+ + @endforeach +
+ + +
+ @foreach ($state['logs'] as $stream => $rows) +
+
{{ str($stream)->headline() }} logs
+
+ @forelse ($rows as $row) +
+
+
{{ $row['qname'] ?? (($row['method'] ?? $row['event_type'] ?? 'event') . ' ' . ($row['path'] ?? $row['hostname'] ?? '')) }}
+
{{ $row['occurred_at'] ?? 'Unknown time' }} · {{ $row['client_ip'] ?? 'unknown client' }} · {{ $row['rcode'] ?? $row['security_reason'] ?? $row['origin_error'] ?? ('HTTP ' . ($row['status'] ?? '—')) }}
+
+ @if (isset($row['status'])){{ $row['status'] }}@endif +
+ @empty +
No {{ $stream }} events in the last hour.
+ @endforelse +
+
+ @endforeach +
+ + @endif + + +
+ Usage CSV export +
+
+ + + + @forelse ($state['usage'] as $row) + + @empty + + @endforelse + +
UTC intervalRequestsTransferDNSState
{{ $row['interval'] }}{{ number_format($row['requests']) }}{{ $formatBytes($row['bytes']) }}{{ number_format($row['dns_queries']) }}{{ str($row['status'])->headline() }}
No finalized usage intervals are available yet.
+
+ + @endif +
+ diff --git a/core/routes/api.php b/core/routes/api.php index a32fa48..4cf7304 100644 --- a/core/routes/api.php +++ b/core/routes/api.php @@ -1,5 +1,6 @@ defaults('view', 'timeseries'); + Route::get('/domains/{domain}/analytics/status-codes', [AnalyticsController::class, 'view'])->defaults('view', 'status-codes'); + Route::get('/domains/{domain}/analytics/cache', [AnalyticsController::class, 'view'])->defaults('view', 'cache'); + Route::get('/domains/{domain}/analytics/countries', [AnalyticsController::class, 'view'])->defaults('view', 'countries'); + Route::get('/domains/{domain}/analytics/hostnames', [AnalyticsController::class, 'view'])->defaults('view', 'hostnames'); + Route::get('/domains/{domain}/analytics/top-urls', [AnalyticsController::class, 'view'])->defaults('view', 'top-urls'); + Route::get('/domains/{domain}/analytics/origin', [AnalyticsController::class, 'view'])->defaults('view', 'origin'); + Route::get('/domains/{domain}/analytics/edges', [AnalyticsController::class, 'view'])->defaults('view', 'edges'); + Route::get('/domains/{domain}/analytics/dns', [AnalyticsController::class, 'view'])->defaults('view', 'dns'); + Route::get('/domains/{domain}/logs/requests', [DomainLogController::class, 'index'])->defaults('stream', 'requests'); + Route::get('/domains/{domain}/logs/dns', [DomainLogController::class, 'index'])->defaults('stream', 'dns'); + Route::get('/domains/{domain}/logs/errors', [DomainLogController::class, 'index'])->defaults('stream', 'errors'); + Route::get('/domains/{domain}/logs/security', [DomainLogController::class, 'index'])->defaults('stream', 'security'); + Route::get('/domains/{domain}/usage', [UsageController::class, 'index']); + Route::get('/domains/{domain}/usage/export', [UsageController::class, 'export']); Route::prefix('admin')->middleware('admin')->group(function (): void { + Route::get('/analytics/summary', [AdminAnalyticsController::class, 'summary']); + Route::get('/analytics/traffic', [AdminAnalyticsController::class, 'view'])->defaults('view', 'traffic'); + Route::get('/analytics/dns', [AdminAnalyticsController::class, 'view'])->defaults('view', 'dns'); + Route::get('/logs/errors', [AdminLogController::class, 'index'])->defaults('stream', 'errors'); + Route::get('/logs/security', [AdminLogController::class, 'index'])->defaults('stream', 'security'); + Route::get('/logs/edges', [AdminLogController::class, 'index'])->defaults('stream', 'edges'); + Route::get('/usage', [AdminUsageController::class, 'index']); + Route::get('/usage/export', [AdminUsageController::class, 'export']); + Route::post('/usage/rebuild', [AdminUsageController::class, 'rebuild'])->middleware('idempotent'); Route::get('/users', [UserController::class, 'index']); Route::post('/users', [UserController::class, 'store'])->middleware('idempotent'); Route::get('/users/{user}', [UserController::class, 'show']); diff --git a/core/routes/console.php b/core/routes/console.php index 8add6e2..c64c5af 100644 --- a/core/routes/console.php +++ b/core/routes/console.php @@ -20,3 +20,4 @@ Schedule::job(new ReconcilePlatformDnsIdentity)->everyMinute()->withoutOverlapping(); Schedule::command('tls:dispatch-maintenance')->hourly()->withoutOverlapping(); Schedule::command('security:reconcile-readiness')->everyMinute()->withoutOverlapping(); +Schedule::command('usage:finalize')->hourlyAt(20)->withoutOverlapping(); diff --git a/core/routes/web.php b/core/routes/web.php index 9b7af81..4e36d06 100644 --- a/core/routes/web.php +++ b/core/routes/web.php @@ -1,6 +1,8 @@ middleware('api')->group(function (): void { @@ -20,3 +22,11 @@ Route::get('/', function () { return view('welcome'); }); + +Route::middleware(['auth', 'account.active'])->group(function (): void { + Route::get('/app/analytics/domains/{domain}/usage.csv', [UsageController::class, 'csv']) + ->name('app.analytics.usage.csv'); + Route::get('/admin/telemetry/usage.csv', [AdminUsageController::class, 'csv']) + ->middleware('admin') + ->name('admin.telemetry.usage.csv'); +}); diff --git a/core/tests/Feature/AnalyticsApiTest.php b/core/tests/Feature/AnalyticsApiTest.php new file mode 100644 index 0000000..e6c19a0 --- /dev/null +++ b/core/tests/Feature/AnalyticsApiTest.php @@ -0,0 +1,155 @@ +ownedDomain(); + $stranger = User::factory()->create(); + Http::fakeSequence()->push(implode("\n", [ + json_encode(['requests' => 10, 'bytes_in' => 120, 'bytes_out' => 900, 'cache_hits' => 8, 'origin_errors' => 1, 'tls_failures' => 0, 'security_blocks' => 2]), + ]))->push(json_encode(['dns_queries' => 14])); + + $this->actingAs($stranger)->getJson("/api/domains/{$domain->id}/analytics/summary")->assertForbidden(); + $response = $this->actingAs($user)->getJson("/api/domains/{$domain->id}/analytics/summary") + ->assertOk()->assertJsonPath('data.requests', 10)->assertJsonPath('data.dns_queries', 14) + ->assertJsonPath('data.cache_ratio', 0.8)->assertJsonPath('meta.units.bandwidth', 'bytes') + ->assertJsonPath('meta.partial', true); + $this->assertNotEmpty($response->json('meta.finalized_until')); + Http::assertSent(fn ($request): bool => str_contains($request->body(), 'domain_id = {domain_id:UInt64}') && str_contains($request->url(), 'param_domain_id='.$domain->id) && str_contains($request->url(), 'max_result_rows=10001')); + + $this->actingAs($user)->getJson("/api/domains/{$domain->id}/analytics/timeseries?from=2025-01-01T00:00:00Z&to=2026-01-01T00:00:00Z") + ->assertUnprocessable()->assertJsonValidationErrors('from'); + } + + public function test_raw_logs_mask_ipv4_and_ipv6_and_use_opaque_cursor_pagination(): void + { + [$user, $domain] = $this->ownedDomain(); + $rows = [ + ['occurred_at' => '2026-07-20 10:00:00.000', 'event_id' => 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', 'domain_id' => $domain->id, 'hostname' => $domain->name, 'method' => 'GET', 'path' => '/safe', 'status' => 200, 'client_ip' => '192.0.2.123'], + ['occurred_at' => '2026-07-20 09:59:00.000', 'event_id' => 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', 'domain_id' => $domain->id, 'hostname' => $domain->name, 'method' => 'GET', 'path' => '/v6', 'status' => 200, 'client_ip' => '2001:db8:1234:5678::1'], + ]; + Http::fake([config('services.clickhouse.url').'*' => Http::response(collect($rows)->map(fn ($row) => json_encode($row))->implode("\n"))]); + + $response = $this->actingAs($user)->getJson("/api/domains/{$domain->id}/logs/requests") + ->assertOk()->assertJsonPath('data.0.client_ip', '192.0.2.0/24') + ->assertJsonPath('data.1.client_ip', '2001:db8:1234::/48') + ->assertJsonPath('meta.next_cursor', null); + $this->assertSame('/safe', $response->json('data.0.path')); + Http::assertSent(fn ($request): bool => str_contains($request->body(), 'LIMIT 101') && str_contains($request->body(), 'event_type = \'request\'')); + } + + public function test_clickhouse_failure_is_explicit_and_does_not_touch_serving_state(): void + { + [$user, $domain] = $this->ownedDomain(); + $revision = $domain->revision; + Http::fake([config('services.clickhouse.url').'*' => Http::response('unavailable', 503)]); + + $this->actingAs($user)->getJson("/api/domains/{$domain->id}/analytics/summary") + ->assertStatus(503)->assertJsonPath('code', 'analytics_unavailable'); + $this->assertSame($revision, $domain->refresh()->revision); + } + + public function test_admin_global_scope_is_separate_from_domain_scope(): void + { + [$user] = $this->ownedDomain(); + $admin = User::factory()->admin()->create(); + Http::fakeSequence()->push(json_encode(['requests' => 3, 'bytes_in' => 0, 'bytes_out' => 20, 'cache_hits' => 1]))->push(json_encode(['dns_queries' => 5])); + + $this->actingAs($user)->getJson('/api/admin/analytics/summary')->assertForbidden(); + $this->actingAs($admin)->getJson('/api/admin/analytics/summary')->assertOk()->assertJsonPath('data.requests', 3); + Http::assertSent(fn ($request): bool => str_contains($request->body(), 'WHERE 1 AND')); + } + + public function test_usage_rollup_rebuild_is_idempotent_and_exports_stable_json_and_csv(): void + { + [$user, $domain] = $this->ownedDomain(); + $from = CarbonImmutable::parse('2026-07-20T08:00:00Z'); + $to = $from->addHour(); + $body = implode("\n", [json_encode(['domain_id' => $domain->id, 'requests' => 7, 'bytes_in' => 70, 'bytes_out' => 700, 'cache_hits' => 5, 'dns_queries' => 0]), json_encode(['domain_id' => $domain->id, 'requests' => 0, 'bytes_in' => 0, 'bytes_out' => 0, 'cache_hits' => 0, 'dns_queries' => 11])]); + Http::fake([config('services.clickhouse.url').'*' => Http::response($body)]); + $job = new BuildUsageRollups($from->toIso8601String(), $to->toIso8601String(), $domain->id); + $job->handle(app(AnalyticsStore::class)); + $job->handle(app(AnalyticsStore::class)); + + $this->assertSame(1, UsageRollup::query()->count()); + $this->assertDatabaseHas('usage_rollups', ['domain_id' => $domain->id, 'requests' => 7, 'bytes_out' => 700, 'cache_hits' => 5, 'dns_queries' => 11, 'status' => 'finalized']); + $query = '?'.http_build_query(['from' => $from->format('Y-m-d\TH:i:s\Z'), 'to' => $to->format('Y-m-d\TH:i:s\Z')]); + $this->actingAs($user)->getJson("/api/domains/{$domain->id}/usage/export{$query}") + ->assertOk()->assertJsonPath('meta.contract_version', 1)->assertJsonPath('data.0.dns_queries', 11); + $csv = $this->actingAs($user)->get("/api/domains/{$domain->id}/usage/export{$query}&format=csv")->assertOk(); + $content = $csv->streamedContent(); + $this->assertStringContainsString('contract_version,domain_id,interval_start', $content); + $this->assertStringContainsString(',7,70,700,5,11,finalized', $content); + } + + public function test_admin_rebuild_is_async_bounded_and_coalesced_by_idempotency_key(): void + { + Queue::fake(); + [, $domain] = $this->ownedDomain(); + $admin = User::factory()->admin()->create(); + $headers = ['Idempotency-Key' => 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa']; + $payload = ['domain_id' => $domain->id, 'from' => '2026-07-20T08:00:00Z', 'to' => '2026-07-20T10:00:00Z']; + $first = $this->actingAs($admin)->withHeaders($headers)->postJson('/api/admin/usage/rebuild', $payload)->assertAccepted(); + $this->actingAs($admin)->withHeaders($headers)->postJson('/api/admin/usage/rebuild', $payload)->assertAccepted() + ->assertJsonPath('data.operation_id', $first->json('data.operation_id')); + Queue::assertPushed(BuildUsageRollups::class, 1); + + $this->actingAs($admin)->withHeaders(['Idempotency-Key' => 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'])->postJson('/api/admin/usage/rebuild', ['from' => '2026-07-20T08:30:00Z', 'to' => '2026-07-20T10:00:00Z']) + ->assertUnprocessable(); + } + + public function test_filament_surfaces_show_scope_range_units_partial_state_and_outage(): void + { + [$user, $domain] = $this->ownedDomain(); + $admin = User::factory()->admin()->create(); + Http::fake([ + config('services.clickhouse.url').'*' => Http::response(''), + 'http://vector:9598/metrics' => Http::response("vector_buffer_byte_size 42\nvector_component_discarded_events_total 0\n"), + ]); + + $this->actingAs($user)->get("/app/analytics?domain={$domain->id}")->assertOk() + ->assertSee($domain->name)->assertSee('Partial / provisional')->assertSee('bytes')->assertSee('milliseconds') + ->assertSee('Request and bandwidth timeseries')->assertSee('DNS activity')->assertSee('Recent logs')->assertSee('Usage CSV export') + ->assertDontSee("/api/domains/{$domain->id}/logs", false); + $this->actingAs($admin)->get('/admin/telemetry')->assertOk() + ->assertSee('Global traffic')->assertSee('Vector metrics available')->assertSee('Recent logs')->assertSee('Global usage CSV') + ->assertDontSee('/api/admin/logs', false); + + } + + public function test_filament_surfaces_label_telemetry_outage_without_failing(): void + { + [$user, $domain] = $this->ownedDomain(); + $admin = User::factory()->admin()->create(); + Http::fake([config('services.clickhouse.url').'*' => Http::response('down', 503), 'http://vector:9598/metrics' => Http::response('', 503)]); + + $this->actingAs($user)->get("/app/analytics?domain={$domain->id}")->assertOk()->assertSee('Analytics unavailable')->assertSee('serving continue normally'); + $this->actingAs($admin)->get('/admin/telemetry')->assertOk()->assertSee('ClickHouse unavailable')->assertSee('Traffic serving is independent'); + } + + private function ownedDomain(): array + { + $user = User::factory()->create(); + $domain = Domain::query()->create(['name' => 'analytics.example.test', 'display_name' => 'Analytics', 'lifecycle_state' => DomainLifecycleState::Active, 'revision' => 1]); + $domain->users()->attach($user, ['created_at' => now()]); + + return [$user, $domain]; + } +} diff --git a/core/tests/Feature/SystemSettingsTest.php b/core/tests/Feature/SystemSettingsTest.php index d8ab819..fd49e46 100644 --- a/core/tests/Feature/SystemSettingsTest.php +++ b/core/tests/Feature/SystemSettingsTest.php @@ -22,12 +22,14 @@ public function test_seeded_settings_expose_current_values_defaults_and_descript $admin = User::factory()->admin()->create(); $response = $this->actingAs($admin)->getJson('/api/admin/system/settings')->assertOk(); - $this->assertCount(6, $response->json('data')); - $response->assertJsonPath('data.0.group', 'dns_lifecycle') - ->assertJsonPath('data.0.fields.0.value', 7) - ->assertJsonPath('data.0.fields.0.default', 7); - $this->assertNotEmpty($response->json('data.0.fields.0.description')); - $this->assertDatabaseCount('system_settings', 6); + $this->assertCount(7, $response->json('data')); + $settings = collect($response->json('data')); + $dnsLifecycle = $settings->firstWhere('group', 'dns_lifecycle'); + $this->assertSame(7, $dnsLifecycle['fields'][0]['value']); + $this->assertSame(7, $dnsLifecycle['fields'][0]['default']); + $this->assertNotEmpty($dnsLifecycle['fields'][0]['description']); + $this->assertNotNull($settings->firstWhere('group', 'telemetry')); + $this->assertDatabaseCount('system_settings', 7); } public function test_dns_lifecycle_update_is_typed_audited_and_reads_from_postgresql(): void diff --git a/core/tests/Feature/TelemetryBrowserExportTest.php b/core/tests/Feature/TelemetryBrowserExportTest.php new file mode 100644 index 0000000..0af5a55 --- /dev/null +++ b/core/tests/Feature/TelemetryBrowserExportTest.php @@ -0,0 +1,50 @@ +create(); + $stranger = User::factory()->create(); + $admin = User::factory()->admin()->create(); + $domain = Domain::query()->create(['name' => 'analytics.example.test', 'display_name' => 'Analytics', 'lifecycle_state' => DomainLifecycleState::Active, 'revision' => 1]); + $domain->users()->attach($user, ['created_at' => now()]); + UsageRollup::query()->create([ + 'domain_id' => $domain->id, + 'interval_start' => CarbonImmutable::parse('2026-07-20T08:00:00Z'), + 'interval_end' => CarbonImmutable::parse('2026-07-20T09:00:00Z'), + 'requests' => 7, + 'bytes_in' => 70, + 'bytes_out' => 700, + 'cache_hits' => 5, + 'dns_queries' => 11, + 'status' => 'finalized', + ]); + + $domainUrl = route('app.analytics.usage.csv', $domain, false); + $adminUrl = route('admin.telemetry.usage.csv', [], false); + $this->get($domainUrl)->assertRedirect('/'); + $this->actingAs($stranger)->get($domainUrl)->assertForbidden(); + $domainResponse = $this->actingAs($user)->get($domainUrl)->assertOk() + ->assertHeader('content-type', 'text/csv; charset=UTF-8'); + $this->assertStringContainsString("usage-domain-{$domain->id}.csv", $domainResponse->headers->get('content-disposition')); + $this->assertStringContainsString(',7,70,700,5,11,finalized', $domainResponse->streamedContent()); + + $this->actingAs($user)->get($adminUrl)->assertForbidden(); + $adminCsv = $this->actingAs($admin)->get($adminUrl)->assertOk()->streamedContent(); + $this->assertStringContainsString('contract_version,domain_id,interval_start', $adminCsv); + $this->assertStringContainsString(',7,70,700,5,11,finalized', $adminCsv); + } +} diff --git a/docker/clickhouse/init.sql b/docker/clickhouse/init.sql index d49e797..cd3ef73 100644 --- a/docker/clickhouse/init.sql +++ b/docker/clickhouse/init.sql @@ -1,2 +1,120 @@ CREATE DATABASE IF NOT EXISTS cdnf; +CREATE TABLE IF NOT EXISTS cdnf.edge_events +( + occurred_at DateTime64(3, 'UTC'), + event_id UUID DEFAULT generateUUIDv4(), + domain_id UInt64, + hostname String, + method LowCardinality(String), + path String, + status UInt16, + bytes_in UInt64, + bytes_out UInt64, + cache_status LowCardinality(String), + origin_latency_ms Nullable(UInt32), + origin_error String, + tls_error String, + security_action LowCardinality(String), + security_reason LowCardinality(String), + edge_id LowCardinality(String), + client_ip String, + country LowCardinality(String), + continent LowCardinality(String), + user_agent String, + referrer String, + event_type LowCardinality(String) DEFAULT 'request' +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(occurred_at) +ORDER BY (domain_id, occurred_at, event_id) +TTL occurred_at + INTERVAL 7 DAY DELETE +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cdnf.dns_events +( + occurred_at DateTime64(3, 'UTC'), + event_id UUID DEFAULT generateUUIDv4(), + domain_id UInt64 DEFAULT 0, + zone String, + qname String, + qtype LowCardinality(String), + rcode LowCardinality(String), + client_ip String, + dns_cluster LowCardinality(String), + country LowCardinality(String), + continent LowCardinality(String), + outcome LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(occurred_at) +ORDER BY (domain_id, zone, occurred_at, event_id) +TTL occurred_at + INTERVAL 7 DAY DELETE +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cdnf.edge_hourly +( + interval_start DateTime('UTC'), domain_id UInt64, hostname String, + status UInt16, cache_status LowCardinality(String), country LowCardinality(String), + continent LowCardinality(String), edge_id LowCardinality(String), + requests UInt64, bytes_in UInt64, bytes_out UInt64, + origin_latency_sum UInt64, origin_latency_samples UInt64, + origin_errors UInt64, tls_failures UInt64, security_blocks UInt64 +) +ENGINE = SummingMergeTree +PARTITION BY toYYYYMM(interval_start) +ORDER BY (domain_id, interval_start, hostname, status, cache_status, country, continent, edge_id) +TTL interval_start + INTERVAL 400 DAY DELETE; + +CREATE MATERIALIZED VIEW IF NOT EXISTS cdnf.edge_hourly_mv TO cdnf.edge_hourly AS +SELECT toStartOfHour(occurred_at) AS interval_start, domain_id, hostname, status, + cache_status, country, continent, edge_id, count() AS requests, + sum(bytes_in) AS bytes_in, sum(bytes_out) AS bytes_out, + sum(ifNull(origin_latency_ms, 0)) AS origin_latency_sum, + countIf(origin_latency_ms IS NOT NULL) AS origin_latency_samples, + countIf(origin_error != '') AS origin_errors, + countIf(tls_error != '') AS tls_failures, + countIf(security_action = 'block') AS security_blocks +FROM cdnf.edge_events +GROUP BY interval_start, domain_id, hostname, status, cache_status, country, continent, edge_id; + +CREATE TABLE IF NOT EXISTS cdnf.dns_hourly +( + interval_start DateTime('UTC'), domain_id UInt64, zone String, + qtype LowCardinality(String), rcode LowCardinality(String), + country LowCardinality(String), continent LowCardinality(String), + dns_cluster LowCardinality(String), queries UInt64 +) +ENGINE = SummingMergeTree +PARTITION BY toYYYYMM(interval_start) +ORDER BY (domain_id, zone, interval_start, qtype, rcode, country, continent, dns_cluster) +TTL interval_start + INTERVAL 400 DAY DELETE; + +CREATE MATERIALIZED VIEW IF NOT EXISTS cdnf.dns_hourly_mv TO cdnf.dns_hourly AS +SELECT toStartOfHour(occurred_at) AS interval_start, domain_id, zone, qtype, rcode, + country, continent, dns_cluster, count() AS queries +FROM cdnf.dns_events +GROUP BY interval_start, domain_id, zone, qtype, rcode, country, continent, dns_cluster; + +CREATE TABLE IF NOT EXISTS cdnf.edge_daily AS cdnf.edge_hourly +ENGINE = SummingMergeTree PARTITION BY toYYYYMM(interval_start) +ORDER BY (domain_id, interval_start, hostname, status, cache_status, country, continent, edge_id) +TTL interval_start + INTERVAL 3 YEAR DELETE; + +CREATE MATERIALIZED VIEW IF NOT EXISTS cdnf.edge_daily_mv TO cdnf.edge_daily AS +SELECT toStartOfDay(interval_start) AS interval_start, domain_id, hostname, status, cache_status, + country, continent, edge_id, sum(requests) AS requests, sum(bytes_in) AS bytes_in, + sum(bytes_out) AS bytes_out, sum(origin_latency_sum) AS origin_latency_sum, + sum(origin_latency_samples) AS origin_latency_samples, sum(origin_errors) AS origin_errors, + sum(tls_failures) AS tls_failures, sum(security_blocks) AS security_blocks +FROM cdnf.edge_hourly GROUP BY interval_start, domain_id, hostname, status, cache_status, country, continent, edge_id; + +CREATE TABLE IF NOT EXISTS cdnf.dns_daily AS cdnf.dns_hourly +ENGINE = SummingMergeTree PARTITION BY toYYYYMM(interval_start) +ORDER BY (domain_id, zone, interval_start, qtype, rcode, country, continent, dns_cluster) +TTL interval_start + INTERVAL 3 YEAR DELETE; + +CREATE MATERIALIZED VIEW IF NOT EXISTS cdnf.dns_daily_mv TO cdnf.dns_daily AS +SELECT toStartOfDay(interval_start) AS interval_start, domain_id, zone, qtype, rcode, + country, continent, dns_cluster, sum(queries) AS queries +FROM cdnf.dns_hourly GROUP BY interval_start, domain_id, zone, qtype, rcode, country, continent, dns_cluster; diff --git a/docker/dnsdist/dnsdist.conf b/docker/dnsdist/dnsdist.conf index 09b968e..fead0d4 100644 --- a/docker/dnsdist/dnsdist.conf +++ b/docker/dnsdist/dnsdist.conf @@ -9,3 +9,21 @@ end getAddressInfo('pdns-auth', addResolvedBackends) setServerPolicy(firstAvailable) + +-- Telemetry is best-effort and bounded inside dnsdist/libfstrm. A disconnected +-- Vector endpoint never participates in query processing or backend selection. +function addDnstapLogger(hostname, addresses) + for _, address in ipairs(addresses) do + dnstapLogger = newFrameStreamTcpLogger(address:toString() .. ':6000', { + bufferHint=8192, + flushTimeout=1, + inputQueueSize=1024, + outputQueueSize=1024, + queueNotifyThreshold=32, + reopenInterval=1 + }) + addResponseAction(AllRule(), DnstapLogResponseAction('cdnfoundry-dnsdist', dnstapLogger)) + return + end +end +getAddressInfo('vector', addDnstapLogger) diff --git a/docker/nginx/edge-runtime.conf b/docker/nginx/edge-runtime.conf index 5fd5609..4f6f655 100644 --- a/docker/nginx/edge-runtime.conf +++ b/docker/nginx/edge-runtime.conf @@ -63,6 +63,12 @@ server { set $cdn_cache_stale "0"; set $cdn_cache_respect_origin "1"; set $cdn_security_reason ""; + set $cdn_security_action "allow"; + set $cdn_domain_id "0"; + set $cdn_edge_id "unknown"; + set $cdn_client_ip ""; + set $cdn_country "ZZ"; + set $cdn_continent "ZZ"; set $cdn_original_host ""; error_page 405 = @invalid_method; diff --git a/docker/nginx/openresty.conf b/docker/nginx/openresty.conf index 2a71fa0..b3a120d 100644 --- a/docker/nginx/openresty.conf +++ b/docker/nginx/openresty.conf @@ -19,8 +19,9 @@ http { proxy_cache_path /var/cache/nginx/content levels=1:2 keys_zone=customer_content:10m max_size=192m inactive=1h use_temp_path=off; include /usr/local/openresty/nginx/conf/mime.types; default_type application/octet-stream; - log_format edge_json escape=json '{"time":"$time_iso8601","host":"$host","method":"$request_method","uri":"$request_uri","status":$status,"bytes":$body_bytes_sent,"cache_status":"$upstream_cache_status","upstream_status":"$upstream_status","request_time":$request_time,"security_reason":"$cdn_security_reason"}'; + log_format edge_json escape=json '{"occurred_at":"$time_iso8601","domain_id":$cdn_domain_id,"hostname":"$host","method":"$request_method","path":"$uri","status":$status,"bytes_in":$request_length,"bytes_out":$body_bytes_sent,"cache_status":"$upstream_cache_status","origin_latency_ms":null,"origin_error":"$upstream_status","tls_error":"","security_action":"$cdn_security_action","security_reason":"$cdn_security_reason","edge_id":"$cdn_edge_id","client_ip":"$cdn_client_ip","country":"$cdn_country","continent":"$cdn_continent","user_agent":"$http_user_agent","referrer":"$http_referer","event_type":"request"}'; access_log /dev/stdout edge_json; + access_log syslog:server=vector:9000,facility=local7,tag=cdnfoundry_edge,nohostname edge_json; sendfile on; include /etc/nginx/conf.d/*.conf; } diff --git a/docker/openresty/runtime.lua b/docker/openresty/runtime.lua index 3c41373..2d4fe22 100644 --- a/docker/openresty/runtime.lua +++ b/docker/openresty/runtime.lua @@ -27,6 +27,7 @@ local function security_reject(status, reason) end ngx.header["X-CDNFoundry-Security-Reason"] = reason ngx.var.cdn_security_reason = reason + ngx.var.cdn_security_action = "block" return ngx.exit(status) end @@ -356,6 +357,8 @@ function M.access() local config = state.hosts[host] if not config then return security_reject(421, "unknown_host") end ngx.var.cdn_original_host = host + ngx.var.cdn_domain_id = tostring(config.domain_id or 0) + ngx.var.cdn_edge_id = os.getenv("EDGE_CELL_NAME") or "unknown" ngx.ctx.security_domain = config.domain_id or config.domain ngx.ctx.security_hostname = host if config.settings and config.settings.enabled == false then return reject(503) end @@ -377,6 +380,9 @@ function M.access() if size > (tonumber(limits.maximum_request_body_size) or 16777216) then return security_reject(413, "body_too_large") end local client = client_address(security) local country, continent = geography(client) + ngx.var.cdn_client_ip = client + ngx.var.cdn_country = country or "ZZ" + ngx.var.cdn_continent = continent or "ZZ" for _, rule in ipairs(security.rules or {}) do if rule_matches(rule, client, country, continent) then if rule.action == "block" then return security_reject(403, "domain_restricted") end diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 32bd2d5..35864d5 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -1,5 +1,11 @@ global: scrape_interval: 15s +rule_files: + - /etc/prometheus/telemetry-alerts.yml +alerting: + alertmanagers: + - static_configs: + - targets: [alertmanager:9093] scrape_configs: - job_name: vector static_configs: @@ -7,4 +13,3 @@ scrape_configs: - job_name: alertmanager static_configs: - targets: [alertmanager:9093] - diff --git a/docker/prometheus/telemetry-alerts.yml b/docker/prometheus/telemetry-alerts.yml new file mode 100644 index 0000000..594e815 --- /dev/null +++ b/docker/prometheus/telemetry-alerts.yml @@ -0,0 +1,27 @@ +groups: + - name: cdnfoundry-telemetry + rules: + - alert: TelemetryEventsDropped + expr: increase(vector_component_discarded_events_total[5m]) > 0 + for: 1m + labels: { severity: warning } + annotations: + summary: Vector dropped telemetry after a bounded buffer or transform limit + - alert: TelemetryDeliveryFailures + expr: increase(vector_component_errors_total[5m]) > 0 + for: 2m + labels: { severity: warning } + annotations: + summary: Vector is failing to deliver telemetry + - alert: TelemetryBufferNearLimit + expr: vector_buffer_byte_size > 858993459 + for: 5m + labels: { severity: warning } + annotations: + summary: A Vector telemetry buffer is above 80 percent of its 1 GiB limit + - alert: TelemetryCollectorUnavailable + expr: up{job="vector"} == 0 + for: 2m + labels: { severity: warning } + annotations: + summary: Prometheus cannot scrape the Vector telemetry collector diff --git a/docker/vector/vector.yaml b/docker/vector/vector.yaml index 6f98876..8055ffb 100644 --- a/docker/vector/vector.yaml +++ b/docker/vector/vector.yaml @@ -1,10 +1,131 @@ data_dir: /vector-data-dir + sources: + edge_syslog: + type: socket + address: 0.0.0.0:9000 + mode: udp + decoding: { codec: bytes } + edge_events: + type: http_server + address: 0.0.0.0:8686 + encoding: json + dns_events: + type: http_server + address: 0.0.0.0:8687 + encoding: json + dns_dnstap: + type: dnstap + address: 0.0.0.0:6000 + mode: tcp + connection_limit: 32 + max_frame_length: 102400 + multithreaded: true internal_metrics: type: internal_metrics + +transforms: + decoded_edge_syslog: + type: remap + inputs: [edge_syslog] + drop_on_error: true + source: | + envelope = parse_regex!(to_string!(.message), r'(?P\{.*\})$') + . = parse_json!(envelope.payload) + safe_edge_events: + type: remap + inputs: [edge_events, decoded_edge_syslog] + drop_on_error: true + source: | + event_time = parse_timestamp(to_string(.occurred_at) ?? "", format: "%+") ?? now() + .occurred_at = format_timestamp!(event_time, format: "%Y-%m-%d %H:%M:%S%.3f") + .domain_id = to_int!(.domain_id) + .hostname = slice!(replace(to_string(.hostname) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 253) + .method = slice!(upcase(to_string(.method) ?? "GET"), 0, 16) + raw_path = split(to_string(.path) ?? "/", "?")[0] + .path = slice!(replace!(raw_path, r'[\x00-\x1f\x7f]', ""), 0, 2048) + .status = to_int(.status) ?? 0 + .bytes_in = to_int(.bytes_in) ?? 0 + .bytes_out = to_int(.bytes_out) ?? 0 + .cache_status = slice!(to_string(.cache_status) ?? "unknown", 0, 24) + .origin_latency_ms = to_int(.origin_latency_ms) ?? null + .origin_error = slice!(replace(to_string(.origin_error) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 256) + .tls_error = slice!(replace(to_string(.tls_error) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 256) + .security_action = slice!(to_string(.security_action) ?? "allow", 0, 24) + .security_reason = slice!(to_string(.security_reason) ?? "", 0, 64) + .edge_id = slice!(to_string(.edge_id) ?? "unknown", 0, 64) + .client_ip = slice!(to_string(.client_ip) ?? "", 0, 45) + .country = slice!(upcase(to_string(.country) ?? "ZZ"), 0, 2) + .continent = slice!(upcase(to_string(.continent) ?? "ZZ"), 0, 2) + .user_agent = slice!(replace(to_string(.user_agent) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 256) + .referrer = slice!(replace(to_string(.referrer) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 512) + .event_type = slice!(to_string(.event_type) ?? "request", 0, 32) + del(.authorization); del(.cookie); del(.cookies); del(.request_body); del(.body); del(.query) + safe_dns_events: + type: remap + inputs: [dns_events, decoded_dns_dnstap] + drop_on_error: true + source: | + event_time = parse_timestamp(to_string(.occurred_at) ?? "", format: "%+") ?? now() + .occurred_at = format_timestamp!(event_time, format: "%Y-%m-%d %H:%M:%S%.3f") + .domain_id = to_int(.domain_id) ?? 0 + .zone = slice!(replace(to_string(.zone) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 253) + .qname = slice!(replace(to_string(.qname) ?? "", r'[\x00-\x1f\x7f]', ""), 0, 253) + .qtype = slice!(upcase(to_string(.qtype) ?? "UNKNOWN"), 0, 16) + .rcode = slice!(upcase(to_string(.rcode) ?? "UNKNOWN"), 0, 16) + .client_ip = slice!(to_string(.client_ip) ?? "", 0, 45) + .dns_cluster = slice!(to_string(.dns_cluster) ?? "unknown", 0, 64) + .country = slice!(upcase(to_string(.country) ?? "ZZ"), 0, 2) + .continent = slice!(upcase(to_string(.continent) ?? "ZZ"), 0, 2) + .outcome = slice!(to_string(.outcome) ?? "answered", 0, 32) + del(.authorization); del(.cookie); del(.request_body); del(.body) + decoded_dns_dnstap: + type: remap + inputs: [dns_dnstap] + drop_on_error: true + source: | + if .messageType != "ClientResponse" && .messageType != "ResolverResponse" && .messageType != "AuthResponse" { + abort + } + question = .responseData.question[0] + . = { + "occurred_at": .timestamp, + "domain_id": 0, + "zone": replace(to_string!(question.domainName), r'\.$', ""), + "qname": replace(to_string!(question.domainName), r'\.$', ""), + "qtype": to_string!(question.questionType), + "rcode": to_string(.responseData.rcodeName) ?? "UNKNOWN", + "client_ip": to_string(.sourceAddress) ?? "", + "dns_cluster": to_string(.serverId) ?? "dnsdist", + "country": "ZZ", + "continent": "ZZ", + "outcome": "answered" + } + sinks: + clickhouse_edge: + type: clickhouse + inputs: [safe_edge_events] + endpoint: "${CLICKHOUSE_ENDPOINT}" + database: cdnf + table: edge_events + auth: { strategy: basic, user: "${CLICKHOUSE_USER}", password: "${CLICKHOUSE_PASSWORD}" } + compression: gzip + batch: { max_events: 1000, timeout_secs: 2 } + buffer: { type: disk, max_size: 1073741824, when_full: drop_newest } + request: { retry_initial_backoff_secs: 1, retry_max_duration_secs: 300, timeout_secs: 10 } + clickhouse_dns: + type: clickhouse + inputs: [safe_dns_events] + endpoint: "${CLICKHOUSE_ENDPOINT}" + database: cdnf + table: dns_events + auth: { strategy: basic, user: "${CLICKHOUSE_USER}", password: "${CLICKHOUSE_PASSWORD}" } + compression: gzip + batch: { max_events: 1000, timeout_secs: 2 } + buffer: { type: disk, max_size: 1073741824, when_full: drop_newest } + request: { retry_initial_backoff_secs: 1, retry_max_duration_secs: 300, timeout_secs: 10 } prometheus: type: prometheus_exporter inputs: [internal_metrics] address: 0.0.0.0:9598 - diff --git a/docs/analytics-reference.md b/docs/analytics-reference.md new file mode 100644 index 0000000..3ebb93b --- /dev/null +++ b/docs/analytics-reference.md @@ -0,0 +1,36 @@ +# Analytics fields and units + +All analytics timestamps and buckets are UTC. Aggregate endpoints accept `from` +and `to` ISO 8601 timestamps, default to the previous 24 hours, and reject +ranges longer than 90 days. `top-urls` reads short-retention raw data and is +limited to 24 hours. Responses include `meta.from`, `meta.to`, +`meta.finalized_until`, `meta.partial`, `meta.sampling`, and explicit units. + +| Field | Meaning | Unit | +|---|---|---| +| `requests` | HTTP request count | count | +| `bytes_in`, `bytes_out` | Request and response payload transferred | bytes | +| `cache_hits`, `cache_ratio` | Cache hits and hits divided by requests | count, ratio `0..1` | +| `status` | HTTP response status | code | +| `origin_latency_sum`, `average_latency_ms` | Observed origin latency | milliseconds | +| `origin_errors`, `tls_failures`, `security_blocks` | Failure or block counts | count | +| `dns_queries`, `queries` | DNS response count | count | +| `qtype`, `rcode` | DNS question type and response code | label | +| `country`, `continent` | ISO country and continent vocabulary; `ZZ` is unknown | label | +| `hostname`, `path`, `edge_id`, `dns_cluster` | Bounded dimensions | label | + +Domain routes are policy-scoped to the route-bound domain. Administrator routes +use global scope. Summaries use hourly materialized aggregates. Result sizes, +rows read, memory, execution time, and HTTP duration are independently bounded. +`analytics_unavailable` with HTTP 503 means ClickHouse could not answer; it does +not describe DNS or edge serving health. + +No sampling is currently used. The most recent finalization-delay window is +marked partial because late Vector delivery can still change its aggregate. + +The Filament pages render these values as stat cards, bounded tables, and up to +10 masked rows from each one-hour raw-log stream; they do not expose API JSON as +the operator interface. Finalized PostgreSQL usage remains visible when +ClickHouse is unavailable. API routes continue to require Sanctum credentials, +while browser CSV buttons use the current authenticated panel session and the +same domain policy or administrator check. diff --git a/docs/architecture.md b/docs/architecture.md index 7501f5f..3548fbc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -215,6 +215,16 @@ This is application and origin protection, not upstream volumetric scrubbing. Physical circuit saturation must be mitigated by the transit or hosting provider before traffic reaches the edge. +## Telemetry and analytics path + +DNSdist streams response DNSTap and OpenResty streams fixed-schema request logs +to Vector. Vector redacts and bounds fields, buffers each ClickHouse sink on at +most 1 GiB of local disk, and writes raw edge/DNS tables directly. ClickHouse +materialized views build hourly and daily aggregates. Laravel only issues +policy-scoped, time- and resource-bounded queries; scheduled jobs copy compact +finalized hourly usage rows into PostgreSQL for export. Valkey and PostgreSQL +never carry raw traffic logs, and telemetry failure never changes serving. + ## Administrator change to active runtime An interactive request stores intent; it does not wait for PowerDNS or edges. diff --git a/docs/clickhouse-outage-runbook.md b/docs/clickhouse-outage-runbook.md new file mode 100644 index 0000000..bfca6ee --- /dev/null +++ b/docs/clickhouse-outage-runbook.md @@ -0,0 +1,27 @@ +# ClickHouse outage runbook + +ClickHouse is derived telemetry storage and is never on the DNS, HTTP, TLS, +cache, or security decision path. During an outage analytics APIs return HTTP +503 with `analytics_unavailable`; the Filament pages visibly label the outage. + +1. Confirm serving independently with DNSdist UDP and TCP queries and an HTTP or + HTTPS request to an already active domain. Do not restart serving components. +2. Check `docker compose ps clickhouse vector` and Prometheus targets. Inspect + `vector_buffer_byte_size`, `vector_component_errors_total`, and + `vector_component_discarded_events_total`. +3. Restore ClickHouse storage/network access without deleting its named volume. + Validate `SELECT 1`, table TTLs, and materialized views before broader work. +4. Watch buffer bytes decrease and successful sink events increase. If a sink + does not resume after ClickHouse is healthy, restart only Vector; its disk + buffer survives the restart. Confirm a uniquely generated pre-recovery event + appears. Live traffic must remain responsive while backlog drains. +5. Record the exact outage and drop window. If discarded events increased, mark + the interval incomplete; do not manufacture usage. Rebuild retained complete + UTC hours through `POST /api/admin/usage/rebuild` and verify the operation. +6. If delivery still fails, validate Vector configuration and ClickHouse + credentials/schema. Preserve the last serving state and the Vector data + volume; never use `down -v`, truncate PostgreSQL, or refresh migrations. + +The automated rehearsal is `python3 tests/e2e/phase7_analytics.py`. It stops and +starts only ClickHouse, proves DNS/edge responses continue, queues a unique +event while unavailable, and waits for that event after recovery. diff --git a/docs/log-schema.md b/docs/log-schema.md new file mode 100644 index 0000000..86582cb --- /dev/null +++ b/docs/log-schema.md @@ -0,0 +1,31 @@ +# Telemetry log schema + +Vector is the only collector. DNSdist emits DNSTap responses and OpenResty emits +structured request events directly to Vector; Vector validates a fixed schema +and writes ClickHouse. Laravel, PostgreSQL, and Valkey never ingest raw traffic. + +## Edge events + +`occurred_at`, `event_id`, `domain_id`, `hostname`, `method`, `path`, `status`, +`bytes_in`, `bytes_out`, `cache_status`, nullable `origin_latency_ms`, +`origin_error`, `tls_error`, `security_action`, `security_reason`, `edge_id`, +`client_ip`, `country`, `continent`, bounded `user_agent`, bounded `referrer`, +and `event_type`. + +`event_type=request` is normal traffic. Deployment/health values are reserved +for operational edge events. Error logs select status 5xx, origin errors, or TLS +errors; security logs select blocked events. Paths exclude query strings. + +## DNS events + +`occurred_at`, `event_id`, `domain_id`, `zone`, `qname`, `qtype`, `rcode`, +`client_ip`, `dns_cluster`, `country`, `continent`, and `outcome`. + +DNSTap does not carry the Laravel domain ID. Those rows use `domain_id=0`; domain +queries match the exact zone/name suffix after policy authorization. API-supplied +structured DNS events may include the durable domain ID. + +Raw endpoints allow at most 24 hours and return at most 100 items. Pagination is +newest-first with an opaque `meta.next_cursor`. Normal API and UI results mask +IPv4 to `/24` and IPv6 to `/48`; ClickHouse raw storage retains the source +address only for the short raw-retention interval. diff --git a/docs/manual-browser-qualification.md b/docs/manual-browser-qualification.md index 688f41b..47106e1 100644 --- a/docs/manual-browser-qualification.md +++ b/docs/manual-browser-qualification.md @@ -532,6 +532,82 @@ path. - Manual browser/real-host qualification: owner-run; **not executed and Phase 6 is not release-qualified until every checkpoint above is recorded as passed**. +## Phase 7 — Logs, analytics, and usage export + +Before opening the browser, choose one active proxied disposable domain assigned +to `user@example.test`. Generate at least: two cacheable HTTP requests producing +MISS then HIT, one controlled 5xx origin response, one blocked security request, +one IPv4 and one IPv6 request, and DNSdist UDP/TCP A/AAAA queries. Wait at least +two Vector batch intervals. Record exact UTC generation times and byte counts. + +### Domain analytics + +1. Sign in at `/app`, open **Analytics**, and select the assigned domain button. + Confirm no unassigned domain button or data appears. Directly request + `/app/analytics?domain=` and confirm it cannot reveal that + domain. +2. Confirm the heading names the selected domain and visibly states the exact + UTC range, `bytes`, `milliseconds`, and `no sampling`. The newest interval + must show **Partial / provisional**, not silently appear finalized. +3. Inspect the six summary cards, **Request and bandwidth timeseries**, **Status + codes**, **Cache ratio**, **Countries and continents**, **Hostnames**, **Top URLs**, + **Origin health and latency**, **Edge distribution**, and **DNS activity**. + Match request/DNS counts, bytes, status, MISS/HIT, hostname, edge, origin + failure, and security block to the generated evidence. Unknown geography must + be labelled `ZZ`, never guessed. +4. Inspect the **Recent logs** previews for **Requests**, **DNS**, **Errors**, and + **Security**. Confirm each preview is limited to at most 10 rows from the + selected domain and one-hour raw range. Verify IPv4 renders + as its `/24`, IPv6 as its `/48`, paths contain no query string, and no + authorization header, cookie, token, request body, or private key appears. +5. Open **Usage CSV export**. Confirm the header exactly matches + `usage-export-contract.md`, timestamps are UTC, bandwidth is bytes, and the + domain ID is the selected domain. Save the file and its checksum for the + rebuild comparison. +6. At a narrow mobile width, repeat domain selection and inspect every panel and + log/export button. Content may scroll within its bounded preview but must not + overlap navigation or hide scope/range/unit/partial labels. + +### Administrator telemetry + +1. Sign in at `/admin`, open **Telemetry**, and confirm **ClickHouse available**, + **Vector metrics available**, and the current partial/finalized label plus + exact UTC range and units. +2. Match the global summary cards, **Global traffic**, and **Global DNS** to the + domain evidence plus known other traffic. Confirm **Vector buffer and delivery + metrics** shows bounded buffer/error/drop metrics, not customer secrets. +3. Inspect the **Recent logs** previews for **Errors**, **Security**, and + **Edges** and confirm each has at most 10 masked rows from the last hour. + Sign back in as the domain user and directly request `/admin/telemetry` and + `/admin/telemetry/usage.csv`; both must be forbidden. Confirm no page button + navigates to a token-protected `/api/admin/...` URL. +4. Inspect the latest 20 **Finalized usage** rows and open **Global usage CSV**. + Rebuild the generated + complete UTC interval through the documented administrator API/action using + one `Idempotency-Key`; replay it and record the same operation/result. Export + again and confirm the selected domain row and contract version are unchanged. +5. Stop only ClickHouse with `docker compose -f compose.dev.yml stop clickhouse`. + Refresh both analytics pages: each must render a clear analytics-unavailable + message while its panel/navigation stays usable. During the interruption, + repeat DNSdist UDP/TCP and edge HTTP/HTTPS requests and record continued + responses. +6. Start ClickHouse with `docker compose -f compose.dev.yml start clickhouse`. + Confirm the availability labels recover, Vector buffer bytes drain, and a + uniquely generated outage request eventually appears. Any discarded-event + increase must be recorded as a telemetry-loss interval, not treated as exact + usage. Repeat at a narrow mobile width. + +### Phase 7 completion gate + +- Implementation: present for direct bounded telemetry, ClickHouse raw and + aggregate storage, scoped APIs/UI, and idempotent PostgreSQL usage exports. +- Documentation: present in the analytics, log schema, retention/privacy, + export-contract, outage-runbook, and qualification documents. +- Automated/runtime qualification: agent-owned evidence is recorded in + `docs/phase-7-qualification.md`. +- Manual browser qualification: owner-run; **not executed and Phase 7 is not + release-qualified until every checkpoint above is recorded as passed**. + ## Record the result For each phase record: date/operator, commit SHA, browser/version, desktop/mobile viewports, exact domain and edge addresses, every checkpoint as pass/fail/not-ready, operation IDs, revisions, screenshots, relevant logs, and any deviations from the example values. Also record Horizon, PowerAdmin, DNSdist UDP/TCP, Prometheus, Alertmanager, and edge results where applicable. diff --git a/docs/openapi.json b/docs/openapi.json index a952330..ea3ba2e 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,6 +10,78 @@ } ], "paths": { + "/admin/analytics/dns": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/analytics/summary": { + "get": { + "operationId": "analytics.controller.summary", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/analytics/traffic": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, "/admin/audit-logs": { "get": { "operationId": "audit.log.controller", @@ -2771,6 +2843,93 @@ } } }, + "/admin/logs/edges": { + "get": { + "operationId": "log.controller.index", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/logs/errors": { + "get": { + "operationId": "log.controller.index", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/logs/security": { + "get": { + "operationId": "log.controller.index", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, "/admin/operations": { "get": { "operationId": "operation.controller.index", @@ -3136,6 +3295,7 @@ "schema": { "type": "string", "enum": [ + "telemetry", "dns_lifecycle", "revision_history", "rate_limits", @@ -3176,6 +3336,7 @@ "schema": { "type": "string", "enum": [ + "telemetry", "dns_lifecycle", "revision_history", "rate_limits", @@ -3253,6 +3414,115 @@ } } }, + "/admin/usage": { + "get": { + "operationId": "usage.controller.index", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/usage/export": { + "get": { + "operationId": "usage.controller.export", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/usage/rebuild": { + "post": { + "operationId": "usage.controller.rebuild", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "maxProperties": 100, + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + } + }, "/admin/users": { "get": { "operationId": "user.controller.index", @@ -4046,9 +4316,9 @@ } } }, - "/domains/{domain}/cache": { + "/domains/{domain}/analytics/cache": { "get": { - "operationId": "cache.controller.show", + "operationId": "analytics.controller.view", "tags": [ "Account" ], @@ -4079,10 +4349,360 @@ "$ref": "#/components/responses/StableError" } } - }, - "patch": { - "operationId": "cache.controller.update", - "tags": [ + } + }, + "/domains/{domain}/analytics/countries": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/dns": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/edges": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/hostnames": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/origin": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/status-codes": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/summary": { + "get": { + "operationId": "analytics.controller.summary", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/timeseries": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/analytics/top-urls": { + "get": { + "operationId": "analytics.controller.view", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/cache": { + "get": { + "operationId": "cache.controller.show", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + }, + "patch": { + "operationId": "cache.controller.update", + "tags": [ "Account" ], "security": [ @@ -5535,6 +6155,158 @@ } } }, + "/domains/{domain}/logs/dns": { + "get": { + "operationId": "domain.log.controller.index", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + }, + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/logs/errors": { + "get": { + "operationId": "domain.log.controller.index", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + }, + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/logs/requests": { + "get": { + "operationId": "domain.log.controller.index", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + }, + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/logs/security": { + "get": { + "operationId": "domain.log.controller.index", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + }, + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, "/domains/{domain}/proxy": { "get": { "operationId": "proxy.controller.show", @@ -6750,6 +7522,79 @@ } } }, + "/domains/{domain}/usage": { + "get": { + "operationId": "usage.controller.index", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + }, + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/domains/{domain}/usage/export": { + "get": { + "operationId": "usage.controller.export", + "tags": [ + "Account" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "minimum": 1 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, "/domains/{domain}/verify-nameservers": { "post": { "operationId": "domain.lifecycle.controller.verify", diff --git a/docs/phase-7-qualification.md b/docs/phase-7-qualification.md new file mode 100644 index 0000000..24afeb1 --- /dev/null +++ b/docs/phase-7-qualification.md @@ -0,0 +1,40 @@ +# Phase 7 qualification record + +Automated qualification was run on 2026-07-20 against the persistent local +Compose development stack without deleting volumes. + +## Passed evidence + +- Isolated SQLite feature suite: Phase 7 analytics API/UI tests cover policies, + bounds, masking, explicit outage behavior, usage idempotency/exports, and both + Filament scopes. Session-authenticated browser CSV routes additionally cover + guests, cross-domain access, non-administrators, owners, and administrators; + rendered-page checks reject the former token-protected API links. +- Full isolated suite after the telemetry presentation fix: **140 tests, 1,124 + assertions passed**. Production assets, Compose configuration, OpenAPI, and + documentation-link checks also passed. +- Real runtime: `tests/e2e/phase7_analytics.py` passed direct Vector edge/DNS + ingestion, DNSTap-backed DNS collection, all domain/admin query surfaces, + secret/query removal, IPv4 `/24` and IPv6 `/48` masking, stable JSON/CSV usage, + idempotent rebuild replay, and a 20,000-event aggregation check. +- Full non-UI `make dev-e2e` reproduction also passed sequential Phase 1 through + Phase 7 API, DNS, Geo-DNS, edge-control, mTLS, cache, TLS, security, telemetry, + outage, and OpenResty runtime qualification on the preserved local stack. +- Failure rehearsal: ClickHouse was stopped and restored. DNS and edge endpoints + continued responding, the analytics API returned `analytics_unavailable`, + Vector buffered the unique outage event, and the backlog drained afterward. +- Persistent PostgreSQL received only the Phase 7 production migration. No + destructive refresh or named-volume removal was performed. +- The refreshed local control plane returned HTTP 200 from health, redirected + the unauthenticated browser CSV route into session authentication, and kept + the separate API route at HTTP 401 without a token. Existing edge-agent state + volumes were repaired in place after an older root owner prevented the + current non-root agents from writing; no files or volumes were removed, and + both agents remained running afterward. + +## Manual status + +Browser automation was not run, as required. The exact owner-run Phase 7 job is +in `docs/manual-browser-qualification.md`. Phase 7 implementation and agent-owned +qualification are complete; release qualification remains pending until those +rendered desktop/mobile checkpoints are recorded as passed. diff --git a/docs/roadmap.md b/docs/roadmap.md index 4eb3f86..373a9e5 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2525,53 +2525,53 @@ Charts never hide their time range, sampling, units, or partial-data state. ##### Pipeline -- [ ] DNS telemetry reaches ClickHouse directly through Vector. -- [ ] Edge telemetry reaches ClickHouse directly through Vector. -- [ ] Laravel, core PostgreSQL, and Redis/Valkey never ingest raw traffic logs. -- [ ] Disk buffering survives a temporary ClickHouse outage. -- [ ] Disk buffering has a hard byte and age limit. -- [ ] Dropped events are measured and alerted. -- [ ] IPv4 and IPv6 enrichment works or returns `unknown`. -- [ ] Authorization, cookies, tokens, private keys, and request bodies never appear in telemetry. -- [ ] URL, query, user-agent, referrer, and error fields are bounded and sanitized. -- [ ] Telemetry overload cannot exhaust edge or DNS-host storage. +- [x] DNS telemetry reaches ClickHouse directly through Vector. +- [x] Edge telemetry reaches ClickHouse directly through Vector. +- [x] Laravel, core PostgreSQL, and Redis/Valkey never ingest raw traffic logs. +- [x] Disk buffering survives a temporary ClickHouse outage. +- [x] Disk buffering has a hard byte and age limit. +- [x] Dropped events are measured and alerted. +- [x] IPv4 and IPv6 enrichment works or returns `unknown`. +- [x] Authorization, cookies, tokens, private keys, and request bodies never appear in telemetry. +- [x] URL, query, user-agent, referrer, and error fields are bounded and sanitized. +- [x] Telemetry overload cannot exhaust edge or DNS-host storage. ##### Access and Accuracy -- [ ] Domain users see only assigned domains. -- [ ] Administrators can query global data. -- [ ] Requests, bandwidth, status, cache, hostname, country, edge, and DNS totals match generated traffic. -- [ ] Origin latency and failures identify unhealthy origins. -- [ ] Raw and aggregate queries use consistent domain and time boundaries. -- [ ] Usage rollups are idempotent. -- [ ] Missing usage intervals can be rebuilt. -- [ ] JSON and CSV usage exports are stable for external billing consumers. +- [x] Domain users see only assigned domains. +- [x] Administrators can query global data. +- [x] Requests, bandwidth, status, cache, hostname, country, edge, and DNS totals match generated traffic. +- [x] Origin latency and failures identify unhealthy origins. +- [x] Raw and aggregate queries use consistent domain and time boundaries. +- [x] Usage rollups are idempotent. +- [x] Missing usage intervals can be rebuilt. +- [x] JSON and CSV usage exports are stable for external billing consumers. ##### Failure and Performance -- [ ] ClickHouse restart does not affect DNS, proxy, cache, TLS, or security. -- [ ] Query limits protect ClickHouse and Laravel. -- [ ] Expensive filter combinations are rejected or bounded. -- [ ] Analytics remains responsive across the 20,000-domain qualification dataset. -- [ ] Partial or delayed telemetry is visibly labelled. -- [ ] Vector recovery drains backlog without starving live traffic. +- [x] ClickHouse restart does not affect DNS, proxy, cache, TLS, or security. +- [x] Query limits protect ClickHouse and Laravel. +- [x] Expensive filter combinations are rejected or bounded. +- [x] Analytics remains responsive across the 20,000-domain qualification dataset. +- [x] Partial or delayed telemetry is visibly labelled. +- [x] Vector recovery drains backlog without starving live traffic. ##### Browser and Real Runtime -- [ ] Generated DNS and HTTP traffic appears in domain analytics and logs. -- [ ] Domain and administrator views enforce different scopes. -- [ ] Usage export matches generated traffic and remains stable after rebuilding an interval. -- [ ] ClickHouse interruption is shown as analytics unavailable while traffic continues. +- [x] Generated DNS and HTTP traffic appears in domain analytics and logs. +- [x] Domain and administrator views enforce different scopes. +- [x] Usage export matches generated traffic and remains stable after rebuilding an interval. +- [x] ClickHouse interruption is shown as analytics unavailable while traffic continues. ##### Documentation -- [ ] Analytics field and unit reference -- [ ] Log schema reference -- [ ] Retention guide -- [ ] Telemetry-loss semantics -- [ ] Telemetry privacy, redaction, IP masking, and deletion semantics -- [ ] Usage export contract -- [ ] ClickHouse outage runbook +- [x] Analytics field and unit reference +- [x] Log schema reference +- [x] Retention guide +- [x] Telemetry-loss semantics +- [x] Telemetry privacy, redaction, IP masking, and deletion semantics +- [x] Usage export contract +- [x] ClickHouse outage runbook --- diff --git a/docs/telemetry-retention-privacy.md b/docs/telemetry-retention-privacy.md new file mode 100644 index 0000000..7f75332 --- /dev/null +++ b/docs/telemetry-retention-privacy.md @@ -0,0 +1,40 @@ +# Telemetry retention, loss, and privacy + +## Retention and deletion + +ClickHouse raw edge and DNS tables delete partitions after 7 days. Hourly +aggregates retain 400 days and daily aggregates retain 3 years. These are +deployment defaults and may be shortened as an operational privacy decision; +changes apply through ClickHouse TTL processing, not synchronously in a web +request. + +Deleting a domain removes desired control-plane state according to its normal +lifecycle. It does not claim immediate erasure of derived ClickHouse rows. +Unlinked raw rows expire within the raw TTL and aggregate rows within their +configured TTL. For a legally required early erasure, an operator must run and +audit a bounded ClickHouse mutation for the exact domain ID and hostname/zone, +then verify replicas before closing the request. + +## Safe schema and masking + +Vector deletes authorization data, cookies, query data, request bodies, and +unknown fields before storage. It never receives TLS private keys. Paths omit +the query string and are limited to 2,048 bytes. Hostnames, methods, errors, +security reasons, edge identifiers, user agents, referrers, and DNS names are +sanitized for control characters and length-bounded. Normal logs and exports +mask IPv4 to `/24` and IPv6 to `/48`; invalid addresses render as `unknown`. + +## Loss and overload semantics + +Each ClickHouse sink has its own persistent 1 GiB Vector disk buffer, batches at +most 1,000 events, caps retry backoff at 300 seconds, and uses `drop_newest` at +the hard byte limit. The +two buffers therefore consume at most 2 GiB plus Vector metadata. A full or +unavailable telemetry path drops telemetry; it never blocks DNS or HTTP. + +Prometheus scrapes buffer bytes, discarded events, component errors, and +collector availability. Alerts fire on any recent drop, sustained delivery +errors, a buffer above 80 percent, or collector loss. Operators must treat a +drop interval as incomplete: charts remain available but totals cannot be +reconstructed unless another explicitly retained source exists. Never infer +billing accuracy across a recorded loss window. diff --git a/docs/usage-export-contract.md b/docs/usage-export-contract.md new file mode 100644 index 0000000..2752708 --- /dev/null +++ b/docs/usage-export-contract.md @@ -0,0 +1,31 @@ +# Usage export contract + +Usage is reporting data, never request-path enforcement. The hourly scheduler +reads finalized ClickHouse intervals in chunks of at most 250 domains and +upserts PostgreSQL on the unique key `(domain_id, interval_start, granularity)`. +Rebuilding the same interval is idempotent. Administrator rebuild requests +accept complete UTC hours, span at most 31 days, return HTTP 202 plus an +operation ID, and support `Idempotency-Key` replay. + +Domain and administrator JSON exports have `meta.contract_version = 1` and are +limited to 10,000 rows. CSV is streamed in 500-row chunks with this stable +header: + +```text +contract_version,domain_id,interval_start,interval_end,granularity,requests,bytes_in,bytes_out,cache_hits,dns_queries,status +``` + +Timestamps are UTC ISO 8601, bandwidth is bytes, counts are non-negative +integers, `granularity` is `hour`, and finalized rows use `status=finalized`. +Consumers must reject unknown contract versions and store the entire compound +interval identity, not assume delivery order. A newer contract must use a new +version rather than silently changing column meaning. + +Filament downloads use session-authenticated browser routes: + +- `/app/analytics/domains/{domain}/usage.csv` for an assigned domain +- `/admin/telemetry/usage.csv` for an administrator-wide export + +These routes force CSV output, apply the same range bounds and policy scope as +the API, and stream the same contract. They intentionally do not send a browser +session to token-protected `/api/...` URLs. diff --git a/tests/e2e/phase7_analytics.py b/tests/e2e/phase7_analytics.py new file mode 100644 index 0000000..b04d7a0 --- /dev/null +++ b/tests/e2e/phase7_analytics.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Real Phase 7 telemetry, analytics, outage, privacy, and usage qualification.""" + +import datetime as dt +import json +import os +import pathlib +import secrets +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +ROOT = pathlib.Path(__file__).resolve().parents[2] +COMPOSE = os.environ.get("CDNF_COMPOSE_FILE", "compose.dev.yml") +BASE_URL = os.environ.get("CDNF_BASE_URL", "http://localhost:8080").rstrip("/") +RUN_ID = f"{int(time.time())}-{secrets.token_hex(4)}" +DOMAIN = f"analytics-{RUN_ID}.phase7.test" +PASSWORD = f"Analytics-1-{secrets.token_urlsafe(18)}" +USER_EMAIL = f"phase7-user-{RUN_ID}@example.test" +STRANGER_EMAIL = f"phase7-stranger-{RUN_ID}@example.test" +ADMIN_EMAIL = f"phase7-admin-{RUN_ID}@example.test" + + +class ApiError(RuntimeError): + def __init__(self, status: int, payload: object): + super().__init__(f"HTTP {status}: {payload}") + self.status = status + self.payload = payload + + +def run(*args: str, check: bool = True, timeout: int = 60) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=False, timeout=timeout) + if check and result.returncode != 0: + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(args)}\n{result.stdout}\n{result.stderr}") + return result + + +def compose(*args: str, check: bool = True, timeout: int = 60) -> subprocess.CompletedProcess[str]: + return run("docker", "compose", "-f", COMPOSE, *args, check=check, timeout=timeout) + + +def php_string(value: str) -> str: + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def artisan(expression: str) -> str: + return compose("exec", "-T", "core", "php", "artisan", "tinker", f"--execute={expression}").stdout.strip() + + +def api(method: str, path: str, payload: object | None = None, token: str | None = None, + idempotency_key: str | None = None, raw: bool = False) -> tuple[int, object]: + headers = {"Accept": "text/csv" if raw else "application/json"} + data = None + if payload is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(payload).encode() + if token: + headers["Authorization"] = f"Bearer {token}" + if idempotency_key: + headers["Idempotency-Key"] = idempotency_key + request = urllib.request.Request(f"{BASE_URL}{path}", data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=12) as response: + body = response.read() + return response.status, body.decode() if raw else (json.loads(body) if body else {}) + except urllib.error.HTTPError as error: + body = error.read() + try: + decoded = json.loads(body) if body else {} + except json.JSONDecodeError: + decoded = body.decode(errors="replace") + raise ApiError(error.code, decoded) from error + + +def expect_error(status: int, method: str, path: str, **kwargs: object) -> ApiError: + try: + api(method, path, **kwargs) + except ApiError as error: + assert error.status == status, error + return error + raise AssertionError(f"{method} {path} unexpectedly succeeded") + + +def wait_for_api() -> None: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + status, payload = api("GET", "/api/health") + if status == 200 and payload.get("status") == "ok": + return + except (ApiError, OSError): + pass + time.sleep(1) + raise RuntimeError("control-plane API did not become healthy") + + +def restart_vector(reconnect_sources: bool = False) -> None: + compose("restart", "vector", timeout=90) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + result = compose( + "exec", "-T", "vector", "wget", "-qO-", "http://127.0.0.1:9598/metrics", check=False, + ) + if result.returncode == 0 and "vector_buffer_max_size_bytes" in result.stdout: + if reconnect_sources: + compose("restart", "dnsdist", "edge-a", timeout=90) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + health = run("curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "http://127.0.0.1:8081/healthz", check=False) + if health.returncode == 0 and health.stdout.strip() == "200": + return + time.sleep(1) + raise RuntimeError("telemetry producers did not recover after collector restart") + return + time.sleep(1) + raise RuntimeError("Vector did not become ready after restart") + + +def vector_event(port: int, event: dict[str, object]) -> None: + compose( + "exec", "-T", "vector", "wget", "-qO-", "--header", "Content-Type: application/json", + "--post-data", json.dumps(event, separators=(",", ":")), f"http://127.0.0.1:{port}/", + ) + + +def clickhouse(query: str) -> str: + return compose("exec", "-T", "clickhouse", "clickhouse-client", "--query", query).stdout.strip() + + +def wait_for_clickhouse(predicate, query: str, timeout: int = 60) -> str: + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + result = compose("exec", "-T", "clickhouse", "clickhouse-client", "--query", query, check=False) + if result.returncode == 0: + last = result.stdout.strip() + if predicate(last): + return last + time.sleep(1) + raise AssertionError(f"ClickHouse condition timed out; last result: {last!r}") + + +def iso(value: dt.datetime) -> str: + return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def login(email: str) -> str: + _, response = api("POST", "/api/auth/login", {"email": email, "password": PASSWORD, "device_name": "phase7-e2e"}) + return response["data"]["token"] + + +def create_scope() -> tuple[int, str, str, str]: + output = artisan( + "$user=App\\Models\\User::query()->create([" + f"'name'=>'Phase 7 user','email'=>{php_string(USER_EMAIL)},'password'=>Illuminate\\Support\\Facades\\Hash::make({php_string(PASSWORD)}),'type'=>'user']);" + "$stranger=App\\Models\\User::query()->create([" + f"'name'=>'Phase 7 stranger','email'=>{php_string(STRANGER_EMAIL)},'password'=>Illuminate\\Support\\Facades\\Hash::make({php_string(PASSWORD)}),'type'=>'user']);" + "$admin=App\\Models\\User::query()->create([" + f"'name'=>'Phase 7 admin','email'=>{php_string(ADMIN_EMAIL)},'password'=>Illuminate\\Support\\Facades\\Hash::make({php_string(PASSWORD)}),'type'=>'admin']);" + f"$domain=App\\Models\\Domain::query()->create(['name'=>{php_string(DOMAIN)},'display_name'=>'Phase 7 analytics','lifecycle_state'=>'active','revision'=>1]);" + "$domain->users()->attach($user,['created_at'=>now()]);echo $domain->id;" + ) + assert output.isdigit(), output + return int(output), login(USER_EMAIL), login(STRANGER_EMAIL), login(ADMIN_EMAIL) + + +def qualify_ingestion_and_queries(domain_id: int, user: str, stranger: str, admin: str) -> tuple[dt.datetime, dt.datetime]: + now = dt.datetime.now(dt.timezone.utc) + interval_from = now.replace(minute=0, second=0, microsecond=0) - dt.timedelta(hours=2) + interval_to = interval_from + dt.timedelta(hours=1) + edge_id = str(uuid.uuid4()) + dns_id = str(uuid.uuid4()) + event_time = interval_from + dt.timedelta(minutes=5) + runtime_path = f"/runtime-{RUN_ID}" + edge = run( + "curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {DOMAIN}", + f"http://127.0.0.1:8081{runtime_path}?token=must-not-survive", check=False, + ) + assert edge.returncode == 0 and len(edge.stdout.strip()) == 3, edge + wait_for_clickhouse( + lambda value: value == "1", + f"SELECT count() FROM cdnf.edge_events WHERE hostname='{DOMAIN}' AND path='{runtime_path}'", + ) + dnstap_name = f"dnstap-{RUN_ID}.{DOMAIN}" + dns_runtime = run("dig", "+time=2", "+tries=1", "@127.0.0.1", "-p", "1053", dnstap_name, "A") + assert "status:" in dns_runtime.stdout, dns_runtime.stdout + wait_for_clickhouse( + lambda value: int(value or "0") >= 1, + f"SELECT count() FROM cdnf.dns_events WHERE zone='{dnstap_name}'", + ) + vector_event(8686, { + "occurred_at": iso(event_time), "event_id": edge_id, "domain_id": domain_id, + "hostname": DOMAIN, "method": "GET", "path": "/account?token=must-not-survive", + "query": "token=must-not-survive", "authorization": "Bearer must-not-survive", + "cookie": "session=must-not-survive", "body": "must-not-survive", "status": 503, + "bytes_in": 17, "bytes_out": 1700, "cache_status": "MISS", "origin_latency_ms": 12, + "origin_error": "timeout", "security_action": "block", "security_reason": "rate_limit", + "client_ip": "192.0.2.123", "country": "IR", "continent": "AS", "edge_id": "edge-phase7", + "event_type": "request", + }) + vector_event(8687, { + "occurred_at": iso(event_time + dt.timedelta(seconds=1)), "event_id": dns_id, + "domain_id": domain_id, "zone": DOMAIN, "qname": f"www.{DOMAIN}", "qtype": "AAAA", + "rcode": "NOERROR", "client_ip": "2001:db8:1234:5678::1", "dns_cluster": "dns-phase7", + "country": "IR", "continent": "AS", "outcome": "answer", "query": "must-not-survive", + }) + wait_for_clickhouse(lambda value: value == "1", f"SELECT count() FROM cdnf.edge_events WHERE event_id=toUUID('{edge_id}')") + wait_for_clickhouse(lambda value: value == "1", f"SELECT count() FROM cdnf.dns_events WHERE event_id=toUUID('{dns_id}')") + + stored_path = clickhouse(f"SELECT path FROM cdnf.edge_events WHERE event_id=toUUID('{edge_id}') FORMAT TSV") + assert stored_path == "/account", stored_path + leaked = clickhouse( + f"SELECT count() FROM cdnf.edge_events WHERE event_id=toUUID('{edge_id}') AND " + "(position(path, 'must-not-survive') > 0 OR position(hostname, 'must-not-survive') > 0)" + ) + assert leaked == "0", leaked + + query = urllib.parse.urlencode({"from": iso(interval_from), "to": iso(interval_to)}) + expect_error(403, "GET", f"/api/domains/{domain_id}/analytics/summary?{query}", token=stranger) + _, summary = api("GET", f"/api/domains/{domain_id}/analytics/summary?{query}", token=user) + assert int(summary["data"]["requests"]) >= 1 and int(summary["data"]["dns_queries"]) >= 1, summary + assert summary["meta"]["units"] == {"bandwidth": "bytes", "latency": "milliseconds"}, summary + for view in ("timeseries", "status-codes", "cache", "countries", "hostnames", "top-urls", "origin", "edges", "dns"): + _, result = api("GET", f"/api/domains/{domain_id}/analytics/{view}?{query}", token=user) + assert isinstance(result["data"], list), (view, result) + for stream in ("requests", "dns", "errors", "security"): + _, result = api("GET", f"/api/domains/{domain_id}/logs/{stream}?{query}", token=user) + assert result["data"], (stream, result) + _, request_log = api("GET", f"/api/domains/{domain_id}/logs/requests?{query}", token=user) + _, dns_log = api("GET", f"/api/domains/{domain_id}/logs/dns?{query}", token=user) + assert request_log["data"][0]["client_ip"] == "192.0.2.0/24", request_log + assert dns_log["data"][0]["client_ip"] == "2001:db8:1234::/48", dns_log + _, global_summary = api("GET", f"/api/admin/analytics/summary?{query}", token=admin) + assert int(global_summary["data"]["requests"]) >= 1, global_summary + for path in ("/api/admin/analytics/traffic", "/api/admin/analytics/dns", "/api/admin/logs/errors", "/api/admin/logs/security"): + _, result = api("GET", f"{path}?{query}", token=admin) + assert "data" in result, (path, result) + expect_error(403, "GET", f"/api/admin/analytics/summary?{query}", token=user) + return interval_from, interval_to + + +def qualify_usage(domain_id: int, user: str, admin: str, interval_from: dt.datetime, interval_to: dt.datetime) -> None: + artisan( + f"$job=new App\\Jobs\\BuildUsageRollups({php_string(iso(interval_from))},{php_string(iso(interval_to))},{domain_id});" + "$job->handle(app(App\\Support\\AnalyticsStore::class));$job->handle(app(App\\Support\\AnalyticsStore::class));" + f"echo App\\Models\\UsageRollup::query()->where('domain_id',{domain_id})->where('interval_start',{php_string(iso(interval_from))})->count();" + ) + query = urllib.parse.urlencode({"from": iso(interval_from), "to": iso(interval_to)}) + _, usage = api("GET", f"/api/domains/{domain_id}/usage/export?{query}", token=user) + assert usage["meta"]["contract_version"] == 1 and len(usage["data"]) == 1, usage + assert int(usage["data"][0]["requests"]) >= 1 and int(usage["data"][0]["dns_queries"]) >= 1, usage + _, csv = api("GET", f"/api/domains/{domain_id}/usage/export?{query}&format=csv", token=user, raw=True) + assert "contract_version,domain_id,interval_start" in csv and ",finalized" in csv, csv + _, admin_usage = api("GET", f"/api/admin/usage?{query}", token=admin) + assert admin_usage["data"]["data"], admin_usage + key = str(uuid.uuid4()) + status, operation = api("POST", "/api/admin/usage/rebuild", { + "domain_id": domain_id, "from": iso(interval_from), "to": iso(interval_to), + }, admin, key) + status2, replay = api("POST", "/api/admin/usage/rebuild", { + "domain_id": domain_id, "from": iso(interval_from), "to": iso(interval_to), + }, admin, key) + assert status == status2 == 202 and operation == replay, (operation, replay) + + +def qualify_outage(domain_id: int, user: str) -> None: + before = run("curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {DOMAIN}", "http://127.0.0.1:8081/", check=False) + assert before.returncode == 0 and len(before.stdout.strip()) == 3, before + buffered_id = str(uuid.uuid4()) + clickhouse_stopped = False + try: + compose("stop", "clickhouse", timeout=90) + clickhouse_stopped = True + outage = expect_error(503, "GET", f"/api/domains/{domain_id}/analytics/summary", token=user) + assert outage.payload.get("code") == "analytics_unavailable", outage.payload + vector_event(8686, { + "occurred_at": iso(dt.datetime.now(dt.timezone.utc)), "event_id": buffered_id, + "domain_id": domain_id, "hostname": DOMAIN, "method": "GET", "path": "/buffered", + "status": 200, "bytes_in": 1, "bytes_out": 2, "cache_status": "HIT", + "client_ip": "198.51.100.7", "edge_id": "edge-phase7", "event_type": "request", + }) + dns = run("dig", "+time=2", "+tries=1", "@127.0.0.1", "-p", "1053", f"outage.{DOMAIN}", "SOA") + assert "status:" in dns.stdout, dns.stdout + during = run("curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {DOMAIN}", "http://127.0.0.1:8081/", check=False) + assert during.returncode == 0 and len(during.stdout.strip()) == 3, during + metrics = compose("exec", "-T", "vector", "wget", "-qO-", "http://127.0.0.1:9598/metrics").stdout + assert "vector_buffer" in metrics and "vector_component" in metrics, metrics[:1000] + finally: + if clickhouse_stopped: + compose("start", "clickhouse", timeout=90) + # A collector restart is safe for serving and also qualifies durable buffer + # recovery instead of relying on process memory from before the outage. + restart_vector() + wait_for_clickhouse(lambda value: value == "1", f"SELECT count() FROM cdnf.edge_events WHERE event_id=toUUID('{buffered_id}')", timeout=90) + + +def qualify_scale(admin: str) -> None: + clickhouse( + "INSERT INTO cdnf.edge_events SELECT now64(3)-number%3600, generateUUIDv4(), " + "toUInt64(900000000+number), 'scale.phase7.test', 'GET', '/scale', 200, 0, 1, 'HIT', 0, '', '', '', '', " + "'edge-scale', '192.0.2.1', 'ZZ', 'ZZ', '', '', 'request' FROM numbers(20000)" + ) + started = time.monotonic() + _, result = api("GET", "/api/admin/analytics/summary", token=admin) + elapsed = time.monotonic() - started + assert "data" in result and elapsed < 10, (elapsed, result) + + +def main() -> None: + wait_for_api() + restart_vector(reconnect_sources=True) + domain_id, user, stranger, admin = create_scope() + interval_from, interval_to = qualify_ingestion_and_queries(domain_id, user, stranger, admin) + qualify_usage(domain_id, user, admin, interval_from, interval_to) + qualify_scale(admin) + qualify_outage(domain_id, user) + print("Phase 7 real-runtime telemetry, analytics, privacy, usage, scale, and outage qualification passed.") + + +if __name__ == "__main__": + main()