Agent/complete phase 6

Makefile

@@ -39,6 +39,7 @@ dev-e2e:
 	python3 tests/e2e/phase4_control_plane.py
 	python3 tests/e2e/phase4_mtls.py
 	python3 tests/e2e/phase5_tls.py
+	python3 tests/e2e/phase6_security.py
 	python3 tests/e2e/phase4_runtime.py
 
 dev-scale-e2e:

compose.dev.yml

@@ -295,15 +295,24 @@ services:
       EDGE_CELL_NAME: shared-default
       EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
+      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
       - dev-pki:/run/edge:ro
       - edge-a-state:/var/lib/cdnfoundry/runtime:ro
+      - mmdb:/mmdb:ro
     tmpfs:
       - /var/cache/nginx:rw,noexec,nosuid,size=256m
+      - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m
+    mem_limit: 512m
+    cpus: 1
+    pids_limit: 128
+    ulimits: { nofile: { soft: 65536, hard: 65536 } }
+    sysctls: { net.ipv4.tcp_syncookies: "1" }
     networks: [edge]
     depends_on:
       dev-pki: { condition: service_completed_successfully }
       origin-http: { condition: service_started }
+      mmdb-updater: { condition: service_healthy }
     restart: unless-stopped
     healthcheck:
       test: [CMD, wget, -qO-, http://127.0.0.1:8080/healthz]
@@ -319,15 +328,24 @@ services:
       EDGE_CELL_NAME: quarantine-default
       EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
+      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
       - dev-pki:/run/edge:ro
       - edge-a-state:/var/lib/cdnfoundry/runtime:ro
+      - mmdb:/mmdb:ro
     tmpfs:
       - /var/cache/nginx:rw,noexec,nosuid,size=256m
+      - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m
+    mem_limit: 256m
+    cpus: 0.5
+    pids_limit: 96
+    ulimits: { nofile: { soft: 65536, hard: 65536 } }
+    sysctls: { net.ipv4.tcp_syncookies: "1" }
     networks: [edge]
     depends_on:
       dev-pki: { condition: service_completed_successfully }
       origin-http: { condition: service_started }
+      mmdb-updater: { condition: service_healthy }
     restart: unless-stopped
 
   edge-agent-a:
@@ -367,15 +385,24 @@ services:
       EDGE_CELL_NAME: shared-default
       EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
+      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
       - dev-pki:/run/edge:ro
       - edge-b-state:/var/lib/cdnfoundry/runtime:ro
+      - mmdb:/mmdb:ro
     tmpfs:
       - /var/cache/nginx:rw,noexec,nosuid,size=256m
+      - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m
+    mem_limit: 512m
+    cpus: 1
+    pids_limit: 128
+    ulimits: { nofile: { soft: 65536, hard: 65536 } }
+    sysctls: { net.ipv4.tcp_syncookies: "1" }
     networks: [edge]
     depends_on:
       dev-pki: { condition: service_completed_successfully }
       origin-http: { condition: service_started }
+      mmdb-updater: { condition: service_healthy }
     restart: unless-stopped
     healthcheck:
       test: [CMD, wget, -qO-, http://127.0.0.1:8080/healthz]
@@ -391,15 +418,24 @@ services:
       EDGE_CELL_NAME: quarantine-default
       EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
+      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
       - dev-pki:/run/edge:ro
       - edge-b-state:/var/lib/cdnfoundry/runtime:ro
+      - mmdb:/mmdb:ro
     tmpfs:
       - /var/cache/nginx:rw,noexec,nosuid,size=256m
+      - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m
+    mem_limit: 256m
+    cpus: 0.5
+    pids_limit: 96
+    ulimits: { nofile: { soft: 65536, hard: 65536 } }
+    sysctls: { net.ipv4.tcp_syncookies: "1" }
     networks: [edge]
     depends_on:
       dev-pki: { condition: service_completed_successfully }
       origin-http: { condition: service_started }
+      mmdb-updater: { condition: service_healthy }
     restart: unless-stopped
 
   edge-agent-b:

compose.prod.yml

@@ -238,10 +238,14 @@ services:
       EDGE_CELL_NAME: shared-default
       EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json
       EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}
+      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
       - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro
       - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro
       - edge-state:/var/lib/cdnfoundry/runtime:ro
+      - mmdb:/mmdb:ro
+    depends_on:
+      mmdb-updater: { condition: service_healthy }
     read_only: true
     tmpfs:
       - /var/cache/nginx:rw,noexec,nosuid,size=512m
@@ -252,6 +256,7 @@ services:
     mem_limit: 2g
     cpus: 2
     pids_limit: 256
+    sysctls: { net.ipv4.tcp_syncookies: "1" }
     ulimits:
       nofile: { soft: 65536, hard: 65536 }
     deploy:
@@ -270,10 +275,14 @@ services:
       EDGE_CELL_NAME: quarantine-default
       EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json
       EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}
+      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
       - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro
       - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro
       - edge-state:/var/lib/cdnfoundry/runtime:ro
+      - mmdb:/mmdb:ro
+    depends_on:
+      mmdb-updater: { condition: service_healthy }
     read_only: true
     tmpfs:
       - /var/cache/nginx:rw,noexec,nosuid,size=256m
@@ -284,6 +293,7 @@ services:
     mem_limit: 512m
     cpus: 0.5
     pids_limit: 128
+    sysctls: { net.ipv4.tcp_syncookies: "1" }
     ulimits:
       nofile: { soft: 32768, hard: 32768 }
     deploy:

core/app/Actions/DispatchEmergencyMode.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Actions;
+
+use App\Models\Edge;
+use App\Models\EdgeCell;
+use App\Models\EdgeTask;
+use App\Models\EmergencyMode;
+use App\Models\Operation;
+use App\Models\User;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Str;
+
+class DispatchEmergencyMode
+{
+    public static function activate(string $targetType, string $targetId, array $actions, ?int $durationMinutes, User $actor): array
+    {
+        return DB::transaction(function () use ($actions, $actor, $durationMinutes, $targetId, $targetType): array {
+            abort_if(EmergencyMode::query()->where('target_type', $targetType)->where('target_id', $targetId)->where('active', true)->exists(), 409, 'This target already has an active emergency mode. Clear it before applying another.');
+            $mode = EmergencyMode::query()->create([
+                'target_type' => $targetType, 'target_id' => $targetId, 'actions' => array_values($actions),
+                'expires_at' => $durationMinutes === null ? null : now()->addMinutes($durationMinutes),
+                'created_by' => $actor->id,
+            ]);
+            $operation = Operation::query()->create([
+                'id' => (string) Str::uuid(), 'actor_id' => $actor->id, 'type' => 'edge.emergency_mode', 'status' => 'pending',
+                'input' => ['emergency_mode_id' => $mode->id, 'target_type' => $targetType, 'target_id' => $targetId, 'active' => true],
+            ]);
+            self::tasks($mode, $operation, true);
+
+            return [$mode, $operation];
+        });
+    }
+
+    public static function deactivateTarget(string $targetType, string $targetId, User|string|null $actor): Operation
+    {
+        return DB::transaction(function () use ($actor, $targetId, $targetType): Operation {
+            $modes = EmergencyMode::query()->where('target_type', $targetType)->where('target_id', $targetId)->where('active', true)->lockForUpdate()->get();
+            abort_if($modes->isEmpty(), 404, 'No active emergency mode exists for this target.');
+            $modes->each->update(['active' => false, 'deactivated_at' => now()]);
+            $actorId = $actor instanceof User ? $actor->id : null;
+            $operation = Operation::query()->create([
+                'id' => (string) Str::uuid(), 'actor_id' => $actorId, 'type' => 'edge.emergency_mode', 'status' => 'pending',
+                'input' => ['emergency_mode_ids' => $modes->pluck('id')->all(), 'target_type' => $targetType, 'target_id' => $targetId, 'active' => false],
+            ]);
+            foreach ($modes as $mode) {
+                self::tasks($mode, $operation, false);
+            }
+
+            return $operation;
+        });
+    }
+
+    private static function tasks(EmergencyMode $mode, Operation $operation, bool $active): void
+    {
+        $edges = self::edges($mode->target_type, $mode->target_id);
+        abort_if($edges->isEmpty(), 409, 'The emergency target has no participating edge.');
+        foreach ($edges as $edge) {
+            $cellNames = match ($mode->target_type) {
+                'cell' => [EdgeCell::query()->findOrFail((int) $mode->target_id)->name],
+                'pool' => $edge->cells()->where('edge_pool_id', (int) $mode->target_id)->orderBy('name')->limit(32)->pluck('name')->all(),
+                default => [],
+            };
+            EdgeTask::query()->create([
+                'id' => (string) Str::uuid(), 'edge_id' => $edge->id, 'type' => 'emergency_mode', 'status' => 'pending',
+                'payload' => [
+                    'operation_id' => $operation->id, 'emergency_mode_id' => $mode->id, 'active' => $active,
+                    'target_type' => $mode->target_type, 'target_id' => $mode->target_id,
+                    'cell_names' => $cellNames,
+                    'actions' => $mode->actions, 'expires_at' => $active ? $mode->expires_at?->timestamp : null,
+                ],
+            ]);
+        }
+        $operation->update(['status' => 'running', 'started_at' => now(), 'result' => ['tasks' => $edges->count(), 'completed' => 0]]);
+    }
+
+    private static function edges(string $targetType, string $targetId): Collection
+    {
+        return match ($targetType) {
+            'edge' => Edge::query()->whereKey($targetId)->where('enabled', true)->get(),
+            'cell' => Edge::query()->whereHas('cells', fn ($query) => $query->whereKey((int) $targetId))->where('enabled', true)->get(),
+            'pool' => Edge::query()->whereHas('cells', fn ($query) => $query->where('edge_pool_id', (int) $targetId))->where('enabled', true)->get(),
+            default => collect(),
+        };
+    }
+}

core/app/Console/Commands/ReconcileSecurityReadiness.php

@@ -0,0 +1,50 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Actions\DispatchEmergencyMode;
+use App\Jobs\ReconcileEdgeDomain;
+use App\Models\AuditLog;
+use App\Models\Domain;
+use App\Models\EmergencyMode;
+use App\Models\Operation;
+use Illuminate\Console\Command;
+
+class ReconcileSecurityReadiness extends Command
+{
+    protected $signature = 'security:reconcile-readiness {--limit=100}';
+
+    protected $description = 'Expire emergency controls and advance quiet domains through bounded security recovery';
+
+    public function handle(): int
+    {
+        $limit = max(1, min(1000, (int) $this->option('limit')));
+        EmergencyMode::query()->where('active', true)->whereNotNull('expires_at')->where('expires_at', '<=', now())
+            ->select(['target_type', 'target_id'])->distinct()->limit($limit)->get()
+            ->each(function (EmergencyMode $mode): void {
+                $operation = DispatchEmergencyMode::deactivateTarget($mode->target_type, $mode->target_id, null);
+                AuditLog::record(null, 'security.emergency_expired', $mode, ['operation_id' => $operation->id]);
+            });
+
+        Domain::query()->whereIn('security_state', ['suspected', 'restricted', 'recovering'])->orderBy('id')->limit($limit)->get()
+            ->each(function (Domain $domain): void {
+                $latest = $domain->securityEvents()->max('occurred_at');
+                if ($latest !== null && now()->diffInMinutes($latest, absolute: true) < 10) {
+                    return;
+                }
+                $next = match ($domain->security_state) {
+                    'suspected' => 'normal', 'restricted' => 'recovering', 'recovering' => 'normal', default => null,
+                };
+                if ($next === null) {
+                    return;
+                }
+                $domain->update(['security_state' => $next, 'security_state_changed_at' => now(), 'revision' => $domain->revision + 1]);
+                Operation::coalesceDomain('edge.domain_reconcile', $domain->id);
+                ReconcileEdgeDomain::dispatch($domain->id);
+                AuditLog::record(null, 'security.state_recovered', $domain, ['state' => $next, 'revision' => $domain->revision]);
+            });
+        $this->info('Security readiness reconciliation completed.');
+
+        return self::SUCCESS;
+    }
+}

core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php

@@ -2,16 +2,19 @@
 
 namespace App\Filament\Admin\Resources\EdgePools;
 
+use App\Actions\DispatchEmergencyMode;
 use App\Filament\Admin\Resources\EdgePools\Pages\CreateEdgePool;
 use App\Filament\Admin\Resources\EdgePools\Pages\EditEdgePool;
 use App\Filament\Admin\Resources\EdgePools\Pages\ListEdgePools;
 use App\Jobs\ReconcilePlatformDnsIdentity;
 use App\Models\AuditLog;
 use App\Models\DomainEdgePlacement;
 use App\Models\EdgePool;
+use App\Models\EmergencyMode;
 use App\Models\PlatformDnsSetting;
 use App\Support\EdgeRoutingCompiler;
 use Filament\Actions\Action;
+use Filament\Forms\Components\CheckboxList;
 use Filament\Forms\Components\Select;
 use Filament\Forms\Components\TextInput;
 use Filament\Notifications\Notification;
@@ -49,6 +52,7 @@ public static function table(Table $table): Table
             TextColumn::make('name')->searchable()->sortable(),
             TextColumn::make('kind')->badge(),
             IconColumn::make('enabled')->boolean(),
+            IconColumn::make('withdrawn')->boolean(),
             TextColumn::make('routing_target')->label('DNS routing target')
                 ->state(fn (EdgePool $record): ?string => $settings === null ? null : EdgeRoutingCompiler::poolHostname($settings, $record))
                 ->copyable()->placeholder('Configure System DNS identity')->wrap(),
@@ -78,6 +82,29 @@ public static function table(Table $table): Table
                 AuditLog::record(auth()->user(), 'edge.pool_disabled', $record, ['revision' => $record->revision], request()->ip());
                 ReconcilePlatformDnsIdentity::dispatchForRoutingChange();
             }),
+            Action::make('withdraw')->color('danger')->requiresConfirmation()->visible(fn (EdgePool $record): bool => ! $record->withdrawn)->action(function (EdgePool $record): void {
+                $record->update(['withdrawn' => true, 'revision' => $record->revision + 1]);
+                AuditLog::record(auth()->user(), 'edge.pool_withdrawn', $record, ['revision' => $record->revision], request()->ip());
+                ReconcilePlatformDnsIdentity::dispatchForRoutingChange();
+            }),
+            Action::make('restore')->color('success')->visible(fn (EdgePool $record): bool => $record->withdrawn)->action(function (EdgePool $record): void {
+                $record->update(['withdrawn' => false, 'revision' => $record->revision + 1]);
+                AuditLog::record(auth()->user(), 'edge.pool_restored', $record, ['revision' => $record->revision], request()->ip());
+                ReconcilePlatformDnsIdentity::dispatchForRoutingChange();
+            }),
+            Action::make('emergencyMode')->label('Emergency')->color('danger')->requiresConfirmation()
+                ->visible(fn (EdgePool $record): bool => ! EmergencyMode::query()->where('target_type', 'pool')->where('target_id', (string) $record->id)->where('active', true)->exists())
+                ->schema([
+                    CheckboxList::make('actions')->options(array_combine(config('security.emergency_actions'), config('security.emergency_actions')))->required()->minItems(1),
+                    TextInput::make('duration_minutes')->numeric()->minValue(1)->maxValue(config('security.emergency_duration_minutes_maximum')),
+                ])->action(function (EdgePool $record, array $data): void {
+                    [$mode, $operation] = DispatchEmergencyMode::activate('pool', (string) $record->id, $data['actions'], filled($data['duration_minutes'] ?? null) ? (int) $data['duration_minutes'] : null, auth()->user());
+                    AuditLog::record(auth()->user(), 'security.emergency_activated', $record, ['mode_id' => $mode->id], request()->ip());
+                    Notification::make()->warning()->title('Pool emergency mode queued')->body("Operation {$operation->id} targets the equivalent cell on each edge.")->send();
+                }),
+            Action::make('clearEmergency')->label('Clear emergency')->color('success')->requiresConfirmation()
+                ->visible(fn (EdgePool $record): bool => EmergencyMode::query()->where('target_type', 'pool')->where('target_id', (string) $record->id)->where('active', true)->exists())
+                ->action(fn (EdgePool $record) => DispatchEmergencyMode::deactivateTarget('pool', (string) $record->id, auth()->user())),
         ])->defaultSort('name');
     }