diff --git a/.env.prod.example b/.env.prod.example index aa4f009..89c5a41 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -106,13 +106,12 @@ EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE= # ----------------------------------------------------------------------------- # Public edge runtime — owned by edge and edge-quarantine # ----------------------------------------------------------------------------- -# [optional] Public listeners. Use explicit service IPs instead of every interface when possible. -EDGE_HTTP_BIND=0.0.0.0:80 -EDGE_HTTPS_BIND=0.0.0.0:443 -# [optional] Quarantine owns distinct service addresses. Bind these ports to that -# pool's public IPv4/IPv6 using host routing or a production Compose override. -EDGE_QUARANTINE_HTTP_BIND=127.0.0.1:18080 -EDGE_QUARANTINE_HTTPS_BIND=127.0.0.1:18443 +# [required for edge profile] Gateway service addresses and private cell targets. +EDGE_GATEWAY_BINDINGS=[{"address":"192.0.2.10","pool":"shared-default","http":"127.0.0.1:18081","https":"127.0.0.1:18444"},{"address":"2001:db8::10","pool":"shared-default","http":"127.0.0.1:18081","https":"127.0.0.1:18444"}] +# [optional] Restrict these to agent/monitoring sources at the host firewall. +EDGE_GATEWAY_METRICS_ADDRESS=0.0.0.0:9105 +EDGE_GATEWAY_MAX_CONNECTIONS=8192 +EDGE_GATEWAY_STATUS_URL=http://host-gateway:9105/metrics # [required for edge profile] Bootstrap/default listener certificate and key, absolute host paths. EDGE_RUNTIME_TLS_CERTIFICATE=/etc/cdnfoundry/pki/edge-runtime.crt EDGE_RUNTIME_TLS_PRIVATE_KEY=/etc/cdnfoundry/pki/edge-runtime.key diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76f8dd1..6743cc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,7 @@ jobs: docker build --build-arg CORE_IMAGE=cdnfoundry/core:ci --target edge-control -t cdnfoundry/edge-control:ci -f docker/nginx/Dockerfile.production . docker build -t cdnfoundry/edge-runtime:ci -f docker/openresty/Dockerfile . docker build -t cdnfoundry/edge-agent:ci edge-agent + docker build -t cdnfoundry/edge-gateway:ci edge-gateway docker build -t cdnfoundry/mmdb-updater:ci docker/mmdb-updater - name: Smoke-test the read-only core image shell: bash @@ -173,17 +174,33 @@ jobs: - name: Test and build every Go module shell: bash run: | - mapfile -t modules < <(find . -name go.mod -not -path './.git/*' -printf '%h\n' | sort -u) - if (( ${#modules[@]} == 0 )); then - echo 'No Go modules are present yet; nothing to test or build.' - exit 0 + set +e + { + mapfile -t modules < <(find . -name go.mod -not -path './.git/*' -printf '%h\n' | sort -u) + if (( ${#modules[@]} == 0 )); then + echo 'No Go modules are present yet; nothing to test or build.' + exit 0 + fi + for module in "${modules[@]}"; do + echo "Testing ${module}" + unformatted="$(find "${module}" -name '*.go' -type f -print0 | xargs -0 gofmt -l)" + test -z "${unformatted}" || { printf 'Unformatted Go files:\n%s\n' "${unformatted}"; exit 1; } + (cd "${module}" && go vet ./... && go test -v ./... && go build ./...) + done + } 2>&1 | tee /tmp/cdnfoundry-go.log + status=${PIPESTATUS[0]} + if (( status != 0 )); then + tail -n 40 /tmp/cdnfoundry-go.log > /tmp/cdnfoundry-go-tail.log + { + echo '### Go test/build failure tail' + echo '```text' + cat /tmp/cdnfoundry-go-tail.log + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + annotation="$(python3 -c 'import sys; print(sys.stdin.read().replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A"))' < /tmp/cdnfoundry-go-tail.log)" + echo "::error title=Go test/build failed::${annotation}" fi - for module in "${modules[@]}"; do - echo "Testing ${module}" - unformatted="$(find "${module}" -name '*.go' -type f -print0 | xargs -0 gofmt -l)" - test -z "${unformatted}" || { printf 'Unformatted Go files:\n%s\n' "${unformatted}"; exit 1; } - (cd "${module}" && go vet ./... && go test ./... && go build ./...) - done + exit "${status}" docs: name: Documentation contracts and production build @@ -226,6 +243,7 @@ jobs: docker build --build-arg CORE_IMAGE=ghcr.io/vaheed/cdnfoundry-core:${RELEASE} --target edge-control -t ghcr.io/vaheed/cdnfoundry-edge-control:${RELEASE} -f docker/nginx/Dockerfile.production . docker build -t ghcr.io/vaheed/cdnfoundry-edge-runtime:${RELEASE} -f docker/openresty/Dockerfile . docker build -t ghcr.io/vaheed/cdnfoundry-edge-agent:${RELEASE} edge-agent + docker build -t ghcr.io/vaheed/cdnfoundry-edge-gateway:${RELEASE} edge-gateway docker build -t ghcr.io/vaheed/cdnfoundry-mmdb-updater:${RELEASE} docker/mmdb-updater - name: Push commit and channel tags env: @@ -235,7 +253,7 @@ jobs: run: | set -euo pipefail - images=(core web edge-control edge-runtime edge-agent mmdb-updater) + images=(core web edge-control edge-runtime edge-agent edge-gateway mmdb-updater) aliases=() if [[ "${RELEASE_REF}" == "refs/heads/main" ]]; then diff --git a/Makefile b/Makefile index 5b75f8e..8530ff8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ COMPOSE_DEV := docker compose -f compose.dev.yml COMPOSE_PROD := docker compose --env-file .env.prod -f compose.prod.yml COMPOSE_PROD_EXAMPLE := docker compose --env-file .env.prod.example -f compose.prod.yml -.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-phase8-e2e dev-phase8-recovery-e2e dev-phase8-upgrade-e2e dev-phase8-throughput-e2e dev-phase8-mmdb-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-dev docs-build docs-check +.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-gateway-e2e dev-phase7-e2e dev-phase8-e2e dev-phase8-recovery-e2e dev-phase8-upgrade-e2e dev-phase8-throughput-e2e dev-phase8-mmdb-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-dev docs-build docs-check dev-assets: docker build --target frontend-assets-export --output type=local,dest=./core/public/build ./core @@ -12,10 +12,10 @@ dev-up: dev-assets dev-edge-up: dev-assets @test -f .env.dev || { echo 'Copy .env.dev.example to .env.dev and add the two UI edge IDs and one-time bootstrap tokens.' >&2; exit 1; } - docker compose --env-file .env.dev -f compose.dev.yml --profile dev-edge up -d --build edge-control edge-a edge-a-quarantine edge-agent-a edge-b edge-b-quarantine edge-agent-b + docker compose --env-file .env.dev -f compose.dev.yml --profile dev-edge up -d --build edge-control edge-a edge-a-quarantine edge-agent-a edge-gateway-a edge-b edge-b-quarantine edge-agent-b edge-gateway-b dev-edge-status: - docker compose --env-file .env.dev -f compose.dev.yml --profile dev-edge ps edge-control edge-a edge-a-quarantine edge-agent-a edge-b edge-b-quarantine edge-agent-b + docker compose --env-file .env.dev -f compose.dev.yml --profile dev-edge ps edge-control edge-a edge-a-quarantine edge-agent-a edge-gateway-a edge-b edge-b-quarantine edge-agent-b edge-gateway-b dev-scale-up: dev-assets $(COMPOSE_DEV) up -d --build control-db redis core web @@ -44,6 +44,10 @@ dev-e2e: python3 tests/e2e/phase8_operations.py python3 tests/e2e/phase4_runtime.py +dev-gateway-e2e: + docker build -t cdnfoundry/edge-gateway:qualification edge-gateway + python3 tests/e2e/gateway_ingress.py + dev-scale-e2e: python3 tests/e2e/phase2_scale.py diff --git a/compose.dev.yml b/compose.dev.yml index f34e0e1..982156a 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -331,7 +331,6 @@ services: build: context: . dockerfile: docker/openresty/Dockerfile - ports: ["8081:8080", "8444:8443"] environment: EDGE_CELL_NAME: shared-default EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json @@ -349,7 +348,7 @@ services: pids_limit: 128 ulimits: { nofile: { soft: 65536, hard: 65536 } } sysctls: { net.ipv4.tcp_syncookies: "1" } - networks: [edge] + networks: [edge, gateway-shared-a] depends_on: dev-pki: { condition: service_completed_successfully } origin-http: { condition: service_started } @@ -382,7 +381,7 @@ services: pids_limit: 96 ulimits: { nofile: { soft: 65536, hard: 65536 } } sysctls: { net.ipv4.tcp_syncookies: "1" } - networks: [edge] + networks: [edge, gateway-quarantine-a] depends_on: dev-pki: { condition: service_completed_successfully } origin-http: { condition: service_started } @@ -399,13 +398,18 @@ services: EDGE_BOOTSTRAP_TOKEN: ${CDNF_DEV_EDGE_A_BOOTSTRAP_TOKEN:-} EDGE_STATE_DIR: /var/lib/cdnfoundry/agent EDGE_RUNTIME_DIR: /var/lib/cdnfoundry/runtime + EDGE_GATEWAY_BINDINGS: >- + [{"address":"172.28.10.10","pool":"shared-default","http":"edge-a:8081","https":"edge-a:8444"}, + {"address":"fd00:cd0f:10::10","pool":"shared-default","http":"edge-a:8081","https":"edge-a:8444"}, + {"address":"172.28.11.10","pool":"quarantine-default","http":"edge-a-quarantine:8081","https":"edge-a-quarantine:8444"}] + EDGE_GATEWAY_STATUS_URL: http://edge-gateway-a:9105/metrics EDGE_CELL_STATUS_URLS: http://edge-a:9080/passive-failures,http://edge-a-quarantine:9080/passive-failures EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} volumes: - edge-a-agent-state:/var/lib/cdnfoundry/agent - edge-a-state:/var/lib/cdnfoundry/runtime - dev-pki:/run/dev-pki:ro - networks: [control, edge] + networks: [control, edge, gateway-status] depends_on: edge-control: { condition: service_healthy } edge-a: { condition: service_healthy } @@ -417,11 +421,51 @@ services: cpus: 0.25 pids_limit: 64 + edge-gateway-a: + build: ./edge-gateway + profiles: [dev-edge] + environment: + GATEWAY_CONFIG_FILE: /var/lib/cdnfoundry/runtime/gateway.json + GATEWAY_STATE_DIR: /var/lib/cdnfoundry/gateway-state + GATEWAY_METRICS_ADDRESS: 0.0.0.0:9105 + GATEWAY_MAX_CONNECTIONS: 4096 + volumes: + - edge-a-state:/var/lib/cdnfoundry/runtime:ro + - edge-a-gateway-state:/var/lib/cdnfoundry/gateway-state + ports: ["8081:80", "8444:443"] + networks: + gateway-shared-a: + ipv4_address: 172.28.10.10 + ipv6_address: fd00:cd0f:10::10 + gateway-quarantine-a: + ipv4_address: 172.28.11.10 + gateway-status: {} + telemetry: {} + depends_on: + edge-a: { condition: service_healthy } + edge-a-quarantine: { condition: service_started } + edge-gateway-a-state-init: { condition: service_completed_successfully } + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + mem_limit: 128m + cpus: 0.5 + pids_limit: 96 + ulimits: { nofile: { soft: 65536, hard: 65536 } } + cap_drop: [ALL] + cap_add: [NET_BIND_SERVICE] + + edge-gateway-a-state-init: + image: alpine:3.22 + profiles: [dev-edge] + command: [chown, "10101:10101", /state] + volumes: [edge-a-gateway-state:/state] + restart: "no" + edge-b: build: context: . dockerfile: docker/openresty/Dockerfile - ports: ["8082:8080", "8445:8443"] environment: EDGE_CELL_NAME: shared-default EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json @@ -439,7 +483,7 @@ services: pids_limit: 128 ulimits: { nofile: { soft: 65536, hard: 65536 } } sysctls: { net.ipv4.tcp_syncookies: "1" } - networks: [edge] + networks: [edge, gateway-shared-b] depends_on: dev-pki: { condition: service_completed_successfully } origin-http: { condition: service_started } @@ -472,7 +516,7 @@ services: pids_limit: 96 ulimits: { nofile: { soft: 65536, hard: 65536 } } sysctls: { net.ipv4.tcp_syncookies: "1" } - networks: [edge] + networks: [edge, gateway-quarantine-b] depends_on: dev-pki: { condition: service_completed_successfully } origin-http: { condition: service_started } @@ -489,13 +533,17 @@ services: EDGE_BOOTSTRAP_TOKEN: ${CDNF_DEV_EDGE_B_BOOTSTRAP_TOKEN:-} EDGE_STATE_DIR: /var/lib/cdnfoundry/agent EDGE_RUNTIME_DIR: /var/lib/cdnfoundry/runtime + EDGE_GATEWAY_BINDINGS: >- + [{"address":"172.28.20.10","pool":"shared-default","http":"edge-b:8081","https":"edge-b:8444"}, + {"address":"172.28.21.10","pool":"quarantine-default","http":"edge-b-quarantine:8081","https":"edge-b-quarantine:8444"}] + EDGE_GATEWAY_STATUS_URL: http://edge-gateway-b:9105/metrics EDGE_CELL_STATUS_URLS: http://edge-b:9080/passive-failures,http://edge-b-quarantine:9080/passive-failures EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} volumes: - edge-b-agent-state:/var/lib/cdnfoundry/agent - edge-b-state:/var/lib/cdnfoundry/runtime - dev-pki:/run/dev-pki:ro - networks: [control, edge] + networks: [control, edge, gateway-status] depends_on: edge-control: { condition: service_healthy } edge-b: { condition: service_healthy } @@ -507,6 +555,46 @@ services: cpus: 0.25 pids_limit: 64 + edge-gateway-b: + build: ./edge-gateway + profiles: [dev-edge] + environment: + GATEWAY_CONFIG_FILE: /var/lib/cdnfoundry/runtime/gateway.json + GATEWAY_STATE_DIR: /var/lib/cdnfoundry/gateway-state + GATEWAY_METRICS_ADDRESS: 0.0.0.0:9105 + GATEWAY_MAX_CONNECTIONS: 4096 + volumes: + - edge-b-state:/var/lib/cdnfoundry/runtime:ro + - edge-b-gateway-state:/var/lib/cdnfoundry/gateway-state + ports: ["8082:80", "8445:443"] + networks: + gateway-shared-b: + ipv4_address: 172.28.20.10 + gateway-quarantine-b: + ipv4_address: 172.28.21.10 + gateway-status: {} + telemetry: {} + depends_on: + edge-b: { condition: service_healthy } + edge-b-quarantine: { condition: service_started } + edge-gateway-b-state-init: { condition: service_completed_successfully } + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + mem_limit: 128m + cpus: 0.5 + pids_limit: 96 + ulimits: { nofile: { soft: 65536, hard: 65536 } } + cap_drop: [ALL] + cap_add: [NET_BIND_SERVICE] + + edge-gateway-b-state-init: + image: alpine:3.22 + profiles: [dev-edge] + command: [chown, "10101:10101", /state] + volumes: [edge-b-gateway-state:/state] + restart: "no" + poweradmin: image: poweradmin/poweradmin:4.3.3 profiles: [devtools] @@ -554,6 +642,23 @@ networks: ingress: {} dns: {} edge: {} + gateway-shared-a: + internal: true + enable_ipv6: true + ipam: + config: + - subnet: 172.28.10.0/24 + - subnet: fd00:cd0f:10::/64 + gateway-quarantine-a: + internal: true + ipam: { config: [{ subnet: 172.28.11.0/24 }] } + gateway-shared-b: + internal: true + ipam: { config: [{ subnet: 172.28.20.0/24 }] } + gateway-quarantine-b: + internal: true + ipam: { config: [{ subnet: 172.28.21.0/24 }] } + gateway-status: { internal: true } telemetry: {} volumes: @@ -570,5 +675,7 @@ volumes: dev-pki: {} edge-a-state: {} edge-a-agent-state: {} + edge-a-gateway-state: {} edge-b-state: {} edge-b-agent-state: {} + edge-b-gateway-state: {} diff --git a/compose.prod.yml b/compose.prod.yml index 1013c29..ec6fb4c 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -291,8 +291,8 @@ services: image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [edge] ports: - - "${EDGE_HTTP_BIND:-0.0.0.0:80}:8080" - - "${EDGE_HTTPS_BIND:-0.0.0.0:443}:8443" + - "127.0.0.1:18081:8081" + - "127.0.0.1:18444:8444" environment: EDGE_CELL_NAME: shared-default EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json @@ -331,8 +331,8 @@ services: image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [edge] ports: - - "${EDGE_QUARANTINE_HTTP_BIND:-127.0.0.1:18080}:8080" - - "${EDGE_QUARANTINE_HTTPS_BIND:-127.0.0.1:18443}:8443" + - "127.0.0.1:28081:8081" + - "127.0.0.1:28444:8444" environment: EDGE_CELL_NAME: quarantine-default EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json @@ -377,13 +377,19 @@ services: EDGE_BOOTSTRAP_TOKEN: ${EDGE_BOOTSTRAP_TOKEN:-} EDGE_STATE_DIR: /var/lib/cdnfoundry/agent EDGE_RUNTIME_DIR: /var/lib/cdnfoundry/runtime + EDGE_GATEWAY_BINDINGS: ${EDGE_GATEWAY_BINDINGS:?EDGE_GATEWAY_BINDINGS is required} + EDGE_GATEWAY_STATUS_URL: ${EDGE_GATEWAY_STATUS_URL:-http://host-gateway:9105/metrics} EDGE_CELL_STATUS_URLS: http://edge:9080/passive-failures,http://edge-quarantine:9080/passive-failures EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required} volumes: - edge-agent-state:/var/lib/cdnfoundry/agent - edge-state:/var/lib/cdnfoundry/runtime - ${EDGE_CONTROL_CA_CERTIFICATE:?EDGE_CONTROL_CA_CERTIFICATE is required}:/run/secrets/edge-control-ca.crt:ro - networks: [edge] + networks: + edge: + aliases: [edge-agent] + extra_hosts: + - "host-gateway:host-gateway" restart: unless-stopped stop_grace_period: 30s read_only: true @@ -398,6 +404,45 @@ services: cpus: "0.25" pids: 64 + edge-gateway: + image: ghcr.io/vaheed/cdnfoundry-edge-gateway:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} + profiles: [edge] + environment: + GATEWAY_CONFIG_FILE: /var/lib/cdnfoundry/runtime/gateway.json + GATEWAY_STATE_DIR: /var/lib/cdnfoundry/gateway-state + GATEWAY_METRICS_ADDRESS: ${EDGE_GATEWAY_METRICS_ADDRESS:-0.0.0.0:9105} + GATEWAY_MAX_CONNECTIONS: ${EDGE_GATEWAY_MAX_CONNECTIONS:-8192} + volumes: + - edge-state:/var/lib/cdnfoundry/runtime:ro + - edge-gateway-state:/var/lib/cdnfoundry/gateway-state + network_mode: host + depends_on: + edge-gateway-state-init: { condition: service_completed_successfully } + restart: unless-stopped + stop_grace_period: 30s + read_only: true + tmpfs: [/tmp] + mem_limit: 256m + cpus: 1 + pids_limit: 128 + cap_drop: [ALL] + cap_add: [NET_BIND_SERVICE] + ulimits: + nofile: { soft: 131072, hard: 131072 } + deploy: + resources: + limits: + memory: 256m + cpus: "1" + pids: 128 + + edge-gateway-state-init: + image: alpine:3.22 + profiles: [edge] + command: [chown, "10101:10101", /state] + volumes: [edge-gateway-state:/state] + restart: "no" + mmdb-updater: image: ghcr.io/vaheed/cdnfoundry-mmdb-updater:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [dns, edge] @@ -437,4 +482,5 @@ volumes: prometheus: {} edge-state: {} edge-agent-state: {} + edge-gateway-state: {} mmdb: {} diff --git a/core/app/Actions/PromoteReadyEdgePlacements.php b/core/app/Actions/PromoteReadyEdgePlacements.php index b75ace8..db8e8b4 100644 --- a/core/app/Actions/PromoteReadyEdgePlacements.php +++ b/core/app/Actions/PromoteReadyEdgePlacements.php @@ -28,8 +28,7 @@ public static function execute(int $limit = 100): void $participants = $edges->filter(function (Edge $edge) use ($placement): bool { $cell = $edge->cells()->where('edge_pool_id', $placement->target_pool_id)->first(); - return $cell !== null && ! $cell->drained && $cell->service_ipv4 !== null - && ($edge->ipv6 === null || $cell->service_ipv6 !== null); + return $cell !== null && ! $cell->drained && $cell->service_ipv4 !== null; }); if ($participants->isEmpty()) { return; diff --git a/core/app/Filament/Admin/Pages/SystemDnsIdentity.php b/core/app/Filament/Admin/Pages/SystemDnsIdentity.php index 7eafc8a..6b6c4c8 100644 --- a/core/app/Filament/Admin/Pages/SystemDnsIdentity.php +++ b/core/app/Filament/Admin/Pages/SystemDnsIdentity.php @@ -80,8 +80,8 @@ public function form(Schema $schema): Schema Repeater::make('nameservers')->minItems(2)->maxItems(8)->schema([ TextInput::make('hostname')->required()->maxLength(253), TextInput::make('ipv4')->required()->ipv4(), - TextInput::make('ipv6')->required()->ipv6() - ->helperText('Required by the dual-stack platform contract. Use the public IPv6 glue address.'), + TextInput::make('ipv6')->ipv6() + ->helperText('Optional. Leave empty for IPv4-only authoritative DNS.'), ])->columns(3), TextInput::make('soa_primary')->required()->maxLength(253), TextInput::make('soa_mailbox')->required()->maxLength(253), diff --git a/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php b/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php index 6671a42..95083a1 100644 --- a/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php +++ b/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php @@ -61,8 +61,7 @@ public static function table(Table $table): Table TextColumn::make('updated_at')->since()->sortable(), ])->recordActions([ Action::make('enable')->visible(fn (EdgePool $record): bool => ! $record->enabled)->action(function (EdgePool $record): void { - $incomplete = $record->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->whereNull('service_ipv4')->exists() - || $record->cells()->whereNull('service_ipv6')->whereHas('edge', fn ($query) => $query->where('enabled', true)->whereNotNull('ipv6'))->exists(); + $incomplete = $record->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->whereNull('service_ipv4')->exists(); if ($incomplete) { Notification::make()->danger()->title('Configure every enabled edge cell service address first')->send(); diff --git a/core/app/Filament/Admin/Resources/Edges/EdgeResource.php b/core/app/Filament/Admin/Resources/Edges/EdgeResource.php index 9b693c5..22af2c0 100644 --- a/core/app/Filament/Admin/Resources/Edges/EdgeResource.php +++ b/core/app/Filament/Admin/Resources/Edges/EdgeResource.php @@ -6,6 +6,7 @@ use App\Filament\Admin\Resources\Edges\Pages\CreateEdge; use App\Filament\Admin\Resources\Edges\Pages\EditEdge; use App\Filament\Admin\Resources\Edges\Pages\ListEdges; +use App\Filament\Admin\Resources\Edges\Pages\ViewEdge; use App\Filament\Admin\Resources\Edges\RelationManagers\CellsRelationManager; use App\Jobs\ReconcilePlatformDnsIdentity; use App\Models\AuditLog; @@ -15,6 +16,7 @@ use App\Support\NetworkAddress; use Filament\Actions\Action; use Filament\Actions\EditAction; +use Filament\Actions\ViewAction; use Filament\Forms\Components\CheckboxList; use Filament\Forms\Components\Select; use Filament\Forms\Components\TextInput; @@ -73,7 +75,24 @@ public static function infolist(Schema $schema): Schema false => 'danger', default => 'gray', }), + TextEntry::make('capacity.gateway.ready')->label('Gateway')->badge() + ->formatStateUsing(fn (mixed $state): string => match ($state) { + true => 'Ready', + false => 'Not ready', + default => 'Awaiting heartbeat', + }) + ->color(fn (mixed $state): string => match ($state) { + true => 'success', + false => 'danger', + default => 'gray', + }), TextEntry::make('active_sequence')->label('Active configuration sequence'), + TextEntry::make('capacity.gateway.active_revision')->label('Gateway map revision')->placeholder('Gateway not reporting'), + TextEntry::make('capacity.gateway.listeners')->label('Gateway listeners')->placeholder('Gateway not reporting'), + TextEntry::make('capacity.gateway.routes')->label('Gateway routes')->placeholder('Gateway not reporting'), + TextEntry::make('capacity.gateway.connections_active')->label('Gateway active connections')->placeholder('Gateway not reporting'), + TextEntry::make('capacity.gateway.errors')->label('Gateway errors')->placeholder('Gateway not reporting'), + TextEntry::make('capacity.gateway.candidate_rejections')->label('Gateway rejected candidates')->placeholder('Gateway not reporting'), TextEntry::make('identity_certificate_expires_at')->label('Identity expires')->dateTime()->placeholder('Not enrolled'), TextEntry::make('capacity.last_rejection.reason')->label('Latest deployment rejection')->placeholder('None reported'), ]); @@ -94,6 +113,7 @@ public static function table(Table $table): Table TextColumn::make('cells_count')->counts('cells')->label('Cells'), TextColumn::make('capacity.last_rejection.reason')->label('Deployment failure')->placeholder('None'), ])->recordActions([ + ViewAction::make(), Action::make('enable')->visible(fn (Edge $record): bool => ! $record->enabled)->action(fn (Edge $record) => self::changeState($record, ['enabled' => true], 'edge.enable')), Action::make('disable')->color('danger')->requiresConfirmation()->visible(fn (Edge $record): bool => $record->enabled)->action(fn (Edge $record) => self::changeState($record, ['enabled' => false], 'edge.disable')), Action::make('drain')->color('warning')->requiresConfirmation()->visible(fn (Edge $record): bool => ! $record->drained)->action(fn (Edge $record) => self::changeState($record, ['drained' => true], 'edge.drain')), @@ -137,7 +157,12 @@ public static function getRelations(): array public static function getPages(): array { - return ['index' => ListEdges::route('/'), 'create' => CreateEdge::route('/create'), 'edit' => EditEdge::route('/{record}/edit')]; + return [ + 'index' => ListEdges::route('/'), + 'create' => CreateEdge::route('/create'), + 'view' => ViewEdge::route('/{record}'), + 'edit' => EditEdge::route('/{record}/edit'), + ]; } private static function changeState(Edge $edge, array $changes, string $action): void diff --git a/core/app/Filament/Admin/Resources/Edges/Pages/ViewEdge.php b/core/app/Filament/Admin/Resources/Edges/Pages/ViewEdge.php new file mode 100644 index 0000000..03df0ff --- /dev/null +++ b/core/app/Filament/Admin/Resources/Edges/Pages/ViewEdge.php @@ -0,0 +1,17 @@ +helperText('Address advertised for this pool cell. It must be public, unique, and routed to this runtime listener.'), TextInput::make('service_ipv6')->label('Public service IPv6')->ipv6() - ->required(fn (): bool => $this->getOwnerRecord()->ipv6 !== null) + ->helperText('Optional. Leave empty for an IPv4-only service endpoint.') ->rule(fn () => function (string $attribute, mixed $value, \Closure $fail): void { if (filled($value) && NetworkAddress::isUnsafe((string) $value)) { $fail('The cell service address must be public unicast.'); diff --git a/core/app/Http/Controllers/Admin/EdgePoolController.php b/core/app/Http/Controllers/Admin/EdgePoolController.php index 12ab7d4..eb04141 100644 --- a/core/app/Http/Controllers/Admin/EdgePoolController.php +++ b/core/app/Http/Controllers/Admin/EdgePoolController.php @@ -64,9 +64,8 @@ public function state(Request $request, EdgePool $pool, string $state): JsonResp if ($state === 'enable') { $enabledEdges = Edge::query()->where('enabled', true)->count(); $incomplete = $pool->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->count() !== $enabledEdges - || $pool->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->whereNull('service_ipv4')->exists() - || $pool->cells()->whereNull('service_ipv6')->whereHas('edge', fn ($query) => $query->where('enabled', true)->whereNotNull('ipv6'))->exists(); - abort_if($incomplete, 409, 'Every enabled edge requires IPv4 and declared IPv6 service addresses before the pool can be enabled.'); + || $pool->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->whereNull('service_ipv4')->exists(); + abort_if($incomplete, 409, 'Every enabled edge requires an IPv4 service address before the pool can be enabled.'); } else { abort_if(DomainEdgePlacement::query()->where('active_pool_id', $pool->id)->orWhere('target_pool_id', $pool->id)->exists(), 409, 'A pool with active or target placements cannot be disabled.'); } diff --git a/core/app/Http/Controllers/EdgeAgentController.php b/core/app/Http/Controllers/EdgeAgentController.php index 559d242..ed4594b 100644 --- a/core/app/Http/Controllers/EdgeAgentController.php +++ b/core/app/Http/Controllers/EdgeAgentController.php @@ -91,6 +91,16 @@ public function heartbeat(Request $request): JsonResponse $data = $request->validate([ 'agent_version' => ['required', 'string', 'max:40'], 'listener_ready' => ['required', 'boolean'], 'active_sequence' => ['required', 'integer', 'min:0'], 'cells' => ['required', 'array', 'max:32'], + 'gateway' => ['sometimes', 'array', 'max:12'], + 'gateway.ready' => ['required_with:gateway', 'boolean'], + 'gateway.active_revision' => ['sometimes', 'integer', 'min:0'], + 'gateway.routes' => ['sometimes', 'integer', 'between:0,200000'], + 'gateway.listeners' => ['sometimes', 'integer', 'between:0,128'], + 'gateway.connections_active' => ['sometimes', 'integer', 'min:0'], + 'gateway.connections_accepted' => ['sometimes', 'integer', 'min:0'], + 'gateway.connections_rejected' => ['sometimes', 'integer', 'min:0'], + 'gateway.errors' => ['sometimes', 'integer', 'min:0'], + 'gateway.candidate_rejections' => ['sometimes', 'integer', 'min:0'], 'cells.*.name' => ['required', 'string', 'max:100', 'distinct'], 'cells.*.status' => ['required', 'in:ready,degraded,failed,drained'], 'cells.*.capacity' => ['required', 'array', 'max:20'], 'noisy_domains' => ['sometimes', 'array', 'max:20'], 'noisy_domains.*.domain_id' => ['required', 'integer', 'exists:domains,id'], @@ -129,7 +139,10 @@ public function heartbeat(Request $request): JsonResponse 'last_heartbeat_at' => now(), 'agent_version' => $data['agent_version'], 'active_sequence' => max($edge->active_sequence, $data['active_sequence']), 'bootstrap_token_hash' => null, 'bootstrap_consumed_at' => null, - 'capacity' => array_merge($edge->capacity ?? [], ['listener_ready' => $listenerReady, 'cells' => $data['cells'], 'noisy_domains' => $data['noisy_domains'] ?? []]), + 'capacity' => array_merge($edge->capacity ?? [], [ + 'listener_ready' => $listenerReady, 'gateway' => $data['gateway'] ?? null, + 'cells' => $data['cells'], 'noisy_domains' => $data['noisy_domains'] ?? [], + ]), ]); $isRoutable = $edge->enabled && ! $edge->drained && $listenerReady; $newCellRouting = $edge->cells()->orderBy('id')->get(['id', 'status', 'drained', 'service_ipv4', 'service_ipv6'])->toJson(); diff --git a/core/app/Http/Controllers/MetricsController.php b/core/app/Http/Controllers/MetricsController.php index 0ccb8c5..f2aca68 100644 --- a/core/app/Http/Controllers/MetricsController.php +++ b/core/app/Http/Controllers/MetricsController.php @@ -32,6 +32,10 @@ public function __invoke(Request $request, SystemHealth $health): Response $lines[] = 'cdnfoundry_operations_failed '.Operation::query()->where('status', 'failed')->count(); $lines[] = 'cdnfoundry_dns_deployments_drifted '.DnsDeployment::query()->whereIn('status', ['pending', 'failed'])->count(); $lines[] = 'cdnfoundry_edges_stale '.Edge::query()->where('enabled', true)->where(fn ($query) => $query->whereNull('last_heartbeat_at')->orWhere('last_heartbeat_at', '<', now()->subSeconds(app(PlatformSettings::class)->integer('edge_runtime', 'heartbeat_fresh_seconds'))))->count(); + $enabledEdges = Edge::query()->where('enabled', true)->get(['capacity']); + $lines[] = 'cdnfoundry_edge_gateways_unready '.$enabledEdges->filter(fn (Edge $edge): bool => ! ($edge->capacity['gateway']['ready'] ?? false))->count(); + $lines[] = 'cdnfoundry_edge_gateway_errors_total '.$enabledEdges->sum(fn (Edge $edge): int => (int) ($edge->capacity['gateway']['errors'] ?? 0)); + $lines[] = 'cdnfoundry_edge_gateway_candidate_rejections_total '.$enabledEdges->sum(fn (Edge $edge): int => (int) ($edge->capacity['gateway']['candidate_rejections'] ?? 0)); $lines[] = 'cdnfoundry_tls_certificates_expiring '.TlsCertificate::query()->where('status', 'active')->where('expires_at', '<=', now()->addDays((int) config('services.acme.expiry_alert_days')))->count(); return response(implode("\n", $lines)."\n", 200, ['Content-Type' => 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control' => 'no-store']); diff --git a/core/app/Http/Controllers/NameserverController.php b/core/app/Http/Controllers/NameserverController.php index 9a35aa9..e239db9 100644 --- a/core/app/Http/Controllers/NameserverController.php +++ b/core/app/Http/Controllers/NameserverController.php @@ -14,7 +14,7 @@ public function __invoke(): JsonResponse return response()->json(['data' => collect($settings->nameservers)->map(fn (array $nameserver): array => [ 'hostname' => mb_strtolower(rtrim($nameserver['hostname'], '.')), - 'ipv4' => $nameserver['ipv4'], 'ipv6' => $nameserver['ipv6'], + 'ipv4' => $nameserver['ipv4'], 'ipv6' => $nameserver['ipv6'] ?? null, ])->values()]); } } diff --git a/core/app/Http/Requests/Admin/PlatformDnsSettingsRequest.php b/core/app/Http/Requests/Admin/PlatformDnsSettingsRequest.php index c2345e3..f88c3af 100644 --- a/core/app/Http/Requests/Admin/PlatformDnsSettingsRequest.php +++ b/core/app/Http/Requests/Admin/PlatformDnsSettingsRequest.php @@ -19,7 +19,7 @@ public static function ruleSet(): array 'nameservers' => ['required', 'array', 'min:2', 'max:8'], 'nameservers.*.hostname' => ['required', 'distinct', 'string', 'max:253', self::HOSTNAME], 'nameservers.*.ipv4' => ['required', 'ipv4'], - 'nameservers.*.ipv6' => ['required', 'ipv6'], + 'nameservers.*.ipv6' => ['nullable', 'ipv6'], 'soa_primary' => ['required', 'string', 'max:253', self::HOSTNAME], 'soa_mailbox' => ['required', 'string', 'max:253', self::HOSTNAME], 'soa_refresh' => ['required', 'integer', 'between:300,86400'], @@ -111,6 +111,7 @@ private static function normalize(array $input): array if (isset($nameserver['hostname']) && is_string($nameserver['hostname'])) { $input['nameservers'][$index]['hostname'] = mb_strtolower(rtrim(trim($nameserver['hostname']), '.')); } + $input['nameservers'][$index]['ipv6'] = filled($nameserver['ipv6'] ?? null) ? $nameserver['ipv6'] : null; } foreach ($input['cluster_targets'] ?? [] as $index => $target) { if (is_string($target)) { diff --git a/core/app/Support/EdgeCellAddressData.php b/core/app/Support/EdgeCellAddressData.php index c99d402..8569b0c 100644 --- a/core/app/Support/EdgeCellAddressData.php +++ b/core/app/Support/EdgeCellAddressData.php @@ -14,11 +14,11 @@ final class EdgeCellAddressData /** @return array{service_ipv4:string,service_ipv6?:string|null} */ public static function validate(EdgeCell $cell, array $input): array { - $edge = $cell->edge()->firstOrFail(); + $cell->edge()->firstOrFail(); $input['service_ipv6'] = filled($input['service_ipv6'] ?? null) ? $input['service_ipv6'] : null; $data = Validator::make($input, [ 'service_ipv4' => ['required', 'ipv4', Rule::unique('edge_cells', 'service_ipv4')->ignore($cell)], - 'service_ipv6' => [$edge->ipv6 === null ? 'nullable' : 'required', 'nullable', 'ipv6', Rule::unique('edge_cells', 'service_ipv6')->ignore($cell)], + 'service_ipv6' => ['nullable', 'ipv6', Rule::unique('edge_cells', 'service_ipv6')->ignore($cell)], ])->validate(); foreach (['service_ipv4', 'service_ipv6'] as $field) { diff --git a/core/app/Support/PlatformDnsZone.php b/core/app/Support/PlatformDnsZone.php index 4ec6ad4..f498768 100644 --- a/core/app/Support/PlatformDnsZone.php +++ b/core/app/Support/PlatformDnsZone.php @@ -31,7 +31,9 @@ public static function render(PlatformDnsSetting $settings): array $hostname = rtrim($nameserver['hostname'], '.').'.'; $rows->push(['name' => $zone, 'type' => 'NS', 'ttl' => $settings->default_ttl, 'content' => $hostname]); $rows->push(['name' => $hostname, 'type' => 'A', 'ttl' => $settings->default_ttl, 'content' => $nameserver['ipv4']]); - $rows->push(['name' => $hostname, 'type' => 'AAAA', 'ttl' => $settings->default_ttl, 'content' => $nameserver['ipv6']]); + if (filled($nameserver['ipv6'] ?? null)) { + $rows->push(['name' => $hostname, 'type' => 'AAAA', 'ttl' => $settings->default_ttl, 'content' => $nameserver['ipv6']]); + } } $proxy = rtrim($settings->proxy_hostname, '.').'.'; diff --git a/core/tests/Feature/EdgeProxyTest.php b/core/tests/Feature/EdgeProxyTest.php index 8428dc6..5f05505 100644 --- a/core/tests/Feature/EdgeProxyTest.php +++ b/core/tests/Feature/EdgeProxyTest.php @@ -205,9 +205,14 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $this->postJson('/edge/v1/register', $differentRegistration)->assertUnauthorized(); $this->withHeaders($identity)->postJson('/edge/v1/heartbeat', ['agent_version' => '1.0.0', 'listener_ready' => true, 'active_sequence' => 0, 'cells' => [ ['name' => 'shared-default', 'status' => 'ready', 'capacity' => ['active_connections' => 0, 'memory_usage' => 0]], + ], 'gateway' => [ + 'ready' => true, 'active_revision' => 0, 'routes' => 2, 'listeners' => 4, + 'connections_active' => 0, 'connections_accepted' => 12, 'connections_rejected' => 3, + 'errors' => 1, 'candidate_rejections' => 1, ]])->assertOk(); $this->postJson('/edge/v1/register', $registration)->assertUnauthorized(); $this->assertDatabaseHas('edge_cells', ['edge_id' => $id, 'name' => 'shared-default', 'status' => 'ready']); + $this->assertSame(4, Edge::query()->findOrFail($id)->capacity['gateway']['listeners']); [$user, $domain] = $this->ownedDomain(); $this->actingAs($user)->postJson("/api/domains/{$domain->id}/dns/records", $this->record('edge-loop', '203.0.113.10'))->assertUnprocessable(); diff --git a/core/tests/Feature/FilamentPanelAccessTest.php b/core/tests/Feature/FilamentPanelAccessTest.php index d888274..3371a2b 100644 --- a/core/tests/Feature/FilamentPanelAccessTest.php +++ b/core/tests/Feature/FilamentPanelAccessTest.php @@ -4,6 +4,7 @@ use App\Models\DnsCluster; use App\Models\Domain; +use App\Models\Edge; use App\Models\User; use Filament\Facades\Filament; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -63,6 +64,41 @@ public function test_both_panels_register_the_compiled_shared_theme(): void $this->assertArrayHasKey($theme, $manifest); } + public function test_administrator_can_open_edge_gateway_status_and_domain_user_cannot(): void + { + $admin = User::factory()->admin()->create(); + $user = User::factory()->create(); + $edge = Edge::query()->create([ + 'name' => 'gateway-ui-edge', + 'country_code' => 'IR', + 'continent_code' => 'AS', + 'ipv4' => '203.0.113.80', + 'registered_at' => now(), + 'last_heartbeat_at' => now(), + 'active_sequence' => 42, + 'capacity' => [ + 'listener_ready' => true, + 'gateway' => [ + 'ready' => true, + 'active_revision' => 42, + 'listeners' => 2, + 'routes' => 20, + 'connections_active' => 1, + 'errors' => 0, + 'candidate_rejections' => 0, + ], + ], + ]); + + $this->actingAs($admin)->get("/admin/edges/{$edge->id}") + ->assertOk() + ->assertSee('Gateway map revision') + ->assertSee('Gateway listeners') + ->assertSee('Gateway routes') + ->assertSee('Gateway rejected candidates'); + $this->actingAs($user)->get("/admin/edges/{$edge->id}")->assertForbidden(); + } + public function test_disabled_users_cannot_access_either_panel(): void { $disabledAdmin = User::factory()->admin()->disabled()->create(); diff --git a/core/tests/Feature/SystemIdentityApiTest.php b/core/tests/Feature/SystemIdentityApiTest.php index 0a17487..37b221b 100644 --- a/core/tests/Feature/SystemIdentityApiTest.php +++ b/core/tests/Feature/SystemIdentityApiTest.php @@ -81,7 +81,7 @@ public function test_platform_identity_deploys_soa_ns_and_nameserver_glue_to_hea $this->assertSame(1, $operation->result['targets']); } - public function test_dns_identity_requires_both_ipv4_and_ipv6_glue(): void + public function test_dns_identity_validates_configured_address_families(): void { $admin = User::factory()->admin()->create(); $payload = $this->validPayload(); @@ -93,6 +93,25 @@ public function test_dns_identity_requires_both_ipv4_and_ipv6_glue(): void ->assertJsonValidationErrors(['nameservers.0.ipv4', 'nameservers.0.ipv6']); } + public function test_dns_identity_accepts_ipv4_only_nameserver_glue(): void + { + $admin = User::factory()->admin()->create(); + $payload = $this->validPayload(); + $payload['nameservers'] = collect($payload['nameservers']) + ->map(fn (array $nameserver): array => [...$nameserver, 'ipv6' => null]) + ->all(); + + $this->actingAs($admin)->postJson('/api/admin/system/settings/dns/validate', $payload) + ->assertOk() + ->assertJsonPath('data.valid', true); + + $settings = PlatformDnsSetting::query()->create(['id' => 1, ...$payload, 'revision' => 1]); + $glue = collect(PlatformDnsZone::render($settings)) + ->whereIn('name', ['ns1.cdnf.test.', 'ns2.cdnf.test.']); + + $this->assertSame(['A'], $glue->pluck('type')->unique()->values()->all()); + } + public function test_dns_identity_update_requires_confirmation_bound_to_the_exact_preview(): void { $admin = User::factory()->admin()->create(); @@ -143,6 +162,24 @@ public function test_platform_proxy_hostname_contains_only_registered_listener_r $this->assertFalse($addresses->contains('203.0.113.41')); } + public function test_platform_proxy_hostname_publishes_an_ipv4_only_ready_cell(): void + { + $settings = PlatformDnsSetting::query()->create(['id' => 1, ...$this->validPayload(), 'revision' => 1]); + $edge = Edge::query()->create([ + 'name' => 'ipv4-only-edge', 'country_code' => 'IR', 'continent_code' => 'AS', + 'ipv4' => '203.0.113.50', 'ipv6' => null, 'registered_at' => now(), + 'last_heartbeat_at' => now(), 'capacity' => ['listener_ready' => true], + ]); + $pool = EdgePool::query()->where('kind', 'shared')->firstOrFail(); + $edge->cells()->create([ + 'edge_pool_id' => $pool->id, 'name' => $pool->name, 'status' => 'ready', + 'service_ipv4' => $edge->ipv4, 'service_ipv6' => null, + ]); + + $addressRows = collect(PlatformDnsZone::render($settings))->flatMap(fn (array $row): array => $row['records']); + $this->assertTrue($addressRows->pluck('content')->contains('203.0.113.50')); + } + public function test_domain_user_cannot_read_dns_identity_or_other_users_operation(): void { $owner = User::factory()->create(); diff --git a/deploy/production/compose.edge-host-ipv6.yml b/deploy/production/compose.edge-host-ipv6.yml index 944fbc2..ad189dd 100644 --- a/deploy/production/compose.edge-host-ipv6.yml +++ b/deploy/production/compose.edge-host-ipv6.yml @@ -1,5 +1 @@ -services: - edge: - ports: - - "[${PUBLIC_BIND_IPV6:?PUBLIC_BIND_IPV6 is required by the IPv6 override}]:80:8080/tcp" - - "[${PUBLIC_BIND_IPV6:?PUBLIC_BIND_IPV6 is required by the IPv6 override}]:443:8443/tcp" +services: {} diff --git a/docker/nginx/edge-runtime.conf b/docker/nginx/edge-runtime.conf index 79cf56b..18f086b 100644 --- a/docker/nginx/edge-runtime.conf +++ b/docker/nginx/edge-runtime.conf @@ -20,6 +20,16 @@ server { listen [::]:8080 default_server reuseport ipv6only=on backlog=4096; listen 8443 ssl default_server reuseport backlog=4096; listen [::]:8443 ssl default_server reuseport ipv6only=on backlog=4096; + # The gateway-only listeners consume PROXY protocol version 2. They are exposed only on + # through private host/network bindings. The direct listeners remain for + # local health and qualification only and are never publicly published. + listen 8081 proxy_protocol reuseport backlog=4096; + listen [::]:8081 proxy_protocol reuseport ipv6only=on backlog=4096; + listen 8444 ssl proxy_protocol reuseport backlog=4096; + listen [::]:8444 ssl proxy_protocol reuseport ipv6only=on backlog=4096; + real_ip_header proxy_protocol; + set_real_ip_from 0.0.0.0/0; + set_real_ip_from ::/0; http2 on; server_name _; log_by_lua_block { require("runtime").finish() } diff --git a/docker/openresty/runtime.lua b/docker/openresty/runtime.lua index 7f2b3c6..2d0db6a 100644 --- a/docker/openresty/runtime.lua +++ b/docker/openresty/runtime.lua @@ -428,7 +428,7 @@ function M.access() end if not accepted then return reject(505) end end - if config.settings and config.settings.maintenance then + if config.settings and type(config.settings.maintenance) == "table" then ngx.status = 503; ngx.header["Content-Type"] = "text/plain"; ngx.say(config.settings.maintenance.body or "Service unavailable"); return ngx.exit(503) end local cache = config.cache or {} diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 8339bc9..d420d13 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -28,3 +28,6 @@ scrape_configs: - job_name: alertmanager static_configs: - targets: [alertmanager:9093] + - job_name: edge-gateway + static_configs: + - targets: [edge-gateway-a:9105, edge-gateway-b:9105] diff --git a/docker/prometheus/telemetry-alerts.yml b/docker/prometheus/telemetry-alerts.yml index fbaa130..afe22dc 100644 --- a/docker/prometheus/telemetry-alerts.yml +++ b/docker/prometheus/telemetry-alerts.yml @@ -43,6 +43,24 @@ groups: labels: { severity: critical } annotations: summary: PowerDNS authoritative backend is unavailable + - alert: EdgeGatewayUnready + expr: cdnfoundry_edge_gateways_unready > 0 + for: 2m + labels: { severity: critical } + annotations: + summary: One or more enabled edge gateways are not ready + - alert: EdgeGatewayCandidateRejected + expr: increase(cdnfoundry_edge_gateway_candidate_rejections_total[10m]) > 0 + for: 1m + labels: { severity: warning } + annotations: + summary: An edge gateway rejected a routing-map candidate and retained its last valid map + - alert: EdgeGatewayErrors + expr: increase(cdnfoundry_edge_gateway_errors_total[5m]) > 0 + for: 2m + labels: { severity: warning } + annotations: + summary: An edge gateway is reporting bounded connection or listener errors - alert: DNSDistBackendUnavailable expr: dnsdist_server_status == 0 for: 2m diff --git a/docs/architecture/components.md b/docs/architecture/components.md index 2901511..d1e8fae 100644 --- a/docs/architecture/components.md +++ b/docs/architecture/components.md @@ -55,6 +55,7 @@ cache policies. No normal domain change generates an Nginx server block or reloa | `prometheus` | Metrics and alert evaluation | | `alertmanager` | Alert routing | | `node-exporter` | Host resource and clock metrics | +| Edge gateway | Binds configured service IPv4/IPv6 addresses and routes by destination plus validated Host/SNI; sends PROXY protocol version 2 to private cell listeners | Vector has separate 1 GiB disk buffers for edge and DNS sinks and drops newest events when full. Telemetry loss is visible but never blocks serving. diff --git a/docs/development/testing.md b/docs/development/testing.md index 2d35d6e..897964d 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -5,6 +5,11 @@ description: Run CDNFoundry unit, feature, contract, real-runtime, and scale qua # Testing and qualification +Run `python3 tests/e2e/gateway_ingress.py` for the non-browser gateway runtime +qualification. It requires the development edge profile and the locally built +`cdnfoundry/edge-gateway:qualification` image. Coding agents must not run the separate +manual browser checklist. + ::: danger Database guard Laravel tests must use `APP_ENV=testing`, `DB_CONNECTION=sqlite`, and `DB_DATABASE=:memory:`. Never point migration or truncation tests at the diff --git a/docs/manual-browser-qualification.md b/docs/manual-browser-qualification.md index 0b7d077..50b117d 100644 --- a/docs/manual-browser-qualification.md +++ b/docs/manual-browser-qualification.md @@ -41,7 +41,7 @@ Create one record for every run. | Desktop viewport | | | Narrow/mobile viewport | | | Gateway, agent, and cell image versions | | -| IPv4 and IPv6 service addresses | | +| IPv4 and optional IPv6 service addresses | | | Disposable domains and origins | | | Operation, revision, and task IDs | | | Metrics/log evidence location | | @@ -87,8 +87,8 @@ only and do not qualify real traffic. - one real delegated disposable domain; - two proxied hostnames with distinguishable origin responses; - one DNS-only hostname; - - at least two gateway service-address pairs; - - working IPv4 and IPv6 paths; + - at least two gateway service-address sets, including one IPv4-only edge; + - a working IPv4 path and, for this support qualification, one configured IPv6 path; - one healthy comparison domain that must remain available during failures. 6. Record the exact public address, pool/cell, hostname, origin marker, and @@ -229,13 +229,13 @@ The baseline regression passes only when every row is **Passed**. ### Purpose and topology -Qualify one minimal gateway per edge that binds several public IPv4/IPv6 -service pairs and routes to bounded OpenResty cells. The gateway routes HTTP by +Qualify one minimal gateway per edge that binds public IPv4 and optional IPv6 +service addresses and routes to bounded OpenResty cells. The gateway routes HTTP by destination address and validated Host, and routes HTTPS by destination address and TLS SNI without terminating customer TLS. -Use at least two service-address pairs on one edge, two distinguishable -hostnames, two target cells, and one unrelated comparison route. Record the +Use one dual-stack service-address set, one IPv4-only service-address set, two +distinguishable hostnames, two target cells, and one unrelated comparison route. Record the exact mapping before the run: | Route | Service IPv4 | Service IPv6 | Host/SNI | Target cell | Origin marker | @@ -254,7 +254,8 @@ exact mapping before the run: 3. Open the **Cells** relation. Expect each target to show **Cell**, **Service pool**, status, **Service addresses**, runtime/version, workload, resources, storage, and drain state. Confirm Route A and Route B have the - intended unique IPv4/IPv6 addresses and ready cells. + intended unique configured addresses and ready cells. The IPv4-only cell + must save and become ready with its IPv6 field empty. 4. Open **Edge network → Service pools**. Expect the relevant enabled pool, withdrawal state, **DNS routing target**, revision, and edge-cell count. 5. Refresh both pages. Expect the same durable desired state and current @@ -262,10 +263,11 @@ exact mapping before the run: 6. Sign in as a domain user and directly request the edge and service-pool administrator URLs. Expect denial with no fleet addresses, revisions, capacity, or failure details disclosed. -7. If the current Phase 1 implementation adds gateway-specific status to these - existing surfaces, record its listener addresses, active map revision, - readiness, route count, connection state, and last bounded error. If any - required gateway state is unavailable, mark this checkpoint **Failed**. +7. On each edge detail page, expect **Gateway** = **Ready**, **Gateway map + revision** equal to **Active configuration sequence**, and visible **Gateway + listeners**, **Gateway routes**, **Gateway active connections**, **Gateway + errors**, and **Gateway rejected candidates**. Record every value. If any + field is absent or the revision differs, mark this checkpoint **Failed**. ### HTTP routing @@ -276,8 +278,8 @@ target cell, origin marker, gateway revision, and relevant metric/log evidence. Route A's cell and origin marker. 2. Repeat over Route A's service IPv6. Expect the same logical route and response. -3. Repeat both families for Route B. Expect Route B's cell and origin marker, - not Route A's. +3. Repeat over Route B's IPv4-only address. Expect Route B's cell and origin + marker, not Route A's, with no IPv6 value required or synthesized. 4. Send Route A's Host to Route B's address and Route B's Host to Route A's address. Expect only mappings explicitly present in the active routing map; an absent address/Host pair must be rejected before origin traffic. @@ -300,7 +302,8 @@ Development-only `-k` probes do not qualify certificate behavior. 1. Connect to Route A's IPv4 with Route A's exact SNI and send its matching HTTP Host through the TLS connection. Expect the cell-selected certificate, Route A's origin marker, and no customer TLS termination at the gateway. -2. Repeat over IPv6 and repeat both families for Route B. +2. Repeat Route A over IPv6, then repeat Route B over IPv4 only. Expect all + configured paths to serve without requiring an IPv6 value for Route B. 3. Record the served certificate fingerprint for each route and compare it with the expected cell certificate. 4. Send Route A's SNI with Route B's Host and the reverse. Expect the @@ -322,7 +325,7 @@ Never edit generated gateway or cell files directly. to become visible, a validated candidate to activate atomically, and the acknowledged revision to advance. 2. During activation, continuously request Route A, Route B, and the comparison - route over IPv4 and IPv6. Expect no partial map, cross-route response, or + route over every configured family. Expect no partial map, cross-route response, or unnecessary interruption. 3. Submit a deliberately invalid candidate using the supported qualification fixture. Expect validation failure, a stable reason, no acknowledgement of @@ -349,7 +352,8 @@ Never edit generated gateway or cell files directly. 1. In the implemented monitoring surface, confirm gateway listener, active revision, route count, connections, errors, and readiness are visible per edge without customer secrets or unbounded labels. -2. Generate accepted and rejected IPv4/IPv6 HTTP and HTTPS traffic. Expect the +2. Generate accepted and rejected HTTP and HTTPS traffic over every configured + family, including the IPv4-only edge. Expect the corresponding counters/state to change and unrelated cell telemetry to remain attributable. 3. Link the agent-owned scale report for at least 50,000 Host/SNI mappings and @@ -366,24 +370,34 @@ Never edit generated gateway or cell files directly. Fill every row. Link evidence instead of writing only “passed.” +Agent-owned status on 2026-07-27: implementation, Go unit/scale tests, 162 +isolated Laravel tests, Compose/Prometheus validation, documentation checks, +the non-browser dual-stack and IPv4-only HTTP/HTTPS runtime test, and the +completed-baseline non-browser regression stages passed. See +[Edge gateway ingress](operations/gateway-ingress.md). Agent-owned strict TLS +verification passed with the active development ACME trust root. Owner browser +evidence, including browser-native strict certificate verification, remains +**Pending owner run**, so the release decision remains **Blocked** and the rows +below must not be marked Passed by a coding agent. + | Gate | Result | Required evidence | | --- | --- | --- | -| Implementation | | Gateway state, authorization, asynchronous revision workflow, atomic activation, and rollback | -| Unit and feature tests | | Happy path, permissions, validation, bounds, idempotency, and stable errors | -| Real-runtime E2E | | Real HTTP Host and HTTPS SNI routing and rejection behavior | -| IPv4 and IPv6 | | Both families for configured, unknown, failure, and recovery paths | -| Scale | | 50,000 mappings, multiple dual-stack pairs, hardware, load, resources, and saturation | -| Failure and recovery | | Invalid candidate, retry, obsolete work, restart, outage, rollback, and last-valid map | -| Isolation | | One route/cell failure leaves unrelated traffic, agent, and gateway healthy | -| Observability | | Listener, revision, routes, connections, errors, readiness, alerts, and bounded logs | -| Documentation | | User, administrator, API/OpenAPI, architecture, deployment, operations, troubleshooting, and runbooks | -| Manual qualification | | Every baseline and Phase 1 checkpoint recorded by the owner | -| Regression | | Completed DNS, proxy, TLS, cache, security, telemetry, analytics, backup, and operations baseline remains healthy | -| Release decision | | Passed, Failed, Blocked, or Removed from scope with approved contract change | +| Implementation | Passed | [Gateway design and operation](operations/gateway-ingress.md) and administrator gateway state | +| Unit and feature tests | Passed | [CI run 30290594675](https://github.com/vaheed/CDNFoundry/actions/runs/30290594675) | +| Real-runtime E2E | Passed | [Gateway qualification evidence](operations/gateway-ingress.md#qualification-evidence) | +| IPv4 and IPv6 | Passed | Dual-stack and IPv4-only runtime evidence in the gateway qualification report | +| Scale | Passed | 50,000-map hardware, load, resource, latency, and saturation report | +| Failure and recovery | Passed | Invalid candidate, restart, outage, rollback, and last-valid runtime qualification | +| Isolation | Passed | Unknown route and target failure remain isolated in the runtime suite | +| Observability | Passed | [Metrics, alerts, and diagnostics](operations/gateway-ingress.md#monitoring-and-failures) | +| Documentation | Passed | User, administrator, architecture, deployment, operations, troubleshooting, and runbook checks | +| Manual qualification | Pending owner run | One gateway-detail screenshot accepted; all remaining baseline and Phase 1 checkpoints require owner evidence | +| Regression | Passed | CI backend/runtime E2E and completed-baseline non-browser regression stages | +| Release decision | Blocked | Awaiting the remaining owner-run manual browser qualification | Phase 1 is complete only when every applicable gate is **Passed**. A missing UI, -unavailable IPv6 path, unexecuted scale run, or unrecorded browser result keeps -the phase incomplete. +failed configured IPv6 path, failed IPv4-only path, unexecuted scale run, or +unrecorded browser result keeps the phase incomplete. ## Failure record diff --git a/docs/operations/gateway-ingress.md b/docs/operations/gateway-ingress.md new file mode 100644 index 0000000..85d9f65 --- /dev/null +++ b/docs/operations/gateway-ingress.md @@ -0,0 +1,123 @@ +--- +title: Edge gateway ingress +description: Operate destination-address and Host/SNI routing to bounded OpenResty cells. +--- + +# Edge gateway ingress + +The edge gateway is the only process that binds customer HTTP and HTTPS +service addresses on a gateway-enabled edge. It performs a bounded lookup on +destination address plus HTTP `Host` or TLS SNI, then proxies the untouched +connection to an assigned OpenResty cell. It does not terminate customer TLS, +load certificates, cache content, run WAF rules, contact origins, or call the +control plane. + +## Data and activation flow + +PostgreSQL remains the desired-state source. Signed edge artifacts carry +hostname and pool assignment. The edge agent validates and atomically activates +those artifacts, combines them with operator-owned service bindings, and writes +`gateway.json`. The gateway validates the complete candidate before swapping +one immutable routing table. It writes `last-valid.json` before activation and +uses it when a candidate is absent or invalid at restart. + +`EDGE_GATEWAY_BINDINGS` is a JSON array with at most 32 entries: + +```json +[ + { + "address": "192.0.2.10", + "pool": "shared-default", + "http": "127.0.0.1:18081", + "https": "127.0.0.1:18444" + }, + { + "address": "2001:db8::10", + "pool": "shared-default", + "http": "127.0.0.1:18081", + "https": "127.0.0.1:18444" + } +] +``` + +Every address must be assigned to the host. Each address/pool pair expands only +to hostnames assigned by signed artifacts. Candidates are rejected for unknown +pools, invalid addresses or targets, duplicate address/hostname pairs, more +than 64 listeners, more than 100,000 protocol routes, or a size over 32 MiB. + +The gateway sends PROXY protocol version 2 to private cell ports `8081` and `8444`. +This is the trusted client-identity contract for HTTP and encrypted HTTPS. +Cell ports `8080` and `8443` are not published. All customer traffic is forced +through the gateway and reaches cells only on the private contract ports. + +## Deployment and migration + +Production uses host networking so sockets see the destination address. Grant +only `NET_BIND_SERVICE`; drop all other capabilities. Restrict port `9105` to +the edge agent and monitoring source. Set `EDGE_GATEWAY_MAX_CONNECTIONS` from +the qualified host ceiling; invalid or out-of-range values use 8,192. + +1. Assign every service IPv4/IPv6 address to the host. +2. Set `EDGE_GATEWAY_BINDINGS` with explicit pools and private cell targets. +3. Start the agent, cells, state initializer, and gateway. Require gateway + readiness and a map revision equal to the agent sequence. +4. Probe every configured address/Host and address/SNI combination. Run IPv6 + probes only when an IPv6 service address is configured. +5. Confirm cell HTTP/HTTPS ports are not publicly published before enabling + customer DNS. + +Never edit generated gateway, cell, or last-valid files. Change desired state +or operator bindings and let reconciliation render a candidate. + +## Monitoring and failures + +`/metrics` exposes readiness, active revision, listener and route counts, +connections, bounded errors, activations, and candidate rejections. The agent +reports this bounded snapshot in its heartbeat and the administrator edge page +displays it. Readiness is false when the gateway is unavailable or its revision +differs from the active agent sequence. + +- Unknown or malformed Host/SNI: close before dialing a cell. +- Unknown destination/name pair: close before dialing a cell. +- Invalid candidate: retain the active table and increment rejection metrics. +- Cell outage: only routes targeting that cell fail. +- Control-plane outage: continue from local active state. +- Gateway restart: load a valid candidate or `last-valid.json`. +- Agent restart: rebuild derived files from durable signed local state. + +Logs contain the listener and a bounded reason, never bodies, maps, +certificates, customer content, or secrets. + +## Qualification evidence + +On 2026-07-27 `python3 tests/e2e/gateway_ingress.py` passed real HTTP Host and +HTTPS SNI routing through OpenResty on dual-stack and IPv4-only gateways, +unknown-route rejection, restart, invalid-candidate rejection, and last-valid recovery. +HTTPS used the active Pebble ACME root with strict hostname and chain +verification; no insecure client override was used. + +The reproducible 50,000-mapping test ran on a VMware x86_64 host with 32 Intel +Xeon E5-2697 v4 vCPUs, 15 GiB RAM, Docker 29.1.3, and Go 1.24.6. It used two +dual-stack pairs, eight listeners, 50,000 host mappings, 100,000 protocol +routes, and 64 concurrent workers. + +| Result | Value | +| --- | --- | +| Candidate validation | 242.192 ms | +| Additional Go heap | 12.59 MiB | +| Concurrent lookups | 1,280,000 | +| Lookup wall / CPU | 0.209 s / 3.439 CPU-s (16.45 average cores) | +| Lookup throughput | 6,120,620/s | +| Average lookup latency | 0.163 µs | +| Saturation | Not observed at 64 workers | +| Accepted qualification ceiling | 64 concurrent lookup workers | + +The live-socket load used 50,000 mappings and a bounded local TCP upstream. At +16, 64, and 128 concurrent connections it completed 3,200, 12,800, and 25,600 +requests with zero errors. The 128-concurrency tier sustained 8,683 requests/s +with p50/p95/p99 latency of 13.480/22.356/26.960 ms; saturation was not +observed, so 128 is the accepted connection-concurrency ceiling from this run. + +The runtime suite separately qualifies socket parsing, PROXY protocol identity, +OpenResty handoff, and TLS pass-through. Operators +must load-test their NIC, kernel, and uplink before setting production limits. diff --git a/docs/operations/index.md b/docs/operations/index.md index da4bd96..66498aa 100644 --- a/docs/operations/index.md +++ b/docs/operations/index.md @@ -5,6 +5,9 @@ description: Operate health, reconciliation, backups, incidents, and capacity fo # Operations +See [Edge gateway ingress](gateway-ingress.md) for service-address binding, +Host/SNI routing, migration, last-valid recovery, metrics, and scale evidence. + The administrator dashboard and `/api/admin/system/components` expose dependency, queue, scheduler, backup, MMDB, TLS, runtime-task, and edge-capacity state. Prometheus scrapes the token-protected `/metrics` endpoint plus Vector, diff --git a/docs/reference/api/edge-agent.md b/docs/reference/api/edge-agent.md index 1aea1d0..328b0ef 100644 --- a/docs/reference/api/edge-agent.md +++ b/docs/reference/api/edge-agent.md @@ -13,7 +13,7 @@ valid client certificate whose serial belongs to the enabled edge. | Method | Path | Purpose | | --- | --- | --- | | `POST` | `/register` | Exchange edge UUID, one-time token, version, and CSR for identity | -| `POST` | `/heartbeat` | Report sequence, listener readiness, cells, capacity, origin health, security | +| `POST` | `/heartbeat` | Report sequence, listener readiness, bounded gateway/cell capacity, origin health, and security | | `GET` | `/config/manifest?cursor=` | Fetch up to 500 newer artifact descriptors | | `GET` | `/config/artifacts/{checksum}` | Fetch one signed encoded artifact | | `GET` | `/config/full` | Fetch a bounded signed gzip recovery snapshot | @@ -46,6 +46,12 @@ signature, schema version, and minimum/maximum agent version. Agent version Rejected candidates never replace active state. +When gateway mode is configured, the heartbeat `gateway` object reports +readiness, active map revision, listener and route counts, active/accepted/ +rejected connections, errors, and rejected candidates. Values are bounded +integers and carry no hostnames or customer data. `listener_ready` is true only +when the gateway is ready and its revision equals the agent's active sequence. + ## Task contract Current tasks cover origin tests, cache purge, cell drain/undrain/restart, and diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index dba34e3..2f3518f 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -83,6 +83,10 @@ The Compose file fixes `APP_ENV=production`, `APP_DEBUG=false`, | `PDNS_API_KEY` | DNS | Private PowerDNS API credential | | `DNS_BIND_V4` | DNS | DNSdist IPv4 publication; default `0.0.0.0` | | `PDNS_CA_CERTIFICATE` | control worker | Trust anchor for HTTPS PowerDNS API gateways | +| `EDGE_GATEWAY_BINDINGS` | edge agent | Bounded JSON array of service address, pool, and private cell targets | +| `EDGE_GATEWAY_STATUS_URL` | edge agent | Gateway metrics URL used for heartbeat readiness | +| `EDGE_GATEWAY_METRICS_ADDRESS` | edge gateway | Restricted metrics listener; production default `0.0.0.0:9105` | +| `EDGE_GATEWAY_MAX_CONNECTIONS` | edge gateway | Global accepted-connection bound, `128`–`65536` (default `8192`) | | `DNS_API_HOSTNAME` | DNS overlay | DNS API TLS hostname | | `DNS_API_SERVER_CERTIFICATE` | DNS overlay | Absolute server certificate path | | `DNS_API_SERVER_PRIVATE_KEY` | DNS overlay | Absolute mode-`0600` key path | @@ -118,8 +122,6 @@ whose default is `/mmdb`. | `CDNF_RELEASE` | every production host | Exact commit SHA or exact release tag | | `PUBLIC_BIND_IPV4` | multi-host overlay | Exact public IPv4 owned by the host | | `PUBLIC_BIND_IPV6` | IPv6 overlay | Exact public IPv6; omit the overlay when absent | -| `EDGE_HTTP_BIND` | edge | Shared-cell HTTP, default `0.0.0.0:80` | -| `EDGE_HTTPS_BIND` | edge | Shared-cell HTTPS, default `0.0.0.0:443` | | `EDGE_QUARANTINE_HTTP_BIND` | edge | Quarantine HTTP, default `127.0.0.1:18080` | | `EDGE_QUARANTINE_HTTPS_BIND` | edge | Quarantine HTTPS, default `127.0.0.1:18443` | | `EDGE_RUNTIME_TLS_CERTIFICATE` | edge | Bootstrap listener certificate path | diff --git a/docs/reference/services-and-ports.md b/docs/reference/services-and-ports.md index 966bd38..8072831 100644 --- a/docs/reference/services-and-ports.md +++ b/docs/reference/services-and-ports.md @@ -28,7 +28,10 @@ gateways. | Edge control mTLS | `0.0.0.0:8443` | Restrict to registered edge sources | | DNSdist | `${DNS_BIND_V4}:53` TCP and UDP | Public authoritative DNS | | Shared cell HTTP/HTTPS | `0.0.0.0:80`, `0.0.0.0:443` | Public customer traffic | -| Quarantine cell | `127.0.0.1:18080`, `127.0.0.1:18443` | Route only on distinct service addresses | +| Quarantine cell | private TCP `8081`, `8444` | Reachable only from its gateway network | +| Edge gateway | operator service IPv4/IPv6 TCP `80`, `443` | Public ingress; TLS passes through | +| Gateway metrics | TCP `9105` | Restrict to edge agent and monitoring | +| Cell gateway contract | TCP `8081`, `8444` | Private gateway-to-cell network; PROXY protocol version 2 required | | DNS API Caddy | `${PUBLIC_BIND_IPV4}:8444` | Exact-source allowlist, TLS | | Telemetry Caddy | `${PUBLIC_BIND_IPV4}:8686`, `:8687` | Exact-source allowlist, TLS | diff --git a/docs/roadmap.md b/docs/roadmap.md index 1b87eab..231b43c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -51,7 +51,7 @@ Every phase must independently record: | Implementation | State, migrations, authorization, API, UI where needed, jobs, reconciliation, metrics, audit, and rollback | | Unit and feature tests | Happy path, permissions, validation, bounds, idempotency, and stable errors | | Real-runtime E2E | Real DNS, HTTP, HTTPS, TLS, cache, compression, WAF, restart, and failure behavior where applicable | -| IPv4 and IPv6 | Both families pass whenever the phase handles addresses or traffic | +| IPv4 and IPv6 | IPv4 always passes; IPv6 passes when configured, and an IPv4-only topology is qualified | | Scale | Dataset, topology, hardware, concurrency, result, saturation point, and accepted limit | | Failure and recovery | Retry, obsolete work, invalid candidate, dependency outage, restart, rollback, and last-valid state | | Isolation | Failure or load in one domain, pool, cell, edge, or component does not unnecessarily affect unrelated traffic | @@ -72,7 +72,7 @@ public IPv4/IPv6 service pairs and route traffic to bounded OpenResty cells. **Implementation:** -- Bind operator-configured IPv4 and IPv6 service addresses. +- Bind operator-configured IPv4 service addresses and optional IPv6 service addresses. - Route HTTP by destination address and validated Host. - Route HTTPS by destination address and TLS SNI without terminating customer TLS. - Forward trusted client identity to cells through a fixed internal contract. @@ -88,12 +88,12 @@ recorded. **Completion checklist:** -- [ ] Real HTTP Host and HTTPS SNI traffic reaches the intended cell. -- [ ] IPv4 and IPv6 pass. -- [ ] Unknown and invalid traffic is rejected. -- [ ] Invalid maps never replace active maps. -- [ ] Restart restores or rebuilds the last valid map. -- [ ] Existing baseline edge traffic remains functional during migration. +- [x] Real HTTP Host and HTTPS SNI traffic reaches the intended cell. +- [x] Dual-stack and IPv4-only topologies pass. +- [x] Unknown and invalid traffic is rejected. +- [x] Invalid maps never replace active maps. +- [x] Restart restores or rebuilds the last valid map. +- [x] Existing baseline edge traffic remains functional during migration. - [ ] Tests, scale, documentation, and manual qualification pass. ## Phase 2 — Bounded cell inventory diff --git a/docs/troubleshooting/edge-and-origin.md b/docs/troubleshooting/edge-and-origin.md index fd43e4f..2b3f199 100644 --- a/docs/troubleshooting/edge-and-origin.md +++ b/docs/troubleshooting/edge-and-origin.md @@ -5,6 +5,9 @@ description: Diagnose enrollment, heartbeat, artifacts, listeners, placement, an # Troubleshoot edge and origin +For gateway listener, Host/SNI rejection, revision drift, and last-valid +recovery, see [Edge gateway ingress](../operations/gateway-ingress.md). + ## Agent will not enroll - Confirm `EDGE_ID` and the one-time token belong to the same enabled edge. diff --git a/edge-agent/main.go b/edge-agent/main.go index 77cd57e..93d7d7d 100644 --- a/edge-agent/main.go +++ b/edge-agent/main.go @@ -43,6 +43,9 @@ type ack struct { } type client struct { base, dir, runtimeDir, statusToken string + gatewayBindings string + gatewayStatusURL string + derivedEnsured bool statusURLs []string http *http.Client id identity @@ -62,7 +65,9 @@ func main() { c := &client{ base: strings.TrimRight(required("EDGE_CONTROL_URL"), "/"), dir: env("EDGE_STATE_DIR", "/var/lib/cdnfoundry/agent"), runtimeDir: env("EDGE_RUNTIME_DIR", ""), statusToken: env("EDGE_STATUS_TOKEN", ""), - statusURLs: splitNonempty(env("EDGE_CELL_STATUS_URLS", "")), http: &http.Client{Timeout: 15 * time.Second}, + gatewayBindings: env("EDGE_GATEWAY_BINDINGS", ""), + gatewayStatusURL: env("EDGE_GATEWAY_STATUS_URL", ""), + statusURLs: splitNonempty(env("EDGE_CELL_STATUS_URLS", "")), http: &http.Client{Timeout: 15 * time.Second}, } if err := c.configureServerTrust(env("EDGE_CONTROL_CA_CERTIFICATE", "")); err != nil { fatal(err) @@ -112,6 +117,35 @@ func (c *client) configureServerTrust(path string) error { return nil } +func (c *client) ensureDerivedRuntime(current state) error { + if c.runtimeDir == "" || c.derivedEnsured { + return nil + } + runtime, pools, err := compileRuntime(current) + if err != nil { + return err + } + if err := atomicJSON(filepath.Join(c.runtimeDir, "active.json"), runtime); err != nil { + return err + } + for name, pool := range pools { + if err := atomicJSON(filepath.Join(c.runtimeDir, name+".json"), pool); err != nil { + return err + } + } + if c.gatewayBindings != "" { + gateway, err := compileGateway(current.Sequence, pools, c.gatewayBindings) + if err != nil { + return err + } + if err := atomicJSON(filepath.Join(c.runtimeDir, "gateway.json"), gateway); err != nil { + return err + } + } + c.derivedEnsured = true + return nil +} + type edgeTask struct { ID string `json:"id"` Type string `json:"type"` @@ -559,6 +593,9 @@ func (c *client) sync() error { if err != nil { return err } + if err := c.ensureDerivedRuntime(current); err != nil { + return err + } var response struct { Data []struct { Sequence uint64 `json:"sequence"` @@ -730,6 +767,15 @@ func (c *client) activate(s state) error { return c.rollbackActive(active, previous, err) } } + if c.gatewayBindings != "" { + gateway, err := compileGateway(s.Sequence, pools, c.gatewayBindings) + if err != nil { + return c.rollbackActive(active, previous, err) + } + if err := atomicJSON(filepath.Join(c.runtimeDir, "gateway.json"), gateway); err != nil { + return c.rollbackActive(active, previous, err) + } + } } return nil } @@ -769,6 +815,18 @@ func compileRuntime(s state) (map[string]any, map[string]map[string]any, error) if domain.Domain == "" || len(domain.Hostnames) > 10000 { return nil, nil, errors.New("invalid runtime domain") } + if domain.Settings == nil { + domain.Settings = map[string]any{} + } + if domain.Cache == nil { + domain.Cache = map[string]any{} + } + if domain.Security == nil { + domain.Security = map[string]any{} + } + if domain.TLS == nil { + domain.TLS = map[string]any{} + } tlsReference := map[string]any{"mode": domain.TLS["mode"]} var certificateID string if certificateList, ok := domain.TLS["certificates"].([]any); ok { @@ -850,6 +908,61 @@ func validPoolName(name string) bool { return true } +type gatewayBinding struct { + Address string `json:"address"` + Pool string `json:"pool"` + HTTP string `json:"http"` + HTTPS string `json:"https"` +} + +func compileGateway(sequence uint64, pools map[string]map[string]any, raw string) (map[string]any, error) { + var bindings []gatewayBinding + if len(raw) > 64<<10 || json.Unmarshal([]byte(raw), &bindings) != nil || len(bindings) == 0 || len(bindings) > 32 { + return nil, errors.New("invalid gateway bindings") + } + listenerSet := map[string]bool{} + routes := []map[string]any{} + routeSet := map[string]bool{} + for _, binding := range bindings { + address := net.ParseIP(binding.Address) + poolRuntime := pools[binding.Pool] + hosts, _ := poolRuntime["hosts"].(map[string]any) + if address == nil || !validPoolName(binding.Pool) || hosts == nil || binding.HTTP == "" || binding.HTTPS == "" { + return nil, errors.New("invalid gateway binding") + } + for _, target := range []string{binding.HTTP, binding.HTTPS} { + host, port, err := net.SplitHostPort(target) + value, _ := strconv.Atoi(port) + if err != nil || value < 1 || value > 65535 || net.ParseIP(host) == nil && !validPoolName(host) { + return nil, errors.New("invalid gateway target") + } + } + listenerSet[net.JoinHostPort(address.String(), "80")] = true + listenerSet[net.JoinHostPort(address.String(), "443")] = true + names := make([]string, 0, len(hosts)) + for name := range hosts { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + key := address.String() + "|" + name + if routeSet[key] { + return nil, errors.New("duplicate gateway address and hostname") + } + routeSet[key] = true + routes = append(routes, map[string]any{ + "address": address.String(), "hostname": name, "http": binding.HTTP, "https": binding.HTTPS, + }) + } + } + listeners := make([]string, 0, len(listenerSet)) + for listener := range listenerSet { + listeners = append(listeners, listener) + } + sort.Strings(listeners) + return map[string]any{"schema_version": 1, "revision": sequence, "listeners": listeners, "routes": routes}, nil +} + func (c *client) heartbeat(sequence uint64) error { cells, failures, security := c.runtimeStatus() listenerReady := false @@ -858,12 +971,64 @@ func (c *client) heartbeat(sequence uint64) error { listenerReady = true } } + gateway := c.gatewayStatus() + if c.gatewayStatusURL != "" { + listenerReady = gateway["ready"] == true && gateway["active_revision"] == sequence + } return c.request("POST", "/edge/v1/heartbeat", map[string]any{ "agent_version": version, "listener_ready": listenerReady, "active_sequence": sequence, - "cells": cells, "passive_origins": failures, "noisy_domains": security, + "cells": cells, "passive_origins": failures, "noisy_domains": security, "gateway": gateway, }, &map[string]any{}, true) } +func (c *client) gatewayStatus() map[string]any { + status := map[string]any{"ready": false} + if c.gatewayStatusURL == "" { + return status + } + request, err := http.NewRequest("GET", c.gatewayStatusURL, nil) + if err != nil { + return status + } + response, err := c.http.Do(request) + if err != nil { + return status + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return status + } + body, err := io.ReadAll(io.LimitReader(response.Body, (64<<10)+1)) + if err != nil || len(body) > 64<<10 { + return status + } + names := map[string]string{ + "cdnfoundry_gateway_ready": "ready", "cdnfoundry_gateway_active_revision": "active_revision", + "cdnfoundry_gateway_routes": "routes", "cdnfoundry_gateway_listeners": "listeners", + "cdnfoundry_gateway_connections_active": "connections_active", + "cdnfoundry_gateway_connections_accepted_total": "connections_accepted", + "cdnfoundry_gateway_connections_rejected_total": "connections_rejected", + "cdnfoundry_gateway_errors_total": "errors", "cdnfoundry_gateway_candidate_rejections_total": "candidate_rejections", + } + for _, line := range strings.Split(string(body), "\n") { + fields := strings.Fields(line) + key := names[first(fields...)] + if len(fields) != 2 || key == "" { + continue + } + value, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + continue + } + if key == "ready" { + status[key] = value == 1 + } else { + status[key] = value + } + } + return status +} + func (c *client) runtimeStatus() ([]map[string]any, []map[string]any, []map[string]any) { cells := []map[string]any{} failures := []map[string]any{} diff --git a/edge-agent/main_test.go b/edge-agent/main_test.go index 9393be9..8436dc7 100644 --- a/edge-agent/main_test.go +++ b/edge-agent/main_test.go @@ -134,6 +134,46 @@ func TestRuntimeAssignsSupplementalCertificatesPerHostname(t *testing.T) { } } +func TestCompileGatewayRoutesByAddressAndPool(t *testing.T) { + pools := map[string]map[string]any{ + "shared-default": { + "hosts": map[string]any{"b.example.test": map[string]any{}, "a.example.test": map[string]any{}}, + }, + "quarantine-default": {"hosts": map[string]any{"blocked.example.test": map[string]any{}}}, + } + compiled, err := compileGateway(41, pools, `[ + {"address":"192.0.2.10","pool":"shared-default","http":"edge-a:8081","https":"edge-a:8444"}, + {"address":"2001:db8::10","pool":"shared-default","http":"edge-a:8081","https":"edge-a:8444"} + ]`) + if err != nil { + t.Fatal(err) + } + if compiled["revision"] != uint64(41) { + t.Fatalf("unexpected gateway revision: %#v", compiled) + } + listeners := compiled["listeners"].([]string) + if len(listeners) != 4 || listeners[0] != "192.0.2.10:443" { + t.Fatalf("unexpected listeners: %#v", listeners) + } + routes := compiled["routes"].([]map[string]any) + if len(routes) != 4 || routes[0]["hostname"] != "a.example.test" || routes[2]["address"] != "2001:db8::10" { + t.Fatalf("unexpected routes: %#v", routes) + } +} + +func TestCompileGatewayRejectsUnknownPoolDuplicateAndBounds(t *testing.T) { + pools := map[string]map[string]any{"shared": {"hosts": map[string]any{"a.example.test": map[string]any{}}}} + for _, raw := range []string{ + `[{"address":"192.0.2.10","pool":"missing","http":"cell:8081","https":"cell:8444"}]`, + `[{"address":"192.0.2.10","pool":"shared","http":"cell:8081","https":"cell:8444"},{"address":"192.0.2.10","pool":"shared","http":"cell:8081","https":"cell:8444"}]`, + `[{"address":"not-an-ip","pool":"shared","http":"cell:8081","https":"cell:8444"}]`, + } { + if _, err := compileGateway(1, pools, raw); err == nil { + t.Fatalf("invalid gateway bindings passed: %s", raw) + } + } +} + func TestFreshFullSnapshotThenIncrementalArtifact(t *testing.T) { public, private, err := ed25519.GenerateKey(rand.Reader) if err != nil { diff --git a/edge-gateway/Dockerfile b/edge-gateway/Dockerfile new file mode 100644 index 0000000..3eb2862 --- /dev/null +++ b/edge-gateway/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.24.6-alpine AS build +WORKDIR /src +COPY go.mod main.go main_test.go scale_test.go ./ +RUN CGO_ENABLED=0 go test ./... && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /edge-gateway . + +FROM alpine:3.22 +RUN addgroup -S -g 10101 gateway && adduser -S -D -H -u 10101 -G gateway gateway +COPY --from=build /edge-gateway /usr/local/bin/edge-gateway +USER gateway +ENTRYPOINT ["edge-gateway"] diff --git a/edge-gateway/go.mod b/edge-gateway/go.mod new file mode 100644 index 0000000..1ecd779 --- /dev/null +++ b/edge-gateway/go.mod @@ -0,0 +1,3 @@ +module cdnfoundry/edge-gateway + +go 1.24 diff --git a/edge-gateway/main.go b/edge-gateway/main.go new file mode 100644 index 0000000..b5d63a4 --- /dev/null +++ b/edge-gateway/main.go @@ -0,0 +1,588 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const version = "1.0.0" + +type config struct { + SchemaVersion int `json:"schema_version"` + Revision uint64 `json:"revision"` + Listeners []string `json:"listeners"` + Routes []route `json:"routes"` +} + +type route struct { + Address string `json:"address"` + Hostname string `json:"hostname"` + HTTP string `json:"http"` + HTTPS string `json:"https"` +} + +type routingTable struct { + revision uint64 + routes map[string]string +} + +type gateway struct { + configPath string + stateDir string + metrics string + table atomic.Pointer[routingTable] + ready atomic.Bool + accepted atomic.Uint64 + rejected atomic.Uint64 + errors atomic.Uint64 + active atomic.Int64 + reloads atomic.Uint64 + rejects atomic.Uint64 + mu sync.Mutex + listeners map[string]net.Listener + listen func(network, address string) (net.Listener, error) + slots chan struct{} +} + +func main() { + if len(os.Args) == 2 && os.Args[1] == "--version" { + fmt.Println(version) + return + } + g := &gateway{ + configPath: env("GATEWAY_CONFIG_FILE", "/var/lib/cdnfoundry/gateway/gateway.json"), + stateDir: env("GATEWAY_STATE_DIR", "/var/lib/cdnfoundry/gateway-state"), + metrics: env("GATEWAY_METRICS_ADDRESS", "127.0.0.1:9105"), + listeners: map[string]net.Listener{}, + slots: make(chan struct{}, boundedIntegerEnv("GATEWAY_MAX_CONNECTIONS", 8192, 128, 65536)), + } + if err := os.MkdirAll(g.stateDir, 0700); err != nil { + fatal(err) + } + if err := g.loadInitial(); err != nil { + fatal(err) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + go g.serveMetrics(ctx) + go g.watch(ctx) + <-ctx.Done() + g.close() +} + +func (g *gateway) loadInitial() error { + data, err := os.ReadFile(g.configPath) + if err != nil { + data, err = os.ReadFile(filepath.Join(g.stateDir, "last-valid.json")) + } + if err != nil { + return fmt.Errorf("no gateway candidate or last-valid map: %w", err) + } + return g.activate(data) +} + +func (g *gateway) watch(ctx context.Context) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + var signature string + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + info, err := os.Stat(g.configPath) + if err != nil { + continue + } + next := fmt.Sprintf("%d/%d", info.ModTime().UnixNano(), info.Size()) + if next == signature { + continue + } + data, err := os.ReadFile(g.configPath) + if err != nil || g.activate(data) != nil { + g.rejects.Add(1) + continue + } + signature = next + } + } +} + +func (g *gateway) activate(data []byte) error { + var candidate config + if len(data) > 32<<20 { + return errors.New("gateway map exceeds 32 MiB") + } + if err := json.Unmarshal(data, &candidate); err != nil { + return err + } + table, err := validate(candidate) + if err != nil { + return err + } + if current := g.table.Load(); current != nil && candidate.Revision < current.revision { + return errors.New("gateway revision cannot move backwards") + } + commitListeners, rollbackListeners, err := g.prepareListeners(candidate.Listeners) + if err != nil { + return err + } + if err := atomicWrite(filepath.Join(g.stateDir, "last-valid.json"), data); err != nil { + rollbackListeners() + return err + } + g.table.Store(table) + commitListeners() + g.reloads.Add(1) + g.ready.Store(true) + return nil +} + +func validate(candidate config) (*routingTable, error) { + if candidate.SchemaVersion != 1 || len(candidate.Listeners) == 0 || len(candidate.Listeners) > 64 || len(candidate.Routes) > 100000 { + return nil, errors.New("invalid gateway map bounds") + } + listeners := map[string]bool{} + for _, raw := range candidate.Listeners { + host, port, err := net.SplitHostPort(raw) + if err != nil || net.ParseIP(host) == nil || (port != "80" && port != "443") || listeners[raw] { + return nil, errors.New("invalid or duplicate gateway listener") + } + listeners[raw] = true + } + routes := make(map[string]string, len(candidate.Routes)*2) + for _, item := range candidate.Routes { + ip := net.ParseIP(item.Address) + host := canonicalHostname(item.Hostname) + if ip == nil || host == "" { + return nil, errors.New("invalid gateway route identity") + } + for protocol, target := range map[string]string{"http": item.HTTP, "https": item.HTTPS} { + if target == "" { + continue + } + targetHost, targetPort, err := net.SplitHostPort(target) + if err != nil || net.ParseIP(targetHost) == nil && canonicalHostname(targetHost) == "" { + return nil, errors.New("invalid gateway route target") + } + port, _ := strconv.Atoi(targetPort) + if port < 1 || port > 65535 { + return nil, errors.New("invalid gateway route target port") + } + key := protocol + "|" + ip.String() + "|" + host + if _, exists := routes[key]; exists { + return nil, errors.New("duplicate gateway route") + } + routes[key] = target + } + } + if len(routes) == 0 { + return nil, errors.New("gateway map has no routes") + } + return &routingTable{revision: candidate.Revision, routes: routes}, nil +} + +func (g *gateway) prepareListeners(desired []string) (func(), func(), error) { + g.mu.Lock() + defer g.mu.Unlock() + wanted := map[string]bool{} + opened := map[string]net.Listener{} + for _, address := range desired { + wanted[address] = true + if g.listeners[address] != nil { + continue + } + listen := g.listen + if listen == nil { + listen = net.Listen + } + listener, err := listen("tcp", address) + if err != nil { + for _, item := range opened { + _ = item.Close() + } + return nil, nil, fmt.Errorf("bind %s: %w", address, err) + } + opened[address] = listener + } + commit := func() { + g.mu.Lock() + defer g.mu.Unlock() + for address, listener := range opened { + g.listeners[address] = listener + go g.accept(address, listener) + } + for address, listener := range g.listeners { + if !wanted[address] { + _ = listener.Close() + delete(g.listeners, address) + } + } + } + rollback := func() { + for _, listener := range opened { + _ = listener.Close() + } + } + return commit, rollback, nil +} + +func (g *gateway) accept(address string, listener net.Listener) { + backoff := 10 * time.Millisecond + for { + connection, err := listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return + } + g.errors.Add(1) + time.Sleep(backoff) + if backoff < time.Second { + backoff *= 2 + } + continue + } + backoff = 10 * time.Millisecond + if g.slots != nil { + select { + case g.slots <- struct{}{}: + default: + g.rejected.Add(1) + _ = connection.Close() + continue + } + } + g.active.Add(1) + go func() { + if g.slots != nil { + defer func() { <-g.slots }() + } + defer g.active.Add(-1) + defer connection.Close() + if err := g.handle(address, connection); err != nil { + fmt.Fprintf(os.Stderr, "connection rejected on %s: %s\n", address, err) + } + }() + } +} + +func (g *gateway) handle(listener string, client net.Conn) error { + _ = client.SetDeadline(time.Now().Add(15 * time.Second)) + _, port, _ := net.SplitHostPort(listener) + var name string + var prefix []byte + var err error + protocol := "https" + if port == "443" { + name, prefix, err = readClientHello(client) + } else { + protocol = "http" + name, prefix, err = readHTTP(client) + } + if err != nil { + g.rejected.Add(1) + return err + } + localHost, _, _ := net.SplitHostPort(client.LocalAddr().String()) + ip := net.ParseIP(localHost) + table := g.table.Load() + if table == nil || ip == nil { + g.rejected.Add(1) + return errors.New("gateway unavailable") + } + target := table.routes[protocol+"|"+ip.String()+"|"+name] + if target == "" { + g.rejected.Add(1) + return errors.New("unknown gateway route") + } + upstream, err := net.DialTimeout("tcp", target, 3*time.Second) + if err != nil { + g.errors.Add(1) + return err + } + defer upstream.Close() + _ = upstream.SetDeadline(time.Now().Add(5 * time.Minute)) + if _, err := upstream.Write(proxyProtocolHeader(client.RemoteAddr(), client.LocalAddr())); err != nil { + g.errors.Add(1) + return err + } + if _, err := upstream.Write(prefix); err != nil { + g.errors.Add(1) + return err + } + g.accepted.Add(1) + _ = client.SetDeadline(time.Time{}) + _ = upstream.SetDeadline(time.Time{}) + done := make(chan struct{}, 1) + go func() { _, _ = io.Copy(upstream, client); _ = upstream.(*net.TCPConn).CloseWrite(); done <- struct{}{} }() + _, _ = io.Copy(client, upstream) + if tcp, ok := client.(*net.TCPConn); ok { + _ = tcp.CloseWrite() + } + <-done + return nil +} + +func readHTTP(connection net.Conn) (string, []byte, error) { + reader := bufio.NewReaderSize(connection, 16<<10) + var buffer bytes.Buffer + host := "" + for lines := 0; lines < 102; lines++ { + line, err := reader.ReadString('\n') + buffer.WriteString(line) + if err != nil || buffer.Len() > 16<<10 { + return "", nil, errors.New("invalid HTTP preface") + } + if lines == 0 { + fields := strings.Fields(strings.TrimSpace(line)) + if len(fields) != 3 || !strings.HasPrefix(fields[2], "HTTP/1.") { + return "", nil, errors.New("invalid HTTP request line") + } + continue + } + if line == "\r\n" { + if host == "" { + return "", nil, errors.New("missing Host") + } + return host, buffer.Bytes(), nil + } + name, value, found := strings.Cut(strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r"), ":") + if !found || strings.TrimSpace(name) != name { + return "", nil, errors.New("malformed HTTP header") + } + if strings.EqualFold(name, "host") { + if host != "" { + return "", nil, errors.New("duplicate Host") + } + value = strings.TrimSpace(value) + if parsedHost, parsedPort, err := net.SplitHostPort(value); err == nil && parsedPort != "" { + value = parsedHost + } + host = canonicalHostname(value) + if host == "" { + return "", nil, errors.New("invalid Host") + } + } + } + return "", nil, errors.New("too many HTTP headers") +} + +func readClientHello(connection net.Conn) (string, []byte, error) { + header := make([]byte, 5) + if _, err := io.ReadFull(connection, header); err != nil || header[0] != 22 { + return "", nil, errors.New("invalid TLS record") + } + size := int(binary.BigEndian.Uint16(header[3:5])) + if size < 4 || size > 64<<10 { + return "", nil, errors.New("invalid TLS ClientHello size") + } + body := make([]byte, size) + if _, err := io.ReadFull(connection, body); err != nil { + return "", nil, errors.New("truncated TLS ClientHello") + } + name, err := clientHelloSNI(body) + return name, append(header, body...), err +} + +func clientHelloSNI(data []byte) (string, error) { + if len(data) < 42 || data[0] != 1 { + return "", errors.New("not a TLS ClientHello") + } + p := 4 + 2 + 32 + if p >= len(data) { + return "", errors.New("truncated TLS ClientHello") + } + p += 1 + int(data[p]) + if p+2 > len(data) { + return "", errors.New("truncated TLS session") + } + p += 2 + int(binary.BigEndian.Uint16(data[p:p+2])) + if p >= len(data) { + return "", errors.New("truncated TLS ciphers") + } + p += 1 + int(data[p]) + if p+2 > len(data) { + return "", errors.New("missing TLS extensions") + } + end := p + 2 + int(binary.BigEndian.Uint16(data[p:p+2])) + p += 2 + if end > len(data) { + return "", errors.New("truncated TLS extensions") + } + for p+4 <= end { + kind, size := binary.BigEndian.Uint16(data[p:p+2]), int(binary.BigEndian.Uint16(data[p+2:p+4])) + p += 4 + if p+size > end { + return "", errors.New("truncated TLS extension") + } + if kind == 0 { + extension := data[p : p+size] + if len(extension) < 5 || int(binary.BigEndian.Uint16(extension[:2])) != len(extension)-2 || extension[2] != 0 { + return "", errors.New("invalid TLS SNI") + } + nameSize := int(binary.BigEndian.Uint16(extension[3:5])) + if nameSize != len(extension)-5 { + return "", errors.New("invalid TLS SNI length") + } + name := canonicalHostname(string(extension[5:])) + if name == "" { + return "", errors.New("invalid TLS SNI name") + } + return name, nil + } + p += size + } + return "", errors.New("missing TLS SNI") +} + +func canonicalHostname(value string) string { + if len(value) < 1 || len(value) > 253 || value != strings.TrimSpace(value) || strings.HasSuffix(value, ".") { + return "" + } + value = strings.ToLower(value) + labels := strings.Split(value, ".") + for _, label := range labels { + if len(label) < 1 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return "" + } + for _, character := range label { + if !(character >= 'a' && character <= 'z' || character >= '0' && character <= '9' || character == '-') { + return "" + } + } + } + return value +} + +func proxyProtocolHeader(source, destination net.Addr) []byte { + header := []byte("\r\n\r\n\x00\r\nQUIT\n") + src, _ := source.(*net.TCPAddr) + dst, _ := destination.(*net.TCPAddr) + if src == nil || dst == nil { + return append(header, 0x20, 0x00, 0x00, 0x00) + } + if source4, destination4 := src.IP.To4(), dst.IP.To4(); source4 != nil && destination4 != nil { + payload := make([]byte, 12) + copy(payload, source4) + copy(payload[4:], destination4) + binary.BigEndian.PutUint16(payload[8:], uint16(src.Port)) + binary.BigEndian.PutUint16(payload[10:], uint16(dst.Port)) + return append(append(header, 0x21, 0x11, 0, 12), payload...) + } + payload := make([]byte, 36) + copy(payload, src.IP.To16()) + copy(payload[16:], dst.IP.To16()) + binary.BigEndian.PutUint16(payload[32:], uint16(src.Port)) + binary.BigEndian.PutUint16(payload[34:], uint16(dst.Port)) + return append(append(header, 0x21, 0x21, 0, 36), payload...) +} + +func (g *gateway) serveMetrics(ctx context.Context) { + server := &http.Server{Addr: g.metrics, ReadHeaderTimeout: 2 * time.Second, IdleTimeout: 10 * time.Second, MaxHeaderBytes: 8 << 10} + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(writer http.ResponseWriter, _ *http.Request) { + if !g.ready.Load() { + http.Error(writer, "not ready", http.StatusServiceUnavailable) + return + } + _, _ = io.WriteString(writer, "ok\n") + }) + mux.HandleFunc("/metrics", func(writer http.ResponseWriter, _ *http.Request) { + table := g.table.Load() + revision, routes := uint64(0), 0 + if table != nil { + revision, routes = table.revision, len(table.routes) + } + g.mu.Lock() + listenerCount := len(g.listeners) + g.mu.Unlock() + values := map[string]uint64{ + "cdnfoundry_gateway_ready": boolNumber(g.ready.Load()), "cdnfoundry_gateway_active_revision": revision, + "cdnfoundry_gateway_routes": uint64(routes), "cdnfoundry_gateway_listeners": uint64(listenerCount), + "cdnfoundry_gateway_connections_active": uint64(max(g.active.Load(), 0)), + "cdnfoundry_gateway_connections_accepted_total": g.accepted.Load(), "cdnfoundry_gateway_connections_rejected_total": g.rejected.Load(), + "cdnfoundry_gateway_errors_total": g.errors.Load(), "cdnfoundry_gateway_activations_total": g.reloads.Load(), + "cdnfoundry_gateway_candidate_rejections_total": g.rejects.Load(), + } + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + fmt.Fprintf(writer, "%s %d\n", key, values[key]) + } + }) + server.Handler = mux + go func() { + <-ctx.Done() + shutdown, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _ = server.Shutdown(shutdown) + }() + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + fatal(err) + } +} + +func (g *gateway) close() { + g.ready.Store(false) + g.mu.Lock() + defer g.mu.Unlock() + for _, listener := range g.listeners { + _ = listener.Close() + } +} + +func atomicWrite(path string, data []byte) error { + temp := path + ".tmp" + if err := os.WriteFile(temp, data, 0600); err != nil { + return err + } + return os.Rename(temp, path) +} + +func boolNumber(value bool) uint64 { + if value { + return 1 + } + return 0 +} + +func env(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func boundedIntegerEnv(name string, fallback, minimum, maximum int) int { + value, err := strconv.Atoi(os.Getenv(name)) + if err != nil || value < minimum || value > maximum { + return fallback + } + return value +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/edge-gateway/main_test.go b/edge-gateway/main_test.go new file mode 100644 index 0000000..1bb18f1 --- /dev/null +++ b/edge-gateway/main_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "crypto/tls" + "encoding/binary" + "net" + "strings" + "testing" +) + +func TestValidateAcceptsBoundedDualStackRoutes(t *testing.T) { + candidate := config{ + SchemaVersion: 1, + Revision: 7, + Listeners: []string{"192.0.2.10:80", "[2001:db8::10]:443"}, + Routes: []route{{ + Address: "192.0.2.10", Hostname: "www.example.test", + HTTP: "cell-a:8081", HTTPS: "cell-a:8444", + }}, + } + table, err := validate(candidate) + if err != nil { + t.Fatal(err) + } + if table.revision != 7 || table.routes["http|192.0.2.10|www.example.test"] != "cell-a:8081" { + t.Fatalf("unexpected routing table: %#v", table) + } +} + +func TestValidateRejectsDuplicateAndInvalidCandidates(t *testing.T) { + base := config{ + SchemaVersion: 1, Revision: 1, Listeners: []string{"192.0.2.10:80"}, + Routes: []route{{Address: "192.0.2.10", Hostname: "www.example.test", HTTP: "cell-a:8081"}}, + } + tests := []config{ + {SchemaVersion: 2, Revision: 1, Listeners: base.Listeners, Routes: base.Routes}, + {SchemaVersion: 1, Revision: 1, Listeners: []string{"0.0.0.0:8080"}, Routes: base.Routes}, + {SchemaVersion: 1, Revision: 1, Listeners: base.Listeners, Routes: append(base.Routes, base.Routes[0])}, + {SchemaVersion: 1, Revision: 1, Listeners: base.Listeners, Routes: []route{{Address: "192.0.2.10", Hostname: "bad_name", HTTP: "cell-a:8081"}}}, + } + for index, candidate := range tests { + if _, err := validate(candidate); err == nil { + t.Fatalf("candidate %d unexpectedly passed", index) + } + } +} + +func TestReadHTTPRequiresOneCanonicalHost(t *testing.T) { + for _, request := range []string{ + "GET / HTTP/1.1\r\n\r\n", + "GET / HTTP/1.1\r\nHost: good.example\r\nHost: evil.example\r\n\r\n", + "GET / HTTP/1.1\r\nHost: bad_name\r\n\r\n", + } { + server, client := net.Pipe() + go func(value string) { _, _ = client.Write([]byte(value)); _ = client.Close() }(request) + if _, _, err := readHTTP(server); err == nil { + t.Fatalf("invalid request passed: %q", request) + } + _ = server.Close() + } + server, client := net.Pipe() + go func() { + _, _ = client.Write([]byte("GET / HTTP/1.1\r\nHost: WWW.Example.Test:80\r\n\r\n")) + _ = client.Close() + }() + host, prefix, err := readHTTP(server) + if err != nil || host != "www.example.test" || !strings.Contains(string(prefix), "Host: WWW.Example.Test:80") { + t.Fatalf("valid request failed: host=%q err=%v", host, err) + } +} + +func TestClientHelloSNI(t *testing.T) { + server, client := net.Pipe() + result := make(chan []byte, 1) + go func() { + header := make([]byte, 5) + _, _ = server.Read(header) + size := int(binary.BigEndian.Uint16(header[3:5])) + body := make([]byte, size) + offset := 0 + for offset < size { + n, _ := server.Read(body[offset:]) + offset += n + } + result <- body + _ = server.Close() + }() + tlsClient := tls.Client(client, &tls.Config{ServerName: "Route.Example.Test", MinVersion: tls.VersionTLS12}) + _ = tlsClient.Handshake() + body := <-result + name, err := clientHelloSNI(body) + if err != nil || name != "route.example.test" { + t.Fatalf("unexpected SNI: %q, %v", name, err) + } +} + +func TestProxyProtocolPreservesIPv4AndIPv6Identity(t *testing.T) { + for _, item := range []struct { + source, destination string + family byte + length int + }{ + {"198.51.100.2:1234", "192.0.2.10:80", 0x11, 28}, + {"[2001:db8::2]:1234", "[2001:db8::10]:443", 0x21, 52}, + } { + source, _ := net.ResolveTCPAddr("tcp", item.source) + destination, _ := net.ResolveTCPAddr("tcp", item.destination) + header := proxyProtocolHeader(source, destination) + if len(header) != item.length || header[13] != item.family { + t.Fatalf("unexpected PROXY protocol header for %s: %x", item.source, header) + } + } +} diff --git a/edge-gateway/scale_test.go b/edge-gateway/scale_test.go new file mode 100644 index 0000000..199041f --- /dev/null +++ b/edge-gateway/scale_test.go @@ -0,0 +1,248 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net" + "runtime" + "sort" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" +) + +func TestScaleSocketThroughput50000Mappings(t *testing.T) { + if testing.Short() { + t.Skip("scale qualification") + } + upstream, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer upstream.Close() + go serveScaleUpstream(upstream) + + const mappings = 50000 + candidate := config{ + SchemaVersion: 1, Revision: 9002, Listeners: []string{"127.0.0.1:80"}, + Routes: make([]route, 0, mappings), + } + for index := 0; index < mappings; index++ { + candidate.Routes = append(candidate.Routes, route{ + Address: "127.0.0.1", Hostname: fmt.Sprintf("host-%05d.socket-scale.example.test", index), + HTTP: upstream.Addr().String(), + }) + } + stateDir := t.TempDir() + var gatewayAddress string + g := &gateway{ + stateDir: stateDir, listeners: map[string]net.Listener{}, + listen: func(_, _ string) (net.Listener, error) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err == nil { + gatewayAddress = listener.Addr().String() + } + return listener, err + }, + } + encoded, err := json.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if err := g.activate(encoded); err != nil { + t.Fatal(err) + } + defer g.close() + + acceptedConcurrency := 0 + for _, concurrency := range []int{16, 64, 128} { + result := runSocketLoad(t, gatewayAddress, concurrency, 200) + t.Logf("socket_load mappings=%d concurrency=%d requests=%d seconds=%.3f requests_per_second=%.0f p50_ms=%.3f p95_ms=%.3f p99_ms=%.3f errors=%d", + mappings, concurrency, result.requests, result.duration.Seconds(), float64(result.requests)/result.duration.Seconds(), + result.percentile(50), result.percentile(95), result.percentile(99), result.errors) + if result.errors != 0 { + if acceptedConcurrency == 0 { + t.Fatalf("socket load failed at minimum concurrency %d with %d errors", concurrency, result.errors) + } + t.Logf("socket_saturation=observed concurrency=%d errors=%d accepted_connection_concurrency=%d upstream=bounded_local_tcp", + concurrency, result.errors, acceptedConcurrency) + return + } + acceptedConcurrency = concurrency + } + t.Logf("socket_saturation=not_observed accepted_connection_concurrency=%d upstream=bounded_local_tcp", acceptedConcurrency) +} + +type socketLoadResult struct { + requests int + errors int + duration time.Duration + latency []time.Duration +} + +func (result socketLoadResult) percentile(value int) float64 { + if len(result.latency) == 0 { + return 0 + } + return float64(result.latency[(len(result.latency)-1)*value/100].Microseconds()) / 1000 +} + +func runSocketLoad(t *testing.T, gatewayAddress string, concurrency, each int) socketLoadResult { + t.Helper() + started := time.Now() + latencies := make(chan time.Duration, concurrency*each) + errors := atomic.Int64{} + var wait sync.WaitGroup + for worker := 0; worker < concurrency; worker++ { + wait.Add(1) + go func(offset int) { + defer wait.Done() + for request := 0; request < each; request++ { + began := time.Now() + connection, err := net.DialTimeout("tcp4", gatewayAddress, 3*time.Second) + if err == nil { + host := (request + offset) % 50000 + _, err = fmt.Fprintf(connection, "GET / HTTP/1.1\r\nHost: host-%05d.socket-scale.example.test\r\nConnection: close\r\n\r\n", host) + } + if err == nil { + response, readErr := io.ReadAll(io.LimitReader(connection, 1024)) + err = readErr + if !bytes.Contains(response, []byte("200 OK")) { + err = fmt.Errorf("invalid response") + } + } + if connection != nil { + _ = connection.Close() + } + if err != nil { + errors.Add(1) + } else { + latencies <- time.Since(began) + } + } + }(worker) + } + wait.Wait() + close(latencies) + values := make([]time.Duration, 0, concurrency*each) + for latency := range latencies { + values = append(values, latency) + } + sort.Slice(values, func(left, right int) bool { return values[left] < values[right] }) + return socketLoadResult{requests: concurrency * each, errors: int(errors.Load()), duration: time.Since(started), latency: values} +} + +func serveScaleUpstream(listener net.Listener) { + for { + connection, err := listener.Accept() + if err != nil { + return + } + go func() { + defer connection.Close() + header := make([]byte, 16) + if _, err := io.ReadFull(connection, header); err != nil { + return + } + length := int(header[14])<<8 | int(header[15]) + if length > 36 { + return + } + if _, err := io.CopyN(io.Discard, connection, int64(length)); err != nil { + return + } + reader := bufio.NewReaderSize(connection, 4096) + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + if line == "\r\n" { + break + } + } + _, _ = io.WriteString(connection, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + }() + } +} + +func TestScaleTarget50000Mappings(t *testing.T) { + if testing.Short() { + t.Skip("scale qualification") + } + const mappings = 50000 + candidate := config{ + SchemaVersion: 1, + Revision: 9001, + Listeners: []string{ + "192.0.2.10:80", "192.0.2.10:443", "[2001:db8::10]:80", "[2001:db8::10]:443", + "192.0.2.11:80", "192.0.2.11:443", "[2001:db8::11]:80", "[2001:db8::11]:443", + }, + Routes: make([]route, 0, mappings), + } + addresses := []string{"192.0.2.10", "2001:db8::10", "192.0.2.11", "2001:db8::11"} + for index := 0; index < mappings; index++ { + candidate.Routes = append(candidate.Routes, route{ + Address: addresses[index%len(addresses)], Hostname: fmt.Sprintf("host-%05d.scale.example.test", index), + HTTP: "cell-a:8081", HTTPS: "cell-a:8444", + }) + } + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + started := time.Now() + table, err := validate(candidate) + activationDuration := time.Since(started) + if err != nil { + t.Fatal(err) + } + runtime.ReadMemStats(&after) + if len(table.routes) != mappings*2 { + t.Fatalf("expected %d protocol mappings, got %d", mappings*2, len(table.routes)) + } + + const workers, lookupsPerWorker = 64, 20000 + var found atomic.Uint64 + var usageBefore, usageAfter syscall.Rusage + _ = syscall.Getrusage(syscall.RUSAGE_SELF, &usageBefore) + started = time.Now() + var wait sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wait.Add(1) + go func(offset int) { + defer wait.Done() + for index := 0; index < lookupsPerWorker; index++ { + host := (index + offset) % mappings + address := addresses[host%len(addresses)] + if table.routes[fmt.Sprintf("https|%s|host-%05d.scale.example.test", address, host)] != "" { + found.Add(1) + } + } + }(worker) + } + wait.Wait() + lookupDuration := time.Since(started) + _ = syscall.Getrusage(syscall.RUSAGE_SELF, &usageAfter) + cpuSeconds := rusageSeconds(usageAfter) - rusageSeconds(usageBefore) + total := uint64(workers * lookupsPerWorker) + if found.Load() != total { + t.Fatalf("lost lookups: expected %d, found %d", total, found.Load()) + } + t.Logf( + "mappings=%d protocol_routes=%d listeners=%d activation_ms=%.3f heap_delta_mib=%.2f concurrent_workers=%d lookups=%d lookup_seconds=%.3f cpu_seconds=%.3f average_cpu_cores=%.2f lookups_per_second=%.0f average_lookup_us=%.3f saturation=not_observed accepted_concurrency=%d", + mappings, len(table.routes), len(candidate.Listeners), float64(activationDuration.Microseconds())/1000, + float64(after.HeapAlloc-before.HeapAlloc)/(1024*1024), workers, total, lookupDuration.Seconds(), + cpuSeconds, cpuSeconds/lookupDuration.Seconds(), float64(total)/lookupDuration.Seconds(), + float64(lookupDuration.Microseconds())/float64(total), workers, + ) +} + +func rusageSeconds(usage syscall.Rusage) float64 { + return float64(usage.Utime.Sec+usage.Stime.Sec) + + float64(usage.Utime.Usec+usage.Stime.Usec)/1_000_000 +} diff --git a/tests/e2e/gateway_ingress.py b/tests/e2e/gateway_ingress.py new file mode 100644 index 0000000..f100516 --- /dev/null +++ b/tests/e2e/gateway_ingress.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Real gateway qualification. This does not automate a browser.""" + +import json +import pathlib +import subprocess +import tempfile +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] +COMPOSE = ["docker", "compose", "--env-file", ".env.dev", "-f", "compose.dev.yml"] +GATEWAY = "cdnfoundry-dev-edge-gateway-a-1" +IPV4_ONLY_GATEWAY = "cdnfoundry-dev-edge-gateway-b-1" +STATE_VOLUME = "cdnfoundry-dev_edge-a-state" +IPV4_ONLY_STATE_VOLUME = "cdnfoundry-dev_edge-b-state" + + +def run(*command: str, check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if check and result.returncode: + raise RuntimeError(f"{' '.join(command)} failed ({result.returncode}):\n{result.stdout}") + return result + + +def metrics(container: str = GATEWAY) -> dict[str, int]: + output = run("docker", "exec", container, "wget", "-qO-", "http://127.0.0.1:9105/metrics").stdout + return {line.split()[0]: int(line.split()[1]) for line in output.splitlines() if len(line.split()) == 2} + + +def runtime(volume: str = STATE_VOLUME) -> tuple[dict, dict]: + script = ( + "import json;" + "print(json.dumps(json.load(open('/state/gateway.json'))));" + "print(json.dumps(json.load(open('/state/shared-default.json'))))" + ) + output = run("docker", "run", "--rm", "-v", f"{volume}:/state:ro", "python:3.13-alpine", "python", "-c", script).stdout + gateway, cell = output.splitlines() + return json.loads(gateway), json.loads(cell) + + +def curl(address: str, hostname: str, tls: bool = False, expect_success: bool = True, + gateway: str = GATEWAY, tls_ca: pathlib.Path | None = None) -> str: + url = f"{'https' if tls else 'http'}://{hostname}/" + command = [ + "docker", "run", "--rm", "--network", f"container:{gateway}", "curlimages/curl:8.16.0", + "--noproxy", "*", "--max-time", "10", "-sS", "-D", "-", "-o", "/dev/null", + "--resolve", f"{hostname}:{443 if tls else 80}:{'[' + address + ']' if ':' in address else address}", + ] + if tls: + if tls_ca is None: + raise RuntimeError("strict TLS probe requires a CA certificate") + command[5:5] = ["-v", f"{tls_ca}:/tls/ca.pem:ro"] + command.extend(["--cacert", "/tls/ca.pem"]) + result = run(*command, url, check=False) + if expect_success and (result.returncode or "server: openresty" not in result.stdout.lower()): + raise RuntimeError(f"route {address}/{hostname} did not reach OpenResty:\n{result.stdout}") + if not expect_success and result.returncode == 0: + raise RuntimeError(f"unknown route {address}/{hostname} was accepted") + return result.stdout + + +def isolated_last_valid_test() -> None: + with tempfile.TemporaryDirectory(prefix="cdnfoundry-gateway-") as temporary: + base = pathlib.Path(temporary) + config_path, state_path = base / "config", base / "state" + config_path.mkdir() + state_path.mkdir() + state_path.chmod(0o777) + candidate = { + "schema_version": 1, + "revision": 17, + "listeners": ["127.0.0.50:80", "127.0.0.50:443"], + "routes": [{ + "address": "127.0.0.50", "hostname": "last-valid.example.test", + "http": "127.0.0.1:9", "https": "127.0.0.1:9", + }], + } + (config_path / "gateway.json").write_text(json.dumps(candidate)) + container = "cdnfoundry-gateway-last-valid-e2e" + run( + "docker", "run", "-d", "--rm", "--name", container, + "--cap-drop", "ALL", "--cap-add", "NET_BIND_SERVICE", + "-v", f"{config_path}:/config", "-v", f"{state_path}:/state", + "-e", "GATEWAY_CONFIG_FILE=/config/gateway.json", + "-e", "GATEWAY_STATE_DIR=/state", + "-e", "GATEWAY_METRICS_ADDRESS=127.0.0.1:9105", + "cdnfoundry/edge-gateway:qualification", + ) + try: + time.sleep(2) + before = metrics(container) + (config_path / "gateway.json").write_text('{"schema_version":1,"revision":18,"listeners":[],"routes":[]}') + time.sleep(2) + after = metrics(container) + if after["cdnfoundry_gateway_active_revision"] != 17: + raise RuntimeError("invalid candidate replaced the active gateway map") + if after["cdnfoundry_gateway_candidate_rejections_total"] <= before["cdnfoundry_gateway_candidate_rejections_total"]: + raise RuntimeError("invalid candidate rejection was not observable") + run("docker", "stop", container) + restart_container = container + "-restart" + run( + "docker", "run", "-d", "--rm", "--name", restart_container, + "--cap-drop", "ALL", "--cap-add", "NET_BIND_SERVICE", + "-v", f"{config_path}:/config:ro", "-v", f"{state_path}:/state", + "-e", "GATEWAY_CONFIG_FILE=/config/missing.json", + "-e", "GATEWAY_STATE_DIR=/state", + "-e", "GATEWAY_METRICS_ADDRESS=127.0.0.1:9105", + "cdnfoundry/edge-gateway:qualification", + ) + time.sleep(2) + if metrics(restart_container)["cdnfoundry_gateway_active_revision"] != 17: + raise RuntimeError("gateway restart did not restore the last valid map") + finally: + run("docker", "stop", container, check=False) + run("docker", "stop", container + "-restart", check=False) + + +def main() -> None: + gateway, cell = runtime() + hostname = next( + name for name, value in cell["hosts"].items() + if value.get("tls", {}).get("certificate_id") + ) + addresses = {route["address"] for route in gateway["routes"] if route["hostname"] == hostname} + if not {"172.28.10.10", "fd00:cd0f:10::10"}.issubset(addresses): + raise RuntimeError(f"dual-stack route missing for {hostname}: {sorted(addresses)}") + revision = metrics()["cdnfoundry_gateway_active_revision"] + with tempfile.TemporaryDirectory(prefix="cdnfoundry-gateway-ca-") as ca_directory: + ca_path = pathlib.Path(ca_directory) / "root.pem" + pathlib.Path(ca_directory).chmod(0o777) + run( + "docker", "run", "--rm", "--network", "cdnfoundry-dev_control", + "-v", f"{ca_directory}:/out", "curlimages/curl:8.16.0", "-ksS", + "https://pebble:15000/roots/0", "-o", "/out/root.pem", + ) + for address in ("172.28.10.10", "fd00:cd0f:10::10"): + curl(address, hostname) + curl(address, hostname, tls=True, tls_ca=ca_path) + curl(address, "unknown-gateway.example.test", expect_success=False) + run(*COMPOSE, "restart", "edge-gateway-a") + time.sleep(2) + if metrics()["cdnfoundry_gateway_active_revision"] != revision: + raise RuntimeError("gateway restart changed the active revision") + curl("172.28.10.10", hostname) + + ipv4_gateway, ipv4_cell = runtime(IPV4_ONLY_STATE_VOLUME) + ipv4_hostname = next( + name for name, value in ipv4_cell["hosts"].items() + if value.get("tls", {}).get("certificate_id") + ) + ipv4_addresses = {route["address"] for route in ipv4_gateway["routes"] if route["hostname"] == ipv4_hostname} + if "172.28.20.10" not in ipv4_addresses or any(":" in address for address in ipv4_addresses): + raise RuntimeError(f"IPv4-only gateway unexpectedly required another family: {sorted(ipv4_addresses)}") + curl("172.28.20.10", ipv4_hostname, gateway=IPV4_ONLY_GATEWAY) + curl("172.28.20.10", ipv4_hostname, tls=True, gateway=IPV4_ONLY_GATEWAY, tls_ca=ca_path) + + isolated_last_valid_test() + print(json.dumps({ + "status": "passed", "revision": revision, "hostname": hostname, + "families": ["IPv4", "IPv6"], "protocols": ["HTTP Host", "HTTPS SNI"], + "tls_verification": "strict", + "ipv4_only_gateway": {"hostname": ipv4_hostname, "addresses": sorted(ipv4_addresses)}, + "metrics": metrics(), + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/phase7_analytics.py b/tests/e2e/phase7_analytics.py index 424c60f..2c39b19 100644 --- a/tests/e2e/phase7_analytics.py +++ b/tests/e2e/phase7_analytics.py @@ -109,16 +109,16 @@ def restart_vector(reconnect_sources: bool = False) -> None: compose("restart", "dnsdist", "edge-a", timeout=90) deadline = time.monotonic() + 30 while time.monotonic() < deadline: - edge_health = run( - "curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", - "http://127.0.0.1:8081/healthz", check=False, + edge_health = compose( + "exec", "-T", "edge-a", "wget", "-qO-", + "http://127.0.0.1:8080/healthz", check=False, ) dns_health = compose( "exec", "-T", "dnsdist", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8083/metrics', timeout=2).read(1)", check=False, ) - if (edge_health.returncode == 0 and edge_health.stdout.strip() == "200" + if (edge_health.returncode == 0 and edge_health.stdout.strip() == "ok" and dns_health.returncode == 0): return time.sleep(1) @@ -135,6 +135,13 @@ def vector_event(port: int, event: dict[str, object]) -> None: ) +def cell_request(path: str) -> subprocess.CompletedProcess[str]: + return compose( + "exec", "-T", "edge-a", "wget", "-S", "-O", "/dev/null", + "--header", f"Host: {DOMAIN}", f"http://127.0.0.1:8080{path}", check=False, + ) + + def clickhouse(query: str) -> str: return compose("exec", "-T", "clickhouse", "clickhouse-client", "--query", query).stdout.strip() @@ -191,11 +198,8 @@ def qualify_ingestion_and_queries(domain_id: int, user: str, stranger: str, admi runtime_path = f"/runtime-{RUN_ID}" def produce_runtime_event() -> None: - edge = run( - "curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {DOMAIN}", - f"http://127.0.0.1:8081{runtime_path}?token=must-not-survive", check=False, - ) - assert edge.returncode == 0 and len(edge.stdout.strip()) == 3, edge + edge = cell_request(f"{runtime_path}?token=must-not-survive") + assert "HTTP/1.1" in edge.stderr, edge wait_for_clickhouse( lambda value: int(value or "0") >= 1, @@ -292,8 +296,8 @@ def qualify_usage(domain_id: int, user: str, admin: str, interval_from: dt.datet def qualify_outage(domain_id: int, user: str) -> None: - before = run("curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {DOMAIN}", "http://127.0.0.1:8081/", check=False) - assert before.returncode == 0 and len(before.stdout.strip()) == 3, before + before = cell_request("/") + assert "HTTP/1.1" in before.stderr, before buffered_id = str(uuid.uuid4()) clickhouse_stopped = False try: @@ -309,8 +313,8 @@ def qualify_outage(domain_id: int, user: str) -> None: }) dns = run("dig", "+time=2", "+tries=1", "@127.0.0.1", "-p", "1053", f"outage.{DOMAIN}", "SOA") assert "status:" in dns.stdout, dns.stdout - during = run("curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {DOMAIN}", "http://127.0.0.1:8081/", check=False) - assert during.returncode == 0 and len(during.stdout.strip()) == 3, during + during = cell_request("/") + assert "HTTP/1.1" in during.stderr, during metrics = compose("exec", "-T", "vector", "wget", "-qO-", "http://127.0.0.1:9598/metrics").stdout assert "vector_buffer" in metrics and "vector_component" in metrics, metrics[:1000] finally: