From 9dd9da779514fce1a9bf15ca9e24a3a4019d863d Mon Sep 17 00:00:00 2001 From: vaheeD Date: Sat, 8 Aug 2026 19:42:42 +0330 Subject: [PATCH 1/3] feat: add configuration-driven production fleet --- .env.prod.example | 3 +- .github/workflows/ci.yml | 5 +- compose.prod.yml | 119 +-- .../production/compose.control-host-ipv6.yml | 6 +- deploy/production/compose.control-host.yml | 10 +- deploy/production/compose.dns-edge-host.yml | 2 +- deploy/production/compose.dns-host-ipv6.yml | 4 +- deploy/production/compose.dns-host.yml | 2 +- .../compose.telemetry-host-ipv6.yml | 4 +- deploy/production/compose.telemetry-host.yml | 9 +- .../examples/multi-region-fleet.json | 35 + deploy/production/examples/starter-fleet.json | 44 + docs/deployment/index.md | 4 + .../production-fleet-config-reference.md | 217 ++++ .../production-fleet-operator-guide.md | 476 +++++++++ docs/deployment/production-fleet.md | 310 ++++++ .../production-quick-start-multi-region.md | 59 ++ docs/deployment/production-quick-start.md | 903 ++--------------- docs/operations/backup-and-recovery.md | 2 +- scripts/README.md | 47 + scripts/cdnfoundry-fleet | 4 + scripts/cdnfoundry_fleet/__init__.py | 3 + scripts/cdnfoundry_fleet/__main__.py | 3 + scripts/cdnfoundry_fleet/certs.py | 189 ++++ scripts/cdnfoundry_fleet/cli.py | 883 ++++++++++++++++ scripts/cdnfoundry_fleet/common.py | 164 +++ scripts/cdnfoundry_fleet/compose.py | 227 +++++ scripts/cdnfoundry_fleet/render.py | 735 ++++++++++++++ scripts/cdnfoundry_fleet/state.py | 499 +++++++++ scripts/install-production-prerequisites.sh | 41 + tests/fleet/test_fleet.py | 950 ++++++++++++++++++ 31 files changed, 5074 insertions(+), 885 deletions(-) create mode 100644 deploy/production/examples/multi-region-fleet.json create mode 100644 deploy/production/examples/starter-fleet.json create mode 100644 docs/deployment/production-fleet-config-reference.md create mode 100644 docs/deployment/production-fleet-operator-guide.md create mode 100644 docs/deployment/production-fleet.md create mode 100644 docs/deployment/production-quick-start-multi-region.md create mode 100644 scripts/README.md create mode 100755 scripts/cdnfoundry-fleet create mode 100644 scripts/cdnfoundry_fleet/__init__.py create mode 100644 scripts/cdnfoundry_fleet/__main__.py create mode 100644 scripts/cdnfoundry_fleet/certs.py create mode 100644 scripts/cdnfoundry_fleet/cli.py create mode 100644 scripts/cdnfoundry_fleet/common.py create mode 100644 scripts/cdnfoundry_fleet/compose.py create mode 100644 scripts/cdnfoundry_fleet/render.py create mode 100644 scripts/cdnfoundry_fleet/state.py create mode 100755 scripts/install-production-prerequisites.sh create mode 100644 tests/fleet/test_fleet.py diff --git a/.env.prod.example b/.env.prod.example index 6fa883f..4c3cd64 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -45,7 +45,7 @@ METRICS_TOKEN_FILE=/etc/cdnfoundry/secrets/metrics-token # The repository location is not a password: Restic encrypts repository contents with # the separate password file. Restrict S3 credentials to this bucket/prefix only. RESTIC_REPOSITORY= -RESTIC_PASSWORD_FILE= +RESTIC_PASSWORD_FILE=/dev/null BACKUP_ACCESS_KEY_ID= BACKUP_SECRET_ACCESS_KEY= BACKUP_DEFAULT_REGION=us-east-1 @@ -84,6 +84,7 @@ GRAFANA_POSTGRES_PASSWORD=replace-with-a-unique-high-entropy-grafana-postgres-pa # [optional] Loopback by default. Terminate public TLS/authentication at a trusted reverse proxy. GRAFANA_BIND=127.0.0.1:3000 GRAFANA_COOKIE_SECURE=true +GRAFANA_LOKI_URL=http://loki:3100 # [optional] Grafana query endpoints. Set verified external hosts for split deployments. GRAFANA_CLICKHOUSE_HOST=clickhouse GRAFANA_CLICKHOUSE_PORT=9000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3488421..41ea216 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - dev tags: - 'v*' pull_request: @@ -310,7 +311,7 @@ jobs: publish-images: name: Publish GHCR images - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')) needs: [core, compose, backend-e2e, scale-e2e, go, docs] runs-on: ubuntu-latest timeout-minutes: 90 @@ -358,6 +359,8 @@ jobs: "${RELEASE_REF_NAME}" "${RELEASE}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${WORKFLOW_RUN_ID}" > release-evidence/release-manifest.json if [[ "${RELEASE_REF}" == "refs/heads/main" ]]; then aliases+=(latest) + elif [[ "${RELEASE_REF}" == "refs/heads/dev" ]]; then + aliases+=(dev dev-latest) elif [[ "${RELEASE_REF}" == refs/tags/v* ]]; then version="${RELEASE_REF_NAME#v}" if [[ ! "${version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then diff --git a/compose.prod.yml b/compose.prod.yml index a48b8fc..629eb10 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -6,32 +6,32 @@ x-core-env: &core-env APP_KEY: ${APP_KEY:?APP_KEY is required} EDGE_ARTIFACT_SIGNING_KEY: ${EDGE_ARTIFACT_SIGNING_KEY:?EDGE_ARTIFACT_SIGNING_KEY is required} APP_URL: ${APP_URL:?APP_URL is required} - SESSION_SECURE_COOKIE: ${SESSION_SECURE_COOKIE:-true} + SESSION_SECURE_COOKIE: ${SESSION_SECURE_COOKIE} DB_CONNECTION: pgsql - DB_URL: ${DB_URL:-} - DB_HOST: ${DB_HOST:-control-db} - DB_PORT: ${DB_PORT:-5432} + DB_URL: ${DB_URL} + DB_HOST: ${DB_HOST} + DB_PORT: ${DB_PORT} DB_DATABASE: cdnf DB_USERNAME: cdnf DB_PASSWORD: ${CONTROL_DB_PASSWORD:?CONTROL_DB_PASSWORD is required} - DB_SSLMODE: ${DB_SSLMODE:-prefer} + DB_SSLMODE: ${DB_SSLMODE} CACHE_STORE: redis SESSION_DRIVER: redis QUEUE_CONNECTION: redis - REDIS_URL: ${REDIS_URL:-} - REDIS_HOST: ${REDIS_HOST:-redis} - REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_URL: ${REDIS_URL} + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} REDIS_CLIENT: predis REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb ACME_ENABLED: "true" - ACME_DIRECTORY_URL: ${ACME_DIRECTORY_URL:-https://acme-v02.api.letsencrypt.org/directory} + ACME_DIRECTORY_URL: ${ACME_DIRECTORY_URL} ACME_CONTACT_EMAIL: ${ACME_CONTACT_EMAIL:?ACME_CONTACT_EMAIL is required} - ACME_ORDER_BUDGET_PER_HOUR: ${ACME_ORDER_BUDGET_PER_HOUR:-20} + ACME_ORDER_BUDGET_PER_HOUR: ${ACME_ORDER_BUDGET_PER_HOUR} EDGE_IDENTITY_CA_CERTIFICATE: /run/secrets/edge-identity-ca.crt EDGE_IDENTITY_CA_PRIVATE_KEY: /run/secrets/edge-identity-ca.key - EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE: ${EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE:-} + EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE: ${EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE} PDNS_CA_CERTIFICATE: /run/secrets/pdns-api-ca.crt CLICKHOUSE_URL: ${CLICKHOUSE_URL:?CLICKHOUSE_URL is required} CLICKHOUSE_DATABASE: cdnf @@ -39,15 +39,15 @@ x-core-env: &core-env CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required} PROMETHEUS_URL: http://prometheus:9090 METRICS_TOKEN_FILE: /run/secrets/metrics-token - RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:-} + RESTIC_REPOSITORY: ${RESTIC_REPOSITORY} RESTIC_PASSWORD_FILE: /run/secrets/restic-password - BACKUP_ACCESS_KEY_ID: ${BACKUP_ACCESS_KEY_ID:-} - BACKUP_SECRET_ACCESS_KEY: ${BACKUP_SECRET_ACCESS_KEY:-} - BACKUP_DEFAULT_REGION: ${BACKUP_DEFAULT_REGION:-us-east-1} + BACKUP_ACCESS_KEY_ID: ${BACKUP_ACCESS_KEY_ID} + BACKUP_SECRET_ACCESS_KEY: ${BACKUP_SECRET_ACCESS_KEY} + BACKUP_DEFAULT_REGION: ${BACKUP_DEFAULT_REGION} LOG_CHANNEL: stderr LOG_STACK: stderr LOG_STDERR_FORMATTER: App\Logging\OperationalJsonFormatter - GRAFANA_EXPLORE_URL: ${GRAFANA_EXPLORE_URL:-} + GRAFANA_EXPLORE_URL: ${GRAFANA_EXPLORE_URL} x-edge-cell: &edge-cell image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} @@ -83,6 +83,7 @@ x-core: &core depends_on: control-db: { condition: service_healthy } redis: { condition: service_healthy } + mmdb-updater: { condition: service_healthy } networks: [control, telemetry, egress] restart: unless-stopped stop_grace_period: 60s @@ -102,7 +103,7 @@ x-core: &core - ${EDGE_IDENTITY_CA_PRIVATE_KEY:?EDGE_IDENTITY_CA_PRIVATE_KEY is required}:/run/secrets/edge-identity-ca.key:ro - ${PDNS_CA_CERTIFICATE:?PDNS_CA_CERTIFICATE is required}:/run/secrets/pdns-api-ca.crt:ro - ${METRICS_TOKEN_FILE:?METRICS_TOKEN_FILE is required}:/run/secrets/metrics-token:ro - - ${RESTIC_PASSWORD_FILE:-/dev/null}:/run/secrets/restic-password:ro + - ${RESTIC_PASSWORD_FILE}:/run/secrets/restic-password:ro services: core: @@ -117,7 +118,7 @@ services: web: image: ghcr.io/vaheed/cdnfoundry-web:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [control] - ports: ["${CONTROL_BIND:-127.0.0.1:8080}:8080"] + ports: ["${CONTROL_BIND}:8080"] depends_on: core: { condition: service_healthy } networks: [control, ingress] @@ -133,7 +134,7 @@ services: edge-control: image: ghcr.io/vaheed/cdnfoundry-edge-control:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [control] - ports: ["${EDGE_CONTROL_BIND:-0.0.0.0:8443}:8443"] + ports: ["${EDGE_CONTROL_BIND}:8443"] volumes: - ${EDGE_CONTROL_SERVER_CERTIFICATE:?EDGE_CONTROL_SERVER_CERTIFICATE is required}:/run/secrets/edge-control-server.crt:ro - ${EDGE_CONTROL_SERVER_PRIVATE_KEY:?EDGE_CONTROL_SERVER_PRIVATE_KEY is required}:/run/secrets/edge-control-server.key:ro @@ -252,8 +253,8 @@ services: image: powerdns/dnsdist-21:2.1.0 profiles: [dns] ports: - - "${DNS_BIND_V4:-0.0.0.0}:53:53/udp" - - "${DNS_BIND_V4:-0.0.0.0}:53:53/tcp" + - "${DNS_BIND_V4}:53:53/udp" + - "${DNS_BIND_V4}:53:53/tcp" volumes: [./docker/dnsdist/dnsdist.conf:/etc/dnsdist/dnsdist.conf:ro] depends_on: pdns-auth: { condition: service_healthy } @@ -307,8 +308,8 @@ services: image: ghcr.io/vaheed/cdnfoundry-loki:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [telemetry] environment: - LOKI_RETENTION_PERIOD: ${LOKI_RETENTION_PERIOD:-336h} - LOKI_MAX_QUERY_LENGTH: ${LOKI_MAX_QUERY_LENGTH:-336h} + LOKI_RETENTION_PERIOD: ${LOKI_RETENTION_PERIOD} + LOKI_MAX_QUERY_LENGTH: ${LOKI_MAX_QUERY_LENGTH} volumes: [loki-data:/loki] depends_on: loki-data-init: { condition: service_completed_successfully } @@ -340,14 +341,14 @@ services: LOG_ROLE: ${LOG_ROLE:?LOG_ROLE is required for the logs profile} LOG_HOST: ${LOG_HOST:?LOG_HOST is required for the logs profile} LOG_COLLECTOR_ID: ${LOG_COLLECTOR_ID:?LOG_COLLECTOR_ID is required for the logs profile} - LOG_BUFFER_BYTES: ${LOG_BUFFER_BYTES:-2147483648} + LOG_BUFFER_BYTES: ${LOG_BUFFER_BYTES} LOG_METRICS_ADDRESS: 0.0.0.0:9599 LOKI_ENDPOINT: ${LOKI_ENDPOINT:?LOKI_ENDPOINT is required for the logs profile} volumes: - ./docker/vector/operational.yaml:/etc/vector/operational.yaml:ro - /var/run/docker.sock:/var/run/docker.sock:ro - operational-vector-data:/vector-data-dir - ports: ["${LOG_METRICS_BIND:-127.0.0.1:9599}:9599"] + ports: ["${LOG_METRICS_BIND}:9599"] networks: [telemetry, egress] restart: unless-stopped read_only: true @@ -366,8 +367,8 @@ services: volumes: - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro - - ${PROMETHEUS_EDGE_TARGETS_FILE:-./docker/prometheus/edge-targets.prod.yml}:/etc/prometheus/edge-targets.yml:ro - - ${PROMETHEUS_LOG_TARGETS_FILE:-./docker/prometheus/operational-log-targets.prod.yml}:/etc/prometheus/operational-log-targets.yml:ro + - ${PROMETHEUS_EDGE_TARGETS_FILE}:/etc/prometheus/edge-targets.yml:ro + - ${PROMETHEUS_LOG_TARGETS_FILE}:/etc/prometheus/operational-log-targets.yml:ro - ${METRICS_TOKEN_FILE:?METRICS_TOKEN_FILE is required}:/run/secrets/metrics-token:ro - prometheus:/prometheus networks: [telemetry, control, dns-private] @@ -377,12 +378,12 @@ services: image: postgres:18.4-alpine profiles: [telemetry] environment: - PGHOST: ${GRAFANA_POSTGRES_PROVISION_HOST:-control-db} - PGPORT: ${GRAFANA_POSTGRES_PROVISION_PORT:-5432} + PGHOST: ${GRAFANA_POSTGRES_PROVISION_HOST} + PGPORT: ${GRAFANA_POSTGRES_PROVISION_PORT} PGDATABASE: cdnf PGUSER: cdnf PGPASSWORD: ${CONTROL_DB_PASSWORD:?CONTROL_DB_PASSWORD is required} - PGSSLMODE: ${DB_SSLMODE:-prefer} + PGSSLMODE: ${DB_SSLMODE} GRAFANA_POSTGRES_PASSWORD: ${GRAFANA_POSTGRES_PASSWORD:?GRAFANA_POSTGRES_PASSWORD is required} volumes: - ./docker/grafana/postgres:/provision:ro @@ -398,9 +399,9 @@ services: grafana: image: ghcr.io/vaheed/cdnfoundry-grafana:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} profiles: [telemetry] - ports: ["${GRAFANA_BIND:-127.0.0.1:3000}:3000"] + ports: ["${GRAFANA_BIND}:3000"] environment: - GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER} GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD is required} GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_ANONYMOUS_ENABLED: "false" @@ -408,22 +409,22 @@ services: GF_ANALYTICS_CHECK_FOR_UPDATES: "false" GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" GF_ANALYTICS_REPORTING_ENABLED: "false" - GF_SECURITY_COOKIE_SECURE: ${GRAFANA_COOKIE_SECURE:-true} + GF_SECURITY_COOKIE_SECURE: ${GRAFANA_COOKIE_SECURE} GF_PATHS_PLUGINS: /usr/share/grafana/cdnfoundry-plugins GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/system-command-center.json - GRAFANA_LOKI_URL: ${GRAFANA_LOKI_URL:-http://loki:3100} - GRAFANA_CLICKHOUSE_HOST: ${GRAFANA_CLICKHOUSE_HOST:-clickhouse} - GRAFANA_CLICKHOUSE_PORT: ${GRAFANA_CLICKHOUSE_PORT:-9000} - GRAFANA_CLICKHOUSE_PROTOCOL: ${GRAFANA_CLICKHOUSE_PROTOCOL:-native} - GRAFANA_CLICKHOUSE_SECURE: ${GRAFANA_CLICKHOUSE_SECURE:-false} - GRAFANA_CLICKHOUSE_USER: ${GRAFANA_CLICKHOUSE_USER:-cdnf_grafana} + GRAFANA_LOKI_URL: ${GRAFANA_LOKI_URL} + GRAFANA_CLICKHOUSE_HOST: ${GRAFANA_CLICKHOUSE_HOST} + GRAFANA_CLICKHOUSE_PORT: ${GRAFANA_CLICKHOUSE_PORT} + GRAFANA_CLICKHOUSE_PROTOCOL: ${GRAFANA_CLICKHOUSE_PROTOCOL} + GRAFANA_CLICKHOUSE_SECURE: ${GRAFANA_CLICKHOUSE_SECURE} + GRAFANA_CLICKHOUSE_USER: ${GRAFANA_CLICKHOUSE_USER} GRAFANA_CLICKHOUSE_PASSWORD: ${GRAFANA_CLICKHOUSE_PASSWORD:?GRAFANA_CLICKHOUSE_PASSWORD is required} - GRAFANA_POSTGRES_HOST: ${GRAFANA_POSTGRES_HOST:-control-db} - GRAFANA_POSTGRES_PORT: ${GRAFANA_POSTGRES_PORT:-5432} - GRAFANA_POSTGRES_DATABASE: ${GRAFANA_POSTGRES_DATABASE:-cdnf} - GRAFANA_POSTGRES_USER: ${GRAFANA_POSTGRES_USER:-cdnf_grafana} + GRAFANA_POSTGRES_HOST: ${GRAFANA_POSTGRES_HOST} + GRAFANA_POSTGRES_PORT: ${GRAFANA_POSTGRES_PORT} + GRAFANA_POSTGRES_DATABASE: ${GRAFANA_POSTGRES_DATABASE} + GRAFANA_POSTGRES_USER: ${GRAFANA_POSTGRES_USER} GRAFANA_POSTGRES_PASSWORD: ${GRAFANA_POSTGRES_PASSWORD:?GRAFANA_POSTGRES_PASSWORD is required} - GRAFANA_POSTGRES_SSLMODE: ${GRAFANA_POSTGRES_SSLMODE:-disable} + GRAFANA_POSTGRES_SSLMODE: ${GRAFANA_POSTGRES_SSLMODE} volumes: - grafana-data:/var/lib/grafana - ./docker/grafana/provisioning:/etc/grafana/provisioning:ro @@ -540,14 +541,14 @@ services: environment: EDGE_CONTROL_URL: ${EDGE_CONTROL_URL:?EDGE_CONTROL_URL is required} EDGE_CONTROL_CA_CERTIFICATE: /run/secrets/edge-control-ca.crt - EDGE_ID: ${EDGE_ID:-} - EDGE_BOOTSTRAP_TOKEN: ${EDGE_BOOTSTRAP_TOKEN:-} + EDGE_ID: ${EDGE_ID} + EDGE_BOOTSTRAP_TOKEN: ${EDGE_BOOTSTRAP_TOKEN} EDGE_STATE_DIR: /var/lib/cdnfoundry/agent EDGE_RUNTIME_DIR: /var/lib/cdnfoundry/runtime - EDGE_GATEWAY_STATUS_URL: ${EDGE_GATEWAY_STATUS_URL:-http://host-gateway:9105/metrics} - EDGE_GATEWAY_ADDRESS_MAP: ${EDGE_GATEWAY_ADDRESS_MAP:-{}} + EDGE_GATEWAY_STATUS_URL: ${EDGE_GATEWAY_STATUS_URL} + EDGE_GATEWAY_ADDRESS_MAP: ${EDGE_GATEWAY_ADDRESS_MAP} EDGE_GATEWAY_REQUIRE_ADDRESS_MAP: "true" - EDGE_RUNTIME_VERSIONS: ${EDGE_RUNTIME_VERSIONS:-{}} + EDGE_RUNTIME_VERSIONS: ${EDGE_RUNTIME_VERSIONS} EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default","cell-03":"","cell-04":"","cell-05":"","cell-06":"","cell-07":"","cell-08":""}' EDGE_CELL_STATUS_URLS: http://cell-01:9080/passive-failures,http://cell-02:9080/passive-failures,http://cell-03:9080/passive-failures,http://cell-04:9080/passive-failures,http://cell-05:9080/passive-failures,http://cell-06:9080/passive-failures,http://cell-07:9080/passive-failures,http://cell-08:9080/passive-failures EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required} @@ -580,8 +581,8 @@ services: environment: GATEWAY_CONFIG_FILE: /var/lib/cdnfoundry/runtime/current/gateway.json GATEWAY_STATE_DIR: /var/lib/cdnfoundry/gateway-state - GATEWAY_METRICS_ADDRESS: ${EDGE_GATEWAY_METRICS_ADDRESS:-0.0.0.0:9105} - GATEWAY_MAX_CONNECTIONS: ${EDGE_GATEWAY_MAX_CONNECTIONS:-8192} + GATEWAY_METRICS_ADDRESS: ${EDGE_GATEWAY_METRICS_ADDRESS} + GATEWAY_MAX_CONNECTIONS: ${EDGE_GATEWAY_MAX_CONNECTIONS} volumes: - edge-state:/var/lib/cdnfoundry/runtime:ro - edge-gateway-state:/var/lib/cdnfoundry/gateway-state @@ -615,15 +616,15 @@ services: mmdb-updater: image: ghcr.io/vaheed/cdnfoundry-mmdb-updater:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} - profiles: [dns, edge] + profiles: [control, dns, edge] environment: - MMDB_PROVIDER: ${MMDB_PROVIDER:-dbip-jsdelivr} - MMDB_TARGET_FILE: ${MMDB_TARGET_FILE:-GeoLite2-City.mmdb} - MMDB_DOWNLOAD_INTERVAL_SECONDS: ${MMDB_DOWNLOAD_INTERVAL_SECONDS:-86400} - MMDB_DOWNLOAD_RETRIES: ${MMDB_DOWNLOAD_RETRIES:-5} - MMDB_EXPECTED_SHA256: ${MMDB_EXPECTED_SHA256:-} - MMDB_DOWNLOAD_URL: ${MMDB_DOWNLOAD_URL:-} - MMDB_DOWNLOAD_HEADER: ${MMDB_DOWNLOAD_HEADER:-} + MMDB_PROVIDER: ${MMDB_PROVIDER} + MMDB_TARGET_FILE: ${MMDB_TARGET_FILE} + MMDB_DOWNLOAD_INTERVAL_SECONDS: ${MMDB_DOWNLOAD_INTERVAL_SECONDS} + MMDB_DOWNLOAD_RETRIES: ${MMDB_DOWNLOAD_RETRIES} + MMDB_EXPECTED_SHA256: ${MMDB_EXPECTED_SHA256} + MMDB_DOWNLOAD_URL: ${MMDB_DOWNLOAD_URL} + MMDB_DOWNLOAD_HEADER: ${MMDB_DOWNLOAD_HEADER} volumes: [mmdb:/mmdb] networks: [edge] restart: unless-stopped diff --git a/deploy/production/compose.control-host-ipv6.yml b/deploy/production/compose.control-host-ipv6.yml index d1dac66..3a99e5b 100644 --- a/deploy/production/compose.control-host-ipv6.yml +++ b/deploy/production/compose.control-host-ipv6.yml @@ -1,6 +1,6 @@ services: caddy: ports: - - "[${HOST_BIND_IPV6:-::}]:80:80/tcp" - - "[${HOST_BIND_IPV6:-::}]:443:443/tcp" - - "[${HOST_BIND_IPV6:-::}]:443:443/udp" + - "[${HOST_BIND_IPV6}]:80:80/tcp" + - "[${HOST_BIND_IPV6}]:443:443/tcp" + - "[${HOST_BIND_IPV6}]:443:443/udp" diff --git a/deploy/production/compose.control-host.yml b/deploy/production/compose.control-host.yml index 672ecc6..2333006 100644 --- a/deploy/production/compose.control-host.yml +++ b/deploy/production/compose.control-host.yml @@ -7,12 +7,12 @@ services: TELEMETRY_HOSTNAME: ${TELEMETRY_HOSTNAME:?TELEMETRY_HOSTNAME is required} ACME_CONTACT_EMAIL: ${ACME_CONTACT_EMAIL:?ACME_CONTACT_EMAIL is required} EDGE_PUBLIC_IPV4_ALLOWLIST: ${EDGE_PUBLIC_IPV4_ALLOWLIST:?EDGE_PUBLIC_IPV4_ALLOWLIST is required} - LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST:-127.0.0.1} + LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST} ports: - - "${HOST_BIND_IPV4:-0.0.0.0}:80:80/tcp" - - "${HOST_BIND_IPV4:-0.0.0.0}:443:443/tcp" - - "${HOST_BIND_IPV4:-0.0.0.0}:443:443/udp" - - "${HOST_BIND_IPV4:-0.0.0.0}:8444:8444/tcp" + - "${HOST_BIND_IPV4}:80:80/tcp" + - "${HOST_BIND_IPV4}:443:443/tcp" + - "${HOST_BIND_IPV4}:443:443/udp" + - "${HOST_BIND_IPV4}:8444:8444/tcp" volumes: - ./deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro - caddy-data:/data diff --git a/deploy/production/compose.dns-edge-host.yml b/deploy/production/compose.dns-edge-host.yml index a05f779..e7c4b46 100644 --- a/deploy/production/compose.dns-edge-host.yml +++ b/deploy/production/compose.dns-edge-host.yml @@ -6,7 +6,7 @@ services: DNS_API_HOSTNAME: ${DNS_API_HOSTNAME:?DNS_API_HOSTNAME is required} CONTROL_PUBLIC_IPV4_ALLOWLIST: ${CONTROL_PUBLIC_IPV4_ALLOWLIST:?CONTROL_PUBLIC_IPV4_ALLOWLIST is required} ports: - - "${HOST_BIND_IPV4:-0.0.0.0}:8444:8444/tcp" + - "${HOST_BIND_IPV4}:8444:8444/tcp" volumes: - ./deploy/production/Caddyfile.dns-api:/etc/caddy/Caddyfile:ro - ${DNS_API_SERVER_CERTIFICATE:?DNS_API_SERVER_CERTIFICATE is required}:/run/secrets/dns-api.crt:ro diff --git a/deploy/production/compose.dns-host-ipv6.yml b/deploy/production/compose.dns-host-ipv6.yml index 7b6d910..321ef35 100644 --- a/deploy/production/compose.dns-host-ipv6.yml +++ b/deploy/production/compose.dns-host-ipv6.yml @@ -1,5 +1,5 @@ services: dnsdist: ports: - - "[${HOST_BIND_IPV6:-::}]:53:53/udp" - - "[${HOST_BIND_IPV6:-::}]:53:53/tcp" + - "[${HOST_BIND_IPV6}]:53:53/udp" + - "[${HOST_BIND_IPV6}]:53:53/tcp" diff --git a/deploy/production/compose.dns-host.yml b/deploy/production/compose.dns-host.yml index a05f779..e7c4b46 100644 --- a/deploy/production/compose.dns-host.yml +++ b/deploy/production/compose.dns-host.yml @@ -6,7 +6,7 @@ services: DNS_API_HOSTNAME: ${DNS_API_HOSTNAME:?DNS_API_HOSTNAME is required} CONTROL_PUBLIC_IPV4_ALLOWLIST: ${CONTROL_PUBLIC_IPV4_ALLOWLIST:?CONTROL_PUBLIC_IPV4_ALLOWLIST is required} ports: - - "${HOST_BIND_IPV4:-0.0.0.0}:8444:8444/tcp" + - "${HOST_BIND_IPV4}:8444:8444/tcp" volumes: - ./deploy/production/Caddyfile.dns-api:/etc/caddy/Caddyfile:ro - ${DNS_API_SERVER_CERTIFICATE:?DNS_API_SERVER_CERTIFICATE is required}:/run/secrets/dns-api.crt:ro diff --git a/deploy/production/compose.telemetry-host-ipv6.yml b/deploy/production/compose.telemetry-host-ipv6.yml index 302b86b..f3c4898 100644 --- a/deploy/production/compose.telemetry-host-ipv6.yml +++ b/deploy/production/compose.telemetry-host-ipv6.yml @@ -1,5 +1,5 @@ services: telemetry-gateway: ports: - - "[${HOST_BIND_IPV6:-::}]:80:80/tcp" - - "[${HOST_BIND_IPV6:-::}]:443:443/tcp" + - "[${HOST_BIND_IPV6}]:80:80/tcp" + - "[${HOST_BIND_IPV6}]:443:443/tcp" diff --git a/deploy/production/compose.telemetry-host.yml b/deploy/production/compose.telemetry-host.yml index 3491065..c46992f 100644 --- a/deploy/production/compose.telemetry-host.yml +++ b/deploy/production/compose.telemetry-host.yml @@ -8,9 +8,9 @@ services: EDGE_PUBLIC_IPV4_ALLOWLIST: ${EDGE_PUBLIC_IPV4_ALLOWLIST:?EDGE_PUBLIC_IPV4_ALLOWLIST is required} LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST:?LOG_SOURCE_IPV4_ALLOWLIST is required} ports: - - "${HOST_BIND_IPV4:-0.0.0.0}:80:80/tcp" - - "${HOST_BIND_IPV4:-0.0.0.0}:443:443/tcp" - - "${HOST_BIND_IPV4:-0.0.0.0}:8444:8444/tcp" + - "${HOST_BIND_IPV4}:80:80/tcp" + - "${HOST_BIND_IPV4}:443:443/tcp" + - "${HOST_BIND_IPV4}:8444:8444/tcp" volumes: - ./deploy/production/Caddyfile.telemetry:/etc/caddy/Caddyfile:ro - telemetry-caddy-data:/data @@ -20,8 +20,7 @@ services: loki: { condition: service_healthy } log-collector: environment: - LOKI_ENDPOINT: http://loki:3100 - networks: [telemetry, egress] + LOKI_ENDPOINT: ${LOKI_ENDPOINT} restart: unless-stopped read_only: true tmpfs: [/tmp] diff --git a/deploy/production/examples/multi-region-fleet.json b/deploy/production/examples/multi-region-fleet.json new file mode 100644 index 0000000..e601c58 --- /dev/null +++ b/deploy/production/examples/multi-region-fleet.json @@ -0,0 +1,35 @@ +{ + "preset": "dedicated-monitoring", + "global": { + "operator_domain": "ops.example.com", + "platform_domain": "cdn.example.com", + "release": "0000000000000000000000000000000000000000", + "acme_email": "cdn-operations@example.com", + "ipv6": false + }, + "nodes": [ + {"name":"control-1","role":"control","region":"global","location":"primary","hostname":"control.ops.example.com","public_ipv4":"192.0.2.10","bind_ipv4":"0.0.0.0"}, + {"name":"monitoring-1","role":"monitoring","region":"global","location":"primary","hostname":"monitoring-1.ops.example.com","public_ipv4":"192.0.2.11","bind_ipv4":"0.0.0.0"}, + {"name":"monitoring-2","role":"monitoring","region":"region-b","location":"site-b","hostname":"monitoring-2.ops.example.com","public_ipv4":"192.0.2.12","bind_ipv4":"0.0.0.0"}, + {"name":"monitoring-3","role":"monitoring","region":"region-c","location":"site-c","hostname":"monitoring-3.ops.example.com","public_ipv4":"192.0.2.13","bind_ipv4":"0.0.0.0"}, + {"name":"dns-1","role":"dns","region":"region-a","location":"site-a","hostname":"dns-1.ops.example.com","public_ipv4":"192.0.2.21","bind_ipv4":"0.0.0.0"}, + {"name":"dns-2","role":"dns","region":"region-b","location":"site-b","hostname":"dns-2.ops.example.com","public_ipv4":"192.0.2.22","bind_ipv4":"0.0.0.0"}, + {"name":"dns-3","role":"dns","region":"region-c","location":"site-c","hostname":"dns-3.ops.example.com","public_ipv4":"192.0.2.23","bind_ipv4":"0.0.0.0"}, + {"name":"dns-4","role":"dns","region":"region-d","location":"site-d","hostname":"dns-4.ops.example.com","public_ipv4":"192.0.2.24","bind_ipv4":"0.0.0.0"}, + {"name":"edge-1","role":"edge","region":"region-a","location":"site-a","hostname":"edge-1.ops.example.com","public_ipv4":"198.51.100.1","bind_ipv4":"0.0.0.0"}, + {"name":"edge-2","role":"edge","region":"region-a","location":"site-e","hostname":"edge-2.ops.example.com","public_ipv4":"198.51.100.2","bind_ipv4":"0.0.0.0"}, + {"name":"edge-3","role":"edge","region":"region-b","location":"site-b","hostname":"edge-3.ops.example.com","public_ipv4":"198.51.100.3","bind_ipv4":"0.0.0.0"}, + {"name":"edge-4","role":"edge","region":"region-b","location":"site-f","hostname":"edge-4.ops.example.com","public_ipv4":"198.51.100.4","bind_ipv4":"0.0.0.0"}, + {"name":"edge-5","role":"edge","region":"region-c","location":"site-c","hostname":"edge-5.ops.example.com","public_ipv4":"198.51.100.5","bind_ipv4":"0.0.0.0"}, + {"name":"edge-6","role":"edge","region":"region-c","location":"site-g","hostname":"edge-6.ops.example.com","public_ipv4":"198.51.100.6","bind_ipv4":"0.0.0.0"}, + {"name":"edge-7","role":"edge","region":"region-d","location":"site-d","hostname":"edge-7.ops.example.com","public_ipv4":"198.51.100.7","bind_ipv4":"0.0.0.0"}, + {"name":"edge-8","role":"edge","region":"region-d","location":"site-h","hostname":"edge-8.ops.example.com","public_ipv4":"198.51.100.8","bind_ipv4":"0.0.0.0"}, + {"name":"edge-9","role":"edge","region":"region-e","location":"site-i","hostname":"edge-9.ops.example.com","public_ipv4":"198.51.100.9","bind_ipv4":"0.0.0.0"}, + {"name":"edge-10","role":"edge","region":"region-e","location":"site-j","hostname":"edge-10.ops.example.com","public_ipv4":"198.51.100.10","bind_ipv4":"0.0.0.0"} + ], + "features": { + "monitoring": {"mode": "dedicated", "host": "monitoring-1"}, + "logs": {"mode": "centralized", "host": "monitoring-1", "endpoint": null}, + "backups": {"mode": "disabled", "repository": null, "region": "us-east-1"} + } +} diff --git a/deploy/production/examples/starter-fleet.json b/deploy/production/examples/starter-fleet.json new file mode 100644 index 0000000..77ea89e --- /dev/null +++ b/deploy/production/examples/starter-fleet.json @@ -0,0 +1,44 @@ +{ + "preset": "control-monitoring", + "global": { + "operator_domain": "ops.example.com", + "platform_domain": "cdn.example.com", + "release": "0000000000000000000000000000000000000000", + "acme_email": "cdn-operations@example.com", + "ipv6": false + }, + "nodes": [ + { + "name": "control-1", + "role": "control", + "region": "global", + "location": "primary", + "hostname": "control.ops.example.com", + "public_ipv4": "192.0.2.10", + "bind_ipv4": "0.0.0.0" + }, + { + "name": "pop-1", + "role": "dns-edge", + "region": "region-a", + "location": "site-a", + "hostname": "pop-1.ops.example.com", + "public_ipv4": "198.51.100.20", + "bind_ipv4": "0.0.0.0" + }, + { + "name": "pop-2", + "role": "dns-edge", + "region": "region-b", + "location": "site-b", + "hostname": "pop-2.ops.example.com", + "public_ipv4": "198.51.100.30", + "bind_ipv4": "0.0.0.0" + } + ], + "features": { + "monitoring": {"mode": "colocated", "host": "control-1"}, + "logs": {"mode": "centralized", "host": "control-1", "endpoint": null}, + "backups": {"mode": "disabled", "repository": null, "region": "us-east-1"} + } +} diff --git a/docs/deployment/index.md b/docs/deployment/index.md index 520a1c7..fe3c8c3 100644 --- a/docs/deployment/index.md +++ b/docs/deployment/index.md @@ -10,6 +10,8 @@ optional files under `deploy/production/`. It does not build application images on production hosts and never migrates a database during container startup. ::: tip Recommended starting point +For a new installation, use the [starter Fleet quick start](production-quick-start.md). It copies a JSON topology, validates it, and generates complete per-node bundles without editing deployment scripts. + Use the [Production quick start](production-quick-start.md) for the complete three-host sequence: bootstrap DNS, private PKI, explicit migrations, cluster qualification, edge enrollment, acceptance checks, and diagnosis. @@ -31,6 +33,8 @@ Before deploying, read: 6. [Configuration](../reference/configuration.md) for every `.env.prod` key. 7. [Upgrade](upgrade.md) for schema, worker, DNS, and edge sequencing. +For separated roles across several failure domains, continue with the [multi-region Fleet quick start](production-quick-start-multi-region.md). The [Fleet operator guide](production-fleet-operator-guide.md) and [configuration reference](production-fleet-config-reference.md) cover lifecycle operations and the JSON schema. + The [Production quick start](production-quick-start.md) is the authoritative first-install procedure. The remaining deployment pages explain individual decisions and are linked from that runbook where they become diff --git a/docs/deployment/production-fleet-config-reference.md b/docs/deployment/production-fleet-config-reference.md new file mode 100644 index 0000000..e7da9dc --- /dev/null +++ b/docs/deployment/production-fleet-config-reference.md @@ -0,0 +1,217 @@ +--- +title: Production fleet configuration reference +description: Reference for CDNFoundry production fleet CLI options, setup config schema, node objects, feature configuration, and generated bundle contract. +--- + +# Production fleet configuration reference + +Copy `deploy/production/examples/starter-fleet.json` or `multi-region-fleet.json` to a protected local `fleet.json`, then change deployment data there. Checked-in examples are templates; repository scripts and generated Compose manifests are not configuration surfaces. + +## Common command options + +These options work before or after a subcommand: + +| Option | Default | Purpose | +| --- | --- | --- | +| `--state-dir` | `/var/lib/cdnfoundry-fleet` | Protected authoritative fleet state | +| `--output-dir` | `/var/lib/cdnfoundry-fleet/bundles` | Generated per-node bundles | +| `--repo-root` | Repository containing the script | Base Compose and production overlays | +| `--config` | none | JSON input for setup or node commands | +| `--non-interactive` | false | Never prompt; fail when required input is absent | +| `--dry-run` | false | Validate intent without writing state or bundles | +| `--yes` | false | Confirm destructive or rotation operations | + +The convenience wrapper uses repository-local defaults unless environment variables override them: + +```text +CDNFOUNDRY_FLEET_STATE_DIR +CDNFOUNDRY_FLEET_OUTPUT_DIR +``` + +## Setup config schema + +```json +{ + "preset": "control-monitoring", + "global": { + "operator_domain": "ops.example.com", + "platform_domain": "example.com", + "release": "v1.0.0", + "acme_email": "operations@example.com", + "ipv6": false + }, + "nodes": [], + "features": { + "monitoring": {"mode": "disabled", "host": null}, + "logs": {"mode": "disabled", "host": null, "endpoint": null}, + "backups": {"mode": "disabled", "repository": null, "region": "us-east-1"} + } +} +``` + +### Presets + +| Preset | Result | +| --- | --- | +| `control-only` | Control node, monitoring disabled | +| `control-monitoring` | Control node with colocated telemetry services | +| `dedicated-monitoring` | Control node plus a monitoring-role node | +| `custom` | Feature configuration comes from `features` or later commands | + +## Node object + +| Field | Required | Description | +| --- | --- | --- | +| `name` | yes | Lowercase stable identifier, letters/digits/hyphens | +| `role` | yes | `control`, `dns`, `edge`, `dns-edge`, or `monitoring` | +| `region` | yes | Routing/operations region label | +| `location` | yes | Human-readable site label | +| `hostname` | no | Defaults to `NAME.OPERATOR_DOMAIN` | +| `public_ipv4` | yes | Public or routed IPv4 used for inventory and policy | +| `public_ipv6` | no | IPv6 service address | +| `bind_ipv4` | no | Local listener bind, defaults to `0.0.0.0` | +| `bind_ipv6` | no | IPv6 bind; defaults to `::` in dual-stack fleets | +| `monitor_ipv4` | no | Private monitoring target; otherwise `public_ipv4` | +| `log_ipv4` | no | Private log-source address metadata | +| `release` | no | Per-node immutable override of the global release | +| `extra_env` | no | Explicit per-node Compose overrides; always preserved in the generated `.env.prod`, including variables that have Compose defaults | +| `enabled` | no | Exclude disabled nodes from rendering and targets | +| `draining` | no | Keep node configured but remove it from preferred routing | + +Example: + +```json +{ + "name": "pop-singapore", + "role": "dns-edge", + "region": "asia", + "location": "singapore", + "hostname": "pop-singapore.ops.example.com", + "public_ipv4": "192.0.2.40", + "public_ipv6": "2001:db8::40", + "bind_ipv4": "0.0.0.0", + "bind_ipv6": "::", + "monitor_ipv4": "10.30.0.40", + "release": "v1.0.0", + "extra_env": {}, + "enabled": true, + "draining": false +} +``` + +## Manual edge registration fields + +Do not put bootstrap tokens in a version-controlled setup JSON. After creating the edge in the running control panel, use: + +```bash +./scripts/cdnfoundry-fleet --state-dir /var/lib/cdnfoundry-fleet \ + configure-edge-registration \ + --node edge-1 \ + --edge-id 11111111-2222-3333-4444-555555555555 \ + --bootstrap-token-file /root/edge-1.bootstrap-token \ + --non-interactive +``` + +This stores: + +- `EDGE_ID` in the protected node state; +- the one-time token in `secrets/nodes/NODE/edge-bootstrap-token` with mode `0600`. + +After successful mTLS enrollment, run `clear-edge-bootstrap-token --node NODE`, rerender, and recreate only `edge-agent`. + +Optional edge overrides such as `EDGE_GATEWAY_ADDRESS_MAP`, `EDGE_RUNTIME_VERSIONS`, MMDB settings, and gateway capacity settings belong in `extra_env`. The renderer preserves explicitly supplied optional variables even when Compose uses `${VAR:-default}`. + +## Control database selection + +Embedded PostgreSQL is used when neither `DB_URL` nor a non-default `DB_HOST` is present. + +Remote PostgreSQL is selected when the control node has either: + +```json +"extra_env": { + "DB_HOST": "postgres.internal.example", + "DB_PORT": "5432", + "DB_SSLMODE": "verify-full" +} +``` + +or a non-empty `DB_URL`. + +Use `set-secret --secret control-db-password --from-file FILE` to replace the generated password with the remote database credential without exposing it in command arguments. In remote mode the control bundle omits `control-db` and its volume. + +## Feature objects + +### Monitoring + +```json +{"mode": "disabled", "host": null} +``` + +- `disabled`: no telemetry stack or node exporters. +- `colocated`: telemetry stack runs on the control node. +- `dedicated`: telemetry stack runs on the named monitoring-role node. + +### Logs + +```json +{"mode": "centralized", "host": "monitoring-1", "endpoint": null} +``` + +- `disabled`: no generated log collector. +- `centralized`: every enabled node receives a generated Vector config and node-specific authentication token. +- `endpoint`: optional explicit Loki-compatible URL; otherwise derived from the configured host. + +### Backups + +```json +{ + "mode": "all-stateful", + "repository": "s3:s3.example.com/cdnfoundry-production", + "region": "us-east-1" +} +``` + +Modes are `disabled`, `control`, and `all-stateful`. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Success | +| `2` | Command-line usage error | +| `3` | Invalid topology, config, or failed doctor check | +| `4` | Missing, locked, or inconsistent fleet state | +| `5` | Compose, PKI, file-copy, or bundle rendering failure | +| `130` | Operator interrupted the command | + +## Generated bundle contract + +Every rendered node directory includes: + +```text +.env.prod +generated Compose manifest +README.md +validate.sh +start.sh +bundle-metadata.json +SHA256SUMS +pki/ +secrets/ +generated/ # when required +referenced docker/... # only runtime files used by selected services +``` + +DNS nodes may additionally receive `reconcile-pdns-password.sh` and a pending password file during a staged rotation. Edge bundles receive `EDGE_ID` plus a one-time token only after `configure-edge-registration`; the token disappears after `clear-edge-bootstrap-token` and rerendering. + +## Security properties + +- State directories use mode `0700`. +- State, secrets, environment files, manifests, and private keys use mode `0600`. +- Node bundles are assembled in temporary directories and activated atomically. +- Normal rendering does not rotate secrets. +- DNS database credentials are node-scoped. +- CA private keys remain in authoritative fleet state, except the edge identity CA key required by the control service in the control bundle. +- Bundle metadata contains hashes and non-secret inventory only. +- Every operator-controlled Compose interpolation value is present in the node's generated `.env.prod`; production Compose and role overlays provide no fallback deployment values. +- Compose `environment` mappings remain explicit per-service allowlists. Replacing them with a shared `env_file` entry would expose unrelated database, PKI, and API credentials to every container, so containers receive only the variables they own while Compose reads values through `--env-file .env.prod`. diff --git a/docs/deployment/production-fleet-operator-guide.md b/docs/deployment/production-fleet-operator-guide.md new file mode 100644 index 0000000..d65169e --- /dev/null +++ b/docs/deployment/production-fleet-operator-guide.md @@ -0,0 +1,476 @@ +--- +title: Production fleet operator guide +description: Complete lifecycle guide for CDNFoundry production fleets including setup, monitoring, DNS, edge nodes, validation, bundle transfer, operation, upgrades, recovery, and troubleshooting. +--- + +# Production fleet operator guide + +Work from an immutable checkout. For a fresh operator host: + +```bash +git clone https://github.com/vaheed/CDNFoundry.git cdnfoundry +cd cdnfoundry +git checkout v1.0.0 +git rev-parse --verify HEAD +``` + +Replace the example tag with the exact release or commit selected for the fleet. Never operate production from a moving branch. + +This guide covers the full lifecycle of a CDNFoundry production fleet: first-time setup, control plus monitoring, additional DNS and edge nodes, validation, bundle transfer, operation, upgrades, recovery, and troubleshooting. + +## What the generator creates + +The generator runs from a trusted checkout of the CDNFoundry repository and writes two protected outputs: + +- **Fleet state**: topology, feature modes, immutable release identifiers, credentials, certificate authorities, node certificates, and state history. +- **Node bundles**: one minimal directory per host containing filtered Compose services, `.env.prod`, only the referenced runtime files, node-specific PKI, generated monitoring/logging files, validation/start scripts, checksums, and a node README. + +Remote nodes do not need a repository clone. They receive only their own bundle. + +## Fastest supported setup + +From the repository root, run: + +```bash +./scripts/generate-production-env.sh +``` + +The wizard now performs the complete workflow instead of only initializing state. It asks for the global domains and release, offers a visible topology menu, adds hosts, validates the fleet, and renders bundles. + +The default local paths are: + +```text +State: ./.cdnfoundry-fleet +Bundles: ./build/fleet-bundles +``` + +Override them without editing the script: + +```bash +CDNFOUNDRY_FLEET_STATE_DIR=/var/lib/cdnfoundry-fleet \ +CDNFOUNDRY_FLEET_OUTPUT_DIR=/var/lib/cdnfoundry-fleet/bundles \ + sudo -E ./scripts/generate-production-env.sh +``` + +## Control plus monitoring + +Choose **Control + monitoring on the same host** in the wizard. This sets monitoring mode to `colocated`, selects both the `control` and `telemetry` profiles for the control node, and creates exporter targets for every enabled host. + +Equivalent non-interactive command: + +```bash +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles \ + --repo-root "$PWD" \ + setup \ + --operator-domain ops.example.com \ + --platform-domain example.com \ + --release v1.0.0 \ + --preset control-monitoring \ + --control-ipv4 192.0.2.10 \ + --non-interactive +``` + +To enable it on an existing fleet: + +```bash +./scripts/cdnfoundry-fleet --state-dir ./.cdnfoundry-fleet \ + configure-monitoring --mode colocated --non-interactive + +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles \ + --repo-root "$PWD" render +``` + +Verify the selected mode and services: + +```bash +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles status + +grep -E '^( )?(grafana|prometheus|clickhouse|loki|node-exporter):' \ + build/fleet-bundles/control-1/compose.yml +``` + +## Config-driven setup + +For repeatable automation, use a JSON file: + +```json +{ + "preset": "control-monitoring", + "global": { + "operator_domain": "ops.example.com", + "platform_domain": "example.com", + "release": "v1.0.0", + "acme_email": "operations@example.com", + "ipv6": false + }, + "nodes": [ + { + "name": "control-1", + "role": "control", + "region": "global", + "location": "ashburn", + "hostname": "control.ops.example.com", + "public_ipv4": "192.0.2.10", + "bind_ipv4": "0.0.0.0" + }, + { + "name": "dns-frankfurt", + "role": "dns", + "region": "europe", + "location": "frankfurt", + "public_ipv4": "192.0.2.20", + "bind_ipv4": "0.0.0.0" + }, + { + "name": "edge-frankfurt", + "role": "edge", + "region": "europe", + "location": "frankfurt", + "public_ipv4": "192.0.2.30", + "bind_ipv4": "0.0.0.0" + } + ], + "features": { + "monitoring": {"mode": "colocated", "host": null}, + "logs": {"mode": "disabled", "host": null, "endpoint": null}, + "backups": {"mode": "disabled", "repository": null, "region": "us-east-1"} + } +} +``` + +Run: + +```bash +./scripts/generate-production-env.sh \ + --config ./fleet-production.json \ + --non-interactive +``` + +The setup command is idempotent for named nodes in the config: existing nodes are updated and missing nodes are added. Secrets remain stable unless an explicit rotation command is used. + +## CLI map + +| Goal | Command | +| --- | --- | +| Full wizard/config setup | `setup` | +| Initialize state only | `init` | +| Add or change a host | `add-node`, `update-node` | +| Store manual edge UUID/token | `configure-edge-registration` | +| Remove one-time edge token | `clear-edge-bootstrap-token` | +| Replace a protected secret from file | `set-secret` | +| See fleet and feature modes | `status` | +| Machine-readable inventory | `status --json`, `list-nodes` | +| Check repository and tools | `doctor` | +| Set monitoring | `configure-monitoring` | +| Set centralized logs | `configure-logs` | +| Set backup metadata | `configure-backups` | +| Test rendering without replacing bundles | `validate` | +| Generate node bundles | `render` | +| Show deployment sequence | `show-start-order` | +| Import existing credentials | `adopt-existing` | +| Rotate a supported secret | `rotate-secret` | + +Every command has command-specific help: + +```bash +./scripts/cdnfoundry-fleet setup --help +./scripts/cdnfoundry-fleet configure-monitoring --help +./scripts/cdnfoundry-fleet render --help +``` + +## Preflight checks + +Run before setup or after updating the repository: + +```bash +./scripts/cdnfoundry-fleet --repo-root "$PWD" doctor +``` + +`doctor` verifies the base Compose file, role overlays, Python, OpenSSL, and existing fleet state. Docker is reported separately because rendering can occur on a generator machine without starting containers, but Docker Compose is required on deployment hosts and for the final host-side validation. + +## Node roles + +### Control + +Runs the application control plane, Valkey, migrations, and the edge-control TLS endpoint. By default it also runs embedded PostgreSQL. When node `extra_env` contains a non-`control-db` `DB_HOST` or a non-empty `DB_URL`, the generated bundle removes embedded `control-db` and points the application at the operator-managed database. In colocated monitoring mode it also runs the telemetry stack. + +### DNS + +Runs node-local PostgreSQL, PowerDNS authoritative service, DNSdist, and required geo/MMDB support. Its database password and API key are unique to that node. + +### Edge + +Runs the edge agent/runtime and gateway services. It receives the edge server CA, a node TLS certificate, and the generated edge-control URL. + +### DNS-edge + +Combines the DNS and edge service sets on one host. It still uses its own local PowerDNS PostgreSQL database. + +### Monitoring + +Runs a dedicated telemetry stack when monitoring mode is `dedicated`. It does not start a second control database. The project-specific Grafana control-database provisioning helper is intentionally omitted on a dedicated host; configure an externally reachable control datasource separately when those dashboards are required. + +## PKI layout + +The generator follows the production repository’s two-CA model: + +- `edge-identity-ca`: used by the control plane for edge identity issuance and verification. +- `edge-server-ca`: signs edge-control, edge runtime, and DNS API TLS certificates. + +CA private keys stay in the protected fleet state directory. Every node bundle receives the edge server CA certificate plus its own certificate and private key. Only the control bundle receives the edge identity CA private key because the control service requires it. + +Important generated environment paths include: + +```text +EDGE_IDENTITY_CA_CERTIFICATE=./pki/edge-identity-ca.crt +EDGE_IDENTITY_CA_PRIVATE_KEY=./pki/edge-identity-ca.key +PDNS_CA_CERTIFICATE=./pki/edge-server-ca.crt +EDGE_CONTROL_SERVER_CERTIFICATE=./pki/node.crt +EDGE_CONTROL_SERVER_PRIVATE_KEY=./pki/node.key +EDGE_CONTROL_CA_CERTIFICATE=./pki/edge-server-ca.crt +EDGE_RUNTIME_TLS_CERTIFICATE=./pki/node.crt +EDGE_RUNTIME_TLS_PRIVATE_KEY=./pki/node.key +DNS_API_SERVER_CERTIFICATE=./pki/node.crt +DNS_API_SERVER_PRIVATE_KEY=./pki/node.key +``` + +## Manual edge registration and mTLS enrollment + +The generator does not create edge records in the control plane. Use this sequence for every edge-capable node: + +1. Start the control plane. +2. In **Edge network → Edges**, create the edge and copy its UUID and one-time bootstrap token. +3. Save the token in a protected mode-`0600` file. +4. Store the UUID/token in fleet state with `configure-edge-registration`. +5. Add the edge's complete `EDGE_GATEWAY_ADDRESS_MAP` with `update-node --extra-env`. +6. Validate and render only that node. +7. Transfer and start the node bundle. +8. Wait for the registered identity and heartbeat. +9. Clear the one-time token, rerender, and recreate only `edge-agent`. + +Example on the control-plane machine: + +```bash +sudo install -m 0600 /dev/null /root/edge-tokyo.bootstrap-token +sudo sh -c 'read -r token; printf "%s\n" "$token" > /root/edge-tokyo.bootstrap-token' + +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + configure-edge-registration \ + --node edge-tokyo \ + --edge-id 11111111-2222-3333-4444-555555555555 \ + --bootstrap-token-file /root/edge-tokyo.bootstrap-token \ + --non-interactive + +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + update-node --node edge-tokyo \ + --extra-env 'EDGE_GATEWAY_ADDRESS_MAP={"203.0.113.40":"10.40.0.40","203.0.113.41":"10.40.0.41"}' \ + --non-interactive + +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + --output-dir /var/lib/cdnfoundry-fleet/bundles \ + --repo-root "$PWD" render --node edge-tokyo +``` + +The edge agent creates its private key locally, sends a CSR during one-time registration, receives an identity certificate from the edge identity CA, and persists the identity in `edge-agent-state`. Do not copy that volume to another host. + +After registration succeeds: + +```bash +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + clear-edge-bootstrap-token --node edge-tokyo --non-interactive + +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + --output-dir /var/lib/cdnfoundry-fleet/bundles \ + --repo-root "$PWD" render --node edge-tokyo + +sudo rm -f /root/edge-tokyo.bootstrap-token +``` + +Transfer the clean bundle and run on the edge: + +```bash +cd /opt/cdnfoundry +docker compose --env-file .env.prod up -d --force-recreate edge-agent +``` + +`EDGE_ID` remains in the generated environment. `EDGE_BOOTSTRAP_TOKEN` is removed. + +## Embedded or remote control PostgreSQL + +Embedded mode is the default. The control bundle contains `control-db`, and `start.sh` waits for `control-db` and `redis` before running the migration. + +For remote mode, set the control node's optional Compose overrides and replace the generated database password with the real remote credential from a protected file: + +```bash +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + update-node --node control-1 \ + --extra-env DB_HOST=postgres.internal.example \ + --extra-env DB_PORT=5432 \ + --extra-env DB_SSLMODE=verify-full \ + --non-interactive + +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + set-secret --secret control-db-password \ + --from-file /root/cdnfoundry-postgres-password \ + --non-interactive +``` + +A non-empty `DB_URL` also selects remote mode. Do not place a password-bearing URL in version control or shell history. + +In remote mode the renderer removes `control-db`, its volume, and every dependency on it. The generated start helper waits only for local Valkey before migration. When telemetry is colocated and `DB_HOST` is supplied, Grafana's control-database provisioning and datasource defaults inherit the same host, port, and SSL mode unless explicitly overridden. + +Before rendering, ensure the external service has: + +- database and application role expected by the project; +- TLS and certificate hostname verification where supported; +- exact-source network allowlists; +- capacity and connection limits for web, Horizon, scheduler, migrations, and Grafana provisioning; +- backups/PITR and an isolated restore test. + +## Validation and rendering + +Use both steps in CI or before a production rollout: + +```bash +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles \ + --repo-root "$PWD" validate + +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles \ + --repo-root "$PWD" render +``` + +Validation renders into a temporary directory and leaves active bundles unchanged. Rendering builds each node in a temporary directory, writes checksums and metadata, then atomically replaces the active bundle while retaining `.previous`. + +## Bundle transfer and activation + +Archive one node from the generator host: + +```bash +tar --numeric-owner --owner=0 --group=0 \ + -C build/fleet-bundles \ + -czf /tmp/edge-frankfurt.tar.gz edge-frankfurt +``` + +On the target host: + +```bash +install -d -m 0700 /opt/cdnfoundry.new +tar -xzf /tmp/edge-frankfurt.tar.gz --strip-components=1 -C /opt/cdnfoundry.new +cd /opt/cdnfoundry.new +sha256sum -c SHA256SUMS +./validate.sh +cd /opt +mv cdnfoundry cdnfoundry.previous 2>/dev/null || true +mv cdnfoundry.new cdnfoundry +cd /opt/cdnfoundry +./start.sh +``` + +Never transfer the entire fleet state or another node’s bundle. + +## Updating the fleet + +Add a host: + +```bash +./scripts/cdnfoundry-fleet --state-dir ./.cdnfoundry-fleet add-node \ + --node edge-tokyo --role edge --region asia --location tokyo \ + --public-ipv4 192.0.2.80 --non-interactive +``` + +Change a release or address: + +```bash +./scripts/cdnfoundry-fleet --state-dir ./.cdnfoundry-fleet update-node \ + --node edge-tokyo --release v1.1.0 --public-ipv4 192.0.2.81 \ + --non-interactive +``` + +Render only that host: + +```bash +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles \ + --repo-root "$PWD" validate --node edge-tokyo + +./scripts/cdnfoundry-fleet \ + --state-dir ./.cdnfoundry-fleet \ + --output-dir ./build/fleet-bundles \ + --repo-root "$PWD" render --node edge-tokyo +``` + +A changed hostname or public IP automatically causes that node certificate to be reissued with updated SANs. + +## Troubleshooting + +### The script prints only initialization output + +Use the full wrapper or `setup`, not the low-level `init` command: + +```bash +./scripts/generate-production-env.sh +# or +./scripts/cdnfoundry-fleet setup --help +``` + +`init` deliberately creates only protected state and secrets. It does not add nodes or render bundles. + +### No interactive questions appear + +Interactive mode needs a real terminal. In CI, a pipe, or a non-TTY shell, provide all values with `--non-interactive --config FILE`. The CLI now returns a clear validation error instead of silently waiting on unavailable input. + +### I cannot find control plus monitoring + +Run the setup wizard and choose **Control + monitoring on the same host**, or configure it explicitly: + +```bash +./scripts/cdnfoundry-fleet --state-dir ./.cdnfoundry-fleet \ + configure-monitoring --mode colocated --non-interactive +``` + +Confirm with `status`. + +### State already exists + +The full setup command reuses existing state. It does not rotate secrets. To inspect it: + +```bash +./scripts/cdnfoundry-fleet --state-dir ./.cdnfoundry-fleet status +``` + +### A required variable is missing + +The renderer derives required variables from the final filtered Compose file. Add project-specific values to a node’s `extra_env` only when the repository introduces a new required variable that the generator does not yet know. Do not place secrets in version-controlled config files. + +### Compose overlay contains `!reset` or `!override` + +The fleet loader supports both Docker Compose tags. Older generator versions used `yaml.safe_load` directly and failed on these overlays. + +### Docker is unavailable on the generator machine + +You can still generate bundles and run Python-level validation. Run each bundle’s `validate.sh` on a host with Docker Compose v2 before activation. + +## Related documents + +- [Production quick start: three nodes](production-quick-start.md) +- [Production quick start: multi-region fleet](production-quick-start-multi-region.md) +- [Production fleet reference](production-fleet.md) +- [Fleet configuration reference](production-fleet-config-reference.md) diff --git a/docs/deployment/production-fleet.md b/docs/deployment/production-fleet.md new file mode 100644 index 0000000..22d6f11 --- /dev/null +++ b/docs/deployment/production-fleet.md @@ -0,0 +1,310 @@ +--- +title: Production fleet reference +description: Understand the design rules, topology, certificate trust, startup order, upgrade behavior, and secret distribution for CDNFoundry production fleets. +--- + +# Production fleet reference + +For the complete command workflow and interactive setup, see [Production fleet operator guide](production-fleet-operator-guide.md). + +## Design rules + +- The Python generator runs only on the control-plane machine. +- A remote host receives one private role-specific bundle and runs Docker Compose locally. +- Each DNS or combined DNS/edge host has its own local PostgreSQL (`pdns-db`) and its own PowerDNS services. +- A DNS host never uses the control-plane application's PostgreSQL database. +- Each DNS host has a separate stable `PDNS_DB_PASSWORD` and `PDNS_API_KEY`. +- Optional monitoring, logging, backups, and IPv6 do not create cross-role variable or secret requirements when disabled. +- Existing DNS and HTTP serving nodes preserve their last valid runtime state during control-plane or regional failures. + +## Global topology + +```mermaid +flowchart TB + CP[Control-plane host\nGenerator + application DB + Valkey] + M[Optional monitoring host\nPrometheus + Grafana + ClickHouse/Loki] + D1[DNS host A\nPowerDNS + local PostgreSQL] + D2[DNS host B\nPowerDNS + local PostgreSQL] + DN[DNS host N\nPowerDNS + local PostgreSQL] + E1[Edge host A] + E2[Edge host B] + EN[Edge host N] + CP -->|signed desired state / DNS API| D1 + CP -->|signed desired state / DNS API| D2 + CP -->|signed desired state / DNS API| DN + CP -->|signed edge configuration| E1 + CP -->|signed edge configuration| E2 + CP -->|signed edge configuration| EN + D1 -. metrics/logs .-> M + D2 -. metrics/logs .-> M + DN -. metrics/logs .-> M + E1 -. metrics/logs .-> M + E2 -. metrics/logs .-> M + EN -. metrics/logs .-> M +``` + +## DNS and geo-routing flow + +```mermaid +flowchart TD + Q[Authoritative DNS query] --> ECS{Valid ECS present?} + ECS -->|Yes| C[ECS client subnet] + ECS -->|No| R[Recursive resolver IP] + C --> P[Country / ASN / region / network overrides] + R --> P + P --> H[Remove disabled, draining, unhealthy, stale, or incompatible edges] + H --> F{Preferred region has healthy address?} + F -->|Yes| S[Deterministic preferred healthy edge] + F -->|No| RF[Ordered regional fallbacks] + RF --> GF[Ordered global fallback] + S --> A[A or AAAA answer] + GF --> A +``` + +A and AAAA health are evaluated independently. Health state uses configurable consecutive-failure and consecutive-success thresholds to reduce flapping. DNS TTL creates eventual consistency; global changes are not instantaneous. + +## Edge request flow + +```mermaid +sequenceDiagram + participant Client + participant DNS as Authoritative DNS + participant Edge + participant Origin + Client->>DNS: A/AAAA query + DNS-->>Client: deterministic healthy edge address + Client->>Edge: HTTP/TLS request + Edge->>Edge: validate host, policy, cache, and signed configuration + alt cache hit + Edge-->>Client: cached response + else cache miss + Edge->>Origin: protected origin request + Origin-->>Edge: response + Edge->>Edge: cache under policy + Edge-->>Client: response + end +``` + +## Monitoring and logging flow + +```mermaid +flowchart LR + C[Control host] -->|node exporter metrics| P[Prometheus] + D[Every DNS host] -->|node exporter metrics| P + E[Every edge host] -->|node exporter metrics| P + MH[Monitoring host] -->|node exporter metrics| P + C -->|Vector: container/system/app logs| L[Loki or central log endpoint] + D -->|Vector: PowerDNS/system/container logs| L + E -->|Vector: access/error/system/container logs| L + MH -->|Vector: observability logs| L + P --> G[Grafana] + L --> G +``` + +Vector uses authenticated transport configuration, bounded disk buffering, retries, health checks, role labels, and basic secret filtering. Every required host receives a unique log credential. Disabling centralized logs removes Vector and its credentials from bundles. + +## Certificate trust + +```mermaid +flowchart TB + ICA[Edge identity CA\nprivate key retained by control services] + SCA[Edge server CA\nprivate key in protected fleet state] + SCA --> CC[Control edge-control TLS certificate] + SCA --> DC[Per-DNS API TLS certificate] + SCA --> EC[Per-edge runtime TLS certificate] + SCA --> MC[Monitoring host certificate] + ICA --> EI[Issued edge identities] +``` + +The generator follows the production repository's two-CA contract. The edge identity CA is used by the control plane for edge identities. The edge server CA signs edge-control, edge runtime, and DNS API TLS certificates. The server CA private key stays in protected fleet state; each node receives only its own keypair and the server CA certificate. The control bundle additionally receives the edge identity CA material required by the control service. Certificate SANs are regenerated when a node hostname or service address changes. + +## Fleet startup order + +```mermaid +flowchart TD + V[Validate every bundle] --> M[Dedicated monitoring data services, if configured] + M --> CDB[Control PostgreSQL + Valkey] + CDB --> CM[Control migrations] + CM --> CS[Control services] + CS --> DDB[Each DNS host: local pdns-db] + DDB --> DM[Each DNS host: pdns-migrate] + DM --> DS[PowerDNS + DNSdist] + DS --> ES[Edge runtime and gateways] + ES --> O[Exporters and log collectors] +``` + +The generated `STARTUP-ORDER.md` lists actual configured nodes in this order. + +## Upgrade, rollback, and regional failure + +```mermaid +flowchart TD + R[Render new immutable release bundle] --> V[Validate Compose, paths, permissions, and certificate] + V --> T[Transfer to host as new directory] + T --> P[Pull images and start] + P --> H{Healthy?} + H -->|Yes| K[Keep previous bundle for retention window] + H -->|No| B[Restore previous bundle] + B --> S[Start previous release without deleting volumes] + RF[Regional failure] --> HF[Health filtering] + HF --> RR[Regional fallback] + RR --> GF[Global fallback] +``` + +Never use `docker compose down -v` during rollback or credential recovery. + +## Fleet state and secret distribution + +Protected control-plane state records global configuration, nodes, addresses, feature modes, release identifiers, secret references, bundle generation, and last successful validation/render timestamps. Writes are atomic and protected by a non-blocking file lock. The previous valid state is retained in bounded history. + +Global application secrets are distributed only when the rendered role's Compose file references them. Node-scoped secrets include: + +| Node capability | Secrets | +| --- | --- | +| DNS | `pdns-db-password`, `pdns-api-key` | +| Edge | `edge-status-token` | +| Monitoring enabled | `node-exporter-token` | +| Centralized logs enabled | `log-auth-token` | + +A DNS node's database password is not shared with any other DNS node or the control plane. + +## Per-node DNS database lifecycle + +On each DNS-capable host: + +1. `pdns-db` starts from that host's persistent PostgreSQL volume. +2. `pdns-migrate` uses the same node-specific stored credential. +3. `pdns-auth` connects to hostname `pdns-db`, database `pdns`, user `pdns`, and that same credential. +4. Health checks and future migrations use the same stable value. +5. A normal render does not generate a new password. + +For an existing installation, use `adopt-existing` with the existing `.env.prod`; the importer stores `PDNS_DB_PASSWORD` under that node without printing it. The source database volume is not deleted. + +Password rotation is staged. `--phase prepare` creates a pending credential while the active one stays unchanged. The rendered `reconcile-pdns-password.sh` changes only the selected DNS host's local PostgreSQL role and local environment. `--phase commit` makes the pending credential active in protected fleet state. `--phase abort` removes an unreconciled pending value. + +## Geo-routing policy + +The generated policy records this decision pipeline: + +```text +valid ECS client subnet +→ resolver IP fallback +→ country and ASN policy +→ health filtering +→ preferred healthy edge IP +``` + +Policy implementations consuming the generated file must support country, ASN, region, and network overrides; deterministic choice; ordered regional/global fallback; draining; stale health; compatibility filters; and separate A/AAAA health. A recommended starting point is three consecutive failures before removal and two consecutive successes before re-entry, with a 90-second stale threshold. Tune these values to probe interval and DNS TTL. + +## MMDB behavior + +MMDB is required only on roles whose selected services actually consume it. A role-specific bundle copies only Compose-referenced runtime files and retains only referenced volumes. A production Compose definition that mounts an MMDB volume must also select a working updater or another documented population mechanism for that same role; an unpopulated MMDB mount is a validation defect and must not be deployed. Roles that do not consume MMDB should have no MMDB environment variable, mount, volume, or dependency. + +Before deployment, inspect the rendered role with: + +```bash +# Run on the control-plane machine against a rendered bundle. +cd /var/lib/cdnfoundry-fleet/bundles/NODE_NAME +grep -Rni -- 'mmdb\|geoip' ./*.yml .env.prod generated 2>/dev/null || true +``` + +Then verify on the **target host** that the populated persistent data exists and the consuming service reports successful database loading. + +## Monitoring modes + +- `disabled`: no full monitoring stack, no node exporter service, and no monitoring credentials in role bundles. +- `colocated`: monitoring data services run on the control host; exporters cover all enabled production hosts. +- `dedicated`: monitoring data services run on a monitoring-role host; exporters cover control, DNS, edge, combined, and monitoring hosts. + +Prometheus targets are regenerated automatically when nodes are added, updated, disabled, or removed. + +## Backup modes + +- `disabled`: no backup-specific credentials or placeholder variables are required. +- `control`: protect the fleet state, CA, application database, and control state. +- `all-stateful`: additionally protect every DNS host's local PostgreSQL state and enabled observability stores. + +Backups must be encrypted, tested by restore, stored outside the host, and protected independently from normal fleet credentials. Do not back up transient bundles instead of authoritative state and data volumes. + +## IPv4 and dual stack + +Initialize with `--dual-stack`, then provide each node's `--public-ipv6` and, where needed, `--bind-ipv6`. Firewalls, authoritative glue, health probes, monitoring, and routing policy must be configured separately for IPv4 and IPv6. An edge may be healthy for A and unhealthy for AAAA without being removed from both families. + +## Hardware sizing + +These are planning baselines; measure actual query rate, request rate, cache working set, log volume, retention, and database growth. + +| Deployment | Control | Each DNS host | Each edge host | Monitoring | +| --- | --- | --- | --- | --- | +| Small | 4 vCPU, 8 GB RAM, 100 GB SSD | 2 vCPU, 4 GB, 40 GB SSD | 4 vCPU, 8 GB, cache-sized NVMe | Colocated or 4 vCPU, 8 GB | +| Medium | 8 vCPU, 16–32 GB, 250 GB SSD | 4 vCPU, 8 GB, 100 GB SSD | 8–16 vCPU, 16–32 GB, NVMe | 8 vCPU, 32 GB, 500 GB+ SSD | +| Multi-region baseline | 16+ vCPU, 64 GB, redundant NVMe | 8 vCPU, 16 GB, redundant SSD | 16–32+ vCPU, 64–128 GB, high-endurance NVMe | 16+ vCPU, 64–128 GB, storage sized to retention | + +## Firewall requirements + +| Destination | Port | Allowed sources | +| --- | --- | --- | +| DNSdist on DNS hosts | UDP/TCP 53 | Internet | +| Edge HTTP/TLS | TCP 80/443 | Internet or configured customer networks | +| Control UI/API | TCP 80/443 | Intended operator/public sources | +| Edge control runtime | TCP 8443 | Configured edge source addresses only | +| DNS API | TCP 8444 | Control-plane source addresses only | +| Node exporter | TCP 9100 | Monitoring host/private monitoring network only | +| PostgreSQL, Valkey, ClickHouse, Loki internal ports | service-specific | Local Docker network or exact private peers only; never Internet | +| SSH | TCP 22 or chosen port | Administrative bastions/VPN only | + +Publish authoritative NS and glue records for every unicast DNS host. Allow both UDP and TCP 53. Keep reverse-path filtering and provider anti-spoofing compatible with the selected unicast design. + +## Multi-region reference topology + +The example contains four authoritative unicast DNS hosts and ten unicast edge hosts. The locations are an example, not generator constants. + +DNS: Ashburn, Frankfurt, Singapore, São Paulo. + +Edges: Ashburn, Los Angeles, São Paulo, Frankfurt, Johannesburg, Dubai, Mumbai, Singapore, Tokyo, Sydney. + +Copy and edit the JSON topology on the **control-plane machine**: + +```bash +install -m 0600 deploy/production/examples/multi-region-fleet.json ./fleet.json +sudo ./scripts/cdnfoundry-fleet --config fleet.json --non-interactive setup +``` + +Replace documentation IP addresses before production use. + +## Troubleshooting + +### PowerDNS cannot authenticate + +On the affected **DNS host**, confirm `pdns-db` and `pdns-auth` are in the same bundle and that PowerDNS points to `pdns-db`: + +```bash +cd /opt/cdnfoundry +docker compose --env-file .env.prod config | grep -A12 -E 'pdns-db:|pdns-auth:' +docker compose --env-file .env.prod ps +docker compose --env-file .env.prod logs --since 15m pdns-db pdns-auth +``` + +Do not print the password. Compare only a local hash when necessary. Use `adopt-existing` for a previously deployed value or the staged rotation procedure. Do not delete the database volume. + +### Compose requests another role's variables + +Render and inspect the selected node again. The node's generated Compose manifest should contain only services for its role and enabled features; `.env.prod` contains only variables referenced by that filtered Compose plus generated role configuration. + +### A bundle render fails + +The destination remains unchanged until the temporary bundle is complete. Correct the error, run `validate`, and render again. The previous state JSON and `.previous` node bundle remain available. + +### A node is removed but remains in monitoring + +Render the monitoring host after state modification. Prometheus target files are generated from current enabled nodes. + +### Generator reports another process + +A control-plane generation process already holds the fleet lock. Find and finish that process; do not delete the lock file while it is active. + +## Deployment runbooks + +- [Three-node production quick start](production-quick-start.md) +- [Multi-region fleet quick start](production-quick-start-multi-region.md) +- [Production fleet operator guide](production-fleet-operator-guide.md) diff --git a/docs/deployment/production-quick-start-multi-region.md b/docs/deployment/production-quick-start-multi-region.md new file mode 100644 index 0000000..1053772 --- /dev/null +++ b/docs/deployment/production-quick-start-multi-region.md @@ -0,0 +1,59 @@ +--- +title: "Production quick start: multi-region fleet" +description: Deploy a separated-role CDNFoundry fleet across multiple regions from a validated JSON topology. +--- + +# Production quick start: multi-region fleet + +This example models one control node, four authoritative DNS nodes, ten edge nodes, and three monitoring-role nodes. “Multi-region” describes its failure-domain design; it is not a special runtime mode or a fixed scale limit. + +Read and complete the [starter fleet quick start](production-quick-start.md) first. The same security, PKI, transfer, migration, enrollment, last-valid-state, backup, and acceptance rules apply. + +## Clone an immutable source revision + +```bash +git clone https://github.com/vaheed/CDNFoundry.git cdnfoundry +cd cdnfoundry +git checkout v1.0.0 +git rev-parse --verify HEAD +sudo ./scripts/install-production-prerequisites.sh +``` + +## Configure the topology without editing scripts + +```bash +install -m 0600 deploy/production/examples/multi-region-fleet.json ./fleet.json +``` + +Edit `fleet.json` and replace the operator/platform domains, exact release, ACME contact, every documentation address, hostname, region, and location. Add private `monitor_ipv4`/`log_ipv4` addresses when monitoring traffic must avoid public paths. For remote PostgreSQL, add typed control-node `extra_env` values for `DB_HOST`, `DB_PORT`, and `DB_SSLMODE`, then replace `control-db-password` through `set-secret --from-file`; never put the password in JSON. + +Validate without writing: + +```bash +python3 -m json.tool fleet.json >/dev/null +./scripts/cdnfoundry-fleet --config fleet.json --non-interactive --dry-run setup +``` + +Create state and all node bundles: + +```bash +sudo ./scripts/cdnfoundry-fleet \ + --config fleet.json \ + --state-dir /var/lib/cdnfoundry-fleet \ + --output-dir /var/lib/cdnfoundry-fleet/bundles \ + --non-interactive \ + setup +``` + +## Roll out in dependency order + +1. Start `control-1`, including its MMDB updater, database dependencies, migrations, control services, and health checks. +2. Start the selected dedicated monitoring host and qualify Prometheus, Grafana, ClickHouse, Loki, and log ingestion. +3. Start all four DNS nodes, configure NS/glue and DNS clusters, and qualify UDP/TCP answers before delegation. +4. Create each edge in the control panel, configure its UUID/token through protected files, rerender, and start edge bundles one at a time. +5. Remove every consumed bootstrap token, rerender, and recreate only the affected agent. +6. Validate regional fallback, draining, node loss, control-plane outage, restart, telemetry loss, and previous-bundle rollback. + +Do not activate all hosts simultaneously. A target must be valid and serving before any source is drained. Generated `.env.prod` is authoritative for every role bundle; do not add deployment defaults to Compose or edit rendered manifests on a host. + +See [Production fleet configuration reference](production-fleet-config-reference.md) for JSON fields and [Production fleet operator guide](production-fleet-operator-guide.md) for lifecycle commands. diff --git a/docs/deployment/production-quick-start.md b/docs/deployment/production-quick-start.md index cd1e716..6f62da4 100644 --- a/docs/deployment/production-quick-start.md +++ b/docs/deployment/production-quick-start.md @@ -1,858 +1,153 @@ --- -title: Production quick start -description: Deploy a small three-host CDNFoundry fleet first, then optionally add monitoring and centralized logs. -keywords: private CDN deployment, production CDN, authoritative DNS, OpenResty CDN, PowerDNS, Grafana monitoring +title: "Production quick start: starter fleet" +description: Deploy CDNFoundry with one control node and two combined DNS and edge nodes from a validated JSON topology. --- -# Production quick start - -This guide deploys the smallest practical CDNFoundry production fleet: - -- one control host, identified as `CONTROL`; -- two combined DNS and edge hosts, `EDGE_1` and `EDGE_2`, in different failure domains; -- one exact CDNFoundry release on every host; -- optional monitoring and centralized logs, enabled only after serving works. - -Follow the numbered steps in order. Commands say exactly which host they run -on. Do not enable the optional `telemetry` or `logs` profiles during the base -installation. - -::: danger Preserve production state -Never run `docker compose down -v`, delete named volumes, regenerate `APP_KEY`, -or replace CA keys during an upgrade. PostgreSQL, application keys, CA keys, -and externally stored TLS material are part of the recovery set. -::: - -::: tip Replace the examples -Replace every uppercase placeholder and documentation IP before running a -command. `198.51.100.0/24` is reserved for documentation and cannot carry real -Internet traffic. -::: - -## Resulting topology - -| Host | Required services | Public listeners | -| --- | --- | --- | -| `CONTROL` | Laravel, web, Horizon, Scheduler, PostgreSQL, Valkey, edge-control, Caddy | TCP `80`, `443`, `8443`, `8444`; UDP `443` | -| `EDGE_1` | DNSdist, PowerDNS, DNS API, OpenResty cells, edge agent, gateway, traffic Vector | TCP/UDP `53`; TCP `80`, `443`, `8444` | -| `EDGE_2` | Same as `EDGE_1` in another provider, rack, or failure domain | TCP/UDP `53`; TCP `80`, `443`, `8444` | - -The optional final step adds ClickHouse, Prometheus, Alertmanager, Grafana, -Loki, node-exporter, and one operational-log collector per host. - -`CONTROL` is a single management failure domain in this minimum -topology. If it is offline, existing DNS and HTTP traffic continue using the -last valid runtime state. Management, deployments, new certificates, and -analytics pause until it recovers. - -```mermaid -flowchart LR - Admin["Administrator"] -->|"HTTPS 443"| Control["CONTROL"] - Control --> Desired[("PostgreSQL desired state")] - Agent1["EDGE_1 agent"] -->|"outbound mTLS 8443"| Control - Agent2["EDGE_2 agent"] -->|"outbound mTLS 8443"| Control - Control -->|"HTTPS 8444"| DNS1["EDGE_1 DNS API"] - Control -->|"HTTPS 8444"| DNS2["EDGE_2 DNS API"] - Resolver["DNS resolver"] -->|"UDP/TCP 53"| DNSdist["DNSdist"] - Visitor["Visitor"] -->|"HTTP/HTTPS"| Cell["OpenResty cell"] - Cell -->|"validated origin"| Origin["Customer origin"] -``` - -Laravel is never in the DNS or HTTP request path. - -## Example values - -Use your real values consistently on every host: - -| Purpose | Example | -| --- | --- | -| Independent operator DNS zone | `ops.example.com` | -| CDNFoundry platform zone | `example.net` | -| `CONTROL` public/NAT IPv4 | `198.51.100.10` | -| `EDGE_1` public/NAT IPv4 | `198.51.100.20` | -| `EDGE_2` public/NAT IPv4 | `198.51.100.30` | -| `EDGE_1` advertised shared/quarantine service IPv4 | `198.51.100.120`, `198.51.100.121` | -| `EDGE_1` assigned local service IPv4 | `10.20.1.120`, `10.20.1.121` | -| `EDGE_2` advertised shared/quarantine service IPv4 | `198.51.100.130`, `198.51.100.131` | -| `EDGE_2` assigned local service IPv4 | `10.20.2.130`, `10.20.2.131` | -| Local IPv4 bind on every host | `0.0.0.0` or an assigned private address | -| Exact release | `v0.9.4` | -| Installation directory | `/opt/cdnfoundry` | -| Protected PKI directory | `/etc/cdnfoundry/pki` | - -Keep `control.ops.example.com`, `edge-control.ops.example.com`, -`telemetry.ops.example.com`, and every `dns-api-N.ops.example.com` at an -independent DNS provider. Do not put management names inside the CDNFoundry -platform zone. - -The public/NAT addresses above are advertised in DNS and used in peer firewall -allowlists. They are not Docker bind addresses. `HOST_BIND_IPV4` and -`DNS_BIND_V4` must name an address that exists locally; the generator defaults -both to `0.0.0.0`, which supports hosts behind DNAT, a provider firewall, or a -load balancer. Configure the external device to forward only the listed ports. -Edge customer traffic is stricter: each advertised pool service address maps -one-to-one to a distinct address actually assigned to the host. The gateway -binds only those local addresses. - -## Step 1: prepare hosts and firewall rules - -Prepare three supported Linux hosts with: - -- Docker Engine and the Docker Compose plugin; -- Git, OpenSSL, curl, and CA certificates; -- accurate system time; -- an operator firewall and a provider firewall; -- optional S3-compatible object storage for encrypted Restic backups; -- console access in case a firewall rule is wrong. - -A reasonable starting size is: - -| Role | CPU | Memory | Disk | -| --- | ---: | ---: | ---: | -| Control | 4 vCPU | 8 GiB | 100 GiB SSD | -| `EDGE_1` and `EDGE_2`, each | 4 vCPU | 6 GiB | 50 GiB SSD plus cache capacity | - -Allow only these inbound connections: - -| Destination | Allowed source | -| --- | --- | -| Control TCP `22` | trusted administrator networks | -| Control TCP `80`, `443`; UDP `443` | public | -| Control TCP `8443`, `8444` | the two edge source addresses seen after NAT | -| Edge TCP/UDP `53` | public | -| Edge TCP `80`, `443` on advertised service addresses | public; forward one-to-one to mapped local service addresses | -| Edge TCP `8444` | the control source address seen after NAT only | -| PostgreSQL, Valkey, ClickHouse, PowerDNS API, Prometheus, Loki, Grafana `3000` | never public | - -Docker-published ports can bypass ordinary UFW rules. Apply the same policy in -the provider firewall and the host `DOCKER-USER` chain. Keep outbound DNS and -HTTPS available for images, ACME, origins, GeoIP, telemetry, and backups. - -## Step 2: create bootstrap DNS and registrar glue - -At the independent provider for `ops.example.com`, create: - -| Record | Value | -| --- | --- | -| `control.ops.example.com` A | control IPv4 | -| `edge-control.ops.example.com` A | control IPv4 | -| `telemetry.ops.example.com` A | control IPv4 | -| `dns-api-1.ops.example.com` A | `EDGE_1` IPv4 | -| `dns-api-2.ops.example.com` A | `EDGE_2` IPv4 | - -Add AAAA records only when that host and its firewall are IPv6-ready. - -At the registrar for `example.net`, register child nameserver glue: - -| Child nameserver | Address | -| --- | --- | -| `ns1.example.net` | `EDGE_1` IPv4 and optional IPv6 | -| `ns2.example.net` | `EDGE_2` IPv4 and optional IPv6 | - -Do not delegate `example.net` yet. - -## Step 3: install the same exact release on every host - -Run on `CONTROL`, `EDGE_1`, and `EDGE_2`: - -```sh -sudo apt-get update -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - ca-certificates curl git openssl - -docker version -docker compose version - -export CDNF_RELEASE=v0.9.4 -sudo install -d -m 0755 /opt/cdnfoundry -sudo git clone --branch "${CDNF_RELEASE}" --depth 1 \ - https://github.com/vaheed/CDNFoundry.git /opt/cdnfoundry -sudo chown -R "$(id -u):$(id -g)" /opt/cdnfoundry -cd /opt/cdnfoundry -git rev-parse HEAD -``` - -The final commit must match on all three hosts. Use an exact release tag or -40-character commit SHA; never deploy `latest` or a moving major/minor alias. -Authenticate to GHCR with a read-only package token if the images are private. - -## Step 4: generate one private environment per host - -The generator creates `.env.prod` with mode `0600` and refuses to overwrite an -existing file. - -On `CONTROL`: - -```sh -cd /opt/cdnfoundry -./scripts/generate-production-env.sh -stat -c '%a %n' .env.prod -``` - -Choose: - -- role `control`; -- your operator and platform domains; -- the exact release from Step 3; -- the control public/NAT IPv4 advertised in DNS; -- local bind `0.0.0.0` or an assigned private control-host address; -- the control source address as the edge firewalls see it; -- the `EDGE_1` and `EDGE_2` source addresses as `CONTROL` sees them; -- one high-entropy shared telemetry password already stored in your password - manager; -- whether to configure optional S3-compatible backups now. - -The generator never uses the advertised public/NAT address as a listener bind. -Do not replace `HOST_BIND_IPV4=0.0.0.0` with an address that is owned only by an -external firewall, NAT gateway, or load balancer. - -If you enable backups, the repository location is a Restic backend address—not -an encryption password. For example: - -```dotenv -RESTIC_REPOSITORY=s3:https://object-storage.example/bucket/cdnfoundry-control -RESTIC_PASSWORD_FILE=/etc/cdnfoundry/secrets/restic-password -``` - -Use an existing bucket, a dedicated prefix, and backup-only S3 credentials that -cannot access unrelated objects. Restic encrypts repository contents using the -separate password file. Leaving the repository and credential fields empty -disables backups without preventing the control plane from starting. Other -Restic backends need deployment-specific credentials or mounts and are outside -the generator's S3 quick-start path. - -Run the generator separately on `EDGE_1` and `EDGE_2`. Choose role `dns-edge` -and use a unique DNS API label: - -- `dns-api-1` on `EDGE_1`; -- `dns-api-2` on `EDGE_2`. - -When prompted, enter the shared telemetry password from the password manager. The -generator sets the remote ClickHouse and Loki endpoints, plus unique log host -and collector identities, so optional monitoring can be enabled later. - -Verify every host: - -```sh -cd /opt/cdnfoundry -test "$(stat -c '%a' .env.prod)" = 600 -if grep -n 'replace-with' .env.prod; then - echo 'Replace every value shown above before continuing.' >&2 - exit 1 -fi -``` - -Never copy the whole control environment to an edge. Database passwords, API -keys, edge tokens, and agent identities have separate trust boundaries. - -## Step 5: create secret files and internal certificates - -Run once on `CONTROL`: - -```sh -sudo install -d -m 0700 /etc/cdnfoundry/secrets -sudo install -d -m 0700 /etc/cdnfoundry/pki - -sudo /opt/cdnfoundry/scripts/generate-production-certificates.sh \ - /etc/cdnfoundry/pki \ - edge-control.ops.example.com \ - proxy.example.net \ - dns-api-1.ops.example.com \ - dns-api-2.ops.example.com - -sudo sh -c 'umask 077; openssl rand -base64 48 > /etc/cdnfoundry/secrets/metrics-token' -sudo chown root:82 /etc/cdnfoundry/pki/edge-identity-ca.key -sudo chmod 0640 /etc/cdnfoundry/pki/edge-identity-ca.key -``` - -If backups were enabled in Step 4, also create the configured Restic password -file. This password encrypts and unlocks the backup repository; losing it makes -the snapshots unrecoverable: - -```sh -sudo sh -c 'umask 077; openssl rand -base64 48 > /etc/cdnfoundry/secrets/restic-password' -``` +# Production quick start: starter fleet -Store a protected recovery copy outside these hosts. Initialize and test the -repository after the control containers are healthy in Step 11. +This runbook creates the smallest practical production CDNFoundry fleet: -Verify certificate names: +- one control-plane node with colocated monitoring and operational logs; +- two combined DNS and edge nodes in separate failure domains; +- one generated, role-filtered bundle per host. -```sh -openssl x509 -in /etc/cdnfoundry/pki/edge-control-server.crt \ - -noout -subject -issuer -ext subjectAltName -openssl verify -CAfile /etc/cdnfoundry/pki/edge-server-ca.crt \ - /etc/cdnfoundry/pki/edge-control-server.crt \ - /etc/cdnfoundry/pki/edge-runtime.crt \ - /etc/cdnfoundry/pki/dns-api-1.crt \ - /etc/cdnfoundry/pki/dns-api-2.crt -``` - -Copy to `EDGE_1` and `EDGE_2` through separate protected channels: +The topology is data, not code. You edit a local JSON file containing your domains, addresses, and locations. You do not edit deployment shell scripts or Compose files. -- `edge-server-ca.crt`; -- `edge-runtime.crt` and `edge-runtime.key`; -- only that host's `dns-api-N.crt` and `dns-api-N.key`. +## 1. Prepare the hosts -Certificates use mode `0644`; private keys use `0600`. Never copy -`edge-server-ca.key` or `edge-identity-ca.key` to an edge. See -[Internal certificates](certificates.md) for the full ownership matrix. +Use supported Linux hosts with Docker Engine, Docker Compose v2, Python 3, PyYAML, OpenSSL, outbound HTTPS, synchronized clocks, and private administrative access. Open only the listeners documented in [Production fleet reference](production-fleet.md). -## Step 6: start the control plane +On an administrative workstation or the future control node, clone an immutable release or commit: -Run on `CONTROL`. Notice that only the required `control` profile is enabled. - -```sh -cd /opt/cdnfoundry - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile control config --quiet - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile control pull - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile control up -d --wait --wait-timeout 120 control-db redis - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile tools run --rm migrate - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile control up -d - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml ps +```bash +git clone https://github.com/vaheed/CDNFoundry.git cdnfoundry +cd cdnfoundry +git checkout v1.0.0 +git rev-parse --verify HEAD +sudo ./scripts/install-production-prerequisites.sh ``` -The first `up` creates the persistent PostgreSQL and Valkey services and waits -for both health checks. The one-shot migration then connects to those already -healthy dependencies. Application startup never performs an implicit migration. - -If `REDIS_PASSWORD` is changed after control containers have already been -created, recreate the control services so every container receives the same -value. A restart alone does not update container environment variables: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile control up -d --force-recreate -``` - -Verify the control database, Valkey, workers, scheduler, web, edge-control, and -Caddy are running. Then check the public API: - -```sh -curl -fsS https://control.ops.example.com/api/health -curl -fsS https://control.ops.example.com/api/ready -``` +Replace `v1.0.0` with a published release tag or exact commit SHA. Do not deploy from a moving branch or mutable image tag. -`health` proves process liveness. `ready` requires PostgreSQL and Valkey and -must return `ready` before continuing. +## 2. Create your topology file -Create the first administrator: +Copy the starter example outside the repository-managed path: -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - exec -u www-data core php artisan cdnf:admin:create \ - --name="CDN Operations" \ - --email="admin@example.com" +```bash +install -m 0600 deploy/production/examples/starter-fleet.json ./fleet.json ``` -Enter the password only at the prompt. Sign in at -`https://control.ops.example.com/admin`. - -## Step 7: start authoritative DNS on `EDGE_1` and `EDGE_2` - -Run these commands independently on `EDGE_1` and `EDGE_2`: - -```sh -cd /opt/cdnfoundry - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile dns --profile edge config --quiet - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile dns --profile edge pull - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile dns up -d --wait --wait-timeout 120 pdns-db - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile tools run --rm pdns-migrate - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile dns up -d - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - ps pdns-db pdns-auth dnsdist dns-api -``` - -The first `up` creates the persistent PowerDNS PostgreSQL service and waits for -its health check. Only then does the separate runtime migration connect to it; -the remaining DNS services start after the migration succeeds. - -From an external workstation, verify UDP and TCP on both hosts: - -```sh -dig @198.51.100.20 example.net SOA -dig @198.51.100.20 example.net SOA +tcp -dig @198.51.100.30 example.net SOA -dig @198.51.100.30 example.net SOA +tcp -``` - -The zone does not exist yet, so an authoritative negative answer is acceptable. -A timeout, refusal, or response from an unrelated DNS server is not. - -## Step 8: configure platform DNS - -In **Control plane → System DNS identity**, enter: - -| Field | Value | -| --- | --- | -| Platform domain | `example.net` | -| Proxy hostname | `proxy.example.net` | -| Nameserver 1 | `ns1.example.net`, `EDGE_1` IPv4, optional IPv6 | -| Nameserver 2 | `ns2.example.net`, `EDGE_2` IPv4, optional IPv6 | -| SOA primary | `ns1.example.net` | -| SOA mailbox | `hostmaster.example.net` | -| Refresh / retry / expire | `3600` / `600` / `1209600` | -| Minimum / default TTL | `300` / `300` | +Edit `fleet.json` and replace every example value: -Choose **Validate and preview**, review the normalized result, and apply the -exact confirmation token. +- `operator_domain`: private operator DNS suffix for control, node, and telemetry names; +- `platform_domain`: customer-facing CDN platform suffix; +- `release`: the exact checked-out tag or 40-character commit SHA; +- `acme_email`: monitored certificate contact; +- every `hostname`, `public_ipv4`, region, and location; +- `public_ipv6` and `bind_ipv6` when deploying dual stack. -Create two disabled DNS clusters: +The checked-in addresses are RFC documentation ranges and cannot serve production traffic. Keep `bind_ipv4` as `0.0.0.0` for normal routed/NAT hosts unless a specific local interface address is required. -| Field | Cluster 1 | Cluster 2 | -| --- | --- | --- | -| API URL | `https://dns-api-1.ops.example.com:8444` | `https://dns-api-2.ops.example.com:8444` | -| API key | `EDGE_1` `PDNS_API_KEY` | `EDGE_2` `PDNS_API_KEY` | -| Server ID | `localhost` | `localhost` | -| Nameserver | `ns1.example.net` | `ns2.example.net` | +Validate the JSON before it can create state: -For each cluster: - -1. save it disabled; -2. run the asynchronous connection test; -3. confirm TLS verification and a healthy result; -4. enable it; -5. reconcile system DNS identity; -6. wait for its candidate checksum to become active. - -Verify the active zone externally: - -```sh -dig @198.51.100.20 example.net SOA +tcp -dig @198.51.100.30 example.net NS -dig @198.51.100.20 ns1.example.net A -dig @198.51.100.30 ns2.example.net A +```bash +python3 -m json.tool fleet.json >/dev/null +./scripts/cdnfoundry-fleet --config fleet.json --non-interactive --dry-run setup ``` -Only now delegate `example.net` to `ns1.example.net` and `ns2.example.net` at -the registrar. - -## Step 9: map service addresses, then enroll `EDGE_1` and `EDGE_2` - -In **Edge network → Edges**, create one edge per host and copy its UUID and -one-time bootstrap token. Each new installation already contains -`shared-default` and `quarantine-default` service pools. On `EDGE_1` and -`EDGE_2`, confirm `cell-01` is assigned to the shared pool and `cell-02` to the -quarantine pool, then create one **Pool endpoint** for each pool using its advertised service -address from the example table. - -On each host, have the network operator assign the two corresponding local -service addresses. Configure the firewall, one-to-one DNAT, or layer-4 load -balancer so each advertised address forwards TCP `80` and `443` to exactly one -local address without terminating TLS. Verify that the local addresses exist; -do not add the advertised addresses to the host: - -```sh -ip -brief address -``` +The dry run performs topology, role, address, feature, and Compose validation without writing Fleet state or bundles. -Add the complete mapping and enrollment values to the matching `.env.prod`. -For `EDGE_1`, the example is: +## 3. Generate protected state and bundles -```dotenv -EDGE_GATEWAY_ADDRESS_MAP={"198.51.100.120":"10.20.1.120","198.51.100.121":"10.20.1.121"} -EDGE_ID=replace-with-edge-uuid -EDGE_BOOTSTRAP_TOKEN=replace-with-one-time-token +```bash +sudo install -d -m 0700 /var/lib/cdnfoundry-fleet +sudo ./scripts/cdnfoundry-fleet \ + --config fleet.json \ + --state-dir /var/lib/cdnfoundry-fleet \ + --output-dir /var/lib/cdnfoundry-fleet/bundles \ + --non-interactive \ + setup ``` -Use `EDGE_2`'s advertised and local pairs on `EDGE_2`. IPv6 pairs use the same map. -Every advertised endpoint must be present, each local value must be distinct, -and both sides of a pair must use the same address family. Production rejects -an incomplete map and preserves the previous valid gateway configuration. +The command creates secrets and private PKI once, validates the complete desired topology, renders bundles atomically, and prints start order. Each node bundle contains its filtered Compose manifest, complete `.env.prod`, required runtime files, certificates, secrets, checksums, and operator scripts. -Start the edge runtime on that host: +Never commit `fleet.json`, Fleet state, generated bundles, `.env.prod`, or private keys. -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile edge up -d +## 4. Inspect before transfer -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - ps cell-01 cell-02 edge-agent edge-gateway vector mmdb-updater +```bash +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + --output-dir /var/lib/cdnfoundry-fleet/bundles \ + validate +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + status +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + show-start-order ``` -The traffic Vector process starts with the edge profile. Until optional -telemetry is enabled, it retries into a bounded disk buffer and may eventually -drop old analytics events. This does not block or slow DNS and HTTP serving. +For every bundle, verify `SHA256SUMS`, review `README.md`, and run `./validate.sh`. Production Compose has no deployment-value defaults: all interpolation comes from that bundle's generated `.env.prod`. -Wait until the panel shows: +## 5. Start the control plane -- registered identity and a fresh heartbeat; -- ready shared and quarantine cells; -- listener-ready gateway status; -- an acknowledged active revision; -- bounded CPU, memory, disk, and connection capacity. +Transfer `bundles/control-1` over an authenticated channel to `/opt/cdnfoundry` on the control host. Preserve modes and do not place the bundle in a public or shared directory. -Remove the bootstrap token after registration and recreate only the agent: - -```sh -sudo sed -i 's/^EDGE_BOOTSTRAP_TOKEN=.*/EDGE_BOOTSTRAP_TOKEN=/' \ - /opt/cdnfoundry/.env.prod - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile edge up -d --force-recreate edge-agent -``` - -Repeat this step on the other edge. Never clone an edge-agent identity volume. - -## Step 10: add and test the first customer domain - -Use this order: - -1. create a domain user; -2. create and assign the domain; -3. ask the owner to delegate to `ns1.example.net` and `ns2.example.net`; -4. verify nameservers asynchronously; -5. activate the domain; -6. wait for both DNS clusters to acknowledge the revision; -7. add DNS-only A/AAAA records and test them; -8. add one proxied hostname with one explicit public origin; -9. wait for edge deployment and DNS-01 certificate completion. - -From an external workstation: - -```sh -dig +trace CUSTOMER_DOMAIN NS -dig @198.51.100.20 CUSTOMER_DOMAIN A -dig @198.51.100.30 CUSTOMER_DOMAIN A +tcp -curl --fail --head \ - --resolve CUSTOMER_DOMAIN:443:198.51.100.20 \ - https://CUSTOMER_DOMAIN/ -curl --fail --head \ - --resolve CUSTOMER_DOMAIN:443:198.51.100.30 \ - https://CUSTOMER_DOMAIN/ -``` - -Origin validation rejects loopback, link-local, multicast, metadata, internal -platform, edge-service, and proxy-loop destinations. Do not weaken this check -to make a private origin work. - -## Step 11, optional: initialize and prove encrypted backups - -The backup integration is optional and an empty `RESTIC_REPOSITORY` does not -block startup. Skipping it leaves the backup health component degraded and -means CDNFoundry has no built-in control-database recovery path. A tested -provider snapshot or another operator-owned recovery system may be used -instead. - -When the S3-compatible Restic settings were enabled in Step 4 and the password -file was created in Step 5, initialize a new repository once from the healthy -control container. The shell maps CDNFoundry's backup-only variables to the -standard names expected by Restic without printing their values: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - exec core sh -eu -c ' - export AWS_ACCESS_KEY_ID="$BACKUP_ACCESS_KEY_ID" - export AWS_SECRET_ACCESS_KEY="$BACKUP_SECRET_ACCESS_KEY" - export AWS_DEFAULT_REGION="$BACKUP_DEFAULT_REGION" - restic init - ' - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - exec core php artisan cdnf:backups:create --wait -``` - -For an existing repository, replace `restic init` with `restic snapshots`. -Then verify the recorded snapshot remotely, restore it in an isolated -environment, and store `APP_KEY`, signing keys, CA keys, the Restic password, -backup credentials, and external TLS material in the protected recovery -system. Record the tested RPO and RTO. See -[Backup and recovery](../operations/backup-and-recovery.md). - -The required serving checklist is: - -- [ ] every host runs the same exact release; -- [ ] every Compose configuration renders successfully; -- [ ] Laravel and PowerDNS migrations completed explicitly; -- [ ] control `/api/health` and `/api/ready` succeed; -- [ ] both DNS servers answer UDP and TCP externally; -- [ ] registrar glue matches the active listener addresses; -- [ ] both DNS clusters have active system-zone revisions; -- [ ] `EDGE_1` and `EDGE_2` identities are registered and bootstrap tokens removed; -- [ ] shared cells and gateways are listener-ready; -- [ ] the test domain resolves through both nameservers; -- [ ] proxied HTTPS works through `EDGE_1` and `EDGE_2`; -- [ ] a failed runtime candidate preserves the previous valid state; -- [ ] the recovery choice is recorded; when built-in backups are enabled, an - off-host snapshot and isolated restore are proven; -- [ ] firewall tests confirm private services are not public. - -## Step 12, optional: enable monitoring and centralized logs - -The CDN can serve without this step. Prometheus, Grafana, Loki, ClickHouse, and -operational log collectors are diagnostic systems: their failure must not stop -DNS, HTTP, queue processing, or runtime activation. - -### 12.1 Start telemetry on `CONTROL` - -The environment generator already created independent Grafana passwords, -ClickHouse credentials, Loki limits, and the source allowlists. - -```sh +```bash cd /opt/cdnfoundry - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile telemetry config --quiet - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile telemetry pull - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile telemetry up -d - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - ps clickhouse prometheus alertmanager grafana loki node-exporter vector -``` - -### 12.2 Start exactly one operational-log collector per host - -On `CONTROL`: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - --profile logs up -d log-collector -``` - -On `EDGE_1` and `EDGE_2`: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - --profile logs up -d log-collector -``` - -The Docker socket is a privileged read boundary. Only the collector receives a -read-only mount; never expose the socket over TCP or mount it into application -containers. Add `deploy/production/compose.host-journal.yml` only on hosts that -use persistent systemd journals and need kernel OOM, disk, and daemon events. - -### 12.3 Configure Prometheus targets and Alertmanager - -Populate deployment-owned copies of: - -- `docker/prometheus/edge-targets.prod.yml` with private gateway metrics - endpoints; -- `docker/prometheus/operational-log-targets.prod.yml` with each private - `host:9599` collector endpoint. - -Configure a real Alertmanager receiver. Keep metrics endpoints private and -require the metrics bearer-token file. - -### 12.4 Open Grafana safely - -Grafana binds to `127.0.0.1:3000`. For the first check, use an SSH tunnel from -an administrator workstation: - -```sh -ssh -L 3000:127.0.0.1:3000 ADMIN_USER@control.ops.example.com +sha256sum -c SHA256SUMS +./validate.sh +./start.sh +docker compose --env-file .env.prod ps ``` -Open `http://127.0.0.1:3000`, sign in with `GRAFANA_ADMIN_USER` and the generated -`GRAFANA_ADMIN_PASSWORD`, and confirm exactly two dashboards exist. For ongoing -use, deploy an authenticated HTTPS reverse proxy or SSO gateway; never expose -port `3000` directly. - -Verify: - -- all four datasources report healthy; -- the System and Domain Command Centers load; -- HTTP and DNS access analytics reach ClickHouse; -- operational logs reach Loki without access-log duplication; -- the request tail shows client and origin status with a bounded refresh; -- stopping monitoring does not interrupt DNS or HTTP traffic. - -See [Grafana](../operations/grafana.md), -[Monitoring](../operations/monitoring.md), and -[Operational logging](../operations/operational-logging.md). - -## Daily status commands - -On `CONTROL`: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml ps - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - logs --tail=200 core horizon scheduler caddy -``` - -On an edge: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml ps - -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - logs --tail=200 dnsdist pdns-auth dns-api cell-01 cell-02 edge-agent edge-gateway -``` - -## Common startup failures - -### Control certificate failure - -Check the independent A/AAAA record, public TCP `80` and `443`, ACME contact, -Caddy logs, outbound HTTPS, and provider rate limits: - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - logs --tail=200 caddy -``` - -### Core is unhealthy - -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - logs --tail=200 core -``` - -Verify migrations, secret-file paths, file permissions, PostgreSQL, Valkey, -and writable tmpfs/storage mounts. Do not make the production root filesystem -writable as a workaround. - -### DNS reconciliation fails - -Check the operation error code, control source IPv4, firewall counters, -`DNS_API_HOSTNAME`, certificate SAN, mounted CA, the edge's unique API key, and -`dns-api`/`pdns-auth` health. Never publish raw PowerDNS API port `8081`. - -### Edge registration fails - -Check edge UUID, one-time token state, server CA, system clock, outbound TCP -`8443`, identity volume, artifact signature, sequence, available disk, gateway, -and cell status. Do not delete the active runtime directory; it is the rollback -state. +The control bundle starts `mmdb-updater` before services that consume GeoIP data. Run migrations only through the generated `start.sh`/tools workflow; container startup never migrates the database. -### Optional monitoring is empty +## 6. Configure DNS desired state -Check `telemetry.ops.example.com`, source allowlists, edge `CLICKHOUSE_URL` and -`LOKI_ENDPOINT`, collector identities, Vector buffers, ClickHouse health, Loki -`/ready`, and the selected Grafana time range. Do not restart serving services -to repair monitoring. +Sign in to the administrator panel, configure platform nameservers and DNS clusters using the two PoP hostnames, and verify registrar glue for their public addresses. DNSdist is the only public authoritative endpoint; PowerDNS and its database remain private. -## Optional IPv6 +Transfer and start each PoP bundle only after its replacement validates. Allow both UDP and TCP 53 and restrict DNS API, metrics, and management listeners to documented control/monitoring sources. -Omit IPv6 overlays on IPv4-only hosts. `HOST_BIND_IPV6=::` is only consumed -when an IPv6 overlay is explicitly included; public/routed AAAA addresses are -configured in DNS and do not need to be Docker bind addresses. +## 7. Enroll both edge nodes -Control with IPv6: +Create each edge in the administrator panel and copy its UUID and one-time bootstrap token to protected local files. On the Fleet authority: -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - -f deploy/production/compose.control-host-ipv6.yml \ - --profile control up -d +```bash +sudo ./scripts/cdnfoundry-fleet \ + --state-dir /var/lib/cdnfoundry-fleet \ + configure-edge-registration \ + --node pop-1 \ + --edge-id EDGE_UUID \ + --bootstrap-token-file /root/pop-1.bootstrap-token \ + --non-interactive ``` -Combined DNS/edge with IPv6: +Repeat for `pop-2`, rerender those bundles, validate, transfer, and activate them. After successful mTLS registration: -```sh -docker compose --env-file .env.prod \ - -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - -f deploy/production/compose.dns-host-ipv6.yml \ - -f deploy/production/compose.edge-host-ipv6.yml \ - --profile dns --profile edge up -d +```bash +sudo ./scripts/cdnfoundry-fleet --state-dir /var/lib/cdnfoundry-fleet \ + clear-edge-bootstrap-token --node pop-1 --non-interactive +sudo ./scripts/cdnfoundry-fleet --state-dir /var/lib/cdnfoundry-fleet \ + render --node pop-1 ``` -Publish AAAA and glue only after external IPv6 DNS and HTTPS checks pass. +Transfer the token-free bundle and recreate only `edge-agent`. Never reuse or retain a consumed bootstrap token. -## Upgrade and rollback +## 8. Acceptance and recovery gate -Back up first. Upgrade one edge at a time, then control components. Run explicit -migrations and keep application/runtime versions inside the documented -compatibility envelope. Never roll back a database after an incompatible -migration. +Confirm: -Use [Upgrade and rollback](upgrade.md) and -[Backup and recovery](../operations/backup-and-recovery.md). +- control health, queues, Scheduler, Horizon, and migrations; +- DNS answers over UDP and TCP from both public nodes; +- edge mTLS enrollment and heartbeat; +- customer HTTP/TLS service through both PoPs; +- MMDB health on control, DNS, and edge roles; +- Prometheus targets, Grafana dashboards, ClickHouse telemetry, and bounded Loki logs; +- encrypted backup and restore rehearsal when backups are enabled; +- restart and previous-bundle rollback without deleting volumes. -## Next steps +Never run `docker compose down -v`, regenerate application/CA keys during an ordinary upgrade, or copy one edge identity volume to another host. -- [Understand the production topology](topology.md) -- [Harden secrets and networks](../security/hardening.md) -- [Practice incident runbooks](../operations/runbooks.md) -- [Scale roles independently](../operations/scaling.md) +Continue with the [Production fleet operator guide](production-fleet-operator-guide.md). For separated roles across several regions, use the [Multi-region fleet quick start](production-quick-start-multi-region.md). diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 508c0f6..160ae3c 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -79,7 +79,7 @@ the Restic password separately from both the repository and S3 credentials. Initialize a new repository once after the control dependencies, migration, and core service are healthy; use `restic snapshots` instead when attaching an existing repository. The exact secret-safe container commands are in the -[Production quick start](../deployment/production-quick-start.md#step-11-optional-initialize-and-prove-encrypted-backups). +[Production quick start](../deployment/production-quick-start.md). Restic also supports SFTP, REST, Azure, Google Cloud Storage, and other backends, but their provider variables, identity files, and mounts are not diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..ab54c83 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,47 @@ +# Production Fleet tooling + +CDNFoundry Fleet turns one validated desired-state document into protected, role-filtered production bundles. PostgreSQL remains authoritative for application desired state; Fleet state is deployment authority for host inventory, PKI, secrets, and generated artifacts. + +## Start here + +Clone an immutable release or commit, then inspect the CLI: + +```bash +git clone https://github.com/vaheed/CDNFoundry.git cdnfoundry +cd cdnfoundry +git checkout v1.0.0 +./scripts/cdnfoundry-fleet doctor +``` + +For a first deployment, copy and edit a JSON topology instead of changing a shell script: + +```bash +install -m 0600 deploy/production/examples/starter-fleet.json ./fleet.json +python3 -m json.tool fleet.json >/dev/null +./scripts/cdnfoundry-fleet --config fleet.json --non-interactive --dry-run setup +sudo ./scripts/cdnfoundry-fleet --config fleet.json --non-interactive setup +``` + +Use `multi-region-fleet.json` when control, DNS, edge, and monitoring roles are separated across failure domains. + +## Commands + +- `cdnfoundry-fleet setup`: create/reuse protected state, validate, and render atomically. +- `validate` and `doctor`: fail closed on invalid topology, missing prerequisites, or broken Compose. +- `add-node`, `update-node`, `remove-node`: mutate bounded host inventory under a lock. +- `configure-monitoring`, `configure-logs`, `configure-backups`: update typed feature state. +- `configure-edge-registration`: import a control-plane-issued UUID and one-time token from a protected file. +- `clear-edge-bootstrap-token`: remove a consumed token before rerendering. +- `render`: deterministically replace complete node bundles while retaining `.previous`. +- `show-start-order`: print dependency-safe rollout order. +- `rotate-secret`: perform explicit staged rotation where supported. + +Generated bundles use `docker compose --env-file .env.prod`; `.env.prod` is complete for that node and production Compose does not supply deployment defaults. Do not hand-edit rendered files. + +## Documentation + +- [Starter fleet quick start](../docs/deployment/production-quick-start.md) +- [Multi-region fleet quick start](../docs/deployment/production-quick-start-multi-region.md) +- [Fleet operator guide](../docs/deployment/production-fleet-operator-guide.md) +- [Fleet configuration reference](../docs/deployment/production-fleet-config-reference.md) +- [Fleet architecture reference](../docs/deployment/production-fleet.md) diff --git a/scripts/cdnfoundry-fleet b/scripts/cdnfoundry-fleet new file mode 100755 index 0000000..b863936 --- /dev/null +++ b/scripts/cdnfoundry-fleet @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from cdnfoundry_fleet.cli import main + +raise SystemExit(main()) diff --git a/scripts/cdnfoundry_fleet/__init__.py b/scripts/cdnfoundry_fleet/__init__.py new file mode 100644 index 0000000..470f81d --- /dev/null +++ b/scripts/cdnfoundry_fleet/__init__.py @@ -0,0 +1,3 @@ +"""Control-plane-only CDNFoundry fleet generator.""" + +__version__ = "1.0.0" diff --git a/scripts/cdnfoundry_fleet/__main__.py b/scripts/cdnfoundry_fleet/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/scripts/cdnfoundry_fleet/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/scripts/cdnfoundry_fleet/certs.py b/scripts/cdnfoundry_fleet/certs.py new file mode 100644 index 0000000..8d8b756 --- /dev/null +++ b/scripts/cdnfoundry_fleet/certs.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Sequence + +from .common import RenderError, atomic_json, ensure_mode, utc_now + + +def run_checked(args: Sequence[str], *, cwd: Path | None = None) -> None: + try: + subprocess.run( + list(args), + cwd=cwd, + check=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except FileNotFoundError as exc: + raise RenderError(f"Required executable not found: {args[0]}") from exc + except subprocess.CalledProcessError as exc: + message = exc.stderr.strip() or exc.stdout.strip() or "unknown error" + raise RenderError(f"Command failed ({args[0]}): {message}") from exc + + +class PKI: + """Manage the certificate layout expected by CDNFoundry production Compose. + + The edge identity CA signs agent identities inside the control plane. The edge + server CA signs the TLS endpoints used by edge-control, edge runtimes and DNS + API servers. CA private keys remain only in the fleet state directory. + """ + + def __init__(self, root: Path, *, dry_run: bool = False) -> None: + self.root = root + self.dry_run = dry_run + + def ensure(self) -> None: + if self.dry_run: + return + self.root.mkdir(parents=True, exist_ok=True, mode=0o700) + self.root.chmod(0o700) + self._ensure_ca("edge-identity-ca", "CDNFoundry Edge Identity CA") + self._ensure_ca("edge-server-ca", "CDNFoundry Edge Server CA") + + def _ensure_ca(self, stem: str, common_name: str) -> None: + key = self.root / f"{stem}.key" + cert = self.root / f"{stem}.crt" + if key.exists() and cert.exists(): + ensure_mode(key, 0o600) + ensure_mode(cert, 0o644) + return + if self.dry_run: + return + with tempfile.TemporaryDirectory(dir=self.root) as tmp_dir: + tmp = Path(tmp_dir) + tmp_key = tmp / key.name + tmp_cert = tmp / cert.name + run_checked(["openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", str(tmp_key)]) + run_checked( + [ + "openssl", + "req", + "-x509", + "-new", + "-sha256", + "-key", + str(tmp_key), + "-out", + str(tmp_cert), + "-days", + "3650", + "-subj", + f"/CN={common_name}", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ] + ) + os.chmod(tmp_key, 0o600) + os.chmod(tmp_cert, 0o644) + os.replace(tmp_key, key) + os.replace(tmp_cert, cert) + + def ensure_node_certificate(self, node: dict[str, object]) -> tuple[Path, Path, Path]: + self.ensure() + name = str(node["name"]) + hostname = str(node["hostname"]) + node_dir = self.root / "nodes" / name + key = node_dir / "node.key" + cert = node_dir / "node.crt" + meta = node_dir / "metadata.json" + ca = self.root / "edge-server-ca.crt" + expected = { + "issuer": "edge-server-ca", + "name": name, + "hostname": hostname, + "public_ipv4": node.get("public_ipv4"), + "public_ipv6": node.get("public_ipv6"), + } + if key.exists() and cert.exists() and meta.exists(): + try: + current = json.loads(meta.read_text(encoding="utf-8")) + except json.JSONDecodeError: + current = {} + if all(current.get(k) == v for k, v in expected.items()): + ensure_mode(key, 0o600) + ensure_mode(cert, 0o644) + return ca, cert, key + if self.dry_run: + return ca, cert, key + node_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + node_dir.chmod(0o700) + with tempfile.TemporaryDirectory(dir=node_dir) as tmp_dir: + tmp = Path(tmp_dir) + tmp_key = tmp / "node.key" + tmp_csr = tmp / "node.csr" + tmp_cert = tmp / "node.crt" + ext = tmp / "ext.cnf" + sans = [f"DNS:{hostname}"] + if node.get("public_ipv4"): + sans.append(f"IP:{node['public_ipv4']}") + if node.get("public_ipv6"): + sans.append(f"IP:{node['public_ipv6']}") + ext.write_text( + "basicConstraints=critical,CA:FALSE\n" + "keyUsage=critical,digitalSignature,keyEncipherment\n" + "extendedKeyUsage=serverAuth,clientAuth\n" + f"subjectAltName={','.join(sans)}\n", + encoding="utf-8", + ) + run_checked(["openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", str(tmp_key)]) + run_checked( + ["openssl", "req", "-new", "-sha256", "-key", str(tmp_key), "-out", str(tmp_csr), "-subj", f"/CN={hostname}"] + ) + run_checked( + [ + "openssl", + "x509", + "-req", + "-sha256", + "-in", + str(tmp_csr), + "-CA", + str(self.root / "edge-server-ca.crt"), + "-CAkey", + str(self.root / "edge-server-ca.key"), + "-CAcreateserial", + "-out", + str(tmp_cert), + "-days", + "825", + "-extfile", + str(ext), + ] + ) + os.chmod(tmp_key, 0o600) + os.chmod(tmp_cert, 0o644) + os.replace(tmp_key, key) + os.replace(tmp_cert, cert) + expected["issued_at"] = utc_now() + atomic_json(meta, expected, 0o600) + return ca, cert, key + + def copy_node_material(self, node: dict[str, object], destination: Path) -> None: + server_ca, cert, key = self.ensure_node_certificate(node) + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + if self.dry_run: + return + + # Generic names preserve backwards compatibility with older generated bundles. + shutil.copy2(server_ca, destination / "edge-server-ca.crt") + shutil.copy2(server_ca, destination / "fleet-ca.crt") + shutil.copy2(cert, destination / "node.crt") + shutil.copy2(key, destination / "node.key") + + if str(node.get("role")) == "control": + shutil.copy2(self.root / "edge-identity-ca.crt", destination / "edge-identity-ca.crt") + shutil.copy2(self.root / "edge-identity-ca.key", destination / "edge-identity-ca.key") + + for path in destination.iterdir(): + path.chmod(0o600 if path.suffix == ".key" else 0o644) diff --git a/scripts/cdnfoundry_fleet/cli.py b/scripts/cdnfoundry_fleet/cli.py new file mode 100644 index 0000000..0dd1092 --- /dev/null +++ b/scripts/cdnfoundry_fleet/cli.py @@ -0,0 +1,883 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from .common import FleetError, StateError, ValidationError, load_json, utc_now +from .render import Renderer +from .state import ( + BACKUP_MODES, + GLOBAL_SECRET_NAMES, + LOG_MODES, + MONITORING_MODES, + NODE_SECRET_NAMES, + FleetState, + ROLES, +) + +EXIT_OK = 0 +EXIT_USAGE = 2 +EXIT_VALIDATION = 3 +EXIT_STATE = 4 +EXIT_RENDER = 5 +EXIT_EXTERNAL = 6 + +SETUP_GLOBAL_FIELDS = {"operator_domain", "platform_domain", "release", "acme_email", "ipv6"} +SETUP_TOP_LEVEL_FIELDS = {"preset", "global", "nodes", "features"} | SETUP_GLOBAL_FIELDS +SETUP_FEATURE_FIELDS = { + "monitoring": {"mode", "host"}, + "logs": {"mode", "host", "endpoint"}, + "backups": {"mode", "repository", "region"}, +} +SETUP_NODE_FIELDS = { + "name", "role", "region", "location", "hostname", "public_ipv4", "public_ipv6", + "bind_ipv4", "bind_ipv6", "monitor_ipv4", "log_ipv4", "release", "extra_env", + "enabled", "draining", "health", +} + + +def _reject_unknown_fields(value: dict[str, Any], allowed: set[str], context: str) -> None: + unknown = sorted(set(value) - allowed) + if unknown: + raise ValidationError(f"Unknown {context} field(s): {', '.join(unknown)}") + + +def _validate_setup_config(config: dict[str, Any]) -> None: + _reject_unknown_fields(config, SETUP_TOP_LEVEL_FIELDS, "setup config") + global_config = config.get("global", {key: config[key] for key in SETUP_GLOBAL_FIELDS if key in config}) + if not isinstance(global_config, dict): + raise ValidationError("setup config field 'global' must be an object") + _reject_unknown_fields(global_config, SETUP_GLOBAL_FIELDS, "global config") + features = config.get("features", {}) + if not isinstance(features, dict): + raise ValidationError("setup config field 'features' must be an object") + _reject_unknown_fields(features, set(SETUP_FEATURE_FIELDS), "feature config") + for name, feature in features.items(): + if not isinstance(feature, dict): + raise ValidationError(f"setup feature '{name}' must be an object") + _reject_unknown_fields(feature, SETUP_FEATURE_FIELDS[name], f"{name} feature") + nodes = config.get("nodes", []) + if not isinstance(nodes, list): + raise ValidationError("setup config field 'nodes' must be a list") + for index, node in enumerate(nodes): + if not isinstance(node, dict): + raise ValidationError(f"setup node at index {index} must be an object") + _reject_unknown_fields(node, SETUP_NODE_FIELDS, f"node {index}") + + +def _common_parser(*, suppressed_defaults: bool) -> argparse.ArgumentParser: + common = argparse.ArgumentParser(add_help=False) + default = argparse.SUPPRESS if suppressed_defaults else None + common.add_argument( + "--state-dir", + default=default if suppressed_defaults else "/var/lib/cdnfoundry-fleet", + ) + common.add_argument( + "--output-dir", + default=default if suppressed_defaults else "/var/lib/cdnfoundry-fleet/bundles", + ) + common.add_argument( + "--repo-root", + default=default if suppressed_defaults else str(Path(__file__).resolve().parents[2]), + ) + common.add_argument("--config", default=default) + common.add_argument("--non-interactive", action="store_true", default=default if suppressed_defaults else False) + common.add_argument("--dry-run", action="store_true", default=default if suppressed_defaults else False) + common.add_argument("--yes", action="store_true", default=default if suppressed_defaults else False) + return common + + +def parser() -> argparse.ArgumentParser: + root_common = _common_parser(suppressed_defaults=False) + command_common = _common_parser(suppressed_defaults=True) + root = argparse.ArgumentParser(prog="cdnfoundry-fleet", parents=[root_common]) + sub = root.add_subparsers(dest="command", required=True) + + setup = sub.add_parser( + "setup", + parents=[command_common], + help="interactive or config-driven full fleet setup, validation, and rendering", + ) + setup.add_argument("--operator-domain") + setup.add_argument("--platform-domain") + setup.add_argument("--release") + setup.add_argument("--acme-email", default="") + setup.add_argument("--dual-stack", action="store_true") + setup.add_argument( + "--preset", + choices=("control-only", "control-monitoring", "dedicated-monitoring", "custom"), + ) + setup.add_argument("--control-name", default="control-1") + setup.add_argument("--control-hostname") + setup.add_argument("--control-ipv4") + setup.add_argument("--control-region", default="global") + setup.add_argument("--control-location", default="primary") + setup.add_argument("--no-render", action="store_true") + + init = sub.add_parser("init", parents=[command_common]) + init.add_argument("--operator-domain") + init.add_argument("--platform-domain") + init.add_argument("--release") + init.add_argument("--acme-email", default="") + init.add_argument("--dual-stack", action="store_true") + + add = sub.add_parser("add-node", parents=[command_common]) + _node_arguments(add, require_name=True) + + update = sub.add_parser("update-node", parents=[command_common]) + update.add_argument("--node", required=True) + _node_arguments(update, require_name=False, optional=True, include_node=False) + + edge_registration = sub.add_parser( + "configure-edge-registration", + parents=[command_common], + help="store a control-plane-created edge UUID and one-time bootstrap token", + ) + edge_registration.add_argument("--node", required=True) + edge_registration.add_argument("--edge-id", required=True) + token_source = edge_registration.add_mutually_exclusive_group(required=True) + token_source.add_argument("--bootstrap-token-file") + token_source.add_argument("--bootstrap-token-stdin", action="store_true") + + clear_edge_token = sub.add_parser( + "clear-edge-bootstrap-token", + parents=[command_common], + help="remove the one-time bootstrap token after successful mTLS enrollment", + ) + clear_edge_token.add_argument("--node", required=True) + + set_secret = sub.add_parser( + "set-secret", + parents=[command_common], + help="replace a supported secret from a protected file without exposing it in argv", + ) + set_secret.add_argument("--secret", required=True) + set_secret.add_argument("--node") + set_secret.add_argument("--from-file", required=True) + + remove = sub.add_parser("remove-node", parents=[command_common]) + remove.add_argument("--node", required=True) + + sub.add_parser("list-nodes", parents=[command_common]) + status = sub.add_parser("status", parents=[command_common]) + status.add_argument("--json", action="store_true") + doctor = sub.add_parser("doctor", parents=[command_common]) + doctor.add_argument("--json", action="store_true") + + mon = sub.add_parser("configure-monitoring", parents=[command_common]) + mon.add_argument("--mode", choices=sorted(MONITORING_MODES), required=True) + mon.add_argument("--host") + + logs = sub.add_parser("configure-logs", parents=[command_common]) + logs.add_argument("--mode", choices=sorted(LOG_MODES), required=True) + logs.add_argument("--host") + logs.add_argument("--endpoint") + + backups = sub.add_parser("configure-backups", parents=[command_common]) + backups.add_argument("--mode", choices=sorted(BACKUP_MODES), required=True) + backups.add_argument("--repository") + backups.add_argument("--region", default="us-east-1") + + render = sub.add_parser("render", parents=[command_common]) + render.add_argument("--node") + + validate = sub.add_parser("validate", parents=[command_common]) + validate.add_argument("--node") + + sub.add_parser("show-start-order", parents=[command_common]) + + adopt = sub.add_parser("adopt-existing", parents=[command_common]) + adopt.add_argument("--node", required=True) + adopt.add_argument("--role", choices=sorted(ROLES), required=True) + adopt.add_argument("--env-file", required=True) + adopt.add_argument("--region", required=True) + adopt.add_argument("--location", required=True) + adopt.add_argument("--hostname") + adopt.add_argument("--public-ipv4", required=True) + adopt.add_argument("--public-ipv6") + adopt.add_argument("--bind-ipv4", default="0.0.0.0") + + rotate = sub.add_parser("rotate-secret", parents=[command_common]) + rotate.add_argument("--node") + rotate.add_argument("--secret", required=True) + rotate.add_argument("--phase", choices=("prepare", "commit", "abort"), default="prepare") + + return root + + +def _node_arguments( + target: argparse.ArgumentParser, + *, + require_name: bool, + optional: bool = False, + include_node: bool = True, +) -> None: + if include_node: + target.add_argument("--node", required=require_name) + target.add_argument("--role", choices=sorted(ROLES), required=require_name) + target.add_argument("--region", required=require_name) + target.add_argument("--location", required=require_name) + target.add_argument("--hostname") + target.add_argument("--public-ipv4", required=require_name) + target.add_argument("--public-ipv6") + target.add_argument("--bind-ipv4", default=None if optional else "0.0.0.0") + target.add_argument("--bind-ipv6") + target.add_argument("--monitor-ipv4") + target.add_argument("--log-ipv4") + target.add_argument("--release") + target.add_argument("--extra-env", action="append", default=[]) + target.add_argument("--disabled", action="store_true", default=None if optional else False) + target.add_argument("--draining", action="store_true", default=None if optional else False) + + +def _config(args: argparse.Namespace) -> dict[str, Any]: + return load_json(Path(args.config)) if args.config else {} + + +def _read_input(prompt: str, *, secret: bool = False) -> str: + import getpass + + try: + return getpass.getpass(prompt) if secret else input(prompt) + except EOFError as exc: + raise ValidationError( + "Interactive input is unavailable. Run in a terminal or use --non-interactive with --config." + ) from exc + + +def _prompt(value: str | None, label: str, *, secret: bool = False, default: str | None = None) -> str: + if value: + return value + while True: + suffix = f" [{default}]" if default else "" + entered = _read_input(f"{label}{suffix}: ", secret=secret).strip() + if entered: + return entered + if default is not None: + return default + + +def _prompt_choice(label: str, choices: list[tuple[str, str]], *, default: str | None = None) -> str: + print(f"\n{label}") + for index, (value, description) in enumerate(choices, 1): + marker = " (default)" if value == default else "" + print(f" {index}) {description}{marker}") + while True: + entered = _read_input("Select an option: ").strip() + if not entered and default is not None: + return default + if entered.isdigit() and 1 <= int(entered) <= len(choices): + return choices[int(entered) - 1][0] + for value, _ in choices: + if entered == value: + return value + print("Invalid selection; enter the number or option name.") + + +def _prompt_yes_no(label: str, *, default: bool = False) -> bool: + suffix = "[Y/n]" if default else "[y/N]" + while True: + entered = _read_input(f"{label} {suffix}: ").strip().lower() + if not entered: + return default + if entered in {"y", "yes"}: + return True + if entered in {"n", "no"}: + return False + print("Please answer yes or no.") + + +def _node_payload(args: argparse.Namespace, config: dict[str, Any], *, update: bool = False) -> dict[str, Any]: + source = config.get("node", config) + payload: dict[str, Any] = {} + mapping = { + "name": "node", + "role": "role", + "region": "region", + "location": "location", + "hostname": "hostname", + "public_ipv4": "public_ipv4", + "public_ipv6": "public_ipv6", + "bind_ipv4": "bind_ipv4", + "bind_ipv6": "bind_ipv6", + "monitor_ipv4": "monitor_ipv4", + "log_ipv4": "log_ipv4", + "release": "release", + } + for target, arg_name in mapping.items(): + value = getattr(args, arg_name, None) + if value is None: + value = source.get(target) + if value is not None: + payload[target] = value + extra = dict(source.get("extra_env", {})) + for item in getattr(args, "extra_env", []) or []: + if "=" not in item: + raise ValidationError("--extra-env must be KEY=VALUE") + key, value = item.split("=", 1) + extra[key] = value + if extra: + payload["extra_env"] = extra + if getattr(args, "disabled", None) is not None: + payload["enabled"] = not args.disabled + if getattr(args, "draining", None) is not None: + payload["draining"] = args.draining + if not update and not args.non_interactive: + payload["name"] = _prompt(payload.get("name"), "Node name") + payload["role"] = _prompt(payload.get("role"), "Role") + payload["region"] = _prompt(payload.get("region"), "Region") + payload["location"] = _prompt(payload.get("location"), "Location") + payload["public_ipv4"] = _prompt(payload.get("public_ipv4"), "Public IPv4") + return payload + + +def _confirm(args: argparse.Namespace, message: str) -> None: + if args.yes: + return + if args.non_interactive: + raise ValidationError(f"{message}; rerun with --yes") + answer = input(f"{message} [y/N]: ").strip().lower() + if answer not in {"y", "yes"}: + raise ValidationError("Operation cancelled") + + +def _interactive_node(state: dict[str, Any], *, role: str | None = None, defaults: dict[str, Any] | None = None) -> dict[str, Any]: + defaults = defaults or {} + if role is None: + role = _prompt_choice( + "Node role", + [ + ("dns", "DNS only"), + ("edge", "Edge only"), + ("dns-edge", "Combined DNS + edge"), + ("monitoring", "Dedicated monitoring"), + ], + ) + name_default = defaults.get("name") or f"{role}-1" + name = _prompt(defaults.get("name"), "Node name", default=name_default) + operator_domain = state["global"]["operator_domain"] + return { + "name": name, + "role": role, + "region": _prompt(defaults.get("region"), "Region", default="global"), + "location": _prompt(defaults.get("location"), "Location", default="primary"), + "hostname": _prompt(defaults.get("hostname"), "Hostname", default=f"{name}.{operator_domain}"), + "public_ipv4": _prompt(defaults.get("public_ipv4"), "Public IPv4"), + "public_ipv6": defaults.get("public_ipv6"), + "bind_ipv4": defaults.get("bind_ipv4") or "0.0.0.0", + "bind_ipv6": defaults.get("bind_ipv6"), + "monitor_ipv4": defaults.get("monitor_ipv4"), + "log_ipv4": defaults.get("log_ipv4"), + "extra_env": defaults.get("extra_env", {}), + } + + +def _apply_setup_features(store: FleetState, state: dict[str, Any], config: dict[str, Any], preset: str) -> dict[str, Any]: + features = config.get("features", {}) + monitoring = features.get("monitoring") + if monitoring: + state = store.configure_feature(state, "monitoring", monitoring) + elif preset == "control-monitoring": + state = store.configure_feature(state, "monitoring", {"mode": "colocated", "host": None}) + elif preset == "control-only": + state = store.configure_feature(state, "monitoring", {"mode": "disabled", "host": None}) + elif preset == "dedicated-monitoring": + monitoring_nodes = [node for node in state["nodes"].values() if node["role"] == "monitoring"] + if not monitoring_nodes: + raise ValidationError("The dedicated-monitoring preset requires a monitoring-role node") + state = store.configure_feature( + state, "monitoring", {"mode": "dedicated", "host": monitoring_nodes[0]["name"]} + ) + + if features.get("logs"): + state = store.configure_feature(state, "logs", features["logs"]) + if features.get("backups"): + state = store.configure_feature(state, "backups", features["backups"]) + return state + + +def _setup(args: argparse.Namespace, store: FleetState, output_dir: Path, config: dict[str, Any]) -> int: + _validate_setup_config(config) + global_cfg = config.get("global", config) + created = not store.exists() + if created: + operator = args.operator_domain or global_cfg.get("operator_domain") + platform = args.platform_domain or global_cfg.get("platform_domain") + release = args.release or global_cfg.get("release") + if not args.non_interactive: + print("CDNFoundry production fleet setup") + print("This wizard creates fleet state, node bundles, certificates, secrets, and operator runbooks.") + operator = _prompt(operator, "Operator domain") + platform = _prompt(platform, "Platform domain") + release = _prompt(release, "Exact release tag or commit") + if not all((operator, platform, release)): + raise ValidationError("operator_domain, platform_domain, and release are required") + state = store.init( + { + "operator_domain": operator, + "platform_domain": platform, + "release": release, + "acme_email": args.acme_email or global_cfg.get("acme_email", ""), + "ipv6": args.dual_stack or bool(global_cfg.get("ipv6", False)), + } + ) + print(f"Initialized fleet state: {store.state_file}") + else: + state = store.load() + print(f"Using existing fleet state: {store.state_file}") + + preset = args.preset or config.get("preset") + if not preset and not args.non_interactive: + preset = _prompt_choice( + "Deployment topology", + [ + ("control-monitoring", "Control + monitoring on the same host"), + ("control-only", "Control plane only"), + ("dedicated-monitoring", "Control plane + dedicated monitoring host"), + ("custom", "Custom fleet"), + ], + default="control-monitoring", + ) + preset = preset or "custom" + + configured_nodes = config.get("nodes", []) + if configured_nodes and not isinstance(configured_nodes, list): + raise ValidationError("setup config field 'nodes' must be a list") + + if configured_nodes: + for payload in configured_nodes: + if not isinstance(payload, dict): + raise ValidationError("Every setup node must be an object") + name = payload.get("name") + if not name: + raise ValidationError("Every setup node requires a name") + current = store.load() + if name in current["nodes"]: + state = store.update_node(current, name, payload) + print(f"Updated node: {name}") + else: + state = store.add_node(current, payload) + print(f"Added node: {name}") + elif created or not state["nodes"]: + control_payload = { + "name": args.control_name, + "role": "control", + "region": args.control_region, + "location": args.control_location, + "hostname": args.control_hostname, + "public_ipv4": args.control_ipv4, + "bind_ipv4": "0.0.0.0", + } + if not args.non_interactive: + control_payload = _interactive_node(state, role="control", defaults=control_payload) + elif not args.control_ipv4: + raise ValidationError("--control-ipv4 or a config nodes list is required in non-interactive setup") + state = store.add_node(store.load(), control_payload) + print(f"Added control node: {control_payload['name']}") + + if preset == "dedicated-monitoring": + if args.non_interactive: + raise ValidationError("Dedicated monitoring in non-interactive mode requires a monitoring node in --config") + monitor_payload = _interactive_node(state, role="monitoring", defaults={"name": "monitoring-1"}) + state = store.add_node(store.load(), monitor_payload) + print(f"Added monitoring node: {monitor_payload['name']}") + + if not args.non_interactive: + while _prompt_yes_no("Add a DNS or edge node now?", default=False): + payload = _interactive_node(state) + state = store.add_node(store.load(), payload) + print(f"Added node: {payload['name']}") + + state = _apply_setup_features(store, store.load(), config, preset) + store.validate(state, require_secrets=not args.dry_run) + print(f"Fleet validation passed ({len(state['nodes'])} node(s)).") + + paths: list[Path] = [] + if not args.no_render: + renderer = Renderer(Path(args.repo_root), store, output_dir, dry_run=args.dry_run) + paths = renderer.render(state) + if not args.dry_run: + with store.transaction() as candidate: + candidate["metadata"]["last_successful_validation"] = utc_now() + candidate["metadata"]["last_successful_render"] = utc_now() + print(f"Generated {len(paths)} node bundle(s) in {output_dir}") + for path in paths: + print(f" - {path}") + result = { + "status": "configured", + "preset": preset, + "state_dir": str(store.state_dir), + "output_dir": str(output_dir), + "nodes": sorted(state["nodes"]), + "bundles": [str(path) for path in paths], + } + print(json.dumps(result)) + return EXIT_OK + + +def _status(state: dict[str, Any], output_dir: Path, *, as_json: bool) -> None: + payload = { + "global": state["global"], + "features": state["features"], + "metadata": state["metadata"], + "nodes": [state["nodes"][name] for name in sorted(state["nodes"])], + "output_dir": str(output_dir), + } + if as_json: + print(json.dumps(payload, indent=2)) + return + print("CDNFoundry fleet status") + print(f" Release: {state['global']['release']}") + print(f" Operator domain: {state['global']['operator_domain']}") + print(f" Monitoring: {state['features']['monitoring']['mode']}") + print(f" Logs: {state['features']['logs']['mode']}") + print(f" Backups: {state['features']['backups']['mode']}") + print(f" Bundles: {output_dir}") + print(" Nodes:") + if not state["nodes"]: + print(" (none)") + for name in sorted(state["nodes"]): + node = state["nodes"][name] + flags = [] + if not node["enabled"]: + flags.append("disabled") + if node["draining"]: + flags.append("draining") + suffix = f" [{', '.join(flags)}]" if flags else "" + print(f" - {name}: {node['role']} / {node['hostname']} / {node['public_ipv4']}{suffix}") + + +def _doctor(args: argparse.Namespace, store: FleetState) -> int: + import shutil + + root = Path(args.repo_root) + required = [ + "compose.prod.yml", + "deploy/production/compose.control-host.yml", + "deploy/production/compose.dns-host.yml", + "deploy/production/compose.edge-host.yml", + "deploy/production/compose.dns-edge-host.yml", + "deploy/production/compose.telemetry-host.yml", + ] + checks: list[dict[str, Any]] = [] + for relative in required: + checks.append({"check": relative, "ok": (root / relative).is_file()}) + for executable in ("python3", "openssl"): + checks.append({"check": executable, "ok": shutil.which(executable) is not None}) + checks.append({"check": "docker", "ok": shutil.which("docker") is not None, "required_for_render": False}) + if store.exists(): + try: + store.validate(store.load(), require_secrets=not args.dry_run) + checks.append({"check": "fleet-state", "ok": True}) + except FleetError as exc: + checks.append({"check": "fleet-state", "ok": False, "detail": str(exc)}) + else: + checks.append({"check": "fleet-state", "ok": True, "detail": "not initialized yet"}) + ok = all(item["ok"] for item in checks if item.get("required_for_render", True)) + if args.json: + print(json.dumps({"ok": ok, "checks": checks}, indent=2)) + else: + for item in checks: + print(f"{'OK' if item['ok'] else 'FAIL'} {item['check']}" + (f" — {item['detail']}" if item.get("detail") else "")) + print("Doctor result: " + ("ready" if ok else "problems found")) + return EXIT_OK if ok else EXIT_VALIDATION + + +def execute(args: argparse.Namespace) -> int: + state_dir = Path(args.state_dir) + output_dir = Path(args.output_dir) + store = FleetState(state_dir, dry_run=args.dry_run) + config = _config(args) + + if args.command == "doctor": + return _doctor(args, store) + + with store.locked(exclusive=args.command not in {"list-nodes", "show-start-order", "status"}): + if args.command == "setup": + return _setup(args, store, output_dir, config) + if args.command == "init": + values = config.get("global", config) + operator = args.operator_domain or values.get("operator_domain") + platform = args.platform_domain or values.get("platform_domain") + release = args.release or values.get("release") + if not args.non_interactive: + operator = _prompt(operator, "Operator domain") + platform = _prompt(platform, "Platform domain") + release = _prompt(release, "Exact release tag or commit") + if not all((operator, platform, release)): + raise ValidationError("operator_domain, platform_domain, and release are required") + state = store.init( + { + "operator_domain": operator, + "platform_domain": platform, + "release": release, + "acme_email": args.acme_email or values.get("acme_email", ""), + "ipv6": args.dual_stack or bool(values.get("ipv6", False)), + } + ) + print(json.dumps({"status": "initialized", "state_dir": str(state_dir), "generation": state["metadata"]["generation"]})) + return EXIT_OK + + state = store.load() + if args.command == "add-node": + state = store.add_node(state, _node_payload(args, config)) + print(json.dumps({"status": "added", "node": args.node or config.get("name")})) + elif args.command == "update-node": + state = store.update_node(state, args.node, _node_payload(args, config, update=True)) + print(json.dumps({"status": "updated", "node": args.node})) + elif args.command == "configure-edge-registration": + if args.node not in state["nodes"]: + raise ValidationError(f"Unknown node: {args.node}") + if state["nodes"][args.node]["role"] not in {"edge", "dns-edge"}: + raise ValidationError("Edge registration is valid only for edge-capable nodes") + from uuid import UUID + + try: + edge_id = str(UUID(args.edge_id)) + except ValueError as exc: + raise ValidationError("--edge-id must be a valid UUID") from exc + if args.bootstrap_token_stdin: + token = sys.stdin.read().rstrip("\r\n") + else: + token_path = Path(args.bootstrap_token_file) + token = token_path.read_text(encoding="utf-8").rstrip("\r\n") + if not token: + raise ValidationError("Bootstrap token must not be empty") + current = state["nodes"][args.node] + extra = dict(current.get("extra_env", {})) + extra.pop("EDGE_BOOTSTRAP_TOKEN", None) + extra["EDGE_ID"] = edge_id + state = store.update_node(state, args.node, {"extra_env": extra}) + store.write_secret("edge-bootstrap-token", token, node=args.node) + print(json.dumps({"status": "configured", "node": args.node, "edge_id": edge_id})) + elif args.command == "clear-edge-bootstrap-token": + if args.node not in state["nodes"]: + raise ValidationError(f"Unknown node: {args.node}") + current = state["nodes"][args.node] + extra = dict(current.get("extra_env", {})) + if "EDGE_BOOTSTRAP_TOKEN" in extra: + extra.pop("EDGE_BOOTSTRAP_TOKEN", None) + state = store.update_node(state, args.node, {"extra_env": extra}) + store.delete_secret("edge-bootstrap-token", node=args.node) + print(json.dumps({"status": "cleared", "node": args.node})) + elif args.command == "set-secret": + if args.node: + if args.node not in state["nodes"]: + raise ValidationError(f"Unknown node: {args.node}") + if args.secret not in NODE_SECRET_NAMES: + raise ValidationError(f"Unsupported node secret: {args.secret}") + elif args.secret not in GLOBAL_SECRET_NAMES: + raise ValidationError(f"Unsupported global secret: {args.secret}") + value = Path(args.from_file).read_text(encoding="utf-8").rstrip("\r\n") + store.write_secret(args.secret, value, node=args.node) + print(json.dumps({"status": "stored", "secret": args.secret, "node": args.node})) + elif args.command == "remove-node": + _confirm(args, f"Remove node {args.node} from fleet state") + store.remove_node(state, args.node) + print(json.dumps({"status": "removed", "node": args.node})) + elif args.command == "list-nodes": + rows = [ + { + "name": node["name"], + "role": node["role"], + "region": node["region"], + "location": node["location"], + "hostname": node["hostname"], + "public_ipv4": node["public_ipv4"], + "enabled": node["enabled"], + "draining": node["draining"], + } + for node in sorted(state["nodes"].values(), key=lambda item: item["name"]) + ] + print(json.dumps(rows, indent=2)) + elif args.command == "status": + _status(state, output_dir, as_json=args.json) + elif args.command == "configure-monitoring": + store.configure_feature(state, "monitoring", {"mode": args.mode, "host": args.host}) + print(json.dumps({"status": "configured", "feature": "monitoring", "mode": args.mode})) + elif args.command == "configure-logs": + store.configure_feature( + state, "logs", {"mode": args.mode, "host": args.host, "endpoint": args.endpoint} + ) + print(json.dumps({"status": "configured", "feature": "logs", "mode": args.mode})) + elif args.command == "configure-backups": + store.configure_feature( + state, + "backups", + {"mode": args.mode, "repository": args.repository, "region": args.region}, + ) + print(json.dumps({"status": "configured", "feature": "backups", "mode": args.mode})) + elif args.command in {"render", "validate"}: + renderer = Renderer(Path(args.repo_root), store, output_dir, dry_run=args.dry_run) + store.validate(state, require_secrets=not args.dry_run) + if args.command == "render": + paths = renderer.render(state, node_name=args.node) + if not args.dry_run: + with store.transaction() as candidate: + candidate["metadata"]["last_successful_render"] = utc_now() + print(json.dumps({"status": "rendered", "bundles": [str(path) for path in paths]})) + else: + # Render to a disposable directory to exercise Compose filtering and minimal environments. + if args.dry_run: + renderer.render(state, node_name=args.node) + else: + import tempfile + + with tempfile.TemporaryDirectory(dir=state_dir) as tmp: + Renderer(Path(args.repo_root), store, Path(tmp), dry_run=False).render(state, node_name=args.node) + with store.transaction() as candidate: + candidate["metadata"]["last_successful_validation"] = utc_now() + print(json.dumps({"status": "valid"})) + elif args.command == "show-start-order": + print(json.dumps(Renderer.start_order(state), indent=2)) + elif args.command == "adopt-existing": + env = parse_env(Path(args.env_file)) + if not store.exists(): + raise StateError("Initialize fleet state before adopting an existing node") + payload = { + "name": args.node, + "role": args.role, + "region": args.region, + "location": args.location, + "hostname": args.hostname, + "public_ipv4": args.public_ipv4, + "public_ipv6": args.public_ipv6, + "bind_ipv4": args.bind_ipv4, + "extra_env": {k: v for k, v in env.items() if k not in secret_env_names()}, + } + state = store.add_node(state, payload) + import_secret_env(store, args.node, args.role, env) + print(json.dumps({"status": "adopted", "node": args.node})) + elif args.command == "rotate-secret": + if args.node: + if args.secret not in NODE_SECRET_NAMES: + raise ValidationError(f"Unsupported node secret: {args.secret}") + if args.node not in state["nodes"]: + raise ValidationError(f"Unknown node: {args.node}") + if args.secret == "pdns-db-password": + if state["nodes"][args.node]["role"] not in {"dns", "dns-edge"}: + raise ValidationError("pdns-db-password is valid only for DNS-capable nodes") + if args.phase == "prepare": + _confirm(args, f"Prepare a local PowerDNS PostgreSQL password rotation for {args.node}") + store.prepare_secret_rotation(args.secret, node=args.node) + elif args.phase == "commit": + _confirm( + args, + f"Commit the PowerDNS PostgreSQL password after reconcile-pdns-password.sh succeeded on {args.node}", + ) + store.commit_secret_rotation(args.secret, node=args.node) + else: + _confirm(args, f"Abort the pending PowerDNS PostgreSQL password rotation for {args.node}") + store.abort_secret_rotation(args.secret, node=args.node) + else: + if args.phase != "prepare": + raise ValidationError("commit and abort phases are used only for pdns-db-password") + _confirm(args, f"Rotate {args.secret} for {args.node}") + store.rotate_secret(args.secret, node=args.node) + else: + if args.phase != "prepare": + raise ValidationError("commit and abort phases require --node and pdns-db-password") + _confirm(args, f"Rotate global secret {args.secret}") + store.rotate_secret(args.secret) + print( + json.dumps( + { + "status": args.phase if args.secret == "pdns-db-password" and args.node else "rotated", + "secret": args.secret, + "node": args.node, + } + ) + ) + else: # pragma: no cover + raise ValidationError(f"Unsupported command: {args.command}") + return EXIT_OK + + +def parse_env(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if "=" not in stripped: + raise ValidationError(f"Invalid env line {number} in {path}") + key, value = stripped.split("=", 1) + values[key] = value.strip().strip('"').strip("'") + return values + + +def secret_env_names() -> set[str]: + return { + "APP_KEY", + "EDGE_ARTIFACT_SIGNING_KEY", + "CONTROL_DB_PASSWORD", + "REDIS_PASSWORD", + "PDNS_DB_PASSWORD", + "PDNS_API_KEY", + "EDGE_STATUS_TOKEN", + "CLICKHOUSE_PASSWORD", + "GRAFANA_ADMIN_PASSWORD", + "GRAFANA_CLICKHOUSE_PASSWORD", + "GRAFANA_POSTGRES_PASSWORD", + } + + +def import_secret_env(store: FleetState, node: str, role: str, env: dict[str, str]) -> None: + global_map = { + "APP_KEY": "app-key", + "EDGE_ARTIFACT_SIGNING_KEY": "artifact-signing-key", + "CONTROL_DB_PASSWORD": "control-db-password", + "REDIS_PASSWORD": "valkey-password", + "CLICKHOUSE_PASSWORD": "clickhouse-password", + "GRAFANA_ADMIN_PASSWORD": "grafana-admin-password", + "GRAFANA_CLICKHOUSE_PASSWORD": "grafana-clickhouse-password", + "GRAFANA_POSTGRES_PASSWORD": "grafana-postgres-password", + } + node_map = { + "PDNS_DB_PASSWORD": "pdns-db-password", + "PDNS_API_KEY": "pdns-api-key", + "EDGE_STATUS_TOKEN": "edge-status-token", + } + for key, name in global_map.items(): + if env.get(key): + path = store.secret_path(name) + from .common import atomic_write + + atomic_write(path, env[key] + "\n", 0o600) + for key, name in node_map.items(): + if env.get(key): + path = store.secret_path(name, node=node) + from .common import atomic_write + + atomic_write(path, env[key] + "\n", 0o600) + + +def main(argv: list[str] | None = None) -> int: + try: + effective = list(sys.argv[1:] if argv is None else argv) + if not effective: + if sys.stdin.isatty(): + effective = ["setup"] + else: + parser().print_help() + return EXIT_OK + return execute(parser().parse_args(effective)) + except ValidationError as exc: + print(f"validation error: {exc}", file=sys.stderr) + return EXIT_VALIDATION + except StateError as exc: + print(f"state error: {exc}", file=sys.stderr) + return EXIT_STATE + except FleetError as exc: + print(f"error: {exc}", file=sys.stderr) + return EXIT_RENDER + except KeyboardInterrupt: + print("cancelled", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/cdnfoundry_fleet/common.py b/scripts/cdnfoundry_fleet/common.py new file mode 100644 index 0000000..d98712a --- /dev/null +++ b/scripts/cdnfoundry_fleet/common.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import json +import os +import re +import secrets +import stat +import tempfile +from pathlib import Path +from typing import Any, Iterable + + +class FleetError(Exception): + """Base class for expected operator errors.""" + + +class ValidationError(FleetError): + pass + + +class StateError(FleetError): + pass + + +class RenderError(FleetError): + pass + + +NODE_RE = re.compile(r"^[a-z][a-z0-9-]{1,62}$") +HOST_RE = re.compile( + r"^(?=.{1,253}$)(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)*" + r"[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$" +) +REGION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_. -]{0,63}$") +ENV_KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") + + +def utc_now() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def random_secret(bytes_: int = 32) -> str: + return secrets.token_hex(bytes_) + + +def ensure_mode(path: Path, mode: int) -> None: + current = stat.S_IMODE(path.stat().st_mode) + if current != mode: + path.chmod(mode) + + +def atomic_write(path: Path, data: str | bytes, mode: int = 0o600) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + path.parent.chmod(0o700) + except PermissionError: + pass + payload = data.encode("utf-8") if isinstance(data, str) else data + fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + tmp = Path(tmp_name) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "wb", closefd=True) as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + os.chmod(path, mode) + dir_fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + finally: + tmp.unlink(missing_ok=True) + + +def atomic_json(path: Path, value: Any, mode: int = 0o600) -> None: + atomic_write(path, json.dumps(value, indent=2, sort_keys=True) + "\n", mode) + + +def load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise StateError(f"Missing file: {path}") from exc + except json.JSONDecodeError as exc: + raise StateError(f"Invalid JSON in {path}: {exc}") from exc + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_node_name(value: str) -> str: + if not NODE_RE.fullmatch(value): + raise ValidationError( + "Node name must start with a lowercase letter and contain only lowercase letters, digits, and hyphens" + ) + return value + +def validate_hostname(value: str) -> str: + value = value.rstrip(".") + if not HOST_RE.fullmatch(value): + raise ValidationError(f"Invalid hostname: {value!r}") + return value.lower() + + +def validate_region(value: str, label: str = "region") -> str: + if not REGION_RE.fullmatch(value): + raise ValidationError(f"Invalid {label}: {value!r}") + return value + + +def validate_ip(value: str | None, *, required: bool = False) -> str | None: + if value in (None, ""): + if required: + raise ValidationError("An IP address is required") + return None + try: + return str(ipaddress.ip_address(value)) + except ValueError as exc: + raise ValidationError(f"Invalid IP address: {value!r}") from exc + + +def validate_release(value: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,127}", value): + raise ValidationError("Release must be an exact tag or commit identifier") + if value in {"latest", "main", "master"}: + raise ValidationError("Moving release identifiers are not allowed") + return value + + +def validate_env_mapping(values: dict[str, Any]) -> dict[str, str]: + clean: dict[str, str] = {} + for key, value in values.items(): + if not ENV_KEY_RE.fullmatch(key): + raise ValidationError(f"Invalid environment key: {key!r}") + text = str(value) + if "\x00" in text or "\n" in text or "\r" in text: + raise ValidationError(f"Environment value for {key} contains a line break or NUL") + clean[key] = text + return clean + + +def unique_nonempty(values: Iterable[str | None]) -> bool: + items = [item for item in values if item] + return len(items) == len(set(items)) + + +def quote_env(value: str) -> str: + # Docker env files accept unquoted values; reject line breaks at validation time. + if value == "" or re.search(r"[\s#'\"\\]", value): + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + return value diff --git a/scripts/cdnfoundry_fleet/compose.py b/scripts/cdnfoundry_fleet/compose.py new file mode 100644 index 0000000..db24e1b --- /dev/null +++ b/scripts/cdnfoundry_fleet/compose.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import copy +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +try: + import yaml + from yaml.nodes import MappingNode, ScalarNode, SequenceNode +except ImportError as exc: # pragma: no cover - handled by prerequisite installer + raise RuntimeError("PyYAML is required; install python3-yaml") from exc + +from .common import RenderError, ValidationError + +VAR_RE = re.compile(r"\$\{([A-Z][A-Z0-9_]*)(?:(:?[-+?])[^}]*)?\}") +ROLE_PROFILES = { + "control": {"control"}, + "dns": {"dns"}, + "edge": {"edge"}, + "dns-edge": {"dns", "edge"}, + "monitoring": {"telemetry"}, +} + + +@dataclass(frozen=True) +class _ResetValue: + value: Any + + +@dataclass(frozen=True) +class _OverrideValue: + value: Any + + +class ComposeLoader(yaml.SafeLoader): + """Safe YAML loader with Docker Compose's !reset and !override tags.""" + + +def _construct_tagged(loader: ComposeLoader, node: yaml.Node, wrapper: type[_ResetValue] | type[_OverrideValue]) -> Any: + if isinstance(node, MappingNode): + value = loader.construct_mapping(node, deep=True) + elif isinstance(node, SequenceNode): + value = loader.construct_sequence(node, deep=True) + elif isinstance(node, ScalarNode): + value = loader.construct_scalar(node) + else: # pragma: no cover - PyYAML currently exposes only these node types + raise yaml.constructor.ConstructorError(None, None, f"Unsupported tagged YAML node: {type(node).__name__}", node.start_mark) + return wrapper(value) + + +ComposeLoader.add_constructor("!reset", lambda loader, node: _construct_tagged(loader, node, _ResetValue)) +ComposeLoader.add_constructor("!override", lambda loader, node: _construct_tagged(loader, node, _OverrideValue)) + + +def load_yaml(path: Path) -> dict[str, Any]: + try: + data = yaml.load(path.read_text(encoding="utf-8"), Loader=ComposeLoader) or {} + except FileNotFoundError as exc: + raise RenderError(f"Missing Compose source: {path}") from exc + except yaml.YAMLError as exc: + raise RenderError(f"Invalid YAML in {path}: {exc}") from exc + if not isinstance(data, dict): + raise RenderError(f"Compose source must be a mapping: {path}") + return data + + +def dump_yaml(data: dict[str, Any]) -> str: + return yaml.safe_dump(data, sort_keys=False, default_flow_style=False, width=120) + + +def merge_compose(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: + return _deep_merge(copy.deepcopy(base), overlay) + + +def _deep_merge(left: Any, right: Any, key: str | None = None) -> Any: + if isinstance(right, (_ResetValue, _OverrideValue)): + return copy.deepcopy(right.value) + if isinstance(left, dict) and isinstance(right, dict): + result = copy.deepcopy(left) + for child_key, value in right.items(): + if child_key in result: + result[child_key] = _deep_merge(result[child_key], value, child_key) + else: + result[child_key] = _deep_merge(None, value, child_key) + return result + if isinstance(left, list) and isinstance(right, list): + if key in {"command", "entrypoint", "healthcheck"}: + return copy.deepcopy(right) + return copy.deepcopy(left) + copy.deepcopy(right) + return copy.deepcopy(right) + + +def profile_set(service: dict[str, Any]) -> set[str]: + raw = service.get("profiles", []) + if isinstance(raw, str): + return {raw} + return {str(item) for item in raw} + + +def select_services( + compose: dict[str, Any], + *, + role: str, + monitoring_enabled: bool, + logs_enabled: bool, + monitoring_host: bool, +) -> dict[str, Any]: + if role not in ROLE_PROFILES: + raise ValidationError(f"Unsupported role: {role}") + services = compose.get("services", {}) + if not isinstance(services, dict): + raise RenderError("Compose file has no services mapping") + + active_profiles = set(ROLE_PROFILES[role]) + if monitoring_host: + active_profiles.add("telemetry") + if logs_enabled: + active_profiles.add("logs") + + selected: set[str] = set() + for name, service in services.items(): + profiles = profile_set(service) + if not profiles or profiles & active_profiles: + if name == "vector" and not monitoring_enabled: + continue + selected.add(name) + + # Tool containers are role-specific and remain behind the tools profile. + if role == "control" and "migrate" in services: + selected.add("migrate") + if role in {"dns", "dns-edge"} and "pdns-migrate" in services: + selected.add("pdns-migrate") + + # Every monitored production host exports node metrics; only the monitoring host runs the full stack. + if monitoring_enabled and "node-exporter" in services: + selected.add("node-exporter") + if logs_enabled and "log-collector" in services: + selected.add("log-collector") + + selected = dependency_closure(services, selected) + rendered_services: dict[str, Any] = {} + for name in sorted(selected): + service = copy.deepcopy(services[name]) + profiles = profile_set(service) + if name not in {"migrate", "pdns-migrate"}: + service.pop("profiles", None) + elif "tools" not in profiles: + service["profiles"] = ["tools"] + rendered_services[name] = service + + result: dict[str, Any] = {"services": rendered_services} + for section in ("networks", "volumes", "configs", "secrets"): + values = compose.get(section) + if isinstance(values, dict): + used = referenced_top_level(rendered_services, section) + result[section] = {k: copy.deepcopy(v) for k, v in values.items() if k in used} + return result + +def prune_top_level(compose: dict[str, Any]) -> None: + services = compose.get("services", {}) + if not isinstance(services, dict): + return + for section in ("networks", "volumes", "configs", "secrets"): + values = compose.get(section) + if isinstance(values, dict): + used = referenced_top_level(services, section) + compose[section] = {key: value for key, value in values.items() if key in used} + + +def dependency_closure(services: dict[str, Any], initial: Iterable[str]) -> set[str]: + selected = set(initial) + changed = True + while changed: + changed = False + for name in list(selected): + service = services.get(name) + if not isinstance(service, dict): + raise RenderError(f"Service {name!r} is missing") + depends = service.get("depends_on", {}) + names = depends.keys() if isinstance(depends, dict) else depends if isinstance(depends, list) else [] + for dep in names: + if dep not in services: + raise RenderError(f"Service {name!r} depends on missing service {dep!r}") + if dep not in selected: + selected.add(dep) + changed = True + return selected + + +def referenced_top_level(services: dict[str, Any], section: str) -> set[str]: + used: set[str] = set() + for service in services.values(): + raw = service.get(section, []) + if isinstance(raw, dict): + used.update(raw) + elif isinstance(raw, list): + for item in raw: + if isinstance(item, str): + source = item.split(":", 1)[0] + if source and not source.startswith((".", "/", "~")) and "${" not in source: + used.add(source) + elif isinstance(item, dict) and item.get("source"): + used.add(str(item["source"])) + return used + + +def required_env(compose: dict[str, Any]) -> set[str]: + text = dump_yaml(compose) + required: set[str] = set() + for name, operator in VAR_RE.findall(text): + if operator in {"", ":?"}: + required.add(name) + return required + + +def bind_mount_sources(compose: dict[str, Any]) -> set[Path]: + result: set[Path] = set() + for service in compose.get("services", {}).values(): + for item in service.get("volumes", []) or []: + if not isinstance(item, str): + continue + source = item.split(":", 1)[0] + if source.startswith("./") and "${" not in source: + result.add(Path(source[2:])) + return result diff --git a/scripts/cdnfoundry_fleet/render.py b/scripts/cdnfoundry_fleet/render.py new file mode 100644 index 0000000..6498611 --- /dev/null +++ b/scripts/cdnfoundry_fleet/render.py @@ -0,0 +1,735 @@ +from __future__ import annotations + +import copy +import json +import os +import shutil +import tempfile +from pathlib import Path +from typing import Any + +from .certs import PKI +from .common import RenderError, atomic_json, atomic_write, quote_env, sha256_file, utc_now +from .compose import ( + bind_mount_sources, + dump_yaml, + load_yaml, + merge_compose, + prune_top_level, + required_env, + select_services, +) +from .state import FleetState + +ROLE_OVERLAYS = { + "control": ["deploy/production/compose.control-host.yml"], + "dns": ["deploy/production/compose.dns-host.yml"], + "edge": ["deploy/production/compose.edge-host.yml"], + "dns-edge": ["deploy/production/compose.dns-edge-host.yml"], + "monitoring": ["deploy/production/compose.telemetry-host.yml"], +} + + +class Renderer: + def __init__(self, repo_root: Path, store: FleetState, output_dir: Path, *, dry_run: bool = False) -> None: + self.repo_root = repo_root.resolve() + self.store = store + self.output_dir = output_dir.resolve() + self.dry_run = dry_run + self.pki = PKI(store.pki_dir, dry_run=dry_run) + + def render(self, state: dict[str, Any], *, node_name: str | None = None) -> list[Path]: + names = [node_name] if node_name else sorted(state["nodes"]) + rendered: list[Path] = [] + for name in names: + if name not in state["nodes"]: + raise RenderError(f"Unknown node: {name}") + node = state["nodes"][name] + if not node.get("enabled", True): + continue + rendered.append(self._render_node(state, node)) + self._render_fleet_files(state) + return rendered + + def _render_node(self, state: dict[str, Any], node: dict[str, Any]) -> Path: + base = load_yaml(self.repo_root / "compose.prod.yml") + merged = base + for overlay in ROLE_OVERLAYS[node["role"]]: + path = self.repo_root / overlay + if path.exists(): + merged = merge_compose(merged, load_yaml(path)) + if state["features"]["logs"]["mode"] == "centralized": + journal = self.repo_root / "deploy/production/compose.host-journal.yml" + if journal.exists(): + merged = merge_compose(merged, load_yaml(journal)) + + monitoring_enabled = state["features"]["monitoring"]["mode"] != "disabled" + monitoring_host = self._is_monitoring_host(state, node) + logs_enabled = state["features"]["logs"]["mode"] == "centralized" + filtered = select_services( + merged, + role=node["role"], + monitoring_enabled=monitoring_enabled, + logs_enabled=logs_enabled, + monitoring_host=monitoring_host, + ) + self._apply_generated_overrides(state, node, filtered) + prune_top_level(filtered) + env = self._environment(state, node, required_env(filtered), monitoring_host=monitoring_host) + + destination = self.output_dir / node["name"] + if self.dry_run: + return destination + self.output_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + self.output_dir.chmod(0o700) + with tempfile.TemporaryDirectory(prefix=f".{node['name']}.", dir=self.output_dir) as tmp_dir: + tmp = Path(tmp_dir) + self._copy_runtime_files(filtered, tmp) + self.pki.copy_node_material(node, tmp / "pki") + atomic_write(tmp / "compose.yml", dump_yaml(filtered), 0o600) + atomic_write(tmp / ".env.prod", self._format_env(env), 0o600) + self._write_generated_configs(state, node, tmp, monitoring_host) + atomic_write(tmp / "README.md", self._node_readme(state, node, filtered), 0o600) + atomic_write(tmp / "validate.sh", self._validate_script(), 0o700) + atomic_write(tmp / "start.sh", self._start_script(node), 0o700) + self._write_manifest(tmp, state, node) + previous = destination.with_name(destination.name + ".previous") + if previous.exists(): + shutil.rmtree(previous) + if destination.exists(): + os.replace(destination, previous) + os.replace(tmp, destination) + return destination + + def _is_monitoring_host(self, state: dict[str, Any], node: dict[str, Any]) -> bool: + feature = state["features"]["monitoring"] + if feature["mode"] == "colocated": + return node["role"] == "control" + if feature["mode"] == "dedicated": + return feature.get("host") == node["name"] + return False + + def _apply_generated_overrides(self, state: dict[str, Any], node: dict[str, Any], compose: dict[str, Any]) -> None: + services = compose.get("services", {}) + if "node-exporter" in services: + bind = node.get("monitor_ipv4") or node["bind_ipv4"] + services["node-exporter"]["ports"] = [f"{bind}:9100:9100/tcp"] + if "log-collector" in services: + service = services["log-collector"] + service["command"] = ["--config", "/etc/vector/generated-node.yaml"] + volumes = [v for v in service.get("volumes", []) if "/etc/vector/operational.yaml" not in str(v)] + volumes.append("./generated/vector-node.yaml:/etc/vector/generated-node.yaml:ro") + service["volumes"] = volumes + service["healthcheck"] = { + "test": ["CMD", "vector", "validate", "--no-environment", "/etc/vector/generated-node.yaml"], + "interval": "30s", + "timeout": "5s", + "retries": 3, + } + if node["role"] in {"dns", "dns-edge"}: + if "pdns-db" not in services or "pdns-auth" not in services: + raise RenderError(f"DNS node {node['name']} does not contain local pdns-db and pdns-auth services") + # Explicitly prevent accidental use of the control database. + pdns_env = services["pdns-auth"].setdefault("environment", {}) + pdns_env["PDNS_gpgsql_host"] = "pdns-db" + pdns_env["PDNS_gpgsql_dbname"] = "pdns" + pdns_env["PDNS_gpgsql_user"] = "pdns" + if node["role"] == "control" and self._uses_remote_control_db(node): + # External PostgreSQL is selected with DB_URL or a DB_HOST other than the + # Compose service name. Remove the embedded database and every dependency + # edge to it; Valkey remains host-local unless REDIS_URL/REDIS_HOST is + # overridden separately. + services.pop("control-db", None) + for service in services.values(): + depends = service.get("depends_on") + if isinstance(depends, dict): + depends.pop("control-db", None) + elif isinstance(depends, list): + service["depends_on"] = [name for name in depends if name != "control-db"] + + if node["role"] == "monitoring": + # A dedicated telemetry host must not accidentally start a second control database. + # The repository's Grafana control-DB provisioning helper depends on control-db, so + # omit that helper on a dedicated host and leave telemetry dashboards operational. + services.pop("grafana-control-db-provision", None) + services.pop("control-db", None) + for service in services.values(): + depends = service.get("depends_on") + if isinstance(depends, dict): + depends.pop("grafana-control-db-provision", None) + depends.pop("control-db", None) + elif isinstance(depends, list): + service["depends_on"] = [ + name for name in depends if name not in {"grafana-control-db-provision", "control-db"} + ] + + def _environment( + self, + state: dict[str, Any], + node: dict[str, Any], + needed: set[str], + *, + monitoring_host: bool, + ) -> dict[str, str]: + operator_domain = state["global"]["operator_domain"] + values: dict[str, str] = { + "CDNF_RELEASE": node["release"], + "HOST_BIND_IPV4": node["bind_ipv4"], + "HOST_BIND_IPV6": node.get("bind_ipv6") or "::", + "DNS_BIND_V4": node["bind_ipv4"], + "APP_URL": f"https://control.{operator_domain}", + "CONTROL_HOSTNAME": f"control.{operator_domain}", + "TELEMETRY_HOSTNAME": f"telemetry.{operator_domain}", + "APP_KEY": self.store.read_secret("app-key"), + "EDGE_ARTIFACT_SIGNING_KEY": self.store.read_secret("artifact-signing-key"), + "CONTROL_DB_PASSWORD": self.store.read_secret("control-db-password"), + "REDIS_PASSWORD": self.store.read_secret("valkey-password"), + "CLICKHOUSE_PASSWORD": self.store.read_secret("clickhouse-password"), + "CLICKHOUSE_URL": self._clickhouse_url(state), + "GRAFANA_ADMIN_PASSWORD": self.store.read_secret("grafana-admin-password"), + "GRAFANA_CLICKHOUSE_PASSWORD": self.store.read_secret("grafana-clickhouse-password"), + "GRAFANA_POSTGRES_PASSWORD": self.store.read_secret("grafana-postgres-password"), + "METRICS_TOKEN_FILE": "./secrets/metrics-token", + # Production Compose PKI contract (control, edge and DNS roles). + "EDGE_IDENTITY_CA_CERTIFICATE": "./pki/edge-identity-ca.crt", + "EDGE_IDENTITY_CA_PRIVATE_KEY": "./pki/edge-identity-ca.key", + "PDNS_CA_CERTIFICATE": "./pki/edge-server-ca.crt", + "EDGE_CONTROL_SERVER_CERTIFICATE": "./pki/node.crt", + "EDGE_CONTROL_SERVER_PRIVATE_KEY": "./pki/node.key", + "EDGE_CONTROL_CA_CERTIFICATE": "./pki/edge-server-ca.crt", + "EDGE_CONTROL_URL": self._edge_control_url(state), + "EDGE_RUNTIME_TLS_CERTIFICATE": "./pki/node.crt", + "EDGE_RUNTIME_TLS_PRIVATE_KEY": "./pki/node.key", + "DNS_API_SERVER_CERTIFICATE": "./pki/node.crt", + "DNS_API_SERVER_PRIVATE_KEY": "./pki/node.key", + "DNS_API_HOSTNAME": node["hostname"], + "CONTROL_PUBLIC_IPV4_ALLOWLIST": self._control_allowlist(state), + "EDGE_PUBLIC_IPV4_ALLOWLIST": self._edge_allowlist(state), + "LOG_SOURCE_IPV4_ALLOWLIST": self._all_public_ipv4(state), + "LOG_ROLE": node["role"], + "LOG_HOST": node["name"], + "LOG_COLLECTOR_ID": node["name"], + "LOKI_ENDPOINT": self._loki_url(state), + "LOG_AUTH_TOKEN": self._optional_node_secret("log-auth-token", node), + "NODE_EXPORTER_TOKEN": self.store.read_secret("node-exporter-token", node=node["name"]), + "EDGE_STATUS_TOKEN": self._optional_node_secret("edge-status-token", node), + "EDGE_ID": node.get("edge_id") or "", + "EDGE_BOOTSTRAP_TOKEN": self._optional_node_secret("edge-bootstrap-token", node), + "PDNS_DB_PASSWORD": self._optional_node_secret("pdns-db-password", node), + "PDNS_API_KEY": self._optional_node_secret("pdns-api-key", node), + "RESTIC_REPOSITORY": state["features"]["backups"].get("repository") or "", + "RESTIC_PASSWORD_FILE": "./secrets/restic-password", + "BACKUP_ACCESS_KEY_ID": self.store.read_secret("backup-access-key"), + "BACKUP_SECRET_ACCESS_KEY": self.store.read_secret("backup-secret-key"), + "BACKUP_DEFAULT_REGION": state["features"]["backups"].get("region") or "us-east-1", + "ACME_CONTACT_EMAIL": state["global"].get("acme_email", ""), + "SESSION_SECURE_COOKIE": "true", + "CONTROL_BIND": "127.0.0.1:8080", + "DB_URL": "", + "DB_HOST": "control-db", + "DB_PORT": "5432", + "DB_SSLMODE": "prefer", + "REDIS_URL": "", + "REDIS_HOST": "redis", + "REDIS_PORT": "6379", + "ACME_DIRECTORY_URL": "https://acme-v02.api.letsencrypt.org/directory", + "ACME_ORDER_BUDGET_PER_HOUR": "20", + "EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE": "", + "GRAFANA_EXPLORE_URL": "", + "EDGE_CONTROL_BIND": "0.0.0.0:8443", + "EDGE_RUNTIME_VERSIONS": "{}", + "EDGE_GATEWAY_METRICS_ADDRESS": "0.0.0.0:9105", + "EDGE_GATEWAY_MAX_CONNECTIONS": "8192", + "EDGE_GATEWAY_STATUS_URL": "http://host-gateway:9105/metrics", + "EDGE_GATEWAY_ADDRESS_MAP": "{}", + "MMDB_PROVIDER": "dbip-jsdelivr", + "MMDB_TARGET_FILE": "GeoLite2-City.mmdb", + "MMDB_DOWNLOAD_INTERVAL_SECONDS": "86400", + "MMDB_DOWNLOAD_RETRIES": "5", + "MMDB_EXPECTED_SHA256": "", + "MMDB_DOWNLOAD_URL": "", + "MMDB_DOWNLOAD_HEADER": "", + "LOG_BUFFER_BYTES": "2147483648", + "LOG_METRICS_BIND": "127.0.0.1:9599", + "LOKI_RETENTION_PERIOD": "336h", + "LOKI_MAX_QUERY_LENGTH": "336h", + "PROMETHEUS_EDGE_TARGETS_FILE": "./docker/prometheus/edge-targets.prod.yml", + "PROMETHEUS_LOG_TARGETS_FILE": "./docker/prometheus/operational-log-targets.prod.yml", + "GRAFANA_ADMIN_USER": "admin", + "GRAFANA_BIND": "127.0.0.1:3000", + "GRAFANA_COOKIE_SECURE": "true", + "GRAFANA_LOKI_URL": "http://loki:3100", + "GRAFANA_CLICKHOUSE_HOST": "clickhouse", + "GRAFANA_CLICKHOUSE_PORT": "9000", + "GRAFANA_CLICKHOUSE_PROTOCOL": "native", + "GRAFANA_CLICKHOUSE_SECURE": "false", + "GRAFANA_CLICKHOUSE_USER": "cdnf_grafana", + "GRAFANA_POSTGRES_HOST": "control-db", + "GRAFANA_POSTGRES_PORT": "5432", + "GRAFANA_POSTGRES_DATABASE": "cdnf", + "GRAFANA_POSTGRES_USER": "cdnf_grafana", + "GRAFANA_POSTGRES_SSLMODE": "disable", + "GRAFANA_POSTGRES_PROVISION_HOST": "control-db", + "GRAFANA_POSTGRES_PROVISION_PORT": "5432", + } + explicit_env = node.get("extra_env", {}) + derived_env: set[str] = set() + values.update(explicit_env) + if node["role"] == "control" and self._uses_remote_control_db(node): + # Keep Grafana's control-database provisioning and datasource pointed at + # the same external PostgreSQL endpoint unless the operator overrides them. + remote_host = values.get("DB_HOST", "") + remote_port = values.get("DB_PORT", "5432") + remote_sslmode = values.get("DB_SSLMODE", "verify-full") + if remote_host: + defaults = { + "GRAFANA_POSTGRES_PROVISION_HOST": remote_host, + "GRAFANA_POSTGRES_PROVISION_PORT": remote_port, + "GRAFANA_POSTGRES_HOST": remote_host, + "GRAFANA_POSTGRES_PORT": remote_port, + "GRAFANA_POSTGRES_SSLMODE": remote_sslmode, + } + for key, value in defaults.items(): + if key not in explicit_env: + values[key] = value + derived_env.add(key) + missing = sorted(key for key in needed if key not in values) + if missing: + raise RenderError( + f"Node {node['name']} is missing required environment values: {', '.join(missing)}; use extra_env" + ) + # Minimal role environment: only variables referenced by the filtered Compose plus generated configs. + always = {"CDNF_RELEASE", "HOST_BIND_IPV4", "HOST_BIND_IPV6"} + if state["features"]["logs"]["mode"] == "centralized": + always |= {"LOG_ROLE", "LOG_HOST", "LOG_COLLECTOR_ID", "LOKI_ENDPOINT", "LOG_AUTH_TOKEN"} + if state["features"]["monitoring"]["mode"] != "disabled": + always |= {"NODE_EXPORTER_TOKEN"} + # Operator-supplied values are deliberate overrides. Include them even when + # Compose gives the variable a default (`${VAR:-...}`), otherwise edge + # enrollment, gateway address maps, MMDB tuning, and remote dependencies are + # silently lost from the generated bundle. + explicit = set(explicit_env) | derived_env + if values.get("EDGE_BOOTSTRAP_TOKEN"): + explicit.add("EDGE_BOOTSTRAP_TOKEN") + return { + key: values[key] + for key in sorted(needed | always | explicit) + if key in values and (values[key] != "" or key in needed) + } + + def _optional_node_secret(self, name: str, node: dict[str, Any]) -> str: + path = self.store.secret_path(name, node=node["name"]) + return self.store.read_secret(name, node=node["name"]) if path.exists() else "" + + @staticmethod + def _uses_remote_control_db(node: dict[str, Any]) -> bool: + extra = node.get("extra_env", {}) + if str(extra.get("DB_URL", "")).strip(): + return True + host = str(extra.get("DB_HOST", "")).strip() + return bool(host and host != "control-db") + + def _edge_control_url(self, state: dict[str, Any]) -> str: + control = next( + ( + item + for item in state["nodes"].values() + if item["role"] == "control" and item.get("enabled", True) + ), + None, + ) + hostname = control["hostname"] if control else f"control.{state['global']['operator_domain']}" + return f"https://{hostname}:8443" + + def _control_allowlist(self, state: dict[str, Any]) -> str: + return " ".join( + node["public_ipv4"] for node in state["nodes"].values() if node["role"] == "control" and node["enabled"] + ) + + def _edge_allowlist(self, state: dict[str, Any]) -> str: + return " ".join( + node["public_ipv4"] + for node in state["nodes"].values() + if node["role"] in {"edge", "dns-edge"} and node["enabled"] + ) + + def _all_public_ipv4(self, state: dict[str, Any]) -> str: + return " ".join(node["public_ipv4"] for node in state["nodes"].values() if node["enabled"]) + + def _feature_host(self, state: dict[str, Any], feature: str) -> dict[str, Any] | None: + cfg = state["features"][feature] + if cfg["mode"] == "disabled": + return None + if cfg["mode"] == "colocated": + return next((node for node in state["nodes"].values() if node["role"] == "control"), None) + name = cfg.get("host") + return state["nodes"].get(name) if name else None + + def _clickhouse_url(self, state: dict[str, Any]) -> str: + host = self._feature_host(state, "monitoring") + return f"https://{host['hostname']}:8444" if host else "http://127.0.0.1:8123" + + def _loki_url(self, state: dict[str, Any]) -> str: + cfg = state["features"]["logs"] + if cfg.get("endpoint"): + return cfg["endpoint"] + host = state["nodes"].get(cfg.get("host")) if cfg.get("host") else None + return f"https://{host['hostname']}:8444" if host else "http://127.0.0.1:3100" + + def _format_env(self, env: dict[str, str]) -> str: + return "".join(f"{key}={quote_env(value)}\n" for key, value in sorted(env.items())) + + def _copy_runtime_files(self, compose: dict[str, Any], destination: Path) -> None: + for relative in sorted(bind_mount_sources(compose)): + if relative.parts and relative.parts[0] in {'generated', 'pki', 'secrets'}: + continue + source = (self.repo_root / relative).resolve() + try: + source.relative_to(self.repo_root) + except ValueError as exc: + raise RenderError(f"Compose bind mount escapes repository: {relative}") from exc + if not source.exists(): + raise RenderError(f"Missing runtime file referenced by Compose: {relative}") + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target, dirs_exist_ok=True) + else: + shutil.copy2(source, target) + + def _write_generated_configs( + self, state: dict[str, Any], node: dict[str, Any], destination: Path, monitoring_host: bool + ) -> None: + secrets_dir = destination / "secrets" + secrets_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + if state["features"]["monitoring"]["mode"] != "disabled": + atomic_write(secrets_dir / "metrics-token", self.store.read_secret("metrics-token") + "\n", 0o600) + if state["features"]["backups"]["mode"] != "disabled": + atomic_write(secrets_dir / "restic-password", self.store.read_secret("backup-password") + "\n", 0o600) + if state["features"]["logs"]["mode"] == "centralized": + atomic_write(destination / "generated/vector-node.yaml", self._vector_config(node), 0o600) + if node["role"] in {"dns", "dns-edge"}: + pending = self.store.pending_secret_path("pdns-db-password", node=node["name"]) + if pending.exists(): + atomic_write( + secrets_dir / "pdns-db-password.next", + pending.read_text(encoding="utf-8"), + 0o600, + ) + atomic_write( + destination / "reconcile-pdns-password.sh", + self._pdns_reconciliation_script(), + 0o700, + ) + if monitoring_host: + generated = destination / "generated" + atomic_write(generated / "prometheus-node-targets.yml", self._prometheus_targets(state), 0o600) + atomic_write(generated / "geo-routing-policy.json", json.dumps(self._geo_policy(state), indent=2) + "\n", 0o600) + + def _pdns_reconciliation_script(self) -> str: + return r'''#!/usr/bin/env sh +set -eu +umask 077 + +cd "$(dirname "$0")" +test -f .env.prod +test -f secrets/pdns-db-password.next + +# The generated env file is shell-compatible and contains the currently active password. +set -a +. ./.env.prod +set +a +next_password=$(cat secrets/pdns-db-password.next) +test -n "$PDNS_DB_PASSWORD" +test -n "$next_password" + +cp -p .env.prod .env.prod.before-pdns-rotation +chmod 600 .env.prod.before-pdns-rotation + +docker compose --env-file .env.prod exec -T \ + -e PGPASSWORD="$PDNS_DB_PASSWORD" \ + -e NEXT_PDNS_DB_PASSWORD="$next_password" \ + pdns-db sh -eu -c ' + psql -v ON_ERROR_STOP=1 \ + -U "${POSTGRES_USER:-pdns}" \ + -d "${POSTGRES_DB:-pdns}" \ + -v next_password="$NEXT_PDNS_DB_PASSWORD" \ + -c "ALTER ROLE pdns PASSWORD :'"'"'next_password'"'"';" + ' + +python3 - "$next_password" <<'PY' +import os +import sys +from pathlib import Path + +value = sys.argv[1] +path = Path(".env.prod") +tmp = path.with_name(".env.prod.next") +lines = path.read_text(encoding="utf-8").splitlines() +updated = False +with tmp.open("w", encoding="utf-8", newline="\n") as handle: + for line in lines: + if line.startswith("PDNS_DB_PASSWORD="): + handle.write(f"PDNS_DB_PASSWORD={value}\n") + updated = True + else: + handle.write(line + "\n") + if not updated: + raise SystemExit("PDNS_DB_PASSWORD is missing from .env.prod") + handle.flush() + os.fsync(handle.fileno()) +os.chmod(tmp, 0o600) +os.replace(tmp, path) +PY + +docker compose --env-file .env.prod config --quiet +printf '%s\n' 'Local PostgreSQL and .env.prod now use the pending password.' +printf '%s\n' 'On the control plane, commit the rotation, rerender this node, transfer the new bundle, and run docker compose up -d.' +''' + + def _vector_config(self, node: dict[str, Any]) -> str: + return f'''data_dir: /vector-data-dir +sources: + docker: + type: docker_logs + journal: + type: journald + journal_directory: /var/log/journal +transforms: + redact: + type: remap + inputs: [docker, journal] + source: | + .fleet_node = "{node['name']}" + .fleet_role = "{node['role']}" + del(.label.com_docker_compose_config_hash) + if exists(.message) {{ .message = redact(string!(.message), filters: [r'(?i)(password|token|secret|api[_-]?key)=\\S+']) }} +sinks: + central: + type: loki + inputs: [redact] + endpoint: "${{LOKI_ENDPOINT}}" + auth: + strategy: bearer + token: "${{LOG_AUTH_TOKEN}}" + encoding: + codec: json + labels: + node: "{node['name']}" + role: "{node['role']}" + buffer: + type: disk + max_size: 1073741824 + when_full: drop_newest + request: + retry_max_duration_secs: 300 + timeout_secs: 10 +healthchecks: + enabled: true +''' + + def _prometheus_targets(self, state: dict[str, Any]) -> str: + groups = [] + for node in sorted(state["nodes"].values(), key=lambda item: item["name"]): + if not node["enabled"]: + continue + address = node.get("monitor_ipv4") or node["public_ipv4"] + groups.append( + { + "targets": [f"{address}:9100"], + "labels": { + "node": node["name"], + "role": node["role"], + "region": node["region"], + "location": node["location"], + }, + } + ) + import yaml + + return yaml.safe_dump(groups, sort_keys=False) + + def _geo_policy(self, state: dict[str, Any]) -> dict[str, Any]: + edges = [] + for node in state["nodes"].values(): + if node["role"] not in {"edge", "dns-edge"}: + continue + edges.append( + { + "name": node["name"], + "region": node["region"], + "location": node["location"], + "ipv4": node["public_ipv4"], + "ipv6": node.get("public_ipv6"), + "enabled": node["enabled"], + "draining": node["draining"], + "failure_threshold": node["health"]["failure_threshold"], + "success_threshold": node["health"]["success_threshold"], + "stale_after_seconds": node["health"]["stale_after_seconds"], + } + ) + return { + "decision_order": [ + "valid_ecs_client_subnet", + "resolver_ip_fallback", + "country_and_asn_policy", + "health_filtering", + "preferred_healthy_edge_ip", + ], + "selection": "deterministic", + "address_families": "independent", + "edges": sorted(edges, key=lambda item: item["name"]), + } + + def _node_readme(self, state: dict[str, Any], node: dict[str, Any], compose: dict[str, Any]) -> str: + services = ", ".join(compose.get("services", {})) + listeners = self._listeners(node) + start_order = self._node_start_order(node) + database_mode = "external PostgreSQL" if self._uses_remote_control_db(node) else "embedded PostgreSQL" + return f"""# CDNFoundry node bundle: {node['name']} + +- Role: `{node['role']}` +- Region: `{node['region']}` +- Location: `{node['location']}` +- Release: `{node['release']}` +- Services: {services} +- Control database mode: {database_mode if node['role'] == 'control' else 'not applicable'} + +This bundle intentionally contains only files and credentials needed by this node. For DNS roles, `pdns-db` is the node-local PostgreSQL service and `pdns-auth` connects only to `pdns-db`; it never uses the control-plane PostgreSQL service. + +## Requirements + +Docker Engine, Docker Compose v2, accurate system time, CA certificates, and sufficient disk for stateful volumes. Keep the directory mode `0700` and `.env.prod`, private keys, and secret files mode `0600`. + +## Validate and start + +If this bundle contains `reconcile-pdns-password.sh`, a PowerDNS database password rotation is pending. Run that script on this DNS host before committing the rotation on the control plane. Normal rerenders do not rotate credentials. + +Run on **{node['name']}**: + +```sh +cd /opt/cdnfoundry +./validate.sh +{start_order} +docker compose --env-file .env.prod up -d +docker compose --env-file .env.prod ps +``` + +## Listeners + +{listeners} + +Restrict management, metrics, logging, database, and DNS API ports to their exact trusted source addresses. Databases must never be publicly reachable. + +## Health checks + +```sh +docker compose --env-file .env.prod ps +docker compose --env-file .env.prod logs --since 10m --no-color +``` + +## Upgrade and rollback + +Replace the bundle atomically, run `./validate.sh`, pull images, then use `docker compose --env-file .env.prod up -d`. If validation or startup fails, restore the `.previous` bundle and rerun the same command. Never use `docker compose down -v` and never delete PostgreSQL, Valkey, ClickHouse, Loki, Prometheus, Grafana, edge-state, cache, or MMDB volumes. + +## Cleanup + +After successful validation and the retention period, securely remove obsolete transferred archives. Do not remove the active directory, `.previous` rollback bundle, state volumes, or protected recovery copies. +""" + + def _listeners(self, node: dict[str, Any]) -> str: + rows = ["| Listener | Exposure |", "| --- | --- |"] + role = node["role"] + if role in {"dns", "dns-edge"}: + rows.append("| UDP/TCP 53 | Public authoritative DNS through DNSdist |") + rows.append("| TCP 8444 | Control-plane source addresses only |") + if role in {"edge", "dns-edge"}: + rows.append("| TCP 80/443 | Public customer traffic on configured service addresses |") + if role == "control": + rows.append("| TCP 80/443 | Public operator UI/API |") + rows.append("| TCP 8443 | Edge nodes only |") + rows.append("| TCP 9100 | Monitoring host/private monitoring network only |") + return "\n".join(rows) + + def _node_start_order(self, node: dict[str, Any]) -> str: + if node["role"] in {"dns", "dns-edge"}: + return ( + "docker compose --env-file .env.prod up -d --wait pdns-db\n" + "docker compose --env-file .env.prod --profile tools run --rm pdns-migrate" + ) + if node["role"] == "control": + dependencies = "redis" if self._uses_remote_control_db(node) else "control-db redis" + return ( + f"docker compose --env-file .env.prod up -d --wait {dependencies}\n" + "docker compose --env-file .env.prod --profile tools run --rm migrate" + ) + return "# No database migration is required for this role." + + def _validate_script(self) -> str: + return """#!/usr/bin/env sh +set -eu +umask 077 +test "$(stat -c '%a' .env.prod)" = 600 +test "$(stat -c '%a' pki/node.key)" = 600 +docker compose --env-file .env.prod config --quiet +openssl verify -CAfile pki/edge-server-ca.crt pki/node.crt +""" + + def _start_script(self, node: dict[str, Any]) -> str: + migration = self._node_start_order(node) + return f"""#!/usr/bin/env sh +set -eu +./validate.sh +{migration} +docker compose --env-file .env.prod up -d +docker compose --env-file .env.prod ps +""" + + def _write_manifest(self, root: Path, state: dict[str, Any], node: dict[str, Any]) -> None: + files = [] + for path in sorted(p for p in root.rglob("*") if p.is_file()): + relative = path.relative_to(root).as_posix() + files.append({"path": relative, "sha256": sha256_file(path), "mode": oct(path.stat().st_mode & 0o777)}) + metadata = { + "schema_version": 1, + "node": node["name"], + "role": node["role"], + "release": node["release"], + "rendered_at": utc_now(), + "fleet_generation": state["metadata"]["generation"], + "files": files, + } + atomic_json(root / "bundle-metadata.json", metadata, 0o600) + checksums = "".join(f"{entry['sha256']} {entry['path']}\n" for entry in files) + atomic_write(root / "SHA256SUMS", checksums, 0o600) + + def _render_fleet_files(self, state: dict[str, Any]) -> None: + if self.dry_run: + return + self.output_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + lines = ["# CDNFoundry fleet startup order", ""] + order = self.start_order(state) + for index, group in enumerate(order, 1): + lines.append(f"## {index}. {group['label']}") + lines.append("") + lines.append(", ".join(group["nodes"]) or "None configured") + lines.append("") + lines.extend( + [ + "Validate each node bundle before startup. Start stateful databases and migrations before dependent services. Existing DNS and HTTP serving nodes keep their last valid state when the control plane is unavailable.", + "", + ] + ) + atomic_write(self.output_dir / "STARTUP-ORDER.md", "\n".join(lines), 0o600) + + @staticmethod + def start_order(state: dict[str, Any]) -> list[dict[str, Any]]: + enabled = [node for node in state["nodes"].values() if node["enabled"]] + groups = [ + {"label": "Dedicated monitoring data services", "nodes": [n["name"] for n in enabled if n["role"] == "monitoring"]}, + {"label": "Control-plane database, Valkey, migrations, and control services", "nodes": [n["name"] for n in enabled if n["role"] == "control"]}, + {"label": "Local PostgreSQL, PowerDNS migrations, and authoritative DNS", "nodes": [n["name"] for n in enabled if n["role"] in {"dns", "dns-edge"}]}, + {"label": "Edge runtime and gateways", "nodes": [n["name"] for n in enabled if n["role"] in {"edge", "dns-edge"}]}, + {"label": "Monitoring exporters and centralized log collectors", "nodes": [n["name"] for n in enabled]}, + ] + return groups diff --git a/scripts/cdnfoundry_fleet/state.py b/scripts/cdnfoundry_fleet/state.py new file mode 100644 index 0000000..a07d53c --- /dev/null +++ b/scripts/cdnfoundry_fleet/state.py @@ -0,0 +1,499 @@ +from __future__ import annotations + +import copy +import fcntl +import os +import shutil +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator + +from .common import ( + StateError, + ValidationError, + atomic_json, + atomic_write, + ensure_mode, + load_json, + random_secret, + utc_now, + validate_env_mapping, + validate_hostname, + validate_ip, + validate_node_name, + validate_region, + validate_release, +) + +SCHEMA_VERSION = 1 +ROLES = {"control", "edge", "dns", "dns-edge", "monitoring"} +MONITORING_MODES = {"disabled", "colocated", "dedicated"} +LOG_MODES = {"disabled", "centralized"} +BACKUP_MODES = {"disabled", "control", "all-stateful"} + +GLOBAL_SECRET_NAMES = { + "app-key", + "artifact-signing-key", + "control-db-password", + "valkey-password", + "grafana-admin-password", + "grafana-clickhouse-password", + "grafana-postgres-password", + "clickhouse-password", + "metrics-token", + "telemetry-token", + "backup-password", + "backup-access-key", + "backup-secret-key", +} + +NODE_SECRET_NAMES = { + "pdns-db-password", + "pdns-api-key", + "edge-status-token", + "edge-bootstrap-token", + "log-auth-token", + "node-exporter-token", +} + + +class FleetState: + def __init__(self, state_dir: Path, *, dry_run: bool = False) -> None: + self.state_dir = state_dir.resolve() + self.state_file = self.state_dir / "fleet.json" + self.secrets_dir = self.state_dir / "secrets" + self.pki_dir = self.state_dir / "pki" + self.lock_file = self.state_dir / ".fleet.lock" + self.backup_dir = self.state_dir / "history" + self.dry_run = dry_run + + @contextmanager + def locked(self, *, exclusive: bool = True) -> Iterator[None]: + if self.dry_run and not self.state_dir.exists(): + yield + return + self.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + self.state_dir.chmod(0o700) + except PermissionError: + pass + fd = os.open(self.lock_file, os.O_CREAT | os.O_RDWR, 0o600) + try: + mode = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + try: + fcntl.flock(fd, mode | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise StateError("Another fleet generator process is already running") from exc + yield + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + + def exists(self) -> bool: + return self.state_file.exists() + + def load(self) -> dict[str, Any]: + state = load_json(self.state_file) + self.validate(state, require_secrets=False) + return state + + def init(self, config: dict[str, Any]) -> dict[str, Any]: + if self.exists(): + raise StateError(f"Fleet state already exists at {self.state_file}") + global_cfg = config.get("global", config) + now = utc_now() + state: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "global": { + "operator_domain": validate_hostname(global_cfg["operator_domain"]), + "platform_domain": validate_hostname(global_cfg["platform_domain"]), + "release": validate_release(global_cfg["release"]), + "acme_email": str(global_cfg.get("acme_email", "")), + "ipv6": bool(global_cfg.get("ipv6", False)), + }, + "features": { + "monitoring": {"mode": "disabled", "host": None}, + "logs": {"mode": "disabled", "host": None}, + "backups": {"mode": "disabled", "repository": None, "region": None}, + }, + "nodes": {}, + "metadata": { + "created_at": now, + "updated_at": now, + "last_successful_validation": None, + "last_successful_render": None, + "generation": 1, + }, + } + self.validate(state, require_secrets=False) + if not self.dry_run: + self._prepare_dirs() + self._ensure_global_secrets() + self._write(state, backup=False) + return state + + def _prepare_dirs(self) -> None: + for directory in (self.state_dir, self.secrets_dir, self.pki_dir, self.backup_dir): + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + ensure_mode(directory, 0o700) + + def _write(self, state: dict[str, Any], *, backup: bool = True) -> None: + self.validate(state, require_secrets=not self.dry_run) + if self.dry_run: + return + self._prepare_dirs() + if backup and self.state_file.exists(): + stamp = utc_now().replace(":", "").replace("+00:00", "Z") + target = self.backup_dir / f"fleet-{stamp}.json" + shutil.copy2(self.state_file, target) + target.chmod(0o600) + state["metadata"]["updated_at"] = utc_now() + state["metadata"]["generation"] = int(state["metadata"].get("generation", 0)) + 1 + atomic_json(self.state_file, state, 0o600) + self._prune_history(20) + + def _prune_history(self, keep: int) -> None: + entries = sorted(self.backup_dir.glob("fleet-*.json"), reverse=True) + for path in entries[keep:]: + path.unlink(missing_ok=True) + + def transaction(self) -> "StateTransaction": + return StateTransaction(self) + + def validate(self, state: dict[str, Any], *, require_secrets: bool = True) -> None: + if state.get("schema_version") != SCHEMA_VERSION: + raise ValidationError(f"Unsupported fleet schema version: {state.get('schema_version')!r}") + global_cfg = state.get("global", {}) + validate_hostname(global_cfg.get("operator_domain", "")) + validate_hostname(global_cfg.get("platform_domain", "")) + validate_release(global_cfg.get("release", "")) + features = state.get("features", {}) + monitoring = features.get("monitoring", {}) + logs = features.get("logs", {}) + backups = features.get("backups", {}) + if monitoring.get("mode") not in MONITORING_MODES: + raise ValidationError("Invalid monitoring mode") + if logs.get("mode") not in LOG_MODES: + raise ValidationError("Invalid logs mode") + if backups.get("mode") not in BACKUP_MODES: + raise ValidationError("Invalid backup mode") + + names: set[str] = set() + hostnames: set[str] = set() + ips: set[str] = set() + monitoring_targets: set[str] = set() + for key, node in state.get("nodes", {}).items(): + name = validate_node_name(key) + if node.get("name") != name: + raise ValidationError(f"Node key/name mismatch for {key}") + if name in names: + raise ValidationError(f"Duplicate node name: {name}") + names.add(name) + if node.get("role") not in ROLES: + raise ValidationError(f"Invalid role for {name}: {node.get('role')}") + validate_region(node.get("region", "")) + validate_region(node.get("location", ""), "location") + hostname = validate_hostname(node.get("hostname", "")) + if hostname in hostnames: + raise ValidationError(f"Duplicate hostname: {hostname}") + hostnames.add(hostname) + for field in ("public_ipv4", "public_ipv6", "bind_ipv4", "bind_ipv6", "monitor_ipv4", "log_ipv4"): + value = validate_ip(node.get(field), required=field in {"public_ipv4", "bind_ipv4"}) + if value and field in {"public_ipv4", "public_ipv6", "monitor_ipv4", "log_ipv4"}: + if value in ips: + raise ValidationError(f"Duplicate fleet IP address: {value}") + ips.add(value) + target = node.get("monitor_ipv4") or node.get("public_ipv4") + if target in monitoring_targets: + raise ValidationError(f"Duplicate monitoring target: {target}") + monitoring_targets.add(target) + validate_env_mapping(node.get("extra_env", {})) + if node.get("release"): + validate_release(node["release"]) + + if require_secrets: + self._validate_node_secrets(node) + + if monitoring.get("mode") == "dedicated": + host = monitoring.get("host") + if host not in names or state["nodes"][host]["role"] != "monitoring": + raise ValidationError("Dedicated monitoring requires an existing monitoring-role node") + if logs.get("mode") == "centralized" and logs.get("host") not in names: + raise ValidationError("Centralized logs require an existing log host") + if require_secrets: + for secret in GLOBAL_SECRET_NAMES: + path = self.secret_path(secret) + if not path.exists(): + raise ValidationError(f"Missing global secret: {secret}") + ensure_mode(path, 0o600) + + def _validate_node_secrets(self, node: dict[str, Any]) -> None: + required = {"node-exporter-token"} + if node["role"] in {"dns", "dns-edge"}: + required |= {"pdns-db-password", "pdns-api-key"} + if node["role"] in {"edge", "dns-edge"}: + required |= {"edge-status-token"} + if self.load_feature_mode("logs") == "centralized": + required |= {"log-auth-token"} + for secret in required: + path = self.secret_path(secret, node=node["name"]) + if not path.exists(): + raise ValidationError(f"Missing {secret} for node {node['name']}") + ensure_mode(path, 0o600) + + def load_feature_mode(self, feature: str) -> str: + try: + return load_json(self.state_file)["features"][feature]["mode"] + except Exception: + return "disabled" + + def add_node(self, state: dict[str, Any], node: dict[str, Any]) -> dict[str, Any]: + name = validate_node_name(node["name"]) + if name in state["nodes"]: + raise ValidationError(f"Node already exists: {name}") + clean = self._normalize_node(state, node) + candidate = copy.deepcopy(state) + candidate["nodes"][name] = clean + self.validate(candidate, require_secrets=False) + if not self.dry_run: + self._ensure_node_secrets(clean) + self._write(candidate) + return candidate + + def update_node(self, state: dict[str, Any], name: str, changes: dict[str, Any]) -> dict[str, Any]: + validate_node_name(name) + if name not in state["nodes"]: + raise ValidationError(f"Unknown node: {name}") + merged = copy.deepcopy(state["nodes"][name]) + merged.update({k: v for k, v in changes.items() if v is not None}) + merged["name"] = name + clean = self._normalize_node(state, merged) + candidate = copy.deepcopy(state) + candidate["nodes"][name] = clean + self.validate(candidate, require_secrets=False) + if not self.dry_run: + self._ensure_node_secrets(clean) + self._write(candidate) + return candidate + + def remove_node(self, state: dict[str, Any], name: str) -> dict[str, Any]: + validate_node_name(name) + if name not in state["nodes"]: + raise ValidationError(f"Unknown node: {name}") + candidate = copy.deepcopy(state) + candidate["nodes"].pop(name) + for feature in ("monitoring", "logs"): + if candidate["features"][feature].get("host") == name: + raise ValidationError(f"Node {name} is configured as the {feature} host") + self.validate(candidate, require_secrets=False) + self._write(candidate) + return candidate + + def _normalize_node(self, state: dict[str, Any], node: dict[str, Any]) -> dict[str, Any]: + role = node.get("role") + if role not in ROLES: + raise ValidationError(f"Invalid role: {role!r}") + name = validate_node_name(node["name"]) + operator_domain = state["global"]["operator_domain"] + hostname = node.get("hostname") or f"{name}.{operator_domain}" + return { + "name": name, + "role": role, + "region": validate_region(node.get("region", "global")), + "location": validate_region(node.get("location", node.get("region", "global")), "location"), + "hostname": validate_hostname(hostname), + "public_ipv4": validate_ip(node.get("public_ipv4"), required=True), + "public_ipv6": validate_ip(node.get("public_ipv6")), + "bind_ipv4": validate_ip(node.get("bind_ipv4") or "0.0.0.0", required=True), + "bind_ipv6": validate_ip(node.get("bind_ipv6") or ("::" if state["global"].get("ipv6") else None)), + "monitor_ipv4": validate_ip(node.get("monitor_ipv4")), + "log_ipv4": validate_ip(node.get("log_ipv4")), + "release": validate_release(node.get("release") or state["global"]["release"]), + "extra_env": validate_env_mapping(node.get("extra_env", {})), + "enabled": bool(node.get("enabled", True)), + "draining": bool(node.get("draining", False)), + "health": { + "failure_threshold": int(node.get("health", {}).get("failure_threshold", 3)), + "success_threshold": int(node.get("health", {}).get("success_threshold", 2)), + "stale_after_seconds": int(node.get("health", {}).get("stale_after_seconds", 90)), + }, + } + + def configure_feature(self, state: dict[str, Any], feature: str, config: dict[str, Any]) -> dict[str, Any]: + candidate = copy.deepcopy(state) + if feature == "monitoring": + mode = config["mode"] + if mode not in MONITORING_MODES: + raise ValidationError("Invalid monitoring mode") + candidate["features"][feature] = {"mode": mode, "host": config.get("host")} + elif feature == "logs": + mode = config["mode"] + if mode not in LOG_MODES: + raise ValidationError("Invalid logs mode") + candidate["features"][feature] = { + "mode": mode, + "host": config.get("host"), + "endpoint": config.get("endpoint"), + } + if mode == "centralized" and not self.dry_run: + for node in candidate["nodes"].values(): + self.ensure_secret("log-auth-token", node=node["name"]) + elif feature == "backups": + mode = config["mode"] + if mode not in BACKUP_MODES: + raise ValidationError("Invalid backup mode") + candidate["features"][feature] = { + "mode": mode, + "repository": config.get("repository"), + "region": config.get("region") or "us-east-1", + } + else: + raise ValidationError(f"Unknown feature: {feature}") + self.validate(candidate, require_secrets=False) + self._write(candidate) + return candidate + + def secret_path(self, name: str, *, node: str | None = None) -> Path: + if node: + validate_node_name(node) + return self.secrets_dir / "nodes" / node / name + return self.secrets_dir / "global" / name + + def ensure_secret(self, name: str, *, node: str | None = None, value: str | None = None) -> Path: + if node: + if name not in NODE_SECRET_NAMES: + raise ValidationError(f"Unsupported node secret: {name}") + elif name not in GLOBAL_SECRET_NAMES: + raise ValidationError(f"Unsupported global secret: {name}") + path = self.secret_path(name, node=node) + if path.exists(): + ensure_mode(path, 0o600) + return path + if self.dry_run: + return path + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + path.parent.chmod(0o700) + generated = value or ("base64:" + random_secret(32) if name == "app-key" else random_secret(32)) + atomic_write(path, generated + "\n", 0o600) + return path + + def read_secret(self, name: str, *, node: str | None = None) -> str: + path = self.secret_path(name, node=node) + try: + ensure_mode(path, 0o600) + return path.read_text(encoding="utf-8").rstrip("\n") + except FileNotFoundError as exc: + raise StateError(f"Missing secret {name}{' for ' + node if node else ''}") from exc + + def write_secret(self, name: str, value: str, *, node: str | None = None) -> Path: + if node: + if name not in NODE_SECRET_NAMES: + raise ValidationError(f"Unsupported node secret: {name}") + validate_node_name(node) + elif name not in GLOBAL_SECRET_NAMES: + raise ValidationError(f"Unsupported global secret: {name}") + clean = value.rstrip("\r\n") + if not clean: + raise ValidationError("Secret value must not be empty") + path = self.secret_path(name, node=node) + if self.dry_run: + return path + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + path.parent.chmod(0o700) + atomic_write(path, clean + "\n", 0o600) + return path + + def delete_secret(self, name: str, *, node: str | None = None) -> None: + path = self.secret_path(name, node=node) + if not self.dry_run: + path.unlink(missing_ok=True) + + def pending_secret_path(self, name: str, *, node: str) -> Path: + if name not in NODE_SECRET_NAMES: + raise ValidationError(f"Unsupported node secret: {name}") + validate_node_name(node) + return self.secrets_dir / "pending" / "nodes" / node / name + + def prepare_secret_rotation(self, name: str, *, node: str) -> Path: + current = self.secret_path(name, node=node) + if not current.exists(): + raise ValidationError(f"Secret does not exist: {name}") + pending = self.pending_secret_path(name, node=node) + if pending.exists(): + raise ValidationError(f"A pending rotation already exists for {name} on {node}") + if self.dry_run: + return pending + pending.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + pending.parent.chmod(0o700) + atomic_write(pending, random_secret(32) + "\n", 0o600) + return pending + + def commit_secret_rotation(self, name: str, *, node: str) -> None: + current = self.secret_path(name, node=node) + pending = self.pending_secret_path(name, node=node) + if not current.exists(): + raise ValidationError(f"Secret does not exist: {name}") + if not pending.exists(): + raise ValidationError(f"No pending rotation exists for {name} on {node}") + if self.dry_run: + return + old = current.read_text(encoding="utf-8") + new = pending.read_text(encoding="utf-8") + archive = current.parent / ".previous" + archive.mkdir(parents=True, exist_ok=True, mode=0o700) + archive.chmod(0o700) + atomic_write(archive / f"{name}-{utc_now().replace(':', '')}", old, 0o600) + atomic_write(current, new, 0o600) + pending.unlink() + + def abort_secret_rotation(self, name: str, *, node: str) -> None: + pending = self.pending_secret_path(name, node=node) + if not pending.exists(): + raise ValidationError(f"No pending rotation exists for {name} on {node}") + if not self.dry_run: + pending.unlink() + + def rotate_secret(self, name: str, *, node: str | None = None) -> None: + path = self.secret_path(name, node=node) + if not path.exists(): + raise ValidationError(f"Secret does not exist: {name}") + if self.dry_run: + return + old = path.read_text(encoding="utf-8") + archive = path.parent / ".previous" + archive.mkdir(parents=True, exist_ok=True, mode=0o700) + archive.chmod(0o700) + atomic_write(archive / f"{name}-{utc_now().replace(':', '')}", old, 0o600) + value = "base64:" + random_secret(32) if name == "app-key" else random_secret(32) + atomic_write(path, value + "\n", 0o600) + + def _ensure_global_secrets(self) -> None: + for name in sorted(GLOBAL_SECRET_NAMES): + self.ensure_secret(name) + + def _ensure_node_secrets(self, node: dict[str, Any]) -> None: + self.ensure_secret("node-exporter-token", node=node["name"]) + if node["role"] in {"dns", "dns-edge"}: + self.ensure_secret("pdns-db-password", node=node["name"]) + self.ensure_secret("pdns-api-key", node=node["name"]) + if node["role"] in {"edge", "dns-edge"}: + self.ensure_secret("edge-status-token", node=node["name"]) + if self.load_feature_mode("logs") == "centralized": + self.ensure_secret("log-auth-token", node=node["name"]) + + +class StateTransaction: + def __init__(self, store: FleetState) -> None: + self.store = store + self.original: dict[str, Any] | None = None + self.candidate: dict[str, Any] | None = None + + def __enter__(self) -> dict[str, Any]: + self.original = self.store.load() + self.candidate = copy.deepcopy(self.original) + return self.candidate + + def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: + if exc_type is None and self.candidate is not None: + self.store._write(self.candidate) + return False diff --git a/scripts/install-production-prerequisites.sh b/scripts/install-production-prerequisites.sh new file mode 100755 index 0000000..ed0eb46 --- /dev/null +++ b/scripts/install-production-prerequisites.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env sh +set -eu + +ready() { + command -v python3 >/dev/null 2>&1 \ + && python3 -c 'import yaml' >/dev/null 2>&1 \ + && command -v openssl >/dev/null 2>&1 \ + && command -v docker >/dev/null 2>&1 \ + && docker compose version >/dev/null 2>&1 +} +ready && exit 0 +[ "$(id -u)" -eq 0 ] || { echo "Run this prerequisite installer as root." >&2; exit 1; } + +if command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3 python3-yaml openssl ca-certificates curl docker.io + if ! docker compose version >/dev/null 2>&1; then + DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-v2 \ + || DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin + fi +elif command -v dnf >/dev/null 2>&1; then + dnf install -y python3 python3-pyyaml openssl ca-certificates curl docker docker-compose-plugin +elif command -v apk >/dev/null 2>&1; then + apk add --no-cache python3 py3-yaml openssl ca-certificates curl docker docker-cli-compose +else + echo "Unsupported package manager. Install Python 3, PyYAML, OpenSSL, Docker Engine, and Docker Compose v2." >&2 + exit 1 +fi + +if command -v systemctl >/dev/null 2>&1; then + systemctl enable --now docker +elif command -v rc-update >/dev/null 2>&1; then + rc-update add docker default || true + rc-service docker start || true +fi + +ready || { + echo "Prerequisite installation is incomplete. Install Docker Engine and Docker Compose v2 from a supported package repository." >&2 + exit 1 +} diff --git a/tests/fleet/test_fleet.py b/tests/fleet/test_fleet.py new file mode 100644 index 0000000..533ca89 --- /dev/null +++ b/tests/fleet/test_fleet.py @@ -0,0 +1,950 @@ +from __future__ import annotations + +import argparse +import json +import os +import stat +import sys +from pathlib import Path + +import pytest +import yaml + +REPO_PATCH = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_PATCH / "scripts")) + +from cdnfoundry_fleet.common import ValidationError +from cdnfoundry_fleet.render import Renderer +from cdnfoundry_fleet.state import FleetState + + +@pytest.fixture() +def source_repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + (root / "deploy/production").mkdir(parents=True) + (root / "docker/postgres").mkdir(parents=True) + (root / "docker/pdns").mkdir(parents=True) + (root / "docker/dnsdist").mkdir(parents=True) + (root / "docker/mmdb").mkdir(parents=True) + (root / "docker/prometheus").mkdir(parents=True) + (root / "docker/vector").mkdir(parents=True) + for path in [ + "docker/postgres/pdns-schema.sql", + "docker/pdns/pdns.conf", + "docker/dnsdist/dnsdist.conf", + "docker/prometheus/prometheus.yml", + "docker/vector/operational.yaml", + ]: + target = root / path + target.write_text("# fixture\n", encoding="utf-8") + + compose = { + "services": { + "control-db": { + "image": "postgres:18-alpine", + "profiles": ["control"], + "environment": {"POSTGRES_PASSWORD": "${CONTROL_DB_PASSWORD:?required}"}, + "volumes": ["control-db:/var/lib/postgresql"], + }, + "redis": { + "image": "valkey:9-alpine", + "profiles": ["control"], + "command": ["valkey-server", "--requirepass", "${REDIS_PASSWORD:?required}"], + "volumes": ["redis:/data"], + }, + "core": { + "image": "core:${CDNF_RELEASE:?required}", + "profiles": ["control"], + "environment": { + "APP_KEY": "${APP_KEY:?required}", + "EDGE_ARTIFACT_SIGNING_KEY": "${EDGE_ARTIFACT_SIGNING_KEY:?required}", + "EDGE_IDENTITY_CA_CERTIFICATE": "${EDGE_IDENTITY_CA_CERTIFICATE:?required}", + "EDGE_IDENTITY_CA_PRIVATE_KEY": "${EDGE_IDENTITY_CA_PRIVATE_KEY:?required}", + "PDNS_CA_CERTIFICATE": "${PDNS_CA_CERTIFICATE:?required}", + }, + "volumes": [ + "${EDGE_IDENTITY_CA_CERTIFICATE:?required}:/run/pki/edge-identity-ca.crt:ro", + "${EDGE_IDENTITY_CA_PRIVATE_KEY:?required}:/run/pki/edge-identity-ca.key:ro", + "${PDNS_CA_CERTIFICATE:?required}:/run/pki/pdns-ca.crt:ro", + ], + "depends_on": {"control-db": {"condition": "service_started"}, "redis": {"condition": "service_started"}}, + }, + "edge-control": { + "image": "edge-control:${CDNF_RELEASE:?required}", + "profiles": ["control"], + "environment": { + "EDGE_CONTROL_SERVER_CERTIFICATE": "${EDGE_CONTROL_SERVER_CERTIFICATE:?required}", + "EDGE_CONTROL_SERVER_PRIVATE_KEY": "${EDGE_CONTROL_SERVER_PRIVATE_KEY:?required}", + }, + "volumes": [ + "${EDGE_CONTROL_SERVER_CERTIFICATE:?required}:/run/pki/server.crt:ro", + "${EDGE_CONTROL_SERVER_PRIVATE_KEY:?required}:/run/pki/server.key:ro", + ], + }, + "migrate": { + "image": "core:${CDNF_RELEASE:?required}", + "profiles": ["tools"], + "environment": {"CONTROL_DB_PASSWORD": "${CONTROL_DB_PASSWORD:?required}"}, + }, + "pdns-db": { + "image": "postgres:18-alpine", + "profiles": ["dns"], + "environment": { + "POSTGRES_DB": "pdns", + "POSTGRES_USER": "pdns", + "POSTGRES_PASSWORD": "${PDNS_DB_PASSWORD:?required}", + }, + "volumes": ["pdns-db:/var/lib/postgresql", "./docker/postgres/pdns-schema.sql:/init.sql:ro"], + }, + "mmdb-updater": { + "image": "mmdb:${CDNF_RELEASE:?required}", + "profiles": ["dns", "edge"], + "environment": {"MMDB_PROVIDER": "${MMDB_PROVIDER:-dbip-jsdelivr}"}, + "volumes": ["mmdb:/mmdb"], + }, + "pdns-auth": { + "image": "pdns:5", + "profiles": ["dns"], + "environment": { + "PDNS_gpgsql_password": "${PDNS_DB_PASSWORD:?required}", + "PDNS_api_key": "${PDNS_API_KEY:?required}", + "DNS_API_SERVER_CERTIFICATE": "${DNS_API_SERVER_CERTIFICATE:?required}", + "DNS_API_SERVER_PRIVATE_KEY": "${DNS_API_SERVER_PRIVATE_KEY:?required}", + }, + "depends_on": {"pdns-db": {"condition": "service_started"}, "mmdb-updater": {"condition": "service_started"}}, + "volumes": ["./docker/pdns/pdns.conf:/etc/pdns.conf:ro", "mmdb:/mmdb:ro"], + }, + "pdns-migrate": { + "image": "postgres:18-alpine", + "profiles": ["tools"], + "environment": {"PGPASSWORD": "${PDNS_DB_PASSWORD:?required}"}, + "volumes": ["./docker/postgres/pdns-schema.sql:/migration.sql:ro"], + }, + "dnsdist": { + "image": "dnsdist:2", + "profiles": ["dns"], + "depends_on": {"pdns-auth": {"condition": "service_started"}}, + "ports": ["${DNS_BIND_V4:-0.0.0.0}:53:53/udp"], + "volumes": ["./docker/dnsdist/dnsdist.conf:/etc/dnsdist.conf:ro"], + }, + "edge-agent": { + "image": "edge:${CDNF_RELEASE:?required}", + "profiles": ["edge"], + "environment": { + "EDGE_STATUS_TOKEN": "${EDGE_STATUS_TOKEN:?required}", + "EDGE_CONTROL_URL": "${EDGE_CONTROL_URL:?required}", + "EDGE_CONTROL_CA_CERTIFICATE": "${EDGE_CONTROL_CA_CERTIFICATE:?required}", + }, + "volumes": [ + "${EDGE_CONTROL_CA_CERTIFICATE:?required}:/run/edge-control-ca.crt:ro", + "${EDGE_RUNTIME_TLS_CERTIFICATE:?required}:/run/node.crt:ro", + "${EDGE_RUNTIME_TLS_PRIVATE_KEY:?required}:/run/node.key:ro", + ], + }, + "cell-01": { + "image": "cell:${CDNF_RELEASE:?required}", + "profiles": ["edge"], + "depends_on": {"mmdb-updater": {"condition": "service_started"}}, + "volumes": ["mmdb:/mmdb:ro", "cell-01-cache:/cache"], + }, + "vector": { + "image": "vector:latest", + "profiles": ["telemetry", "edge"], + "environment": { + "CLICKHOUSE_ENDPOINT": "${CLICKHOUSE_URL:?required}", + "CLICKHOUSE_PASSWORD": "${CLICKHOUSE_PASSWORD:?required}", + }, + }, + "clickhouse": { + "image": "clickhouse:latest", + "profiles": ["telemetry"], + "environment": {"CLICKHOUSE_PASSWORD": "${CLICKHOUSE_PASSWORD:?required}"}, + "volumes": ["clickhouse:/var/lib/clickhouse"], + }, + "prometheus": { + "image": "prometheus:latest", + "profiles": ["telemetry"], + "volumes": ["./docker/prometheus/prometheus.yml:/etc/prometheus.yml:ro", "prometheus:/prometheus"], + }, + "node-exporter": { + "image": "node-exporter:latest", + "profiles": ["telemetry"], + }, + "log-collector": { + "image": "vector:latest", + "profiles": ["logs"], + "environment": { + "LOG_ROLE": "${LOG_ROLE:?required}", + "LOG_HOST": "${LOG_HOST:?required}", + "LOG_COLLECTOR_ID": "${LOG_COLLECTOR_ID:?required}", + "LOKI_ENDPOINT": "${LOKI_ENDPOINT:?required}", + "LOG_AUTH_TOKEN": "${LOG_AUTH_TOKEN:?required}", + }, + "volumes": ["./docker/vector/operational.yaml:/etc/vector/operational.yaml:ro"], + }, + }, + "volumes": { + "control-db": {}, + "redis": {}, + "pdns-db": {}, + "mmdb": {}, + "cell-01-cache": {}, + "clickhouse": {}, + "prometheus": {}, + }, + } + (root / "compose.prod.yml").write_text(yaml.safe_dump(compose, sort_keys=False), encoding="utf-8") + for name in [ + "compose.control-host.yml", + "compose.dns-host.yml", + "compose.edge-host.yml", + "compose.dns-edge-host.yml", + "compose.telemetry-host.yml", + "compose.external-control-data.yml", + "compose.host-journal.yml", + ]: + (root / "deploy/production" / name).write_text("services: {}\n", encoding="utf-8") + return root + + +@pytest.fixture() +def store(tmp_path: Path) -> FleetState: + state = FleetState(tmp_path / "state") + with state.locked(): + state.init( + { + "operator_domain": "ops.example.com", + "platform_domain": "example.net", + "release": "v1.0.0", + "acme_email": "ops@example.com", + } + ) + return state + + +def node(name: str, role: str, ip: str, *, region: str = "eu", location: str | None = None) -> dict[str, object]: + return { + "name": name, + "role": role, + "region": region, + "location": location or name, + "public_ipv4": ip, + "bind_ipv4": "0.0.0.0", + } + + +def add(store: FleetState, payload: dict[str, object]) -> dict[str, object]: + with store.locked(): + return store.add_node(store.load(), payload) + + +def env_values(path: Path) -> dict[str, str]: + return dict(line.split("=", 1) for line in path.read_text(encoding="utf-8").splitlines() if line) + + +def test_dns_hosts_get_unique_stable_local_database_credentials(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("dns-ashburn", "dns", "192.0.2.11")) + add(store, node("dns-frankfurt", "dns", "192.0.2.12")) + output = tmp_path / "bundles" + renderer = Renderer(source_repo, store, output) + renderer.render(store.load()) + + first = env_values(output / "dns-ashburn/.env.prod") + second = env_values(output / "dns-frankfurt/.env.prod") + assert first["PDNS_DB_PASSWORD"] != second["PDNS_DB_PASSWORD"] + assert first["PDNS_API_KEY"] != second["PDNS_API_KEY"] + + renderer.render(store.load()) + rerendered = env_values(output / "dns-ashburn/.env.prod") + assert rerendered["PDNS_DB_PASSWORD"] == first["PDNS_DB_PASSWORD"] + assert rerendered["PDNS_API_KEY"] == first["PDNS_API_KEY"] + + +def test_dns_bundle_uses_local_postgres_and_has_no_control_secrets(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("dns-singapore", "dns", "192.0.2.20")) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load()) + compose = yaml.safe_load((output / "dns-singapore/compose.yml").read_text(encoding="utf-8")) + env = env_values(output / "dns-singapore/.env.prod") + + assert "pdns-db" in compose["services"] + assert "pdns-auth" in compose["services"] + assert "control-db" not in compose["services"] + assert compose["services"]["pdns-auth"]["environment"]["PDNS_gpgsql_host"] == "pdns-db" + assert "PDNS_DB_PASSWORD" in env + assert "CONTROL_DB_PASSWORD" not in env + assert "APP_KEY" not in env + assert "GRAFANA_ADMIN_PASSWORD" not in env + + +def test_disabled_monitoring_does_not_require_clickhouse_credentials_on_edge(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("edge-dubai", "edge", "192.0.2.30")) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load()) + compose = yaml.safe_load((output / "edge-dubai/compose.yml").read_text(encoding="utf-8")) + env = env_values(output / "edge-dubai/.env.prod") + assert "vector" not in compose["services"] + assert "CLICKHOUSE_PASSWORD" not in env + + +def test_duplicate_ip_and_malicious_node_input_are_rejected(store: FleetState) -> None: + add(store, node("edge-one", "edge", "192.0.2.40")) + with pytest.raises(ValidationError, match="Duplicate fleet IP"): + add(store, node("edge-two", "edge", "192.0.2.40")) + with pytest.raises(ValidationError): + add(store, node("bad;rm-rf", "edge", "192.0.2.41")) + + +def test_failed_update_preserves_previous_valid_state(store: FleetState) -> None: + add(store, node("edge-one", "edge", "192.0.2.50")) + before = store.state_file.read_bytes() + with store.locked(): + with pytest.raises(ValidationError): + store.update_node(store.load(), "edge-one", {"public_ipv4": "not-an-ip"}) + assert store.state_file.read_bytes() == before + + +def test_monitoring_targets_cover_every_host_and_update_after_removal(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("control-1", "control", "192.0.2.60")) + add(store, node("monitor-1", "monitoring", "192.0.2.61")) + add(store, node("dns-1", "dns", "192.0.2.62")) + add(store, node("edge-1", "edge", "192.0.2.63")) + with store.locked(): + store.configure_feature(store.load(), "monitoring", {"mode": "dedicated", "host": "monitor-1"}) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load()) + targets = yaml.safe_load((output / "monitor-1/generated/prometheus-node-targets.yml").read_text(encoding="utf-8")) + assert {group["labels"]["node"] for group in targets} == {"control-1", "monitor-1", "dns-1", "edge-1"} + + with store.locked(): + store.remove_node(store.load(), "edge-1") + Renderer(source_repo, store, output).render(store.load(), node_name="monitor-1") + targets = yaml.safe_load((output / "monitor-1/generated/prometheus-node-targets.yml").read_text(encoding="utf-8")) + assert {group["labels"]["node"] for group in targets} == {"control-1", "monitor-1", "dns-1"} + + +def test_multi_region_four_dns_ten_edge_topology_validates(store: FleetState) -> None: + for index, location in enumerate(["ashburn", "frankfurt", "singapore", "sao-paulo"], 1): + add(store, node(f"dns-{location}", "dns", f"192.0.2.{70 + index}", region=f"r{index}", location=location)) + edges = ["ashburn", "los-angeles", "sao-paulo", "frankfurt", "johannesburg", "dubai", "mumbai", "singapore", "tokyo", "sydney"] + for index, location in enumerate(edges, 1): + add(store, node(f"edge-{location}", "edge", f"198.51.100.{index}", region=f"r{index % 4}", location=location)) + state = store.load() + store.validate(state) + assert len([n for n in state["nodes"].values() if n["role"] == "dns"]) == 4 + assert len([n for n in state["nodes"].values() if n["role"] == "edge"]) == 10 + + +def test_file_permissions_and_redacted_metadata(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("dns-sydney", "dns", "192.0.2.90")) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load()) + bundle = output / "dns-sydney" + assert stat.S_IMODE((bundle / ".env.prod").stat().st_mode) == 0o600 + assert stat.S_IMODE((bundle / "pki/node.key").stat().st_mode) == 0o600 + assert stat.S_IMODE(store.state_file.stat().st_mode) == 0o600 + metadata = (bundle / "bundle-metadata.json").read_text(encoding="utf-8") + pdns_password = env_values(bundle / ".env.prod")["PDNS_DB_PASSWORD"] + assert pdns_password not in metadata + + +def test_dry_run_does_not_create_state_or_secrets(tmp_path: Path) -> None: + store = FleetState(tmp_path / "dry-state", dry_run=True) + state = store.init({"operator_domain": "ops.example.com", "platform_domain": "example.net", "release": "v1.0.0"}) + assert state["schema_version"] == 1 + assert not store.state_dir.exists() + + +def test_explicit_rotation_changes_only_target_dns_node(store: FleetState) -> None: + add(store, node("dns-one", "dns", "192.0.2.101")) + add(store, node("dns-two", "dns", "192.0.2.102")) + one_before = store.read_secret("pdns-db-password", node="dns-one") + two_before = store.read_secret("pdns-db-password", node="dns-two") + store.rotate_secret("pdns-db-password", node="dns-one") + assert store.read_secret("pdns-db-password", node="dns-one") != one_before + assert store.read_secret("pdns-db-password", node="dns-two") == two_before + + +def test_geo_policy_records_independent_address_families_and_flap_thresholds(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, {**node("edge-tokyo", "edge", "192.0.2.110"), "public_ipv6": "2001:db8::110"}) + add(store, node("monitor-1", "monitoring", "192.0.2.111")) + with store.locked(): + store.configure_feature(store.load(), "monitoring", {"mode": "dedicated", "host": "monitor-1"}) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load(), node_name="monitor-1") + policy = json.loads((output / "monitor-1/generated/geo-routing-policy.json").read_text(encoding="utf-8")) + assert policy["address_families"] == "independent" + assert policy["decision_order"][0] == "valid_ecs_client_subnet" + edge = policy["edges"][0] + assert edge["failure_threshold"] == 3 + assert edge["success_threshold"] == 2 + + +def test_cli_parser_exposes_all_commands_and_help() -> None: + from cdnfoundry_fleet.cli import parser + + cli = parser() + commands = { + "setup", + "status", + "doctor", + "init", + "add-node", + "update-node", + "configure-edge-registration", + "clear-edge-bootstrap-token", + "set-secret", + "remove-node", + "list-nodes", + "configure-monitoring", + "configure-logs", + "configure-backups", + "render", + "validate", + "show-start-order", + "adopt-existing", + "rotate-secret", + } + subparsers = next(action for action in cli._actions if isinstance(action, argparse._SubParsersAction)) + assert set(subparsers.choices) == commands + args = cli.parse_args(["update-node", "--node", "dns-one", "--location", "New Location"]) + assert args.command == "update-node" + assert args.node == "dns-one" + assert args.location == "New Location" + + +@pytest.mark.parametrize( + "command", + [ + "setup", + "status", + "doctor", + "init", + "add-node", + "update-node", + "configure-edge-registration", + "clear-edge-bootstrap-token", + "set-secret", + "remove-node", + "list-nodes", + "configure-monitoring", + "configure-logs", + "configure-backups", + "render", + "validate", + "show-start-order", + "adopt-existing", + "rotate-secret", + ], +) +def test_every_command_supports_help(command: str) -> None: + from cdnfoundry_fleet.cli import parser + + with pytest.raises(SystemExit) as exc: + parser().parse_args([command, "--help"]) + assert exc.value.code == 0 + + +def test_pdns_rotation_is_prepared_reconciled_and_committed(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("dns-rotate", "dns", "192.0.2.120")) + current = store.read_secret("pdns-db-password", node="dns-rotate") + pending = store.prepare_secret_rotation("pdns-db-password", node="dns-rotate") + assert pending.exists() + assert pending.read_text(encoding="utf-8").strip() != current + assert store.read_secret("pdns-db-password", node="dns-rotate") == current + + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load(), node_name="dns-rotate") + bundle = output / "dns-rotate" + assert (bundle / "reconcile-pdns-password.sh").exists() + assert stat.S_IMODE((bundle / "reconcile-pdns-password.sh").stat().st_mode) == 0o700 + assert (bundle / "secrets/pdns-db-password.next").read_text(encoding="utf-8") == pending.read_text(encoding="utf-8") + assert env_values(bundle / ".env.prod")["PDNS_DB_PASSWORD"] == current + + next_value = pending.read_text(encoding="utf-8").strip() + store.commit_secret_rotation("pdns-db-password", node="dns-rotate") + assert store.read_secret("pdns-db-password", node="dns-rotate") == next_value + assert not pending.exists() + + +def test_pdns_rotation_can_be_aborted_without_changing_current_secret(store: FleetState) -> None: + add(store, node("dns-abort", "dns", "192.0.2.121")) + current = store.read_secret("pdns-db-password", node="dns-abort") + pending = store.prepare_secret_rotation("pdns-db-password", node="dns-abort") + store.abort_secret_rotation("pdns-db-password", node="dns-abort") + assert not pending.exists() + assert store.read_secret("pdns-db-password", node="dns-abort") == current + + +def test_disabled_monitoring_bundle_does_not_receive_metrics_secret(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("edge-no-metrics", "edge", "192.0.2.122")) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load()) + assert not (output / "edge-no-metrics/secrets/metrics-token").exists() + + +def test_common_cli_options_work_before_or_after_subcommand() -> None: + from cdnfoundry_fleet.cli import parser + + before = parser().parse_args( + [ + "--state-dir", + "/tmp/state-before", + "--repo-root", + "/tmp/repo-before", + "validate", + ] + ) + after = parser().parse_args( + [ + "validate", + "--state-dir", + "/tmp/state-after", + "--repo-root", + "/tmp/repo-after", + ] + ) + assert before.state_dir == "/tmp/state-before" + assert before.repo_root == "/tmp/repo-before" + assert after.state_dir == "/tmp/state-after" + assert after.repo_root == "/tmp/repo-after" + + +def test_cli_end_to_end_dns_nodes_keep_separate_local_credentials(source_repo: Path, tmp_path: Path) -> None: + import subprocess + + cli = Path(__file__).resolve().parents[2] / "scripts/cdnfoundry-fleet" + state_dir = tmp_path / "cli-state" + output_dir = tmp_path / "cli-bundles" + + def run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + str(cli), + "--state-dir", + str(state_dir), + "--output-dir", + str(output_dir), + "--repo-root", + str(source_repo), + *args, + ], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + run( + "init", + "--operator-domain", + "ops.example.com", + "--platform-domain", + "example.net", + "--release", + "v1.0.0", + "--non-interactive", + ) + run( + "add-node", + "--node", + "dns-one", + "--role", + "dns", + "--region", + "eu", + "--location", + "frankfurt", + "--public-ipv4", + "192.0.2.130", + "--non-interactive", + ) + run( + "add-node", + "--node", + "dns-two", + "--role", + "dns", + "--region", + "asia", + "--location", + "singapore", + "--public-ipv4", + "192.0.2.131", + "--non-interactive", + ) + run("validate") + run("render") + + one = env_values(output_dir / "dns-one/.env.prod") + two = env_values(output_dir / "dns-two/.env.prod") + assert one["PDNS_DB_PASSWORD"] != two["PDNS_DB_PASSWORD"] + assert "CONTROL_DB_PASSWORD" not in one + assert "CONTROL_DB_PASSWORD" not in two + + +def test_production_docs_match_generated_bundle_workflow() -> None: + root = Path(__file__).resolve().parents[2] + quick = (root / "docs/deployment/production-quick-start.md").read_text(encoding="utf-8") + reference = (root / "docs/deployment/production-fleet.md").read_text(encoding="utf-8") + assert reference.count("```mermaid") >= 7 + assert "git clone https://github.com/vaheed/CDNFoundry.git" in quick + assert "starter-fleet.json" in quick + assert "--config fleet.json" in quick + assert "own local PostgreSQL" in reference + assert "never uses the control-plane" in reference + assert "docker compose down -v" in quick + + +def test_multi_region_json_example_builds_complete_fleet(source_repo: Path, tmp_path: Path) -> None: + import shutil + import subprocess + + patch_root = Path(__file__).resolve().parents[2] + shutil.copytree(patch_root / "scripts", source_repo / "scripts", dirs_exist_ok=True) + example_target = source_repo / "deploy/production/examples/multi-region-fleet.json" + example_target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(patch_root / "deploy/production/examples/multi-region-fleet.json", example_target) + state_dir = tmp_path / "multi-region-state" + subprocess.run( + [str(source_repo / "scripts/cdnfoundry-fleet"), "--config", str(example_target), + "--state-dir", str(state_dir), "--output-dir", str(state_dir / "bundles"), + "--repo-root", str(source_repo), "--non-interactive", "setup"], + cwd=source_repo, check=True, text=True, capture_output=True, + ) + state = json.loads((state_dir / "fleet.json").read_text(encoding="utf-8")) + assert len(state["nodes"]) == 18 + assert len([n for n in state["nodes"].values() if n["role"] == "dns"]) == 4 + assert len([n for n in state["nodes"].values() if n["role"] == "edge"]) == 10 + assert len([n for n in state["nodes"].values() if n["role"] == "monitoring"]) == 3 + assert len([p for p in (state_dir / "bundles").iterdir() if p.is_dir() and not p.name.endswith(".previous")]) == 18 + + + +def test_compose_loader_supports_reset_and_override_tags(tmp_path: Path) -> None: + from cdnfoundry_fleet.compose import load_yaml, merge_compose + + base_path = tmp_path / "base.yml" + overlay_path = tmp_path / "overlay.yml" + base_path.write_text( + "services:\n app:\n command: [old]\n ports: [80:80]\n environment:\n OLD: value\n", + encoding="utf-8", + ) + overlay_path.write_text( + "services:\n app:\n command: !override [new]\n ports: !reset []\n environment: !override\n NEW: value\n", + encoding="utf-8", + ) + merged = merge_compose(load_yaml(base_path), load_yaml(overlay_path)) + assert merged["services"]["app"]["command"] == ["new"] + assert merged["services"]["app"]["ports"] == [] + assert merged["services"]["app"]["environment"] == {"NEW": "value"} + + +def test_control_monitoring_bundle_uses_project_pki_contract(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("control-1", "control", "192.0.2.140")) + with store.locked(): + store.configure_feature(store.load(), "monitoring", {"mode": "colocated", "host": None}) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load()) + bundle = output / "control-1" + env = env_values(bundle / ".env.prod") + assert env["EDGE_IDENTITY_CA_CERTIFICATE"] == "./pki/edge-identity-ca.crt" + assert env["EDGE_IDENTITY_CA_PRIVATE_KEY"] == "./pki/edge-identity-ca.key" + assert env["PDNS_CA_CERTIFICATE"] == "./pki/edge-server-ca.crt" + assert env["EDGE_CONTROL_SERVER_CERTIFICATE"] == "./pki/node.crt" + assert (bundle / "pki/edge-identity-ca.crt").exists() + assert (bundle / "pki/edge-identity-ca.key").exists() + assert (bundle / "pki/edge-server-ca.crt").exists() + compose = yaml.safe_load((bundle / "compose.yml").read_text(encoding="utf-8")) + assert "core" in compose["services"] + assert "clickhouse" in compose["services"] + assert "prometheus" in compose["services"] + + +def test_edge_bundle_has_control_url_and_server_ca(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("control-1", "control", "192.0.2.141")) + add(store, node("edge-1", "edge", "192.0.2.142")) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load(), node_name="edge-1") + env = env_values(output / "edge-1/.env.prod") + assert env["EDGE_CONTROL_URL"] == "https://control-1.ops.example.com:8443" + assert env["EDGE_CONTROL_CA_CERTIFICATE"] == "./pki/edge-server-ca.crt" + assert (output / "edge-1/pki/edge-server-ca.crt").exists() + + +def test_setup_command_generates_control_monitoring_fleet(source_repo: Path, tmp_path: Path) -> None: + import subprocess + + cli = Path(__file__).resolve().parents[2] / "scripts/cdnfoundry-fleet" + state_dir = tmp_path / "setup-state" + output_dir = tmp_path / "setup-bundles" + result = subprocess.run( + [ + str(cli), + "--state-dir", str(state_dir), + "--output-dir", str(output_dir), + "--repo-root", str(source_repo), + "setup", + "--operator-domain", "ops.example.com", + "--platform-domain", "example.net", + "--release", "v1.0.0", + "--preset", "control-monitoring", + "--control-ipv4", "192.0.2.150", + "--non-interactive", + ], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert "Generated 1 node bundle" in result.stdout + assert (output_dir / "control-1/compose.yml").exists() + state = json.loads((state_dir / "fleet.json").read_text(encoding="utf-8")) + assert state["features"]["monitoring"]["mode"] == "colocated" + + +def test_status_and_doctor_are_actionable(source_repo: Path, tmp_path: Path) -> None: + import subprocess + + cli = REPO_PATCH / "scripts/cdnfoundry-fleet" + state_dir = tmp_path / "status-state" + output_dir = tmp_path / "status-bundles" + common = [ + str(cli), + "--state-dir", str(state_dir), + "--output-dir", str(output_dir), + "--repo-root", str(source_repo), + ] + subprocess.run( + common + + [ + "setup", + "--operator-domain", "ops.example.com", + "--platform-domain", "example.net", + "--release", "v1.0.0", + "--preset", "control-monitoring", + "--control-ipv4", "192.0.2.161", + "--non-interactive", + ], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + status = subprocess.run(common + ["status"], check=True, text=True, stdout=subprocess.PIPE) + assert "Monitoring: colocated" in status.stdout + assert "control-1: control" in status.stdout + doctor = subprocess.run(common + ["doctor"], check=True, text=True, stdout=subprocess.PIPE) + assert "Doctor result: ready" in doctor.stdout + + +def test_optional_extra_env_is_preserved_for_manual_edge_registration(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + add(store, node("control-1", "control", "192.0.2.170")) + payload = node("edge-1", "edge", "192.0.2.171") + payload["extra_env"] = { + "EDGE_ID": "11111111-2222-3333-4444-555555555555", + "EDGE_BOOTSTRAP_TOKEN": "one-time-token", + "EDGE_GATEWAY_ADDRESS_MAP": '{"198.51.100.10":"10.20.0.10"}', + } + add(store, payload) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load(), node_name="edge-1") + env = env_values(output / "edge-1/.env.prod") + assert env["EDGE_ID"] == "11111111-2222-3333-4444-555555555555" + assert env["EDGE_BOOTSTRAP_TOKEN"] == "one-time-token" + assert json.loads(env["EDGE_GATEWAY_ADDRESS_MAP"]) == '{"198.51.100.10":"10.20.0.10"}' + + +def test_edge_registration_command_uses_protected_token_file(source_repo: Path, tmp_path: Path) -> None: + import subprocess + + cli = REPO_PATCH / "scripts/cdnfoundry-fleet" + state_dir = tmp_path / "edge-registration-state" + output_dir = tmp_path / "edge-registration-bundles" + common = [ + str(cli), + "--state-dir", str(state_dir), + "--output-dir", str(output_dir), + "--repo-root", str(source_repo), + ] + subprocess.run( + common + [ + "init", "--operator-domain", "ops.example.com", "--platform-domain", "example.net", + "--release", "v1.0.0", "--non-interactive", + ], + check=True, + ) + subprocess.run( + common + [ + "add-node", "--node", "control-1", "--role", "control", "--region", "global", + "--location", "primary", "--public-ipv4", "192.0.2.172", "--non-interactive", + ], + check=True, + ) + subprocess.run( + common + [ + "add-node", "--node", "edge-1", "--role", "edge", "--region", "eu", + "--location", "ams", "--public-ipv4", "192.0.2.173", "--non-interactive", + ], + check=True, + ) + token_file = tmp_path / "bootstrap-token" + token_file.write_text("protected-one-time-token\n", encoding="utf-8") + token_file.chmod(0o600) + subprocess.run( + common + [ + "configure-edge-registration", "--node", "edge-1", + "--edge-id", "11111111-2222-3333-4444-555555555555", + "--bootstrap-token-file", str(token_file), "--non-interactive", + ], + check=True, + ) + subprocess.run(common + ["render", "--node", "edge-1"], check=True) + env = env_values(output_dir / "edge-1/.env.prod") + assert env["EDGE_ID"] == "11111111-2222-3333-4444-555555555555" + assert env["EDGE_BOOTSTRAP_TOKEN"] == "protected-one-time-token" + secret_path = state_dir / "secrets/nodes/edge-1/edge-bootstrap-token" + assert stat.S_IMODE(secret_path.stat().st_mode) == 0o600 + + subprocess.run(common + ["clear-edge-bootstrap-token", "--node", "edge-1", "--non-interactive"], check=True) + subprocess.run(common + ["render", "--node", "edge-1"], check=True) + env = env_values(output_dir / "edge-1/.env.prod") + assert env["EDGE_ID"] == "11111111-2222-3333-4444-555555555555" + assert "EDGE_BOOTSTRAP_TOKEN" not in env + assert not secret_path.exists() + + +def test_remote_control_postgres_removes_embedded_database(store: FleetState, source_repo: Path, tmp_path: Path) -> None: + payload = node("control-1", "control", "192.0.2.174") + payload["extra_env"] = { + "DB_HOST": "postgres.internal.example", + "DB_PORT": "5432", + "DB_SSLMODE": "verify-full", + } + add(store, payload) + with store.locked(): + store.configure_feature(store.load(), "monitoring", {"mode": "colocated", "host": None}) + output = tmp_path / "bundles" + Renderer(source_repo, store, output).render(store.load(), node_name="control-1") + compose = yaml.safe_load((output / "control-1/compose.yml").read_text(encoding="utf-8")) + assert "control-db" not in compose["services"] + for service in compose["services"].values(): + depends = service.get("depends_on", {}) + if isinstance(depends, dict): + assert "control-db" not in depends + elif isinstance(depends, list): + assert "control-db" not in depends + env = env_values(output / "control-1/.env.prod") + assert env["DB_HOST"] == "postgres.internal.example" + assert env["DB_PORT"] == "5432" + assert env["DB_SSLMODE"] == "verify-full" + assert env["GRAFANA_POSTGRES_HOST"] == "postgres.internal.example" + assert env["GRAFANA_POSTGRES_PROVISION_HOST"] == "postgres.internal.example" + start = (output / "control-1/start.sh").read_text(encoding="utf-8") + assert "up -d --wait redis" in start + assert "up -d --wait control-db redis" not in start + + +def test_set_secret_replaces_external_database_password(source_repo: Path, tmp_path: Path) -> None: + import subprocess + + cli = REPO_PATCH / "scripts/cdnfoundry-fleet" + state_dir = tmp_path / "secret-state" + common = [str(cli), "--state-dir", str(state_dir), "--repo-root", str(source_repo)] + subprocess.run( + common + [ + "init", "--operator-domain", "ops.example.com", "--platform-domain", "example.net", + "--release", "v1.0.0", "--non-interactive", + ], + check=True, + ) + password_file = tmp_path / "postgres-password" + password_file.write_text("remote-database-password\n", encoding="utf-8") + password_file.chmod(0o600) + subprocess.run( + common + [ + "set-secret", "--secret", "control-db-password", "--from-file", str(password_file), + "--non-interactive", + ], + check=True, + ) + stored = state_dir / "secrets/global/control-db-password" + assert stored.read_text(encoding="utf-8").strip() == "remote-database-password" + assert stat.S_IMODE(stored.stat().st_mode) == 0o600 + + +def test_starter_json_example_builds_control_and_two_combined_pops(source_repo: Path, tmp_path: Path) -> None: + import shutil + import subprocess + + shutil.copytree(REPO_PATCH / "scripts", source_repo / "scripts", dirs_exist_ok=True) + target = source_repo / "deploy/production/examples/starter-fleet.json" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPO_PATCH / "deploy/production/examples/starter-fleet.json", target) + state_dir = tmp_path / "three-node-state" + subprocess.run( + [str(source_repo / "scripts/cdnfoundry-fleet"), "--config", str(target), + "--state-dir", str(state_dir), "--output-dir", str(state_dir / "bundles"), + "--repo-root", str(source_repo), "--non-interactive", "setup"], + cwd=source_repo, check=True, text=True, capture_output=True, + ) + state = json.loads((state_dir / "fleet.json").read_text(encoding="utf-8")) + assert len(state["nodes"]) == 3 + assert len([n for n in state["nodes"].values() if n["role"] == "control"]) == 1 + assert len([n for n in state["nodes"].values() if n["role"] == "dns-edge"]) == 2 + assert state["features"]["monitoring"]["mode"] == "colocated" + assert state["features"]["logs"]["host"] == "control-1" + + +def test_quick_starts_document_json_setup_mtls_and_remote_postgres() -> None: + root = Path(__file__).resolve().parents[2] + small = (root / "docs/deployment/production-quick-start.md").read_text(encoding="utf-8") + multi_region = (root / "docs/deployment/production-quick-start-multi-region.md").read_text(encoding="utf-8") + assert "configure-edge-registration" in small + assert "clear-edge-bootstrap-token" in small + assert "starter-fleet.json" in small + assert "multi-region-fleet.json" in multi_region + assert "four authoritative DNS nodes" in multi_region + assert "ten edge nodes" in multi_region + assert "three monitoring-role nodes" in multi_region + assert "remote PostgreSQL" in multi_region + assert "set-secret --from-file" in multi_region + + +def test_production_compose_uses_env_file_contract_and_control_has_mmdb_updater() -> None: + root = Path(__file__).resolve().parents[2] + compose_text = (root / "compose.prod.yml").read_text(encoding="utf-8") + for overlay in (root / "deploy/production").glob("*.yml"): + compose_text += overlay.read_text(encoding="utf-8") + assert "${" in compose_text + assert ":-" not in compose_text + + compose = yaml.safe_load((root / "compose.prod.yml").read_text(encoding="utf-8")) + updater = compose["services"]["mmdb-updater"] + assert set(updater["profiles"]) == {"control", "dns", "edge"} + assert "mmdb-updater" in compose["services"]["core"]["depends_on"] + + +def test_setup_config_rejects_unknown_fields(source_repo: Path, tmp_path: Path) -> None: + import subprocess + + config = tmp_path / "fleet.json" + config.write_text(json.dumps({ + "global": { + "operator_domain": "ops.example.com", + "platform_domain": "example.net", + "release": "v1.0.0", + }, + "nodes": [{ + "name": "control-1", "role": "control", "region": "global", "location": "primary", + "public_ipv4": "192.0.2.10", "public_ip4": "192.0.2.11", + }], + }), encoding="utf-8") + result = subprocess.run( + [str(REPO_PATCH / "scripts/cdnfoundry-fleet"), "--config", str(config), + "--state-dir", str(tmp_path / "state"), "--output-dir", str(tmp_path / "bundles"), + "--repo-root", str(source_repo), "--non-interactive", "setup"], + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + assert result.returncode == 3 + assert "Unknown node 0 field(s): public_ip4" in result.stderr From d8ca31427826cfb982a77c30eb121865ad5ec900 Mon Sep 17 00:00:00 2001 From: vaheeD Date: Sat, 8 Aug 2026 19:49:41 +0330 Subject: [PATCH 2/3] chore: resolve dev pipeline advisories --- core/composer.lock | 26 +++++++++++++------------- docs/package-lock.json | 20 ++++++++++---------- docs/package.json | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/core/composer.lock b/core/composer.lock index 7a97b8e..0d34559 100644 --- a/core/composer.lock +++ b/core/composer.lock @@ -2706,16 +2706,16 @@ }, { "name": "league/commonmark", - "version": "2.8.3", + "version": "2.9.0", "source": { "type": "git", "url": "https://github.com/thephpleague/commonmark.git", - "reference": "1902f60f984235023acbe03db6ad614a37b3c3e7" + "reference": "5703d83ba3da3b2e356a5fedc848ed6d8ffb6529" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/commonmark/zipball/1902f60f984235023acbe03db6ad614a37b3c3e7", - "reference": "1902f60f984235023acbe03db6ad614a37b3c3e7", + "url": "https://api.github.com/repos/thephpleague/commonmark/zipball/5703d83ba3da3b2e356a5fedc848ed6d8ffb6529", + "reference": "5703d83ba3da3b2e356a5fedc848ed6d8ffb6529", "shasum": "" }, "require": { @@ -2752,7 +2752,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "2.9-dev" + "dev-main": "2.10-dev" } }, "autoload": { @@ -2809,7 +2809,7 @@ "type": "tidelift" } ], - "time": "2026-07-12T15:29:16+00:00" + "time": "2026-08-03T13:42:31+00:00" }, { "name": "league/config", @@ -3865,16 +3865,16 @@ }, { "name": "nette/utils", - "version": "v4.1.4", + "version": "v4.1.5", "source": { "type": "git", "url": "https://github.com/nette/utils.git", - "reference": "7da6c396d7ebe142bc857c20479d5e70a5e1aac7" + "reference": "b043439dbdf954e6c28b5ea7e34b0100f83165e0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nette/utils/zipball/7da6c396d7ebe142bc857c20479d5e70a5e1aac7", - "reference": "7da6c396d7ebe142bc857c20479d5e70a5e1aac7", + "url": "https://api.github.com/repos/nette/utils/zipball/b043439dbdf954e6c28b5ea7e34b0100f83165e0", + "reference": "b043439dbdf954e6c28b5ea7e34b0100f83165e0", "shasum": "" }, "require": { @@ -3894,7 +3894,7 @@ }, "suggest": { "ext-gd": "to use Image", - "ext-iconv": "to use Strings::webalize(), toAscii(), chr() and reverse()", + "ext-iconv": "to use Strings::chr(), ord() and reverse()", "ext-intl": "to use Strings::webalize(), toAscii(), normalize() and compare()", "ext-json": "to use Nette\\Utils\\Json", "ext-mbstring": "to use Strings::lower() etc...", @@ -3950,9 +3950,9 @@ ], "support": { "issues": "https://github.com/nette/utils/issues", - "source": "https://github.com/nette/utils/tree/v4.1.4" + "source": "https://github.com/nette/utils/tree/v4.1.5" }, - "time": "2026-05-11T20:49:54+00:00" + "time": "2026-07-17T23:02:45+00:00" }, { "name": "nikic/php-parser", diff --git a/docs/package-lock.json b/docs/package-lock.json index 8585988..60d758f 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -7,7 +7,7 @@ "name": "cdnfoundry-documentation", "devDependencies": { "markdownlint-cli2": "0.23.1", - "mermaid": "11.16.0", + "mermaid": "11.16.1", "vitepress": "1.6.4" } }, @@ -2899,9 +2899,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.12", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.12.tgz", - "integrity": "sha512-zQvGet8Z2sWbQhCmfFz/T5QWH2oBmjnqK3qvOjaqaNLrLEF912WamU+ohnTp0TCep/MFVHpdJuCZEdFOdTnEFg==", + "version": "3.4.13", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", + "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", "dev": true, "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { @@ -3581,9 +3581,9 @@ } }, "node_modules/mermaid": { - "version": "11.16.0", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.16.0.tgz", - "integrity": "sha512-Zvm3kbstgdpvIJPPItlL7fppIZ3kibvc1oZIGxdvk9t6UFz6flv+Jw7FtRGKwfcI8OckmH04LqG6LlS6X4B1pA==", + "version": "11.16.1", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.16.1.tgz", + "integrity": "sha512-TQsq6u22fAn3rek5VOubrhKPo1g5hwC3FXUN9hiyupTckcYiGuuKGkNQrKYwGJkXUxZdojwRG46gsSCFZMDp4g==", "dev": true, "license": "MIT", "dependencies": { @@ -4182,9 +4182,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { diff --git a/docs/package.json b/docs/package.json index 973b610..8225ac4 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,7 +10,7 @@ }, "devDependencies": { "markdownlint-cli2": "0.23.1", - "mermaid": "11.16.0", + "mermaid": "11.16.1", "vitepress": "1.6.4" }, "overrides": { From 5d338761403effb2f02ab99def4d0960e796c28c Mon Sep 17 00:00:00 2001 From: vaheeD Date: Sat, 8 Aug 2026 20:12:42 +0330 Subject: [PATCH 3/3] feat: unify dual-stack fleet deployment --- .env.prod.example | 18 +- Makefile | 10 +- compose.dev.yml | 10 - compose.prod.yml | 63 ++++ deploy/production/Caddyfile | 9 +- deploy/production/Caddyfile.dns-api | 2 +- deploy/production/Caddyfile.telemetry | 9 +- .../production/compose.control-host-ipv6.yml | 6 - deploy/production/compose.control-host.yml | 33 -- deploy/production/compose.dns-edge-host.yml | 30 -- deploy/production/compose.dns-host-ipv6.yml | 5 - deploy/production/compose.dns-host.yml | 30 -- deploy/production/compose.edge-host-ipv6.yml | 1 - deploy/production/compose.edge-host.yml | 1 - .../compose.external-control-data.yml | 15 - .../compose.external-telemetry-data.yml | 8 - deploy/production/compose.host-journal.yml | 8 - .../compose.telemetry-host-ipv6.yml | 5 - deploy/production/compose.telemetry-host.yml | 35 -- .../examples/multi-region-fleet.json | 307 ++++++++++++++++-- deploy/production/examples/starter-fleet.json | 26 +- .../production-fleet-config-reference.md | 15 +- .../production-fleet-operator-guide.md | 14 +- .../production-quick-start-multi-region.md | 11 + docs/deployment/production-quick-start.md | 39 +++ docs/deployment/topology.md | 22 +- docs/legacy/production-quick-start.md | 48 +-- docs/legacy/production-scaling.md | 39 ++- docs/operations/grafana.md | 4 +- docs/operations/operational-logging.md | 6 +- docs/operations/scaling.md | 6 +- docs/reference/configuration.md | 4 + docs/reference/services-and-ports.md | 4 +- scripts/cdnfoundry_fleet/cli.py | 18 +- scripts/cdnfoundry_fleet/compose.py | 4 + scripts/cdnfoundry_fleet/render.py | 65 ++-- scripts/cdnfoundry_fleet/state.py | 6 +- scripts/validate-production-overrides.sh | 44 +-- tests/fleet/test_fleet.py | 10 - .../test_operational_logging_contract.py | 2 +- 40 files changed, 610 insertions(+), 382 deletions(-) delete mode 100644 deploy/production/compose.control-host-ipv6.yml delete mode 100644 deploy/production/compose.control-host.yml delete mode 100644 deploy/production/compose.dns-edge-host.yml delete mode 100644 deploy/production/compose.dns-host-ipv6.yml delete mode 100644 deploy/production/compose.dns-host.yml delete mode 100644 deploy/production/compose.edge-host-ipv6.yml delete mode 100644 deploy/production/compose.edge-host.yml delete mode 100644 deploy/production/compose.external-control-data.yml delete mode 100644 deploy/production/compose.external-telemetry-data.yml delete mode 100644 deploy/production/compose.host-journal.yml delete mode 100644 deploy/production/compose.telemetry-host-ipv6.yml delete mode 100644 deploy/production/compose.telemetry-host.yml diff --git a/.env.prod.example b/.env.prod.example index 4c3cd64..7b8ce8a 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -10,21 +10,24 @@ APP_KEY=base64:replace-with-php-artisan-key-generate-show # [required] Separate high-entropy signing seed; never reuse APP_KEY or a password. EDGE_ARTIFACT_SIGNING_KEY=replace-with-a-separate-high-entropy-signing-seed # [required] Public canonical control-panel URL. HTTPS is strongly recommended. -APP_URL=https://control.example.com +APP_URL=https://control.ops.example.com # [required behind HTTPS] Prevent browser sessions from being sent over plaintext HTTP. SESSION_SECURE_COOKIE=true # [optional] Host listener for the web UI/API. Keep loopback when a host reverse proxy terminates public TLS. CONTROL_BIND=127.0.0.1:8080 # [required with deploy/production/compose.control-host.yml] Public control-panel hostname served by Caddy. -CONTROL_HOSTNAME=control.example.com +CONTROL_HOSTNAME=control.ops.example.com # [required with the control-host override] Public TLS hostname used only for edge telemetry ingestion. -TELEMETRY_HOSTNAME=telemetry.example.com +TELEMETRY_HOSTNAME=telemetry.ops.example.com # [required on DNS API gateways] Control/worker sources allowed to reconcile DNS. CONTROL_PUBLIC_IPV4_ALLOWLIST= +CONTROL_PUBLIC_IPV6_ALLOWLIST= # [required on telemetry gateways] Edge/Vector sources allowed to ingest events. EDGE_PUBLIC_IPV4_ALLOWLIST= +EDGE_PUBLIC_IPV6_ALLOWLIST= # [required on telemetry gateways] Control, DNS and telemetry-host sources allowed to push operational logs. LOG_SOURCE_IPV4_ALLOWLIST= +LOG_SOURCE_IPV6_ALLOWLIST= # [required] Unique high-entropy secrets owned by the named database/service. CONTROL_DB_PASSWORD=replace-with-a-unique-high-entropy-control-db-password REDIS_PASSWORD=replace-with-a-unique-high-entropy-valkey-password @@ -110,14 +113,15 @@ PROMETHEUS_LOG_TARGETS_FILE=./docker/prometheus/operational-log-targets.prod.yml LOG_ROLE=telemetry LOG_HOST=telemetry-01 LOG_COLLECTOR_ID=telemetry-01 -LOKI_ENDPOINT=https://telemetry.example.com:8444 +LOKI_ENDPOINT=https://telemetry.ops.example.com:8444 LOG_BUFFER_BYTES=2147483648 LOG_METRICS_BIND=127.0.0.1:9599 # Loki defaults to 14 days in production; query range is independently bounded. LOKI_RETENTION_PERIOD=336h LOKI_MAX_QUERY_LENGTH=336h # Admin-only external navigation. Leave empty to hide Live Logs. -GRAFANA_EXPLORE_URL=https://grafana.example.com/explore?left=%7B%22datasource%22:%22loki%22%7D +GRAFANA_HOSTNAME=grafana.ops.example.com +GRAFANA_EXPLORE_URL=https://grafana.ops.example.com/explore?left=%7B%22datasource%22:%22loki%22%7D # ----------------------------------------------------------------------------- # Container images — owned by Compose on hosts running the corresponding profile @@ -138,7 +142,7 @@ HOST_BIND_IPV6=:: # See docs/deployment/certificates.md and scripts/generate-production-certificates.sh. # ----------------------------------------------------------------------------- # [required for control profile] Public URL agents use; certificate SAN must match its hostname. -EDGE_CONTROL_URL=https://edge-control.example.com:8443 +EDGE_CONTROL_URL=https://control.ops.example.com:8443 # [optional] Agent-control listener. Restrict with a firewall to registered edge networks where practical. EDGE_CONTROL_BIND=0.0.0.0:8443 # [required] Absolute host paths. Keep private keys mode 0600 and outside the repository. @@ -175,7 +179,7 @@ EDGE_RUNTIME_TLS_PRIVATE_KEY=/etc/cdnfoundry/pki/edge-runtime.key # [required for edge profile] Separate high-entropy token protecting the private cell-status endpoint. EDGE_STATUS_TOKEN=replace-with-a-separate-high-entropy-status-token # [required with deploy/production/compose.dns-edge-host.yml] Per-host DNS API TLS identity. -DNS_API_HOSTNAME=dns-api.example.com +DNS_API_HOSTNAME=dns-api.ops.example.com DNS_API_SERVER_CERTIFICATE=/etc/cdnfoundry/pki/dns-api.crt DNS_API_SERVER_PRIVATE_KEY=/etc/cdnfoundry/pki/dns-api.key diff --git a/Makefile b/Makefile index 79cf86e..7a2fc32 100644 --- a/Makefile +++ b/Makefile @@ -19,14 +19,14 @@ dev-control-up: dev-assets $(COMPOSE_DEV) up -d --build control-db redis core web dev-up: dev-assets - $(COMPOSE_DEV) --profile devtools up -d --build + $(COMPOSE_DEV) up -d --build dev-edge-up: dev-assets @test -f .env.dev || { echo 'Copy .env.dev.example to .env.dev and add the two UI edge IDs and one-time bootstrap tokens.' >&2; exit 1; } - docker compose --env-file .env.dev -f compose.dev.yml --profile dev-edge up -d --build edge-control edge-a edge-a-quarantine edge-agent-a edge-gateway-a edge-b edge-b-quarantine edge-agent-b edge-gateway-b + docker compose --env-file .env.dev -f compose.dev.yml up -d --build edge-control edge-a edge-a-quarantine edge-agent-a edge-gateway-a edge-b edge-b-quarantine edge-agent-b edge-gateway-b dev-edge-status: - docker compose --env-file .env.dev -f compose.dev.yml --profile dev-edge ps edge-control edge-a edge-a-quarantine edge-agent-a edge-gateway-a edge-b edge-b-quarantine edge-agent-b edge-gateway-b + docker compose --env-file .env.dev -f compose.dev.yml ps edge-control edge-a edge-a-quarantine edge-agent-a edge-gateway-a edge-b edge-b-quarantine edge-agent-b edge-gateway-b dev-scale-up: dev-control-up @@ -34,10 +34,10 @@ dev-down: $(COMPOSE_DEV) down dev-migrate: - $(COMPOSE_DEV) --profile tools run --rm migrate + $(COMPOSE_DEV) run --rm migrate dev-pdns-migrate: - $(COMPOSE_DEV) --profile tools run --rm pdns-migrate + $(COMPOSE_DEV) run --rm pdns-migrate dev-test: dev-assets $(COMPOSE_DEV) run --rm -e APP_ENV=testing -e APP_CONFIG_CACHE=/tmp/cdnfoundry-test-config.php -e DB_CONNECTION=sqlite -e DB_DATABASE=:memory: -e CACHE_STORE=array -e QUEUE_CONNECTION=sync core php artisan test diff --git a/compose.dev.yml b/compose.dev.yml index 58a44bb..7eed7cd 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -78,7 +78,6 @@ x-dev-cell: &dev-cell build: context: . dockerfile: docker/openresty/Dockerfile - profiles: [dev-edge] tmpfs: - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m mem_limit: 512m @@ -179,7 +178,6 @@ services: <<: *core command: [php, artisan, migrate, --force] restart: "no" - profiles: [tools] control-db: image: postgres:18.4-alpine @@ -248,7 +246,6 @@ services: pdns-db: { condition: service_healthy } networks: [dns] restart: "no" - profiles: [tools] dnsdist: image: powerdns/dnsdist-21:2.1.0 @@ -572,7 +569,6 @@ services: edge-agent-a: build: ./edge-agent - profiles: [dev-edge] environment: EDGE_CONTROL_URL: https://edge-control:8443 EDGE_CONTROL_CA_CERTIFICATE: /run/dev-pki/edge-server-ca.crt @@ -610,7 +606,6 @@ services: edge-gateway-a: build: ./edge-gateway - profiles: [dev-edge] environment: GATEWAY_CONFIG_FILE: /var/lib/cdnfoundry/runtime/current/gateway.json GATEWAY_STATE_DIR: /var/lib/cdnfoundry/gateway-state @@ -644,7 +639,6 @@ services: edge-gateway-a-state-init: image: alpine:3.22 - profiles: [dev-edge] command: [chown, "10101:10101", /state] volumes: [edge-a-gateway-state:/state] restart: "no" @@ -742,7 +736,6 @@ services: edge-agent-b: build: ./edge-agent - profiles: [dev-edge] environment: EDGE_CONTROL_URL: https://edge-control:8443 EDGE_CONTROL_CA_CERTIFICATE: /run/dev-pki/edge-server-ca.crt @@ -780,7 +773,6 @@ services: edge-gateway-b: build: ./edge-gateway - profiles: [dev-edge] environment: GATEWAY_CONFIG_FILE: /var/lib/cdnfoundry/runtime/current/gateway.json GATEWAY_STATE_DIR: /var/lib/cdnfoundry/gateway-state @@ -813,14 +805,12 @@ services: edge-gateway-b-state-init: image: alpine:3.22 - profiles: [dev-edge] command: [chown, "10101:10101", /state] volumes: [edge-b-gateway-state:/state] restart: "no" poweradmin: image: poweradmin/poweradmin:4.3.3 - profiles: [devtools] ports: ["9191:80"] environment: DB_TYPE: pgsql diff --git a/compose.prod.yml b/compose.prod.yml index 629eb10..8a346ae 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -106,6 +106,63 @@ x-core: &core - ${RESTIC_PASSWORD_FILE}:/run/secrets/restic-password:ro services: + caddy: + image: caddy:2.11.4-alpine + profiles: [control] + environment: + CONTROL_HOSTNAME: ${CONTROL_HOSTNAME:?CONTROL_HOSTNAME is required} + TELEMETRY_HOSTNAME: ${TELEMETRY_HOSTNAME:?TELEMETRY_HOSTNAME is required} + GRAFANA_HOSTNAME: ${GRAFANA_HOSTNAME:?GRAFANA_HOSTNAME is required} + ACME_CONTACT_EMAIL: ${ACME_CONTACT_EMAIL:?ACME_CONTACT_EMAIL is required} + EDGE_PUBLIC_IPV4_ALLOWLIST: ${EDGE_PUBLIC_IPV4_ALLOWLIST} + EDGE_PUBLIC_IPV6_ALLOWLIST: ${EDGE_PUBLIC_IPV6_ALLOWLIST} + LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST} + LOG_SOURCE_IPV6_ALLOWLIST: ${LOG_SOURCE_IPV6_ALLOWLIST} + ports: ["${HOST_BIND_IPV4}:80:80/tcp", "${HOST_BIND_IPV4}:443:443/tcp", "${HOST_BIND_IPV4}:443:443/udp", "${HOST_BIND_IPV4}:8444:8444/tcp"] + volumes: [./deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro, caddy-data:/data, caddy-config:/config] + depends_on: { web: { condition: service_healthy } } + networks: [ingress, telemetry] + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + healthcheck: { test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz], interval: 10s, timeout: 3s, retries: 10 } + + dns-api: + image: caddy:2.11.4-alpine + profiles: [dns] + environment: + DNS_API_HOSTNAME: ${DNS_API_HOSTNAME:?DNS_API_HOSTNAME is required} + CONTROL_PUBLIC_IPV4_ALLOWLIST: ${CONTROL_PUBLIC_IPV4_ALLOWLIST} + CONTROL_PUBLIC_IPV6_ALLOWLIST: ${CONTROL_PUBLIC_IPV6_ALLOWLIST} + ports: ["${HOST_BIND_IPV4}:8444:8444/tcp"] + volumes: [./deploy/production/Caddyfile.dns-api:/etc/caddy/Caddyfile:ro, "${DNS_API_SERVER_CERTIFICATE:?required}:/run/secrets/dns-api.crt:ro", "${DNS_API_SERVER_PRIVATE_KEY:?required}:/run/secrets/dns-api.key:ro", dns-api-caddy-data:/data, dns-api-caddy-config:/config] + depends_on: { pdns-auth: { condition: service_healthy } } + networks: [dns-private] + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + healthcheck: { test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz], interval: 10s, timeout: 3s, retries: 10 } + + telemetry-gateway: + image: caddy:2.11.4-alpine + profiles: [telemetry] + environment: + TELEMETRY_HOSTNAME: ${TELEMETRY_HOSTNAME:?TELEMETRY_HOSTNAME is required} + GRAFANA_HOSTNAME: ${GRAFANA_HOSTNAME:?GRAFANA_HOSTNAME is required} + ACME_CONTACT_EMAIL: ${ACME_CONTACT_EMAIL:?ACME_CONTACT_EMAIL is required} + EDGE_PUBLIC_IPV4_ALLOWLIST: ${EDGE_PUBLIC_IPV4_ALLOWLIST} + EDGE_PUBLIC_IPV6_ALLOWLIST: ${EDGE_PUBLIC_IPV6_ALLOWLIST} + LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST} + LOG_SOURCE_IPV6_ALLOWLIST: ${LOG_SOURCE_IPV6_ALLOWLIST} + ports: ["${HOST_BIND_IPV4}:80:80/tcp", "${HOST_BIND_IPV4}:443:443/tcp", "${HOST_BIND_IPV4}:8444:8444/tcp"] + volumes: [./deploy/production/Caddyfile.telemetry:/etc/caddy/Caddyfile:ro, telemetry-caddy-data:/data, telemetry-caddy-config:/config] + depends_on: { clickhouse: { condition: service_started }, loki: { condition: service_healthy }, grafana: { condition: service_healthy } } + networks: [telemetry] + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + healthcheck: { test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz], interval: 10s, timeout: 3s, retries: 10 } + core: <<: *core profiles: [control] @@ -644,6 +701,12 @@ networks: egress: {} volumes: + caddy-data: {} + caddy-config: {} + dns-api-caddy-data: {} + dns-api-caddy-config: {} + telemetry-caddy-data: {} + telemetry-caddy-config: {} core-storage: {} control-db: {} redis: {} diff --git a/deploy/production/Caddyfile b/deploy/production/Caddyfile index 7b6dca0..e8e9545 100644 --- a/deploy/production/Caddyfile +++ b/deploy/production/Caddyfile @@ -25,7 +25,7 @@ https://{$TELEMETRY_HOSTNAME}:8444 { @log_sources { - remote_ip {$LOG_SOURCE_IPV4_ALLOWLIST} + remote_ip {$LOG_SOURCE_IPV4_ALLOWLIST} {$LOG_SOURCE_IPV6_ALLOWLIST} path /loki/api/v1/push } @@ -33,7 +33,7 @@ https://{$TELEMETRY_HOSTNAME}:8444 { reverse_proxy loki:3100 } - @edge_sources remote_ip {$EDGE_PUBLIC_IPV4_ALLOWLIST} + @edge_sources remote_ip {$EDGE_PUBLIC_IPV4_ALLOWLIST} {$EDGE_PUBLIC_IPV6_ALLOWLIST} handle @edge_sources { reverse_proxy clickhouse:8123 @@ -42,6 +42,11 @@ https://{$TELEMETRY_HOSTNAME}:8444 { respond 403 } +{$GRAFANA_HOSTNAME} { + reverse_proxy grafana:3000 + header { Strict-Transport-Security "max-age=31536000"; X-Content-Type-Options "nosniff"; -Server } +} + http://127.0.0.1:2019 { respond /healthz 200 } diff --git a/deploy/production/Caddyfile.dns-api b/deploy/production/Caddyfile.dns-api index bd64ba2..fc3830a 100644 --- a/deploy/production/Caddyfile.dns-api +++ b/deploy/production/Caddyfile.dns-api @@ -6,7 +6,7 @@ https://{$DNS_API_HOSTNAME}:8444 { tls /run/secrets/dns-api.crt /run/secrets/dns-api.key - @control_source remote_ip {$CONTROL_PUBLIC_IPV4_ALLOWLIST} + @control_source remote_ip {$CONTROL_PUBLIC_IPV4_ALLOWLIST} {$CONTROL_PUBLIC_IPV6_ALLOWLIST} handle @control_source { reverse_proxy pdns-auth:8081 diff --git a/deploy/production/Caddyfile.telemetry b/deploy/production/Caddyfile.telemetry index af46274..a927524 100644 --- a/deploy/production/Caddyfile.telemetry +++ b/deploy/production/Caddyfile.telemetry @@ -4,10 +4,10 @@ } https://{$TELEMETRY_HOSTNAME}:8444 { - @edge_sources remote_ip {$EDGE_PUBLIC_IPV4_ALLOWLIST} + @edge_sources remote_ip {$EDGE_PUBLIC_IPV4_ALLOWLIST} {$EDGE_PUBLIC_IPV6_ALLOWLIST} @log_sources { path /loki/api/v1/push - remote_ip {$EDGE_PUBLIC_IPV4_ALLOWLIST} {$LOG_SOURCE_IPV4_ALLOWLIST} + remote_ip {$EDGE_PUBLIC_IPV4_ALLOWLIST} {$EDGE_PUBLIC_IPV6_ALLOWLIST} {$LOG_SOURCE_IPV4_ALLOWLIST} {$LOG_SOURCE_IPV6_ALLOWLIST} } handle @log_sources { @@ -21,6 +21,11 @@ https://{$TELEMETRY_HOSTNAME}:8444 { respond 403 } +{$GRAFANA_HOSTNAME} { + reverse_proxy grafana:3000 + header { Strict-Transport-Security "max-age=31536000"; X-Content-Type-Options "nosniff"; -Server } +} + http://127.0.0.1:2019 { respond /healthz 200 } diff --git a/deploy/production/compose.control-host-ipv6.yml b/deploy/production/compose.control-host-ipv6.yml deleted file mode 100644 index 3a99e5b..0000000 --- a/deploy/production/compose.control-host-ipv6.yml +++ /dev/null @@ -1,6 +0,0 @@ -services: - caddy: - ports: - - "[${HOST_BIND_IPV6}]:80:80/tcp" - - "[${HOST_BIND_IPV6}]:443:443/tcp" - - "[${HOST_BIND_IPV6}]:443:443/udp" diff --git a/deploy/production/compose.control-host.yml b/deploy/production/compose.control-host.yml deleted file mode 100644 index 2333006..0000000 --- a/deploy/production/compose.control-host.yml +++ /dev/null @@ -1,33 +0,0 @@ -services: - caddy: - image: caddy:2.11.4-alpine - profiles: [control] - environment: - CONTROL_HOSTNAME: ${CONTROL_HOSTNAME:?CONTROL_HOSTNAME is required} - TELEMETRY_HOSTNAME: ${TELEMETRY_HOSTNAME:?TELEMETRY_HOSTNAME is required} - ACME_CONTACT_EMAIL: ${ACME_CONTACT_EMAIL:?ACME_CONTACT_EMAIL is required} - EDGE_PUBLIC_IPV4_ALLOWLIST: ${EDGE_PUBLIC_IPV4_ALLOWLIST:?EDGE_PUBLIC_IPV4_ALLOWLIST is required} - LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST} - ports: - - "${HOST_BIND_IPV4}:80:80/tcp" - - "${HOST_BIND_IPV4}:443:443/tcp" - - "${HOST_BIND_IPV4}:443:443/udp" - - "${HOST_BIND_IPV4}:8444:8444/tcp" - volumes: - - ./deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro - - caddy-data:/data - - caddy-config:/config - depends_on: - web: { condition: service_healthy } - networks: [ingress, telemetry] - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - healthcheck: - test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz] - interval: 10s - timeout: 3s - retries: 10 -volumes: - caddy-data: {} - caddy-config: {} diff --git a/deploy/production/compose.dns-edge-host.yml b/deploy/production/compose.dns-edge-host.yml deleted file mode 100644 index e7c4b46..0000000 --- a/deploy/production/compose.dns-edge-host.yml +++ /dev/null @@ -1,30 +0,0 @@ -services: - dns-api: - image: caddy:2.11.4-alpine - profiles: [dns] - environment: - DNS_API_HOSTNAME: ${DNS_API_HOSTNAME:?DNS_API_HOSTNAME is required} - CONTROL_PUBLIC_IPV4_ALLOWLIST: ${CONTROL_PUBLIC_IPV4_ALLOWLIST:?CONTROL_PUBLIC_IPV4_ALLOWLIST is required} - ports: - - "${HOST_BIND_IPV4}:8444:8444/tcp" - volumes: - - ./deploy/production/Caddyfile.dns-api:/etc/caddy/Caddyfile:ro - - ${DNS_API_SERVER_CERTIFICATE:?DNS_API_SERVER_CERTIFICATE is required}:/run/secrets/dns-api.crt:ro - - ${DNS_API_SERVER_PRIVATE_KEY:?DNS_API_SERVER_PRIVATE_KEY is required}:/run/secrets/dns-api.key:ro - - dns-api-caddy-data:/data - - dns-api-caddy-config:/config - depends_on: - pdns-auth: { condition: service_healthy } - networks: [dns-private] - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - healthcheck: - test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz] - interval: 10s - timeout: 3s - retries: 10 - -volumes: - dns-api-caddy-data: {} - dns-api-caddy-config: {} diff --git a/deploy/production/compose.dns-host-ipv6.yml b/deploy/production/compose.dns-host-ipv6.yml deleted file mode 100644 index 321ef35..0000000 --- a/deploy/production/compose.dns-host-ipv6.yml +++ /dev/null @@ -1,5 +0,0 @@ -services: - dnsdist: - ports: - - "[${HOST_BIND_IPV6}]:53:53/udp" - - "[${HOST_BIND_IPV6}]:53:53/tcp" diff --git a/deploy/production/compose.dns-host.yml b/deploy/production/compose.dns-host.yml deleted file mode 100644 index e7c4b46..0000000 --- a/deploy/production/compose.dns-host.yml +++ /dev/null @@ -1,30 +0,0 @@ -services: - dns-api: - image: caddy:2.11.4-alpine - profiles: [dns] - environment: - DNS_API_HOSTNAME: ${DNS_API_HOSTNAME:?DNS_API_HOSTNAME is required} - CONTROL_PUBLIC_IPV4_ALLOWLIST: ${CONTROL_PUBLIC_IPV4_ALLOWLIST:?CONTROL_PUBLIC_IPV4_ALLOWLIST is required} - ports: - - "${HOST_BIND_IPV4}:8444:8444/tcp" - volumes: - - ./deploy/production/Caddyfile.dns-api:/etc/caddy/Caddyfile:ro - - ${DNS_API_SERVER_CERTIFICATE:?DNS_API_SERVER_CERTIFICATE is required}:/run/secrets/dns-api.crt:ro - - ${DNS_API_SERVER_PRIVATE_KEY:?DNS_API_SERVER_PRIVATE_KEY is required}:/run/secrets/dns-api.key:ro - - dns-api-caddy-data:/data - - dns-api-caddy-config:/config - depends_on: - pdns-auth: { condition: service_healthy } - networks: [dns-private] - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - healthcheck: - test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz] - interval: 10s - timeout: 3s - retries: 10 - -volumes: - dns-api-caddy-data: {} - dns-api-caddy-config: {} diff --git a/deploy/production/compose.edge-host-ipv6.yml b/deploy/production/compose.edge-host-ipv6.yml deleted file mode 100644 index ad189dd..0000000 --- a/deploy/production/compose.edge-host-ipv6.yml +++ /dev/null @@ -1 +0,0 @@ -services: {} diff --git a/deploy/production/compose.edge-host.yml b/deploy/production/compose.edge-host.yml deleted file mode 100644 index ad189dd..0000000 --- a/deploy/production/compose.edge-host.yml +++ /dev/null @@ -1 +0,0 @@ -services: {} diff --git a/deploy/production/compose.external-control-data.yml b/deploy/production/compose.external-control-data.yml deleted file mode 100644 index 07273ff..0000000 --- a/deploy/production/compose.external-control-data.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Use only when PostgreSQL and Valkey are provided by external TLS-protected -# endpoints. This prevents the control profile from starting local primaries. -services: - core: - depends_on: !reset {} - horizon: - depends_on: !reset {} - scheduler: - depends_on: !reset {} - migrate: - depends_on: !reset {} - control-db: - profiles: !override [local-control-data] - redis: - profiles: !override [local-control-data] diff --git a/deploy/production/compose.external-telemetry-data.yml b/deploy/production/compose.external-telemetry-data.yml deleted file mode 100644 index 8f90059..0000000 --- a/deploy/production/compose.external-telemetry-data.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Use when ClickHouse is an external TLS-protected service. Grafana and Vector -# receive their external endpoints from .env.prod; this prevents the telemetry -# profile from starting the local ClickHouse primary. -services: - clickhouse: - profiles: !override [local-telemetry-data] - grafana: - depends_on: !reset {} diff --git a/deploy/production/compose.host-journal.yml b/deploy/production/compose.host-journal.yml deleted file mode 100644 index cf6abda..0000000 --- a/deploy/production/compose.host-journal.yml +++ /dev/null @@ -1,8 +0,0 @@ -services: - log-collector: - command: [--config, /etc/vector/operational.yaml, --config, /etc/vector/host-journal.yaml] - volumes: - - ./docker/vector/host-journal.yaml:/etc/vector/host-journal.yaml:ro - - /var/log/journal:/var/log/journal:ro - - /run/log/journal:/run/log/journal:ro - - /etc/machine-id:/etc/machine-id:ro diff --git a/deploy/production/compose.telemetry-host-ipv6.yml b/deploy/production/compose.telemetry-host-ipv6.yml deleted file mode 100644 index f3c4898..0000000 --- a/deploy/production/compose.telemetry-host-ipv6.yml +++ /dev/null @@ -1,5 +0,0 @@ -services: - telemetry-gateway: - ports: - - "[${HOST_BIND_IPV6}]:80:80/tcp" - - "[${HOST_BIND_IPV6}]:443:443/tcp" diff --git a/deploy/production/compose.telemetry-host.yml b/deploy/production/compose.telemetry-host.yml deleted file mode 100644 index c46992f..0000000 --- a/deploy/production/compose.telemetry-host.yml +++ /dev/null @@ -1,35 +0,0 @@ -services: - telemetry-gateway: - image: caddy:2.11.4-alpine - profiles: [telemetry] - environment: - TELEMETRY_HOSTNAME: ${TELEMETRY_HOSTNAME:?TELEMETRY_HOSTNAME is required} - ACME_CONTACT_EMAIL: ${ACME_CONTACT_EMAIL:?ACME_CONTACT_EMAIL is required} - EDGE_PUBLIC_IPV4_ALLOWLIST: ${EDGE_PUBLIC_IPV4_ALLOWLIST:?EDGE_PUBLIC_IPV4_ALLOWLIST is required} - LOG_SOURCE_IPV4_ALLOWLIST: ${LOG_SOURCE_IPV4_ALLOWLIST:?LOG_SOURCE_IPV4_ALLOWLIST is required} - ports: - - "${HOST_BIND_IPV4}:80:80/tcp" - - "${HOST_BIND_IPV4}:443:443/tcp" - - "${HOST_BIND_IPV4}:8444:8444/tcp" - volumes: - - ./deploy/production/Caddyfile.telemetry:/etc/caddy/Caddyfile:ro - - telemetry-caddy-data:/data - - telemetry-caddy-config:/config - depends_on: - clickhouse: { condition: service_started } - loki: { condition: service_healthy } - log-collector: - environment: - LOKI_ENDPOINT: ${LOKI_ENDPOINT} - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - healthcheck: - test: [CMD, wget, -qO-, http://127.0.0.1:2019/healthz] - interval: 10s - timeout: 3s - retries: 10 - -volumes: - telemetry-caddy-data: {} - telemetry-caddy-config: {} diff --git a/deploy/production/examples/multi-region-fleet.json b/deploy/production/examples/multi-region-fleet.json index e601c58..49ff8b7 100644 --- a/deploy/production/examples/multi-region-fleet.json +++ b/deploy/production/examples/multi-region-fleet.json @@ -2,34 +2,297 @@ "preset": "dedicated-monitoring", "global": { "operator_domain": "ops.example.com", - "platform_domain": "cdn.example.com", + "platform_domain": "example.net", "release": "0000000000000000000000000000000000000000", "acme_email": "cdn-operations@example.com", "ipv6": false }, "nodes": [ - {"name":"control-1","role":"control","region":"global","location":"primary","hostname":"control.ops.example.com","public_ipv4":"192.0.2.10","bind_ipv4":"0.0.0.0"}, - {"name":"monitoring-1","role":"monitoring","region":"global","location":"primary","hostname":"monitoring-1.ops.example.com","public_ipv4":"192.0.2.11","bind_ipv4":"0.0.0.0"}, - {"name":"monitoring-2","role":"monitoring","region":"region-b","location":"site-b","hostname":"monitoring-2.ops.example.com","public_ipv4":"192.0.2.12","bind_ipv4":"0.0.0.0"}, - {"name":"monitoring-3","role":"monitoring","region":"region-c","location":"site-c","hostname":"monitoring-3.ops.example.com","public_ipv4":"192.0.2.13","bind_ipv4":"0.0.0.0"}, - {"name":"dns-1","role":"dns","region":"region-a","location":"site-a","hostname":"dns-1.ops.example.com","public_ipv4":"192.0.2.21","bind_ipv4":"0.0.0.0"}, - {"name":"dns-2","role":"dns","region":"region-b","location":"site-b","hostname":"dns-2.ops.example.com","public_ipv4":"192.0.2.22","bind_ipv4":"0.0.0.0"}, - {"name":"dns-3","role":"dns","region":"region-c","location":"site-c","hostname":"dns-3.ops.example.com","public_ipv4":"192.0.2.23","bind_ipv4":"0.0.0.0"}, - {"name":"dns-4","role":"dns","region":"region-d","location":"site-d","hostname":"dns-4.ops.example.com","public_ipv4":"192.0.2.24","bind_ipv4":"0.0.0.0"}, - {"name":"edge-1","role":"edge","region":"region-a","location":"site-a","hostname":"edge-1.ops.example.com","public_ipv4":"198.51.100.1","bind_ipv4":"0.0.0.0"}, - {"name":"edge-2","role":"edge","region":"region-a","location":"site-e","hostname":"edge-2.ops.example.com","public_ipv4":"198.51.100.2","bind_ipv4":"0.0.0.0"}, - {"name":"edge-3","role":"edge","region":"region-b","location":"site-b","hostname":"edge-3.ops.example.com","public_ipv4":"198.51.100.3","bind_ipv4":"0.0.0.0"}, - {"name":"edge-4","role":"edge","region":"region-b","location":"site-f","hostname":"edge-4.ops.example.com","public_ipv4":"198.51.100.4","bind_ipv4":"0.0.0.0"}, - {"name":"edge-5","role":"edge","region":"region-c","location":"site-c","hostname":"edge-5.ops.example.com","public_ipv4":"198.51.100.5","bind_ipv4":"0.0.0.0"}, - {"name":"edge-6","role":"edge","region":"region-c","location":"site-g","hostname":"edge-6.ops.example.com","public_ipv4":"198.51.100.6","bind_ipv4":"0.0.0.0"}, - {"name":"edge-7","role":"edge","region":"region-d","location":"site-d","hostname":"edge-7.ops.example.com","public_ipv4":"198.51.100.7","bind_ipv4":"0.0.0.0"}, - {"name":"edge-8","role":"edge","region":"region-d","location":"site-h","hostname":"edge-8.ops.example.com","public_ipv4":"198.51.100.8","bind_ipv4":"0.0.0.0"}, - {"name":"edge-9","role":"edge","region":"region-e","location":"site-i","hostname":"edge-9.ops.example.com","public_ipv4":"198.51.100.9","bind_ipv4":"0.0.0.0"}, - {"name":"edge-10","role":"edge","region":"region-e","location":"site-j","hostname":"edge-10.ops.example.com","public_ipv4":"198.51.100.10","bind_ipv4":"0.0.0.0"} + { + "name": "control-1", + "role": "control", + "region": "global", + "location": "primary", + "hostname": "control.ops.example.com", + "public_ipv4": "192.0.2.10", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "monitoring-1", + "role": "monitoring", + "region": "global", + "location": "primary", + "hostname": "monitoring-1.ops.example.com", + "public_ipv4": "192.0.2.11", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "monitoring-2", + "role": "monitoring", + "region": "region-b", + "location": "site-b", + "hostname": "monitoring-2.ops.example.com", + "public_ipv4": "192.0.2.12", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "monitoring-3", + "role": "monitoring", + "region": "region-c", + "location": "site-c", + "hostname": "monitoring-3.ops.example.com", + "public_ipv4": "192.0.2.13", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "dns-1", + "role": "dns", + "region": "region-a", + "location": "site-a", + "hostname": "dns-1.ops.example.com", + "public_ipv4": "192.0.2.21", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "dns-2", + "role": "dns", + "region": "region-b", + "location": "site-b", + "hostname": "dns-2.ops.example.com", + "public_ipv4": "192.0.2.22", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "dns-3", + "role": "dns", + "region": "region-c", + "location": "site-c", + "hostname": "dns-3.ops.example.com", + "public_ipv4": "192.0.2.23", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "dns-4", + "role": "dns", + "region": "region-d", + "location": "site-d", + "hostname": "dns-4.ops.example.com", + "public_ipv4": "192.0.2.24", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-1", + "role": "edge", + "region": "region-a", + "location": "site-a", + "hostname": "edge-1.ops.example.com", + "public_ipv4": "198.51.100.1", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-2", + "role": "edge", + "region": "region-a", + "location": "site-e", + "hostname": "edge-2.ops.example.com", + "public_ipv4": "198.51.100.2", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-3", + "role": "edge", + "region": "region-b", + "location": "site-b", + "hostname": "edge-3.ops.example.com", + "public_ipv4": "198.51.100.3", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-4", + "role": "edge", + "region": "region-b", + "location": "site-f", + "hostname": "edge-4.ops.example.com", + "public_ipv4": "198.51.100.4", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-5", + "role": "edge", + "region": "region-c", + "location": "site-c", + "hostname": "edge-5.ops.example.com", + "public_ipv4": "198.51.100.5", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-6", + "role": "edge", + "region": "region-c", + "location": "site-g", + "hostname": "edge-6.ops.example.com", + "public_ipv4": "198.51.100.6", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-7", + "role": "edge", + "region": "region-d", + "location": "site-d", + "hostname": "edge-7.ops.example.com", + "public_ipv4": "198.51.100.7", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-8", + "role": "edge", + "region": "region-d", + "location": "site-h", + "hostname": "edge-8.ops.example.com", + "public_ipv4": "198.51.100.8", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-9", + "role": "edge", + "region": "region-e", + "location": "site-i", + "hostname": "edge-9.ops.example.com", + "public_ipv4": "198.51.100.9", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + }, + { + "name": "edge-10", + "role": "edge", + "region": "region-e", + "location": "site-j", + "hostname": "edge-10.ops.example.com", + "public_ipv4": "198.51.100.10", + "bind_ipv4": "0.0.0.0", + "public_ipv6": null, + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null + } ], "features": { - "monitoring": {"mode": "dedicated", "host": "monitoring-1"}, - "logs": {"mode": "centralized", "host": "monitoring-1", "endpoint": null}, - "backups": {"mode": "disabled", "repository": null, "region": "us-east-1"} + "monitoring": { + "mode": "dedicated", + "host": "monitoring-1" + }, + "logs": { + "mode": "centralized", + "host": "monitoring-1", + "endpoint": null + }, + "backups": { + "mode": "disabled", + "repository": null, + "region": "us-east-1" + } } } diff --git a/deploy/production/examples/starter-fleet.json b/deploy/production/examples/starter-fleet.json index 77ea89e..e155ffb 100644 --- a/deploy/production/examples/starter-fleet.json +++ b/deploy/production/examples/starter-fleet.json @@ -2,7 +2,7 @@ "preset": "control-monitoring", "global": { "operator_domain": "ops.example.com", - "platform_domain": "cdn.example.com", + "platform_domain": "example.net", "release": "0000000000000000000000000000000000000000", "acme_email": "cdn-operations@example.com", "ipv6": false @@ -15,7 +15,13 @@ "location": "primary", "hostname": "control.ops.example.com", "public_ipv4": "192.0.2.10", - "bind_ipv4": "0.0.0.0" + "public_ipv6": null, + "bind_ipv4": "0.0.0.0", + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null }, { "name": "pop-1", @@ -24,7 +30,13 @@ "location": "site-a", "hostname": "pop-1.ops.example.com", "public_ipv4": "198.51.100.20", - "bind_ipv4": "0.0.0.0" + "public_ipv6": null, + "bind_ipv4": "0.0.0.0", + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null }, { "name": "pop-2", @@ -33,7 +45,13 @@ "location": "site-b", "hostname": "pop-2.ops.example.com", "public_ipv4": "198.51.100.30", - "bind_ipv4": "0.0.0.0" + "public_ipv6": null, + "bind_ipv4": "0.0.0.0", + "bind_ipv6": null, + "monitor_ipv4": null, + "monitor_ipv6": null, + "log_ipv4": null, + "log_ipv6": null } ], "features": { diff --git a/docs/deployment/production-fleet-config-reference.md b/docs/deployment/production-fleet-config-reference.md index e7da9dc..a40e0b7 100644 --- a/docs/deployment/production-fleet-config-reference.md +++ b/docs/deployment/production-fleet-config-reference.md @@ -5,6 +5,15 @@ description: Reference for CDNFoundry production fleet CLI options, setup config # Production fleet configuration reference +```mermaid +flowchart LR + JSON[Fleet JSON including nullable IPv6] --> STATE[Protected desired state] + STATE --> C[Control bundle] + STATE --> D[DNS bundles] + STATE --> E[Edge bundles] + STATE --> M[Monitoring bundle] +``` + Copy `deploy/production/examples/starter-fleet.json` or `multi-region-fleet.json` to a protected local `fleet.json`, then change deployment data there. Checked-in examples are templates; repository scripts and generated Compose manifests are not configuration surfaces. ## Common command options @@ -35,7 +44,7 @@ CDNFOUNDRY_FLEET_OUTPUT_DIR "preset": "control-monitoring", "global": { "operator_domain": "ops.example.com", - "platform_domain": "example.com", + "platform_domain": "example.net", "release": "v1.0.0", "acme_email": "operations@example.com", "ipv6": false @@ -71,8 +80,8 @@ CDNFOUNDRY_FLEET_OUTPUT_DIR | `public_ipv6` | no | IPv6 service address | | `bind_ipv4` | no | Local listener bind, defaults to `0.0.0.0` | | `bind_ipv6` | no | IPv6 bind; defaults to `::` in dual-stack fleets | -| `monitor_ipv4` | no | Private monitoring target; otherwise `public_ipv4` | -| `log_ipv4` | no | Private log-source address metadata | +| `monitor_ipv4` / `monitor_ipv6` | no | Private monitoring addresses; IPv4 otherwise uses `public_ipv4` | +| `log_ipv4` / `log_ipv6` | no | Private log-source address metadata | | `release` | no | Per-node immutable override of the global release | | `extra_env` | no | Explicit per-node Compose overrides; always preserved in the generated `.env.prod`, including variables that have Compose defaults | | `enabled` | no | Exclude disabled nodes from rendering and targets | diff --git a/docs/deployment/production-fleet-operator-guide.md b/docs/deployment/production-fleet-operator-guide.md index d65169e..1d61cfd 100644 --- a/docs/deployment/production-fleet-operator-guide.md +++ b/docs/deployment/production-fleet-operator-guide.md @@ -5,6 +5,16 @@ description: Complete lifecycle guide for CDNFoundry production fleets including # Production fleet operator guide +```mermaid +flowchart LR + A[Fleet authority] --> C[Control-only env and PKI] + A --> D[DNS-only env and PKI] + A --> E1[Edge A env and identity] + A --> E2[Edge B env and identity] +``` + +Each bundle is a security boundary. Never use a shared `env_file`: database, PowerDNS, bootstrap, identity, backup, and telemetry credentials are emitted only when that node's filtered services require them. Never copy a bundle, `.env.prod`, `pki/`, or `secrets/` directory between nodes. + Work from an immutable checkout. For a fresh operator host: ```bash @@ -65,7 +75,7 @@ Equivalent non-interactive command: --repo-root "$PWD" \ setup \ --operator-domain ops.example.com \ - --platform-domain example.com \ + --platform-domain example.net \ --release v1.0.0 \ --preset control-monitoring \ --control-ipv4 192.0.2.10 \ @@ -104,7 +114,7 @@ For repeatable automation, use a JSON file: "preset": "control-monitoring", "global": { "operator_domain": "ops.example.com", - "platform_domain": "example.com", + "platform_domain": "example.net", "release": "v1.0.0", "acme_email": "operations@example.com", "ipv6": false diff --git a/docs/deployment/production-quick-start-multi-region.md b/docs/deployment/production-quick-start-multi-region.md index 1053772..bc39e25 100644 --- a/docs/deployment/production-quick-start-multi-region.md +++ b/docs/deployment/production-quick-start-multi-region.md @@ -5,6 +5,17 @@ description: Deploy a separated-role CDNFoundry fleet across multiple regions fr # Production quick start: multi-region fleet +```mermaid +flowchart TB + CF[Cloudflare: ops.example.com] --> CP[Control and Grafana] + REG[example.net delegation] --> D1[DNS region A] + REG --> D2[DNS region B] + CP --> D1 + CP --> D2 + CP --> E1[Edge region A] + CP --> E2[Edge region B] +``` + This example models one control node, four authoritative DNS nodes, ten edge nodes, and three monitoring-role nodes. “Multi-region” describes its failure-domain design; it is not a special runtime mode or a fixed scale limit. Read and complete the [starter fleet quick start](production-quick-start.md) first. The same security, PKI, transfer, migration, enrollment, last-valid-state, backup, and acceptance rules apply. diff --git a/docs/deployment/production-quick-start.md b/docs/deployment/production-quick-start.md index 6f62da4..ad76802 100644 --- a/docs/deployment/production-quick-start.md +++ b/docs/deployment/production-quick-start.md @@ -5,6 +5,20 @@ description: Deploy CDNFoundry with one control node and two combined DNS and ed # Production quick start: starter fleet +```mermaid +flowchart LR + CF[Cloudflare DNS: ops.example.com] --> C[control.ops.example.com] + CF --> G[grafana.ops.example.com] + CF --> P1[pop-1.ops.example.com] + CF --> P2[pop-2.ops.example.com] + R[example.net delegation] --> P1 + R --> P2 + C -->|mTLS control| P1 + C -->|mTLS control| P2 +``` + +`ops.example.com` and `example.net` are intentionally unrelated zones. Cloudflare remains authoritative for the operational zone: create DNS-only A and optional AAAA records for control, Grafana, telemetry, and every node. PowerDNS owns `example.net` and enrolled customer zones; never delegate `ops.example.com` to CDNFoundry. + This runbook creates the smallest practical production CDNFoundry fleet: - one control-plane node with colocated monitoring and operational logs; @@ -46,6 +60,8 @@ Edit `fleet.json` and replace every example value: - every `hostname`, `public_ipv4`, region, and location; - `public_ipv6` and `bind_ipv6` when deploying dual stack. +Keep `public_ipv6`, `bind_ipv6`, `monitor_ipv6`, and `log_ipv6` in every node object and set unavailable paths to JSON `null`. Set global `ipv6` to `true` only after Cloudflare AAAA records, host routes, firewalls, and external reachability are ready. + The checked-in addresses are RFC documentation ranges and cannot serve production traffic. Keep `bind_ipv4` as `0.0.0.0` for normal routed/NAT hosts unless a specific local interface address is required. Validate the JSON before it can create state: @@ -108,6 +124,15 @@ The control bundle starts `mmdb-updater` before services that consume GeoIP data Sign in to the administrator panel, configure platform nameservers and DNS clusters using the two PoP hostnames, and verify registrar glue for their public addresses. DNSdist is the only public authoritative endpoint; PowerDNS and its database remain private. +Use this exact order: + +1. Open `https://control.ops.example.com/admin`. In **Control plane → System settings**, configure `example.net`, `ns1.example.net`, `ns2.example.net`, and their A/optional AAAA glue. +2. In **Infrastructure → DNS clusters**, create each PoP disabled with its generated `https://pop-N.ops.example.com:8444` endpoint and node-local API key. Test it, then enable it. +3. In **Domains → Create domain**, add a delegated customer zone and its first DNS-only A/AAAA record. Wait for both cluster acknowledgements before registrar delegation. +4. Verify UDP and TCP answers. Only then use **Edge network → Edges** to create edge inventory, capture each UUID and one-time bootstrap token, enroll it as described below, create/assign a service pool, add the origin endpoint, and enable proxying for the hostname. + +API automation follows the same sequence. Authenticate with `POST /api/v1/admin/login`, protect the returned bearer token, and use the DNS-cluster, domain, record, and edge endpoints in the live OpenAPI document. Send `Idempotency-Key` on mutations and poll the operation returned by `202 Accepted`. Never store an API token in Fleet JSON. + Transfer and start each PoP bundle only after its replacement validates. Allow both UDP and TCP 53 and restrict DNS API, metrics, and management listeners to documented control/monitoring sources. ## 7. Enroll both edge nodes @@ -137,6 +162,20 @@ Transfer the token-free bundle and recreate only `edge-agent`. Never reuse or re ## 8. Acceptance and recovery gate +Check public endpoints before delegation or traffic: + +```bash +curl --fail https://control.ops.example.com/health +curl --fail https://grafana.ops.example.com/api/health +dig +short A control.ops.example.com @1.1.1.1 +dig +short AAAA control.ops.example.com @1.1.1.1 # empty is valid when IPv6 is null +dig +tcp SOA example.net @ns1.example.net +dig SOA example.net @ns2.example.net +curl --fail --resolve www.example.net:443:EDGE_IP https://www.example.net/ +``` + +Run `docker compose --env-file .env.prod ps` on every node. Long-running services must be healthy; completed migration helpers may be exited. Ports 8443/8444, metrics, PostgreSQL, Valkey, PowerDNS API, ClickHouse, and Loki are restricted interfaces, not general public endpoints. + Confirm: - control health, queues, Scheduler, Horizon, and migrations; diff --git a/docs/deployment/topology.md b/docs/deployment/topology.md index d6cbe2c..14759fa 100644 --- a/docs/deployment/topology.md +++ b/docs/deployment/topology.md @@ -122,15 +122,15 @@ and only then start application processes: ```sh docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile control up -d --wait --wait-timeout 120 control-db redis docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile tools run --rm migrate docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile control up -d ``` @@ -152,15 +152,15 @@ migration, then start the profiles: ```sh docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile dns up -d --wait --wait-timeout 120 pdns-db docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile tools run --rm pdns-migrate docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile dns --profile edge up -d ``` @@ -204,11 +204,11 @@ topology, operation IDs, certificate fingerprints, checks, and deviations. ## Split-role overlays -- `compose.dns-host.yml` adds the DNS API gateway to a DNS-only host. -- `compose.edge-host.yml` documents the base edge-only role; the base file owns its listeners. -- `compose.telemetry-host.yml` adds public, source-restricted telemetry TLS. -- `compose.external-control-data.yml` disables local PostgreSQL and Valkey. -- `compose.external-telemetry-data.yml` disables local ClickHouse while Grafana +- `compose.prod.yml` adds the DNS API gateway to a DNS-only host. +- `compose.prod.yml` documents the base edge-only role; the base file owns its listeners. +- `compose.prod.yml` adds public, source-restricted telemetry TLS. +- `compose.prod.yml` disables local PostgreSQL and Valkey. +- `compose.prod.yml` disables local ClickHouse while Grafana and Vector use the configured external telemetry endpoint. - `*-ipv6.yml` files explicitly add IPv6 publications. diff --git a/docs/legacy/production-quick-start.md b/docs/legacy/production-quick-start.md index 4f26f52..ed1c275 100644 --- a/docs/legacy/production-quick-start.md +++ b/docs/legacy/production-quick-start.md @@ -310,23 +310,23 @@ runs before long-lived workers start. Named volumes are preserved. cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile control --profile telemetry config --quiet docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile control --profile telemetry pull docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile tools run --rm migrate docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile control --profile telemetry up -d docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml ps + -f compose.prod.yml ps ``` Check the public path: @@ -345,7 +345,7 @@ Create the first administrator; the command prompts for the password: ```sh docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ exec -u www-data core php artisan cdnf:admin:create \ --name="CDN Operations" --email="admin@example.com" ``` @@ -364,19 +364,19 @@ authoritative service without affecting public resolvers. cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile dns config --quiet docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile dns --profile edge pull docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile tools run --rm pdns-migrate docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile dns up -d ``` @@ -453,11 +453,11 @@ Start the runtime on each edge: cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile edge up -d docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ ps edge edge-quarantine edge-agent vector mmdb-updater ``` @@ -469,7 +469,7 @@ sudo sed -i 's/^EDGE_BOOTSTRAP_TOKEN=.*/EDGE_BOOTSTRAP_TOKEN=/' \ /opt/cdnfoundry/.env.prod docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ + -f compose.prod.yml \ --profile edge up -d --force-recreate edge-agent ``` @@ -503,13 +503,13 @@ curl -I --resolve CUSTOMER_DOMAIN:443:198.51.100.20 https://CUSTOMER_DOMAIN/ ```sh docker compose --env-file .env.prod -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml ps + -f compose.prod.yml ps docker compose --env-file .env.prod -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml logs --tail=200 core caddy + -f compose.prod.yml logs --tail=200 core caddy ``` On an edge, replace the override with -`deploy/production/compose.dns-edge-host.yml` and inspect `dnsdist`, +`compose.prod.yml` and inspect `dnsdist`, `pdns-auth`, `edge`, `edge-agent`, and `vector`. ### Upgrade @@ -527,7 +527,7 @@ logs. Do not expose Caddy's administration endpoint. ```sh docker compose --env-file .env.prod -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml logs --tail=200 caddy + -f compose.prod.yml logs --tail=200 caddy ``` If `core` is unhealthy, inspect its startup output and health result. The @@ -536,10 +536,10 @@ making it writable. ```sh docker compose --env-file .env.prod -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml logs --tail=200 core + -f compose.prod.yml logs --tail=200 core docker inspect --format '{{json .State.Health}}' \ "$(docker compose --env-file .env.prod -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml ps -q core)" + -f compose.prod.yml ps -q core)" ``` Verify `/etc/cdnfoundry/pki/edge-identity-ca.key` is readable by container @@ -596,16 +596,16 @@ matching AAAA/glue records, and append the relevant opt-in override: # Control host: docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ - -f deploy/production/compose.control-host-ipv6.yml \ + -f compose.prod.yml \ + -f compose.prod.yml \ --profile control --profile telemetry up -d # Combined DNS/edge host: docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-edge-host.yml \ - -f deploy/production/compose.dns-host-ipv6.yml \ - -f deploy/production/compose.edge-host-ipv6.yml \ + -f compose.prod.yml \ + -f compose.prod.yml \ + -f compose.prod.yml \ --profile dns --profile edge up -d ``` diff --git a/docs/legacy/production-scaling.md b/docs/legacy/production-scaling.md index a132f75..053219e 100644 --- a/docs/legacy/production-scaling.md +++ b/docs/legacy/production-scaling.md @@ -85,16 +85,15 @@ database topology and are not inferred by CDNFoundry. | Host role | Compose files | Start target/profile | |---|---|---| -| Three-host combined controller/telemetry | base + `compose.control-host.yml` | `control`, `telemetry` | -| Control replica using external PostgreSQL/Valkey | base + `compose.external-control-data.yml` | selected `core`, `web`, `edge-control` services | -| Worker host using external PostgreSQL/Valkey | base + `compose.external-control-data.yml` | selected `horizon`; one host may run `scheduler` | -| DNS-only host | base + `compose.dns-host.yml` | `dns` | -| Edge-only host | base + `compose.edge-host.yml` | `edge` | -| Combined DNS/edge starter host | base + `compose.dns-edge-host.yml` | `dns`, `edge` | -| Dedicated telemetry host | base + `compose.telemetry-host.yml` | `telemetry` | +| Three-host combined controller/telemetry | `compose.prod.yml` | `control`, `telemetry` | +| Control replica using external PostgreSQL/Valkey | base + `compose.prod.yml` | selected `core`, `web`, `edge-control` services | +| Worker host using external PostgreSQL/Valkey | base + `compose.prod.yml` | selected `horizon`; one host may run `scheduler` | +| DNS-only host | base + `compose.prod.yml` | `dns` | +| Edge-only host | base + `compose.prod.yml` | `edge` | +| Combined DNS/edge starter host | `compose.prod.yml` | `dns`, `edge` | +| Dedicated telemetry host | base + `compose.prod.yml` | `telemetry` | -Here, "base" means `compose.prod.yml`; override files are under -`deploy/production/`. +Production roles are selected from the single `compose.prod.yml` profile contract. ## Example fleet sizes @@ -173,11 +172,11 @@ and IPv6 listeners. cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.edge-host.yml \ + -f compose.prod.yml \ --profile edge config --quiet docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.edge-host.yml \ + -f compose.prod.yml \ --profile edge up -d ``` @@ -199,7 +198,7 @@ cd /opt/cdnfoundry sudoedit .env.prod docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.control-host.yml \ + -f compose.prod.yml \ --profile control --profile telemetry up -d --force-recreate caddy ``` @@ -239,15 +238,15 @@ Horizon/control workers. Mirror the list in UFW, provider firewall, and cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-host.yml \ + -f compose.prod.yml \ --profile dns config --quiet docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-host.yml \ + -f compose.prod.yml \ --profile tools run --rm pdns-migrate docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.dns-host.yml \ + -f compose.prod.yml \ --profile dns up -d ``` @@ -273,11 +272,11 @@ ClickHouse port `8123`. cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.telemetry-host.yml \ + -f compose.prod.yml \ --profile telemetry config --quiet docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.telemetry-host.yml \ + -f compose.prod.yml \ --profile telemetry up -d ``` @@ -313,11 +312,11 @@ services. The public load balancer is operator-owned and must health-check cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.external-control-data.yml \ + -f compose.prod.yml \ --profile control config --quiet docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.external-control-data.yml \ + -f compose.prod.yml \ --profile control up -d core web edge-control ``` @@ -333,7 +332,7 @@ scheduler. cd /opt/cdnfoundry docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.external-control-data.yml \ + -f compose.prod.yml \ --profile control up -d horizon ``` diff --git a/docs/operations/grafana.md b/docs/operations/grafana.md index 5fbde0a..21ca33f 100644 --- a/docs/operations/grafana.md +++ b/docs/operations/grafana.md @@ -83,8 +83,8 @@ provider's approved role workflow. `docker/grafana/postgres/provision.sql` is the exact PostgreSQL grant contract. Apply the ClickHouse grants listed above, including the bounded profile, then set the `GRAFANA_*_HOST`, port, protocol, TLS, user, and password variables. Use -`deploy/production/compose.external-control-data.yml` for external PostgreSQL -and `deploy/production/compose.external-telemetry-data.yml` for external +`compose.prod.yml` for external PostgreSQL +and `compose.prod.yml` for external ClickHouse. Verified TLS is required when endpoints cross hosts. ## Dashboard behavior diff --git a/docs/operations/operational-logging.md b/docs/operations/operational-logging.md index f67dac2..71bd83c 100644 --- a/docs/operations/operational-logging.md +++ b/docs/operations/operational-logging.md @@ -39,7 +39,7 @@ adding the `logs` profile to that host's normal role command: ```sh docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.edge-host.yml \ + -f compose.prod.yml \ --profile edge --profile logs up -d ``` @@ -61,8 +61,8 @@ On hosts with persistent systemd journals, add the overlay: ```sh docker compose --env-file .env.prod \ -f compose.prod.yml \ - -f deploy/production/compose.edge-host.yml \ - -f deploy/production/compose.host-journal.yml \ + -f compose.prod.yml \ + -f compose.prod.yml \ --profile edge --profile logs up -d ``` diff --git a/docs/operations/scaling.md b/docs/operations/scaling.md index 76ab212..174be71 100644 --- a/docs/operations/scaling.md +++ b/docs/operations/scaling.md @@ -28,7 +28,7 @@ cache directory, or reload per domain. 1. prepare host firewall, immutable release, `.env.prod`, runtime certificate, server CA, status token, advertised service identities, and private local gateway address mappings; -2. validate `compose.prod.yml` plus `compose.edge-host.yml`; +2. validate `compose.prod.yml` plus `compose.prod.yml`; 3. create the edge row and configure its cells; 4. enroll its agent with the one-time token; 5. wait for fresh heartbeat and ready cells; @@ -50,14 +50,14 @@ Adding DNS capacity does not create an edge runtime. ## Move telemetry -Use `compose.telemetry-host.yml` with an exact edge-source allowlist. Set control +Use `compose.prod.yml` with an exact edge-source allowlist. Set control `CLICKHOUSE_URL` and edge Vector endpoints to verified TLS. Keep ClickHouse and Prometheus private. Prove that telemetry outage and backlog drain do not affect serving. ## External control data -`compose.external-control-data.yml` disables local PostgreSQL and Valkey. Set +`compose.prod.yml` disables local PostgreSQL and Valkey. Set `DB_URL` and `REDIS_URL` to owner-operated replicated services with verified TLS, exact-source firewalls, backup, failover, and capacity plans. The repository does not configure database replication or automatic failover. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c65ac29..454ff53 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -31,6 +31,9 @@ Runtime product policy is not an environment variable. Manage it through | `CONTROL_HOSTNAME` | control overlay | Public browser/API hostname | | `TELEMETRY_HOSTNAME` | control overlay | Public telemetry-ingest hostname | | `CONTROL_PUBLIC_IPV4_ALLOWLIST` | DNS API gateway | Exact control/worker sources allowed to call PowerDNS | +| `CONTROL_PUBLIC_IPV6_ALLOWLIST` | DNS API gateway | Optional IPv6 control/worker sources allowed to call PowerDNS | +| `EDGE_PUBLIC_IPV6_ALLOWLIST` | telemetry gateway | Optional IPv6 edge sources allowed to submit telemetry | +| `LOG_SOURCE_IPV6_ALLOWLIST` | telemetry gateway | Optional IPv6 node sources allowed to submit operational logs | | `EDGE_PUBLIC_IPV4_ALLOWLIST` | telemetry gateway | Exact edge/Vector ingestion sources | | `CONTROL_DB_PASSWORD` | local control DB | PostgreSQL password for database `cdnf` | | `REDIS_PASSWORD` | local Valkey | Required Valkey password | @@ -147,6 +150,7 @@ Grafana telemetry variables are: | `PROMETHEUS_EDGE_TARGETS_FILE` | telemetry | Private file_sd target file; production default is empty | | `PROMETHEUS_LOG_TARGETS_FILE` | telemetry | Private file_sd targets for remote collector metrics; production default is empty | | `GRAFANA_EXPLORE_URL` | control | Optional deployment fallback for the admin-only Live Logs link. The PostgreSQL-backed **Platform settings → Observability links → Grafana Explore URL** overrides it. Laravel supplies Loki, a safe selector, and a one-hour range when the chosen URL has no query; both empty hides the link | +| `GRAFANA_HOSTNAME` | control/telemetry | Public Grafana hostname under the Cloudflare-managed operational domain | | `GRAFANA_LOKI_URL` | telemetry | Private Grafana-to-Loki endpoint; default `http://loki:3100` | | `LOKI_RETENTION_PERIOD` | telemetry | Loki retention; production default `336h` | | `LOKI_MAX_QUERY_LENGTH` | telemetry | Maximum query range; production default `336h` | diff --git a/docs/reference/services-and-ports.md b/docs/reference/services-and-ports.md index fc00df0..19e6f88 100644 --- a/docs/reference/services-and-ports.md +++ b/docs/reference/services-and-ports.md @@ -22,11 +22,11 @@ telemetry internals off public networks. | `logs` | one `log-collector` on the current host; combine once with its role profile | | `tools` | explicit `migrate` and `pdns-migrate` one-shot services | -`deploy/production/compose.external-control-data.yml` replaces local +`compose.prod.yml` replaces local `control-db` and `redis` with configured external endpoints. The control, DNS, edge, and telemetry host overlays add only role-specific publication and gateways. -`deploy/production/compose.external-telemetry-data.yml` disables local +`compose.prod.yml` disables local ClickHouse when a verified external endpoint is configured. ## Production listeners diff --git a/scripts/cdnfoundry_fleet/cli.py b/scripts/cdnfoundry_fleet/cli.py index 0dd1092..bf8b19f 100644 --- a/scripts/cdnfoundry_fleet/cli.py +++ b/scripts/cdnfoundry_fleet/cli.py @@ -34,7 +34,7 @@ } SETUP_NODE_FIELDS = { "name", "role", "region", "location", "hostname", "public_ipv4", "public_ipv6", - "bind_ipv4", "bind_ipv6", "monitor_ipv4", "log_ipv4", "release", "extra_env", + "bind_ipv4", "bind_ipv6", "monitor_ipv4", "monitor_ipv6", "log_ipv4", "log_ipv6", "release", "extra_env", "enabled", "draining", "health", } @@ -226,7 +226,9 @@ def _node_arguments( target.add_argument("--bind-ipv4", default=None if optional else "0.0.0.0") target.add_argument("--bind-ipv6") target.add_argument("--monitor-ipv4") + target.add_argument("--monitor-ipv6") target.add_argument("--log-ipv4") + target.add_argument("--log-ipv6") target.add_argument("--release") target.add_argument("--extra-env", action="append", default=[]) target.add_argument("--disabled", action="store_true", default=None if optional else False) @@ -304,7 +306,9 @@ def _node_payload(args: argparse.Namespace, config: dict[str, Any], *, update: b "bind_ipv4": "bind_ipv4", "bind_ipv6": "bind_ipv6", "monitor_ipv4": "monitor_ipv4", + "monitor_ipv6": "monitor_ipv6", "log_ipv4": "log_ipv4", + "log_ipv6": "log_ipv6", "release": "release", } for target, arg_name in mapping.items(): @@ -370,7 +374,9 @@ def _interactive_node(state: dict[str, Any], *, role: str | None = None, default "bind_ipv4": defaults.get("bind_ipv4") or "0.0.0.0", "bind_ipv6": defaults.get("bind_ipv6"), "monitor_ipv4": defaults.get("monitor_ipv4"), + "monitor_ipv6": defaults.get("monitor_ipv6"), "log_ipv4": defaults.get("log_ipv4"), + "log_ipv6": defaults.get("log_ipv6"), "extra_env": defaults.get("extra_env", {}), } @@ -556,11 +562,6 @@ def _doctor(args: argparse.Namespace, store: FleetState) -> int: root = Path(args.repo_root) required = [ "compose.prod.yml", - "deploy/production/compose.control-host.yml", - "deploy/production/compose.dns-host.yml", - "deploy/production/compose.edge-host.yml", - "deploy/production/compose.dns-edge-host.yml", - "deploy/production/compose.telemetry-host.yml", ] checks: list[dict[str, Any]] = [] for relative in required: @@ -746,6 +747,11 @@ def execute(args: argparse.Namespace) -> int: "public_ipv4": args.public_ipv4, "public_ipv6": args.public_ipv6, "bind_ipv4": args.bind_ipv4, + "bind_ipv6": args.bind_ipv6, + "monitor_ipv4": args.monitor_ipv4, + "monitor_ipv6": args.monitor_ipv6, + "log_ipv4": args.log_ipv4, + "log_ipv6": args.log_ipv6, "extra_env": {k: v for k, v in env.items() if k not in secret_env_names()}, } state = store.add_node(state, payload) diff --git a/scripts/cdnfoundry_fleet/compose.py b/scripts/cdnfoundry_fleet/compose.py index db24e1b..4362fd8 100644 --- a/scripts/cdnfoundry_fleet/compose.py +++ b/scripts/cdnfoundry_fleet/compose.py @@ -125,6 +125,10 @@ def select_services( if not profiles or profiles & active_profiles: if name == "vector" and not monitoring_enabled: continue + # The control gateway already serves colocated telemetry and Grafana. + # The dedicated gateway owns the same public ports only on monitoring-role hosts. + if name == "telemetry-gateway" and role == "control": + continue selected.add(name) # Tool containers are role-specific and remain behind the tools profile. diff --git a/scripts/cdnfoundry_fleet/render.py b/scripts/cdnfoundry_fleet/render.py index 6498611..1ade086 100644 --- a/scripts/cdnfoundry_fleet/render.py +++ b/scripts/cdnfoundry_fleet/render.py @@ -14,22 +14,12 @@ bind_mount_sources, dump_yaml, load_yaml, - merge_compose, prune_top_level, required_env, select_services, ) from .state import FleetState -ROLE_OVERLAYS = { - "control": ["deploy/production/compose.control-host.yml"], - "dns": ["deploy/production/compose.dns-host.yml"], - "edge": ["deploy/production/compose.edge-host.yml"], - "dns-edge": ["deploy/production/compose.dns-edge-host.yml"], - "monitoring": ["deploy/production/compose.telemetry-host.yml"], -} - - class Renderer: def __init__(self, repo_root: Path, store: FleetState, output_dir: Path, *, dry_run: bool = False) -> None: self.repo_root = repo_root.resolve() @@ -53,21 +43,11 @@ def render(self, state: dict[str, Any], *, node_name: str | None = None) -> list def _render_node(self, state: dict[str, Any], node: dict[str, Any]) -> Path: base = load_yaml(self.repo_root / "compose.prod.yml") - merged = base - for overlay in ROLE_OVERLAYS[node["role"]]: - path = self.repo_root / overlay - if path.exists(): - merged = merge_compose(merged, load_yaml(path)) - if state["features"]["logs"]["mode"] == "centralized": - journal = self.repo_root / "deploy/production/compose.host-journal.yml" - if journal.exists(): - merged = merge_compose(merged, load_yaml(journal)) - monitoring_enabled = state["features"]["monitoring"]["mode"] != "disabled" monitoring_host = self._is_monitoring_host(state, node) logs_enabled = state["features"]["logs"]["mode"] == "centralized" filtered = select_services( - merged, + base, role=node["role"], monitoring_enabled=monitoring_enabled, logs_enabled=logs_enabled, @@ -113,7 +93,12 @@ def _apply_generated_overrides(self, state: dict[str, Any], node: dict[str, Any] services = compose.get("services", {}) if "node-exporter" in services: bind = node.get("monitor_ipv4") or node["bind_ipv4"] - services["node-exporter"]["ports"] = [f"{bind}:9100:9100/tcp"] + ports = [f"{bind}:9100:9100/tcp"] + if node.get("monitor_ipv6"): + ports.append(f"[{node['monitor_ipv6']}]:9100:9100/tcp") + services["node-exporter"]["ports"] = ports + if node.get("bind_ipv6"): + self._add_ipv6_publications(services, node["bind_ipv6"]) if "log-collector" in services: service = services["log-collector"] service["command"] = ["--config", "/etc/vector/generated-node.yaml"] @@ -180,6 +165,7 @@ def _environment( "APP_URL": f"https://control.{operator_domain}", "CONTROL_HOSTNAME": f"control.{operator_domain}", "TELEMETRY_HOSTNAME": f"telemetry.{operator_domain}", + "GRAFANA_HOSTNAME": f"grafana.{operator_domain}", "APP_KEY": self.store.read_secret("app-key"), "EDGE_ARTIFACT_SIGNING_KEY": self.store.read_secret("artifact-signing-key"), "CONTROL_DB_PASSWORD": self.store.read_secret("control-db-password"), @@ -206,6 +192,9 @@ def _environment( "CONTROL_PUBLIC_IPV4_ALLOWLIST": self._control_allowlist(state), "EDGE_PUBLIC_IPV4_ALLOWLIST": self._edge_allowlist(state), "LOG_SOURCE_IPV4_ALLOWLIST": self._all_public_ipv4(state), + "CONTROL_PUBLIC_IPV6_ALLOWLIST": self._control_allowlist(state, family=6), + "EDGE_PUBLIC_IPV6_ALLOWLIST": self._edge_allowlist(state, family=6), + "LOG_SOURCE_IPV6_ALLOWLIST": self._all_public_ipv6(state), "LOG_ROLE": node["role"], "LOG_HOST": node["name"], "LOG_COLLECTOR_ID": node["name"], @@ -235,7 +224,7 @@ def _environment( "ACME_DIRECTORY_URL": "https://acme-v02.api.letsencrypt.org/directory", "ACME_ORDER_BUDGET_PER_HOUR": "20", "EDGE_IDENTITY_CA_PRIVATE_KEY_PASSPHRASE": "", - "GRAFANA_EXPLORE_URL": "", + "GRAFANA_EXPLORE_URL": f"https://grafana.{operator_domain}/explore?left=%7B%22datasource%22:%22loki%22%7D", "EDGE_CONTROL_BIND": "0.0.0.0:8443", "EDGE_RUNTIME_VERSIONS": "{}", "EDGE_GATEWAY_METRICS_ADDRESS": "0.0.0.0:9105", @@ -341,21 +330,41 @@ def _edge_control_url(self, state: dict[str, Any]) -> str: hostname = control["hostname"] if control else f"control.{state['global']['operator_domain']}" return f"https://{hostname}:8443" - def _control_allowlist(self, state: dict[str, Any]) -> str: + @staticmethod + def _add_ipv6_publications(services: dict[str, Any], bind: str) -> None: + mappings = { + "caddy": [(80, "tcp"), (443, "tcp"), (443, "udp"), (8444, "tcp")], + "dns-api": [(8444, "tcp")], + "telemetry-gateway": [(80, "tcp"), (443, "tcp"), (8444, "tcp")], + "dnsdist": [(53, "tcp"), (53, "udp")], + "edge-gateway": [(80, "tcp"), (443, "tcp"), (443, "udp")], + } + for service_name, ports in mappings.items(): + if service_name in services: + current = services[service_name].setdefault("ports", []) + current.extend(f"[{bind}]:{port}:{port}/{protocol}" for port, protocol in ports) + + def _control_allowlist(self, state: dict[str, Any], *, family: int = 4) -> str: + field = f"public_ipv{family}" return " ".join( - node["public_ipv4"] for node in state["nodes"].values() if node["role"] == "control" and node["enabled"] + node[field] for node in state["nodes"].values() + if node["role"] == "control" and node["enabled"] and node.get(field) ) - def _edge_allowlist(self, state: dict[str, Any]) -> str: + def _edge_allowlist(self, state: dict[str, Any], *, family: int = 4) -> str: + field = f"public_ipv{family}" return " ".join( - node["public_ipv4"] + node[field] for node in state["nodes"].values() - if node["role"] in {"edge", "dns-edge"} and node["enabled"] + if node["role"] in {"edge", "dns-edge"} and node["enabled"] and node.get(field) ) def _all_public_ipv4(self, state: dict[str, Any]) -> str: return " ".join(node["public_ipv4"] for node in state["nodes"].values() if node["enabled"]) + def _all_public_ipv6(self, state: dict[str, Any]) -> str: + return " ".join(node["public_ipv6"] for node in state["nodes"].values() if node["enabled"] and node.get("public_ipv6")) + def _feature_host(self, state: dict[str, Any], feature: str) -> dict[str, Any] | None: cfg = state["features"][feature] if cfg["mode"] == "disabled": diff --git a/scripts/cdnfoundry_fleet/state.py b/scripts/cdnfoundry_fleet/state.py index a07d53c..18f6e67 100644 --- a/scripts/cdnfoundry_fleet/state.py +++ b/scripts/cdnfoundry_fleet/state.py @@ -197,9 +197,9 @@ def validate(self, state: dict[str, Any], *, require_secrets: bool = True) -> No if hostname in hostnames: raise ValidationError(f"Duplicate hostname: {hostname}") hostnames.add(hostname) - for field in ("public_ipv4", "public_ipv6", "bind_ipv4", "bind_ipv6", "monitor_ipv4", "log_ipv4"): + for field in ("public_ipv4", "public_ipv6", "bind_ipv4", "bind_ipv6", "monitor_ipv4", "monitor_ipv6", "log_ipv4", "log_ipv6"): value = validate_ip(node.get(field), required=field in {"public_ipv4", "bind_ipv4"}) - if value and field in {"public_ipv4", "public_ipv6", "monitor_ipv4", "log_ipv4"}: + if value and field in {"public_ipv4", "public_ipv6", "monitor_ipv4", "monitor_ipv6", "log_ipv4", "log_ipv6"}: if value in ips: raise ValidationError(f"Duplicate fleet IP address: {value}") ips.add(value) @@ -307,7 +307,9 @@ def _normalize_node(self, state: dict[str, Any], node: dict[str, Any]) -> dict[s "bind_ipv4": validate_ip(node.get("bind_ipv4") or "0.0.0.0", required=True), "bind_ipv6": validate_ip(node.get("bind_ipv6") or ("::" if state["global"].get("ipv6") else None)), "monitor_ipv4": validate_ip(node.get("monitor_ipv4")), + "monitor_ipv6": validate_ip(node.get("monitor_ipv6")), "log_ipv4": validate_ip(node.get("log_ipv4")), + "log_ipv6": validate_ip(node.get("log_ipv6")), "release": validate_release(node.get("release") or state["global"]["release"]), "extra_env": validate_env_mapping(node.get("extra_env", {})), "enabled": bool(node.get("enabled", True)), diff --git a/scripts/validate-production-overrides.sh b/scripts/validate-production-overrides.sh index 40a2871..e119b7e 100755 --- a/scripts/validate-production-overrides.sh +++ b/scripts/validate-production-overrides.sh @@ -5,9 +5,13 @@ cd "$(dirname "$0")/.." export CONTROL_HOSTNAME=control.ops.example.com export TELEMETRY_HOSTNAME=telemetry.ops.example.com +export GRAFANA_HOSTNAME=grafana.ops.example.com export CONTROL_PUBLIC_IPV4_ALLOWLIST="198.51.100.10 198.51.100.11" export EDGE_PUBLIC_IPV4_ALLOWLIST="198.51.100.20 198.51.100.30 198.51.100.40" export LOG_SOURCE_IPV4_ALLOWLIST="198.51.100.10 198.51.100.20 198.51.100.30 198.51.100.40 198.51.100.50" +export CONTROL_PUBLIC_IPV6_ALLOWLIST="2001:db8::10" +export EDGE_PUBLIC_IPV6_ALLOWLIST="2001:db8::20 2001:db8::30" +export LOG_SOURCE_IPV6_ALLOWLIST="2001:db8::10 2001:db8::20 2001:db8::30" export HOST_BIND_IPV4=0.0.0.0 export HOST_BIND_IPV6=:: export EDGE_CONTROL_BIND=0.0.0.0:8443 @@ -21,39 +25,9 @@ compose() { docker compose --env-file .env.prod.example -f compose.prod.yml "$@" } -# IPv4-only configuration must not require a fake IPv6 value. -unset HOST_BIND_IPV6 -compose -f deploy/production/compose.control-host.yml --profile control --profile telemetry --profile logs config --quiet -compose -f deploy/production/compose.control-host.yml --profile tools config --quiet +compose --profile control --profile telemetry --profile logs config --quiet +compose --profile dns --profile edge --profile logs config --quiet +compose --profile tools config --quiet -compose -f deploy/production/compose.dns-edge-host.yml --profile dns --profile edge --profile logs config --quiet -compose -f deploy/production/compose.dns-edge-host.yml --profile tools config --quiet -compose -f deploy/production/compose.dns-host.yml --profile dns --profile logs config --quiet -compose -f deploy/production/compose.edge-host.yml --profile edge --profile logs config --quiet - -compose -f deploy/production/compose.telemetry-host.yml --profile telemetry --profile logs config --quiet - -# IPv6 publication is explicit and validated independently. -export HOST_BIND_IPV6=:: -compose -f deploy/production/compose.control-host.yml \ - -f deploy/production/compose.control-host-ipv6.yml \ - --profile control --profile telemetry config --quiet -compose -f deploy/production/compose.dns-edge-host.yml \ - -f deploy/production/compose.dns-host-ipv6.yml \ - -f deploy/production/compose.edge-host-ipv6.yml \ - --profile dns --profile edge config --quiet -compose -f deploy/production/compose.telemetry-host.yml \ - -f deploy/production/compose.telemetry-host-ipv6.yml \ - --profile telemetry config --quiet - -export DB_URL='postgresql://cdnf:password@db.ops.example.com:5432/cdnf?sslmode=verify-full' -export REDIS_URL='tls://:password@redis.ops.example.com:6379' -compose -f deploy/production/compose.external-control-data.yml --profile control config --quiet - -if compose -f deploy/production/compose.external-control-data.yml --profile control config --services \ - | grep -Eq '^(control-db|redis)$'; then - echo "External control-data override unexpectedly enables a local data service." >&2 - exit 1 -fi - -echo "production_overrides=passed" +test "$(find deploy/production -maxdepth 1 -name 'compose*.yml' -print -quit)" = "" +echo "production_compose=passed" diff --git a/tests/fleet/test_fleet.py b/tests/fleet/test_fleet.py index 533ca89..b98fa92 100644 --- a/tests/fleet/test_fleet.py +++ b/tests/fleet/test_fleet.py @@ -194,16 +194,6 @@ def source_repo(tmp_path: Path) -> Path: }, } (root / "compose.prod.yml").write_text(yaml.safe_dump(compose, sort_keys=False), encoding="utf-8") - for name in [ - "compose.control-host.yml", - "compose.dns-host.yml", - "compose.edge-host.yml", - "compose.dns-edge-host.yml", - "compose.telemetry-host.yml", - "compose.external-control-data.yml", - "compose.host-journal.yml", - ]: - (root / "deploy/production" / name).write_text("services: {}\n", encoding="utf-8") return root diff --git a/tests/observability/test_operational_logging_contract.py b/tests/observability/test_operational_logging_contract.py index da43130..c337359 100644 --- a/tests/observability/test_operational_logging_contract.py +++ b/tests/observability/test_operational_logging_contract.py @@ -48,7 +48,7 @@ def test_one_host_collector_is_bounded_and_independent(self) -> None: def test_combined_control_host_can_enable_monitoring_after_serving(self) -> None: caddy = (ROOT / "deploy/production/Caddyfile").read_text() - overlay = (ROOT / "deploy/production/compose.control-host.yml").read_text() + overlay = (ROOT / "compose.prod.yml").read_text() generator = (ROOT / "scripts/generate-production-env.sh").read_text() self.assertIn("path /loki/api/v1/push", caddy) self.assertIn("reverse_proxy loki:3100", caddy)