feat: mTLS AWS <> Cloudflare & DNSSEC

.github/workflows/ci.yml

@@ -7,43 +7,53 @@
   cancel-in-progress: true
 
 jobs:
+  should-run:
+    runs-on: ubuntu-latest
+    if: |
+      !contains(github.event.head_commit.message, 'skip-ci')
+    steps:
+      - run: echo "Running tests"
+
   # azure:
+  #   needs: should-run
   #   strategy:
   #     matrix:
   #       module:
   #         - blob-storage
   #         - container-app
   #         - container-registry
   #         - database/postgres
   #         - observability/alerts
   #         - observability/monitors
   #         - virtual-network
   #     fail-fast: false
   #   runs-on: ubuntu-latest
   #   steps:
   #     - uses: actions/checkout@v4
   #     - uses: opentofu/setup-opentofu@v1
 
   #     - name: init
   #       run: |
   #         cd azure
   #         cd ${{ matrix.module }}
   #         tofu init
   #     - name: validate
   #       run: |
   #         cd azure
   #         cd ${{ matrix.module }}
   #         tofu validate
   #     - name: fmt -check
   #       run: |
   #         cd azure
   #         cd ${{ matrix.module }}
   #         tofu fmt -check
 
   aws:
+    needs: should-run
     strategy:
       matrix:
         module:
+          - alb-mtls
           - autoscaling/ecs
           - aws-transfer
           - aws-transfer/iam-role
@@ -83,6 +93,7 @@
           - multi-stack/setup
           - nat
           - rds
+          - route53domains-dnssec
           - s3
           - s3-private
           - s3-public
@@ -112,9 +123,13 @@
         run: |
           cd aws
           cd ${{ matrix.module }}
+          # stack/app cannot be pinned to the matrix version: its aurora child module
+          # requires aws >= 6.0, so let terraform resolve the real constraints instead.
+          if [[ "${{ matrix.module }}" == "stack/app" ]]; then exit 0; fi
           FILE=versions.tf
           if test -f "$FILE"; then
-            sed -i '/aws = {/,/}/ s/version = "[^"]*/version = "${{ matrix.aws-version }}/' $FILE
+            # tolerate column-aligned assignments, e.g. `version   = "..."`
+            sed -i '/aws[[:space:]]*=[[:space:]]*{/,/}/ s/version[[:space:]]*=[[:space:]]*"[^"]*/version = "${{ matrix.aws-version }}/' $FILE
 
             echo "$FILE exists."
           else
@@ -131,6 +146,10 @@
           EOF
           fi
 
+          # restore canonical formatting (e.g. column alignment) after the rewrite,
+          # otherwise the fmt -check step fails on the file this step just edited
+          terraform fmt $FILE
+
           cat $FILE
 
       - name: init
@@ -146,6 +165,7 @@
           # the stack/app module must have a provider block because the sub-module "vpc-peering" requires a non-standard "provider
           # see: https://github.com/hashicorp/terraform/issues/28490
           if [[ "${{ matrix.module }}" == "stack/app" ]]; then echo "provider \"aws\" { alias = \"peer\" }" >> vpc-peering.tf; fi
+          if [[ "${{ matrix.module }}" == "stack/app" ]]; then echo "provider \"aws\" { alias = \"us-east-1\" }" >> dnssec.tf; fi
           if [[ "${{ matrix.module }}" == "stack/global" ]]; then echo "provider \"aws\" { alias = \"us-east-1\" }" >> versions.tf; fi
           if [[ "${{ matrix.module }}" == "vpc-peering" ]]; then echo "provider \"aws\" { alias = \"peer\" }" >> main.tf; fi
           terraform validate
@@ -156,55 +176,78 @@
           cd ${{ matrix.module }}
           terraform fmt -check
 
+  tests:
+    needs: should-run
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v3
+
+      # terraform test with mock_provider requires Terraform >= 1.7
+      - name: module tests (route53domains-dnssec)
+        run: |
+          cd aws/route53domains-dnssec
+          terraform init
+          terraform test
+
+      # validates the stack as a consumer calls it (provider aliases, mTLS + DNSSEC inputs)
+      - name: stack/app consumer validation
+        run: |
+          cd aws/stack/app/tests/validate
+          terraform init
+          terraform validate
+
   misc:
+    needs: should-run
     strategy:
       matrix:
         module:
           - cloudflare
           - fivetran
           - fivetran/connectors/google_analytics
           - fivetran/connectors/lambda
           - script/database-roles
           - snowflake/cloud
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v3
 
       - name: init
         run: |
           cd ${{ matrix.module }}
           terraform init
 
       - name: validate
         run: |
           cd ${{ matrix.module }}
           terraform validate
 
       - name: fmt -check
         run: |
           cd ${{ matrix.module }}
           terraform fmt -check
 
   security:
+    needs: should-run
     name: Trivy Security Scan
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Run Trivy vulnerability scanner in repo mode
         uses: aquasecurity/trivy-action@master
         with:
           scan-type: "config"
           hide-progress: true
           format: "sarif"
           output: "trivy-results.sarif"
           exit-code: "0"
           ignore-unfixed: true
           severity: "CRITICAL,HIGH"
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "trivy-results.sarif"

.gitignore

@@ -17,6 +17,7 @@
 
 # Special\Personal dev setup
 /.idea
+docs/
 
 # os
 .DS_Store
@@ -31,3 +32,7 @@
 cloudwatch-snowflake/dist/tracker.zip
 
 .env
+
+# Local mTLS certificate material
+script/mtls_certs/
+/mtls_certs/

aws/alb-mtls/README.md

@@ -0,0 +1,173 @@
+# AWS ALB mTLS (Trust Store) Module
+
+This module provisions an AWS ALB Trust Store to enable mTLS (mutual TLS) authentication on an ALB listener. This is particularly useful for implementing **Cloudflare Authenticated Origin Pulls**.
+
+## Overview
+
+AWS ALB Trust Stores (`aws_lb_trust_store`) are required for verifying incoming client certificates (such as the certificate Cloudflare presents when proxying traffic to your origin). 
+
+**Important limitation:** Trust Stores cannot use AWS Certificate Manager (ACM). ACM only stores *server* certificates (what the ALB presents to clients). Trust Stores *must* read CA certificate bundles from S3. This module creates a minimal, locked-down S3 bucket to store the provided CA bundle and creates the Trust Store from it.
+
+## ⚠️ Rollout Warning
+
+When enabling mTLS on an *existing* ALB, **deployment order matters**:
+Adding `mutual_authentication` with `mode = "verify"` to an active ALB listener **immediately rejects all requests without a valid client certificate**. 
+
+The safe rollout strategy is:
+1. Generate your CA and leaf certificates (e.g., using `script/generate-mtls-certs.sh`).
+2. Deploy the **Cloudflare** side first (upload the leaf cert, enable Authenticated Origin Pulls for the zone/hostname).
+3. Deploy the **AWS** side with `alb_mtls_mode = "passthrough"`. Passthrough does not validate the client certificate; it forwards the presented certificate chain to the target in `X-Amzn-Mtls-Clientcert`.
+4. Confirm through a diagnostic target or ALB connection logs that Cloudflare presents the expected client certificate. Do not treat passthrough as an access-control boundary.
+5. Switch the AWS config to `alb_mtls_mode = "verify"` to enforce protection.
+
+In `verify` mode the ALB terminates an invalid or certificate-less TLS connection before it becomes an HTTP request. Keep Cloudflare IP allowlisting as defense in depth, and keep the Cloudflare SSL mode at **Full (strict)**.
+
+The CA private key is not required by Terraform or AWS, but it is required whenever a new Cloudflare client leaf certificate is issued. Keep it encrypted in a restricted administrator vault. The CA certificate is public material and is the only CA material uploaded to the ALB trust store.
+
+## CA storage
+
+`script/generate-mtls-certs.sh` encrypts `rootca.key` with AES-256. Set `MTLS_CA_PASSPHRASE` for non-interactive use; when run from a terminal, the script prompts for it instead:
+
+```shell
+MTLS_CA_PASSPHRASE="$passphrase" ./script/generate-mtls-certs.sh "*.example.com"
+```
+
+For projects using the restricted `${project}/terraform/${environment}` Secrets Manager vault, store these values there:
+
+- `cloudflare_aop_ca_private_key_pem`: the encrypted contents of `rootca.key`.
+- `cloudflare_aop_ca_private_key_passphrase`: the encryption passphrase.
+- `cloudflare_aop_ca_certificate_pem`: the public `rootca.crt`, so CI can create renewed leaf certificates without another storage dependency.
+
+The encrypted key and its passphrase may share this vault when access is already limited to the administrators and CI role authorized to manage infrastructure-level credentials.
+
+### Upload without storing the values in Terraform state
+
+Terraform 1.11 and AWS provider 5.99.1 introduced write-only arguments. `secret_string_wo` sends the local value to Secrets Manager but omits it from Terraform plan and state files. `sensitive` only redacts terminal output; it does not provide this state protection by itself.
+
+Add these paths to the consuming project's `.gitignore`:
+
+```gitignore
+/.terraform-secrets.json
+/certificates/cloudflare-aop/
+```
+
+Create `.terraform-secrets.json` locally with mode `0600`:
+
+```json
+{
+  "cloudflare_aop_ca_private_key_passphrase": "replace-with-a-strong-passphrase"
+}
+```
+
+Place `rootca.key` and `rootca.crt` in `certificates/cloudflare-aop/`, then create and populate the restricted vault:
+
+```terraform
+terraform {
+  required_version = ">= 1.11"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.99.1"
+    }
+  }
+}
+
+locals {
+  terraform_secret_upload = merge(
+    jsondecode(file("${path.root}/.terraform-secrets.json")),
+    {
+      cloudflare_aop_ca_private_key_pem = file("${path.root}/certificates/cloudflare-aop/rootca.key")
+      cloudflare_aop_ca_certificate_pem = file("${path.root}/certificates/cloudflare-aop/rootca.crt")
+    }
+  )
+}
+
+module "terraform_secrets" {
+  source = "github.com/dbl-works/terraform//aws/secrets?ref=main"
+
+  project        = var.project
+  environment    = var.environment
+  application    = "terraform"
+  create_kms_key = true
+  description    = "Infrastructure-level credentials restricted to administrators and Terraform CI."
+}
+
+resource "aws_secretsmanager_secret_version" "terraform" {
+  secret_id = module.terraform_secrets.id
+
+  secret_string_wo         = jsonencode(sensitive(local.terraform_secret_upload))
+  secret_string_wo_version = 1
+}
+```
+
+Increment `secret_string_wo_version` whenever the local values are intentionally changed. Terraform cannot compare a write-only value with the previous version because that value is never stored in state.
+
+Terraform evaluates the local files whenever this configuration is planned. Keep this upload configuration in a dedicated bootstrap root that administrators run only when synchronizing the vault, rather than in every application's normal deployment root. Apply the upload once before adding a consumer that reads the secret ephemerally.
+
+### Read without storing the values in Terraform state
+
+Use the ephemeral Secrets Manager resource rather than the ordinary `data "aws_secretsmanager_secret_version"` data source. The ordinary data source marks `secret_string` as sensitive but still serializes it into state.
+
+```terraform
+ephemeral "aws_secretsmanager_secret_version" "terraform" {
+  secret_id = module.terraform_secrets.id
+}
+
+locals {
+  credentials_terraform = jsondecode(
+    ephemeral.aws_secretsmanager_secret_version.terraform.secret_string
+  )
+}
+```
+
+Values derived from this local remain ephemeral. Terraform permits them only in ephemeral-compatible contexts such as provider configuration, write-only resource arguments, ephemeral child-module inputs and outputs, or provisioner environments. Passing a value into an ordinary resource argument still requires Terraform to persist it and is therefore rejected. See HashiCorp's [sensitive and ephemeral data guidance](https://developer.hashicorp.com/terraform/language/manage-sensitive-data).
+
+## Leaf certificate rotation
+
+Cloudflare does not replace an expired custom Authenticated Origin Pulls certificate automatically. Configure Cloudflare's zone-level AOP certificate expiration notification, which warns 30 and 14 days before expiry, and rotate well before the first warning becomes urgent.
+
+`script/rotate-mtls-certificates.sh` is a portable helper intended to be copied into a project's deployment repository or invoked from that repository's CI job:
+
+```shell
+export MTLS_CA_PASSPHRASE="$passphrase"
+./script/rotate-mtls-certificates.sh \
+  "*.example.com" \
+  /tmp/cloudflare-aop/rootca.key \
+  /tmp/cloudflare-aop/rootca.crt \
+  /tmp/cloudflare-aop/renewed
+```
+
+An administrator-triggered or scheduled CI rotation should:
+
+1. Assume a narrowly scoped deployment role and read the three CA values from `${project}/terraform/${environment}`.
+2. Write the CA key and certificate to a private temporary directory, without logging their contents.
+3. Run the rotation script to produce a new `cert.crt` and `cert.key` signed by the existing CA.
+4. Persist the new leaf certificate and key wherever the normal Terraform CI job hydrates them, make those files available to the Terraform configuration, and run `terraform apply`. The Cloudflare certificate resource uses `create_before_destroy`, so it uploads and selects the new certificate before deleting the old one.
+5. Verify proxied traffic succeeds and a direct request without a client certificate is rejected by the ALB.
+6. Delete all temporary files.
+
+Automate leaf rotation only. Root CA rotation requires an overlapping ALB trust-store rollout and remains a deliberate operation. The rotation script refuses to issue a leaf whose requested lifetime exceeds the remaining CA lifetime. See [Cloudflare's AOP certificate management guidance](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/set-up/manage-certificates/).
+
+## Architecture
+
+```mermaid
+sequenceDiagram
+    participant User
+    participant CF as Cloudflare Edge<br/>(your account)
+    participant ALB as AWS ALB
+    participant ECS as ECS Containers
+
+    User->>CF: HTTPS request to api.example.com
+    Note over CF: Cloudflare presents YOUR<br/>custom leaf certificate
+    CF->>ALB: mTLS handshake (leaf cert signed by your CA)
+    ALB->>ALB: Verify cert chain against Trust Store<br/>(your root CA only)
+    ALB->>ECS: Forward to target group
+    ECS-->>User: Response (via CF)
+
+    Note over User,ALB: Attack scenario — blocked
+    User->>CF: Attacker routes evil.com → ALB origin
+    Note over CF: Cloudflare presents a certificate<br/>that was not signed by your CA
+    CF->>ALB: mTLS handshake (wrong CA)
+    ALB--xCF: ❌ Trust Store rejects<br/>(not signed by your CA)
+```

aws/alb-mtls/main.tf

@@ -0,0 +1,48 @@
+locals {
+  name = var.name != null ? var.name : "mtls"
+}
+
+# S3 bucket — only exists because aws_lb_trust_store requires S3
+module "s3_bucket" {
+  source = "../s3-private"
+
+  project     = var.project
+  environment = var.environment
+  bucket_name = "${var.project}-${var.environment}-${local.name}"
+
+  # ALB Trust Store only needs to read the file, no special features required
+  versioning = true
+
+  # No user-uploaded files, only a static CA cert managed by Terraform.
+  # Disable malware scanning to avoid unnecessary cost and a potential
+  # race condition where the Trust Store cannot read an unscanned object.
+  file_malwarescanning = {
+    enabled                           = false
+    allow_downloading_unscanned_files = true
+  }
+}
+
+resource "aws_s3_object" "ca_bundle" {
+  bucket       = module.s3_bucket.id
+  key          = "ca-certificates-bundle.pem"
+  content      = var.ca_certificates_pem
+  content_type = "application/x-pem-file"
+
+  # Referencing only module.s3_bucket.id would not wait for the bucket's
+  # versioning configuration, and an object created before versioning is
+  # enabled has no version_id for the trust store to pin.
+  depends_on = [module.s3_bucket]
+}
+
+# ALB Trust Store — references the CA bundle in S3
+resource "aws_lb_trust_store" "main" {
+  name                                     = "${var.project}-${var.environment}-${local.name}"
+  ca_certificates_bundle_s3_bucket         = module.s3_bucket.id
+  ca_certificates_bundle_s3_key            = aws_s3_object.ca_bundle.key
+  ca_certificates_bundle_s3_object_version = aws_s3_object.ca_bundle.version_id
+  tags = {
+    Name        = "${var.project}-${var.environment}-${local.name}"
+    Project     = var.project
+    Environment = var.environment
+  }
+}

aws/alb-mtls/outputs.tf

@@ -0,0 +1,9 @@
+output "trust_store_arn" {
+  description = "ARN of the ALB Trust Store."
+  value       = aws_lb_trust_store.main.arn
+}
+
+output "trust_store_name" {
+  description = "Name of the ALB Trust Store."
+  value       = aws_lb_trust_store.main.name
+}

aws/alb-mtls/variables.tf

@@ -0,0 +1,20 @@
+variable "project" {
+  type = string
+}
+
+variable "environment" {
+  type = string
+}
+
+# Not marked sensitive: a CA certificate is public material, and sensitivity
+# would propagate into plan output, hiding the trust store and listener diffs.
+variable "ca_certificates_pem" {
+  description = "PEM-encoded root CA certificate that signed the Cloudflare client certificate."
+  type        = string
+}
+
+variable "name" {
+  description = "Name prefix for trust store resources. Defaults to mTLS."
+  type        = string
+  default     = "mtls"
+}

aws/alb-mtls/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+      # aws_lb_trust_store was added in 5.30; kept consistent with the ecs and route53domains-dnssec modules
+      version = ">= 5.34"
+    }
+  }
+  required_version = ">= 1.0"
+}