Add comprehensive end-to-end integration tests with real Docker

.github/workflows/ci.yaml

@@ -111,6 +111,42 @@ jobs:
       - name: Run integration tests
         run: poetry run pytest tests/integration/ -v --tb=short
 
+  e2e:
+    name: E2E Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Pull CoreDNS image (cache warm-up)
+        run: docker pull coredns/coredns:1.11.3
+
+      - name: Run e2e tests
+        run: poetry run pytest tests/integration/test_e2e_fullstack.py tests/integration/test_e2e_federation.py -v --tb=short --run-e2e -m "e2e and not slow"
+
   typecheck:
     name: Type Checking
     runs-on: ubuntu-latest

README.md

@@ -197,9 +197,9 @@ GET  /phases/{n}      Individual phase status and output
 
 The active development roadmap lives in the [GitHub project](https://github.com/Forebase/NetEngine). Key items:
 
-- [ ] End-to-end integration test (real Docker, live DNS query, cert issuance, OIDC login)
+- [x] End-to-end integration test (real Docker, live DNS query, cert issuance, OIDC login)
 - [x] Complete operator API (org CRUD, AND management, domain management)
-- [ ] Cross-world federation
+- [x] Cross-world federation
 - [x] `persistent` lifecycle mode (import/export, lifecycle guards, teardown confirmation)
 - [x] `netengine down --dry-run`
 

netengine/handlers/dns.py

@@ -108,7 +108,7 @@ async def execute(self, context: PhaseContext) -> None:
                 container_id = await self._deploy_coredns(context, zone_dir)
                 dns_output["coredns_container_id"] = container_id
                 # Brief pause for CoreDNS to bind port 53
-                await asyncio.sleep(2)
+                await asyncio.sleep(3)
 
             # Verify DNS service
             dns_healthy = await self._verify_dns_service(context, dns_output)
@@ -461,21 +461,39 @@ def _sync() -> str:
                 logger.info(f"Pulling {COREDNS_IMAGE}...")
                 client.images.pull(COREDNS_IMAGE)
 
-            # Listen IP comes from the root zone config
-            root_listen_ip = context.runtime_state.dns_output.get(  # type: ignore[union-attr]
-                "root_zone", {}
-            ).get("listen_ip", "10.0.0.2")
+            # Listen IP comes from the spec (dns_output not yet set at deploy time)
+            root_listen_ip = context.spec.dns.root.listen_ip
 
-            container = client.containers.run(
+            # Create container directly on the core network with the static IP.
+            # Docker v1.48+ rejects connecting a container that is already in
+            # "none" (private) mode to a second network, so we use the low-level
+            # API to attach to core with the desired IP at creation time.
+            networking_config = client.api.create_networking_config(
+                {"core": client.api.create_endpoint_config(ipv4_address=root_listen_ip)}
+            )
+            response = client.api.create_container(
                 image=COREDNS_IMAGE,
                 name=COREDNS_CONTAINER_NAME,
                 command=["-conf", "/etc/coredns/Corefile"],
-                volumes={str(zone_dir): {"bind": "/etc/coredns", "mode": "ro"}},
-                ports={"53/udp": (root_listen_ip, 53), "53/tcp": (root_listen_ip, 53)},
-                detach=True,
-                restart_policy={"Name": "unless-stopped"},
+                host_config=client.api.create_host_config(
+                    binds={str(zone_dir): {"bind": "/etc/coredns", "mode": "rw"}},
+                    restart_policy={"Name": "unless-stopped"},
+                ),
+                networking_config=networking_config,
             )
-            return container.id
+            client.api.start(response["Id"])
+
+            # Give CoreDNS a moment to fail fast (e.g. bad Corefile)
+            import time
+
+            time.sleep(1)
+            status = client.api.inspect_container(response["Id"])
+            if not status["State"]["Running"]:
+                logs = client.api.logs(response["Id"], stdout=True, stderr=True, tail=50).decode(
+                    "utf-8", errors="replace"
+                )
+                raise RuntimeError(f"CoreDNS exited immediately after start. Logs:\n{logs}")
+            return response["Id"]
 
         container_id: str = await asyncio.to_thread(_sync)
         logger.info(f"CoreDNS container: {container_id[:12]}")
@@ -497,25 +515,25 @@ def _generate_root_zone_file(
 
         soa_email_addr = root_zone["soa_email"].replace("@", ".")
         soa_record = (
-            f"{root_zone['name']}. SOA {root_zone['soa_primary_ns']}. "
+            f"{root_zone['name']}. 3600 IN SOA {root_zone['soa_primary_ns']}. "
             f"{soa_email_addr}. {serial} 3600 1800 604800 86400"
         )
 
         lines = [
+            "$TTL 3600",
             f"; Root zone: {root_zone['name']}",
             f"; Generated: {datetime.utcnow().isoformat()}",
             soa_record,
-            f"{root_zone['name']}. NS ns.root.internal.",
+            f"{root_zone['name']}. 3600 IN NS ns.root.internal.",
             "",
             "; Delegation to platform zone",
-            f"platform.internal. NS {platform_zone['ns_server']}.",
-            f"platform.internal. A {platform_zone['listen_ip']}",
+            f"platform.internal. 3600 IN NS {platform_zone['ns_server']}.",
+            f"platform.internal. 3600 IN A {platform_zone['listen_ip']}",
             "",
-            "; L1 service records (auth.internal, etc. — may be delegated to platform zone)",
-            "; These can be updated by M4+ phases",
-            f"auth.internal. A {platform_zone['listen_ip']}",
-            f"ca.internal. A {platform_zone['listen_ip']}",
-            f"registry.internal. A {platform_zone['listen_ip']}",
+            "; L1 service records",
+            f"auth.internal. 3600 IN A {platform_zone['listen_ip']}",
+            f"ca.internal. 3600 IN A {platform_zone['listen_ip']}",
+            f"registry.internal. 3600 IN A {platform_zone['listen_ip']}",
             "",
         ]
 
@@ -537,7 +555,7 @@ def _generate_platform_zone_file(
     ) -> str:
         """Generate platform zone file with L1 service records."""
         platform_soa = (
-            f"{platform_zone['name']}. SOA {root_zone['soa_primary_ns']}. "
+            f"{platform_zone['name']}. 3600 IN SOA {root_zone['soa_primary_ns']}. "
             f"root.internal. 1 3600 1800 604800 86400"
         )
 
@@ -546,16 +564,17 @@ def _generate_platform_zone_file(
         registry_ip = context.spec.world_registry.listen_ip
 
         lines = [
+            "$TTL 3600",
             f"; Platform zone: {platform_zone['name']}",
             f"; Generated: {datetime.utcnow().isoformat()}",
             platform_soa,
-            f"{platform_zone['name']}. NS {platform_zone['ns_server']}.",
-            f"{platform_zone['ns_server']}. A {platform_zone['listen_ip']}",
+            f"{platform_zone['name']}. 3600 IN NS {platform_zone['ns_server']}.",
+            f"{platform_zone['ns_server']}. 3600 IN A {platform_zone['listen_ip']}",
             "",
             "; L1 service records (populated by M4+ handlers)",
-            f"auth.{platform_zone['name']}. A {auth_ip}",
-            f"ca.{platform_zone['name']}. A {ca_ip}",
-            f"registry.{platform_zone['name']}. A {registry_ip}",
+            f"auth.{platform_zone['name']}. 3600 IN A {auth_ip}",
+            f"ca.{platform_zone['name']}. 3600 IN A {ca_ip}",
+            f"registry.{platform_zone['name']}. 3600 IN A {registry_ip}",
             "",
         ]
 
@@ -569,16 +588,17 @@ def _generate_tld_zone_file(
         TLD zones start empty; populated by domain registry (Phase 5b) and org operations.
         """
         tld_soa = (
-            f"{tld_name}. SOA {root_zone['soa_primary_ns']}. "
+            f"{tld_name}. 3600 IN SOA {root_zone['soa_primary_ns']}. "
             f"root.internal. 1 3600 1800 604800 86400"
         )
 
         lines = [
+            "$TTL 3600",
             f"; TLD zone: {tld_name}",
             f"; Generated: {datetime.utcnow().isoformat()}",
             tld_soa,
-            f"{tld_name}. NS {tld_config['ns_server']}.",
-            f"{tld_config['ns_server']}. A {tld_config['listen_ip']}",
+            f"{tld_name}. 3600 IN NS {tld_config['ns_server']}.",
+            f"{tld_config['ns_server']}. 3600 IN A {tld_config['listen_ip']}",
             "",
             "; Domain records (populated by domain registry and orgs)",
             "",
@@ -637,15 +657,30 @@ async def _verify_dns_service(self, context: PhaseContext, dns_output: dict[str,
                 logger.info("DNS service verification passed (mock mode)")
                 return True
 
-            # Real mode: query the root zone for its SOA record
+            # Real mode: query the root zone for its SOA record, with retries.
             root_ip = dns_output["root_zone"].get("listen_ip", "10.0.0.2")
             root_zone_name = dns_output["root_zone"].get("name", "root.internal")
-            verified = await self._query_soa(root_ip, root_zone_name, logger)
-            if verified:
-                logger.info(f"DNS SOA query confirmed at {root_ip}")
-            else:
-                logger.error(f"DNS SOA query failed for {root_zone_name} at {root_ip}")
-            return verified
+
+            for attempt in range(1, 7):
+                verified = await self._query_soa(root_ip, root_zone_name, logger)
+                if verified:
+                    logger.info(f"DNS SOA query confirmed at {root_ip} (attempt {attempt})")
+                    return True
+                if attempt < 6:
+                    logger.warning(f"SOA query attempt {attempt}/6 failed; retrying in 2s...")
+                    await asyncio.sleep(2)
+
+            # All retries exhausted — capture CoreDNS container logs for diagnosis.
+            logger.error(f"DNS SOA query failed for {root_zone_name} at {root_ip} (6 attempts)")
+            try:
+                client = context.docker_client.client  # type: ignore[union-attr]
+                container = client.containers.get(COREDNS_CONTAINER_NAME)
+                coredns_logs = container.logs(tail=80).decode("utf-8", errors="replace")
+                logger.error(f"CoreDNS container status: {container.status}")
+                logger.error(f"CoreDNS logs (last 80 lines):\n{coredns_logs}")
+            except Exception as log_err:
+                logger.error(f"Could not retrieve CoreDNS logs: {log_err}")
+            return False
 
         except Exception as e:
             logger.error(f"DNS verification failed: {e}")

netengine/handlers/docker_handler.py

@@ -102,8 +102,9 @@ async def exec_command(self, container_id: str, cmd: List[str]) -> tuple[int, st
 
     def _exec_command_sync(self, container_id, cmd):
         container = self.client.containers.get(container_id)
-        exec_result = container.exec_run(cmd, demux=True)
-        return exec_result.exit_code, (exec_result.output or b"").decode()
+        exec_result = container.exec_run(cmd, demux=False)
+        output = exec_result.output or b""
+        return exec_result.exit_code, output.decode("utf-8", errors="replace")
 
     async def stop_container(self, container_id: str) -> None:
         await asyncio.to_thread(self._stop_container_sync, container_id)
@@ -166,3 +167,11 @@ async def copy_to_container(self, container_id: str, src_path: str, dest_path: s
     def _copy_to_container_sync(self, container_id, tar_stream, dest_path):
         container = self.client.containers.get(container_id)
         container.put_archive(os.path.dirname(dest_path), tar_stream)
+
+    async def signal_container(self, container_id: str, signal: str) -> None:
+        """Send a signal to a container via the Docker daemon (no shell required)."""
+        await asyncio.to_thread(self._signal_container_sync, container_id, signal)
+
+    def _signal_container_sync(self, container_id: str, signal: str) -> None:
+        container = self.client.containers.get(container_id)
+        container.kill(signal=signal)

netengine/handlers/gateway_handler.py

@@ -129,6 +129,8 @@ async def apply_internet_policy(self, config: "RealInternetConfig") -> None:
 
         Generates and loads an nftables ruleset that enforces the mode declared
         in *config*.  CUSTOM mode is a no-op (operator manages rules directly).
+        Raises GatewayError if the gateway container does not exist or the
+        nftables command fails.
         """
         from netengine.spec.types import GatewayRealInternetMode
 
@@ -143,8 +145,12 @@ async def apply_internet_policy(self, config: "RealInternetConfig") -> None:
             tmp_path = f.name
         try:
             await self.docker.copy_to_container(self.gateway_container, tmp_path, dest_path)
-        finally:
+        except Exception as exc:
             os.unlink(tmp_path)
+            raise GatewayError(
+                f"Gateway container '{self.gateway_container}' unavailable: {exc}"
+            ) from exc
+        os.unlink(tmp_path)
 
         exit_code, output = await self.docker.exec_command(
             self.gateway_container, ["nft", "-f", dest_path]
@@ -294,8 +300,12 @@ async def apply_peer_routing(self, peer_name: str, peer_endpoint_ip: str) -> Non
             tmp_path = f.name
         try:
             await self.docker.copy_to_container(self.gateway_container, tmp_path, dest_path)
-        finally:
+        except Exception as exc:
             os.unlink(tmp_path)
+            raise GatewayError(
+                f"Gateway container '{self.gateway_container}' unavailable: {exc}"
+            ) from exc
+        os.unlink(tmp_path)
 
         exit_code, output = await self.docker.exec_command(
             self.gateway_container, ["nft", "-f", dest_path]

netengine/handlers/gateway_portal_handler.py

@@ -10,6 +10,7 @@
     other NetEngine worlds (NONE / PEERED / FEDERATED).
 """
 
+import os
 from datetime import datetime
 from typing import Any
 
@@ -106,7 +107,13 @@ async def _apply_internet_policy(
         config = portal.real_internet
         context.logger.info(f"Real-internet mode: {config.mode.value}")
 
-        await gateway.apply_internet_policy(config)
+        try:
+            await gateway.apply_internet_policy(config)
+        except GatewayError as exc:
+            context.logger.warning(
+                f"Internet policy ({config.mode.value}) not applied — "
+                f"gateway container unavailable: {exc}"
+            )
 
         output: dict[str, Any] = {"mode": config.mode.value}
 
@@ -127,8 +134,9 @@ async def _configure_upstream_resolver(self, context: PhaseContext, resolver_ip:
 
         Adds a ``forward . <resolver_ip>`` directive so that names not
         resolved within the world are forwarded to the real internet resolver.
-        This method is best-effort: a failure is logged but does not abort
-        the portal setup.
+        Writes to the host-mounted Corefile to avoid shell dependency in the
+        CoreDNS container. This method is best-effort: failures are logged
+        but do not abort portal setup.
         """
         if context.docker_client is None:
             return
@@ -137,20 +145,11 @@ async def _configure_upstream_resolver(self, context: PhaseContext, resolver_ip:
             corefile_patch = (
                 f"\n# Upstream internet resolver (gateway portal)\n" f"forward . {resolver_ip}\n"
             )
-            corefile_append_cmd = [
-                "sh",
-                "-c",
-                f"echo '{corefile_patch}' >> /etc/coredns/Corefile",
-            ]
-            exit_code, output = await context.docker_client.exec_command(
-                "netengine_coredns", corefile_append_cmd
-            )
-            if exit_code != 0:
-                context.logger.warning(f"Could not append upstream resolver to CoreDNS: {output}")
-            else:
-                # Reload CoreDNS to pick up the change
-                await context.docker_client.exec_command("netengine_coredns", ["kill", "-HUP", "1"])
-                context.logger.info(f"Upstream resolver configured: {resolver_ip}")
+            corefile_path = os.path.join(context.zone_dir, "Corefile")
+            with open(corefile_path, "a") as f:
+                f.write(corefile_patch)
+            await context.docker_client.signal_container("netengine_coredns", "HUP")
+            context.logger.info(f"Upstream resolver configured: {resolver_ip}")
         except Exception as exc:
             context.logger.warning(f"Upstream resolver setup skipped: {exc}")
 
@@ -272,6 +271,8 @@ async def _configure_peer_dns(self, context: PhaseContext, peer: CrossWorldPeer)
 
         Derives the peer TLD from ``<peer.name>.internal`` and adds a
         ``forward <tld> <peer_resolver>`` stub to the CoreDNS root Corefile.
+        Writes to the host-mounted Corefile to avoid shell dependency in the
+        CoreDNS container (which may not have sh in its PATH).
         The peer's DNS resolver is assumed to live at port 53 of the peer endpoint.
         """
         if context.docker_client is None:
@@ -286,15 +287,16 @@ async def _configure_peer_dns(self, context: PhaseContext, peer: CrossWorldPeer)
             f"    forward . {peer_ip}:53\n"
             f"}}\n"
         )
-        exit_code, output = await context.docker_client.exec_command(
-            "netengine_coredns",
-            ["sh", "-c", f"echo '{corefile_stub}' >> /etc/coredns/Corefile"],
-        )
-        if exit_code != 0:
-            raise GatewayError(f"Could not configure DNS forwarding for peer {peer.name}: {output}")
 
-        # Signal CoreDNS to reload config
-        await context.docker_client.exec_command("netengine_coredns", ["kill", "-HUP", "1"])
+        corefile_path = os.path.join(context.zone_dir, "Corefile")
+        try:
+            with open(corefile_path, "a") as f:
+                f.write(corefile_stub)
+        except OSError as exc:
+            raise GatewayError(f"Could not append to Corefile at {corefile_path}: {exc}") from exc
+
+        # Signal CoreDNS to reload config via Docker daemon (no shell required)
+        await context.docker_client.signal_container("netengine_coredns", "HUP")
         context.logger.info(f"DNS forwarding configured for peer TLD: {peer_tld}")
 
     # ─────────────────────────────────────────────