diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 585b2cf..9746b52 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -111,6 +111,42 @@ jobs: - name: Run integration tests run: poetry run pytest tests/integration/ -v --tb=short + e2e: + name: E2E Tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.13"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: latest + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Cache Poetry dependencies + uses: actions/cache@v3 + with: + path: .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + run: poetry install + + - name: Pull CoreDNS image (cache warm-up) + run: docker pull coredns/coredns:1.11.3 + + - name: Run e2e tests + run: poetry run pytest tests/integration/test_e2e_fullstack.py tests/integration/test_e2e_federation.py -v --tb=short --run-e2e -m "e2e and not slow" + typecheck: name: Type Checking runs-on: ubuntu-latest diff --git a/README.md b/README.md index 703b28a..a577032 100644 --- a/README.md +++ b/README.md @@ -197,9 +197,9 @@ GET /phases/{n} Individual phase status and output The active development roadmap lives in the [GitHub project](https://github.com/Forebase/NetEngine). Key items: -- [ ] End-to-end integration test (real Docker, live DNS query, cert issuance, OIDC login) +- [x] End-to-end integration test (real Docker, live DNS query, cert issuance, OIDC login) - [x] Complete operator API (org CRUD, AND management, domain management) -- [ ] Cross-world federation +- [x] Cross-world federation - [x] `persistent` lifecycle mode (import/export, lifecycle guards, teardown confirmation) - [x] `netengine down --dry-run` diff --git a/netengine/handlers/dns.py b/netengine/handlers/dns.py index c1a3230..0210440 100644 --- a/netengine/handlers/dns.py +++ b/netengine/handlers/dns.py @@ -108,7 +108,7 @@ async def execute(self, context: PhaseContext) -> None: container_id = await self._deploy_coredns(context, zone_dir) dns_output["coredns_container_id"] = container_id # Brief pause for CoreDNS to bind port 53 - await asyncio.sleep(2) + await asyncio.sleep(3) # Verify DNS service dns_healthy = await self._verify_dns_service(context, dns_output) @@ -461,21 +461,39 @@ def _sync() -> str: logger.info(f"Pulling {COREDNS_IMAGE}...") client.images.pull(COREDNS_IMAGE) - # Listen IP comes from the root zone config - root_listen_ip = context.runtime_state.dns_output.get( # type: ignore[union-attr] - "root_zone", {} - ).get("listen_ip", "10.0.0.2") + # Listen IP comes from the spec (dns_output not yet set at deploy time) + root_listen_ip = context.spec.dns.root.listen_ip - container = client.containers.run( + # Create container directly on the core network with the static IP. + # Docker v1.48+ rejects connecting a container that is already in + # "none" (private) mode to a second network, so we use the low-level + # API to attach to core with the desired IP at creation time. + networking_config = client.api.create_networking_config( + {"core": client.api.create_endpoint_config(ipv4_address=root_listen_ip)} + ) + response = client.api.create_container( image=COREDNS_IMAGE, name=COREDNS_CONTAINER_NAME, command=["-conf", "/etc/coredns/Corefile"], - volumes={str(zone_dir): {"bind": "/etc/coredns", "mode": "ro"}}, - ports={"53/udp": (root_listen_ip, 53), "53/tcp": (root_listen_ip, 53)}, - detach=True, - restart_policy={"Name": "unless-stopped"}, + host_config=client.api.create_host_config( + binds={str(zone_dir): {"bind": "/etc/coredns", "mode": "rw"}}, + restart_policy={"Name": "unless-stopped"}, + ), + networking_config=networking_config, ) - return container.id + client.api.start(response["Id"]) + + # Give CoreDNS a moment to fail fast (e.g. bad Corefile) + import time + + time.sleep(1) + status = client.api.inspect_container(response["Id"]) + if not status["State"]["Running"]: + logs = client.api.logs(response["Id"], stdout=True, stderr=True, tail=50).decode( + "utf-8", errors="replace" + ) + raise RuntimeError(f"CoreDNS exited immediately after start. Logs:\n{logs}") + return response["Id"] container_id: str = await asyncio.to_thread(_sync) logger.info(f"CoreDNS container: {container_id[:12]}") @@ -497,25 +515,25 @@ def _generate_root_zone_file( soa_email_addr = root_zone["soa_email"].replace("@", ".") soa_record = ( - f"{root_zone['name']}. SOA {root_zone['soa_primary_ns']}. " + f"{root_zone['name']}. 3600 IN SOA {root_zone['soa_primary_ns']}. " f"{soa_email_addr}. {serial} 3600 1800 604800 86400" ) lines = [ + "$TTL 3600", f"; Root zone: {root_zone['name']}", f"; Generated: {datetime.utcnow().isoformat()}", soa_record, - f"{root_zone['name']}. NS ns.root.internal.", + f"{root_zone['name']}. 3600 IN NS ns.root.internal.", "", "; Delegation to platform zone", - f"platform.internal. NS {platform_zone['ns_server']}.", - f"platform.internal. A {platform_zone['listen_ip']}", + f"platform.internal. 3600 IN NS {platform_zone['ns_server']}.", + f"platform.internal. 3600 IN A {platform_zone['listen_ip']}", "", - "; L1 service records (auth.internal, etc. — may be delegated to platform zone)", - "; These can be updated by M4+ phases", - f"auth.internal. A {platform_zone['listen_ip']}", - f"ca.internal. A {platform_zone['listen_ip']}", - f"registry.internal. A {platform_zone['listen_ip']}", + "; L1 service records", + f"auth.internal. 3600 IN A {platform_zone['listen_ip']}", + f"ca.internal. 3600 IN A {platform_zone['listen_ip']}", + f"registry.internal. 3600 IN A {platform_zone['listen_ip']}", "", ] @@ -537,7 +555,7 @@ def _generate_platform_zone_file( ) -> str: """Generate platform zone file with L1 service records.""" platform_soa = ( - f"{platform_zone['name']}. SOA {root_zone['soa_primary_ns']}. " + f"{platform_zone['name']}. 3600 IN SOA {root_zone['soa_primary_ns']}. " f"root.internal. 1 3600 1800 604800 86400" ) @@ -546,16 +564,17 @@ def _generate_platform_zone_file( registry_ip = context.spec.world_registry.listen_ip lines = [ + "$TTL 3600", f"; Platform zone: {platform_zone['name']}", f"; Generated: {datetime.utcnow().isoformat()}", platform_soa, - f"{platform_zone['name']}. NS {platform_zone['ns_server']}.", - f"{platform_zone['ns_server']}. A {platform_zone['listen_ip']}", + f"{platform_zone['name']}. 3600 IN NS {platform_zone['ns_server']}.", + f"{platform_zone['ns_server']}. 3600 IN A {platform_zone['listen_ip']}", "", "; L1 service records (populated by M4+ handlers)", - f"auth.{platform_zone['name']}. A {auth_ip}", - f"ca.{platform_zone['name']}. A {ca_ip}", - f"registry.{platform_zone['name']}. A {registry_ip}", + f"auth.{platform_zone['name']}. 3600 IN A {auth_ip}", + f"ca.{platform_zone['name']}. 3600 IN A {ca_ip}", + f"registry.{platform_zone['name']}. 3600 IN A {registry_ip}", "", ] @@ -569,16 +588,17 @@ def _generate_tld_zone_file( TLD zones start empty; populated by domain registry (Phase 5b) and org operations. """ tld_soa = ( - f"{tld_name}. SOA {root_zone['soa_primary_ns']}. " + f"{tld_name}. 3600 IN SOA {root_zone['soa_primary_ns']}. " f"root.internal. 1 3600 1800 604800 86400" ) lines = [ + "$TTL 3600", f"; TLD zone: {tld_name}", f"; Generated: {datetime.utcnow().isoformat()}", tld_soa, - f"{tld_name}. NS {tld_config['ns_server']}.", - f"{tld_config['ns_server']}. A {tld_config['listen_ip']}", + f"{tld_name}. 3600 IN NS {tld_config['ns_server']}.", + f"{tld_config['ns_server']}. 3600 IN A {tld_config['listen_ip']}", "", "; Domain records (populated by domain registry and orgs)", "", @@ -637,15 +657,30 @@ async def _verify_dns_service(self, context: PhaseContext, dns_output: dict[str, logger.info("DNS service verification passed (mock mode)") return True - # Real mode: query the root zone for its SOA record + # Real mode: query the root zone for its SOA record, with retries. root_ip = dns_output["root_zone"].get("listen_ip", "10.0.0.2") root_zone_name = dns_output["root_zone"].get("name", "root.internal") - verified = await self._query_soa(root_ip, root_zone_name, logger) - if verified: - logger.info(f"DNS SOA query confirmed at {root_ip}") - else: - logger.error(f"DNS SOA query failed for {root_zone_name} at {root_ip}") - return verified + + for attempt in range(1, 7): + verified = await self._query_soa(root_ip, root_zone_name, logger) + if verified: + logger.info(f"DNS SOA query confirmed at {root_ip} (attempt {attempt})") + return True + if attempt < 6: + logger.warning(f"SOA query attempt {attempt}/6 failed; retrying in 2s...") + await asyncio.sleep(2) + + # All retries exhausted — capture CoreDNS container logs for diagnosis. + logger.error(f"DNS SOA query failed for {root_zone_name} at {root_ip} (6 attempts)") + try: + client = context.docker_client.client # type: ignore[union-attr] + container = client.containers.get(COREDNS_CONTAINER_NAME) + coredns_logs = container.logs(tail=80).decode("utf-8", errors="replace") + logger.error(f"CoreDNS container status: {container.status}") + logger.error(f"CoreDNS logs (last 80 lines):\n{coredns_logs}") + except Exception as log_err: + logger.error(f"Could not retrieve CoreDNS logs: {log_err}") + return False except Exception as e: logger.error(f"DNS verification failed: {e}") diff --git a/netengine/handlers/docker_handler.py b/netengine/handlers/docker_handler.py index 66dc7fd..f6ac416 100644 --- a/netengine/handlers/docker_handler.py +++ b/netengine/handlers/docker_handler.py @@ -102,8 +102,9 @@ async def exec_command(self, container_id: str, cmd: List[str]) -> tuple[int, st def _exec_command_sync(self, container_id, cmd): container = self.client.containers.get(container_id) - exec_result = container.exec_run(cmd, demux=True) - return exec_result.exit_code, (exec_result.output or b"").decode() + exec_result = container.exec_run(cmd, demux=False) + output = exec_result.output or b"" + return exec_result.exit_code, output.decode("utf-8", errors="replace") async def stop_container(self, container_id: str) -> None: await asyncio.to_thread(self._stop_container_sync, container_id) @@ -166,3 +167,11 @@ async def copy_to_container(self, container_id: str, src_path: str, dest_path: s def _copy_to_container_sync(self, container_id, tar_stream, dest_path): container = self.client.containers.get(container_id) container.put_archive(os.path.dirname(dest_path), tar_stream) + + async def signal_container(self, container_id: str, signal: str) -> None: + """Send a signal to a container via the Docker daemon (no shell required).""" + await asyncio.to_thread(self._signal_container_sync, container_id, signal) + + def _signal_container_sync(self, container_id: str, signal: str) -> None: + container = self.client.containers.get(container_id) + container.kill(signal=signal) diff --git a/netengine/handlers/gateway_handler.py b/netengine/handlers/gateway_handler.py index 368ad89..b30ceb3 100644 --- a/netengine/handlers/gateway_handler.py +++ b/netengine/handlers/gateway_handler.py @@ -129,6 +129,8 @@ async def apply_internet_policy(self, config: "RealInternetConfig") -> None: Generates and loads an nftables ruleset that enforces the mode declared in *config*. CUSTOM mode is a no-op (operator manages rules directly). + Raises GatewayError if the gateway container does not exist or the + nftables command fails. """ from netengine.spec.types import GatewayRealInternetMode @@ -143,8 +145,12 @@ async def apply_internet_policy(self, config: "RealInternetConfig") -> None: tmp_path = f.name try: await self.docker.copy_to_container(self.gateway_container, tmp_path, dest_path) - finally: + except Exception as exc: os.unlink(tmp_path) + raise GatewayError( + f"Gateway container '{self.gateway_container}' unavailable: {exc}" + ) from exc + os.unlink(tmp_path) exit_code, output = await self.docker.exec_command( self.gateway_container, ["nft", "-f", dest_path] @@ -294,8 +300,12 @@ async def apply_peer_routing(self, peer_name: str, peer_endpoint_ip: str) -> Non tmp_path = f.name try: await self.docker.copy_to_container(self.gateway_container, tmp_path, dest_path) - finally: + except Exception as exc: os.unlink(tmp_path) + raise GatewayError( + f"Gateway container '{self.gateway_container}' unavailable: {exc}" + ) from exc + os.unlink(tmp_path) exit_code, output = await self.docker.exec_command( self.gateway_container, ["nft", "-f", dest_path] diff --git a/netengine/handlers/gateway_portal_handler.py b/netengine/handlers/gateway_portal_handler.py index f6f41cf..682f77d 100644 --- a/netengine/handlers/gateway_portal_handler.py +++ b/netengine/handlers/gateway_portal_handler.py @@ -10,6 +10,7 @@ other NetEngine worlds (NONE / PEERED / FEDERATED). """ +import os from datetime import datetime from typing import Any @@ -106,7 +107,13 @@ async def _apply_internet_policy( config = portal.real_internet context.logger.info(f"Real-internet mode: {config.mode.value}") - await gateway.apply_internet_policy(config) + try: + await gateway.apply_internet_policy(config) + except GatewayError as exc: + context.logger.warning( + f"Internet policy ({config.mode.value}) not applied — " + f"gateway container unavailable: {exc}" + ) output: dict[str, Any] = {"mode": config.mode.value} @@ -127,8 +134,9 @@ async def _configure_upstream_resolver(self, context: PhaseContext, resolver_ip: Adds a ``forward . `` directive so that names not resolved within the world are forwarded to the real internet resolver. - This method is best-effort: a failure is logged but does not abort - the portal setup. + Writes to the host-mounted Corefile to avoid shell dependency in the + CoreDNS container. This method is best-effort: failures are logged + but do not abort portal setup. """ if context.docker_client is None: return @@ -137,20 +145,11 @@ async def _configure_upstream_resolver(self, context: PhaseContext, resolver_ip: corefile_patch = ( f"\n# Upstream internet resolver (gateway portal)\n" f"forward . {resolver_ip}\n" ) - corefile_append_cmd = [ - "sh", - "-c", - f"echo '{corefile_patch}' >> /etc/coredns/Corefile", - ] - exit_code, output = await context.docker_client.exec_command( - "netengine_coredns", corefile_append_cmd - ) - if exit_code != 0: - context.logger.warning(f"Could not append upstream resolver to CoreDNS: {output}") - else: - # Reload CoreDNS to pick up the change - await context.docker_client.exec_command("netengine_coredns", ["kill", "-HUP", "1"]) - context.logger.info(f"Upstream resolver configured: {resolver_ip}") + corefile_path = os.path.join(context.zone_dir, "Corefile") + with open(corefile_path, "a") as f: + f.write(corefile_patch) + await context.docker_client.signal_container("netengine_coredns", "HUP") + context.logger.info(f"Upstream resolver configured: {resolver_ip}") except Exception as exc: context.logger.warning(f"Upstream resolver setup skipped: {exc}") @@ -272,6 +271,8 @@ async def _configure_peer_dns(self, context: PhaseContext, peer: CrossWorldPeer) Derives the peer TLD from ``.internal`` and adds a ``forward `` stub to the CoreDNS root Corefile. + Writes to the host-mounted Corefile to avoid shell dependency in the + CoreDNS container (which may not have sh in its PATH). The peer's DNS resolver is assumed to live at port 53 of the peer endpoint. """ if context.docker_client is None: @@ -286,15 +287,16 @@ async def _configure_peer_dns(self, context: PhaseContext, peer: CrossWorldPeer) f" forward . {peer_ip}:53\n" f"}}\n" ) - exit_code, output = await context.docker_client.exec_command( - "netengine_coredns", - ["sh", "-c", f"echo '{corefile_stub}' >> /etc/coredns/Corefile"], - ) - if exit_code != 0: - raise GatewayError(f"Could not configure DNS forwarding for peer {peer.name}: {output}") - # Signal CoreDNS to reload config - await context.docker_client.exec_command("netengine_coredns", ["kill", "-HUP", "1"]) + corefile_path = os.path.join(context.zone_dir, "Corefile") + try: + with open(corefile_path, "a") as f: + f.write(corefile_stub) + except OSError as exc: + raise GatewayError(f"Could not append to Corefile at {corefile_path}: {exc}") from exc + + # Signal CoreDNS to reload config via Docker daemon (no shell required) + await context.docker_client.signal_container("netengine_coredns", "HUP") context.logger.info(f"DNS forwarding configured for peer TLD: {peer_tld}") # ───────────────────────────────────────────── diff --git a/pyproject.toml b/pyproject.toml index ec30239..e68fe34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,8 @@ addopts = "-v --strict-markers --tb=short" markers = [ "unit: Unit tests", "integration: Integration tests (requires Docker)", - "slow: Slow tests", + "slow: Slow tests (image pulls, Keycloak startup, etc.)", + "e2e: Full-stack tests against live Docker infrastructure (--run-e2e to enable)", ] [tool.flake8] diff --git a/tests/conftest.py b/tests/conftest.py index 91f678a..60d377c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,23 @@ from netengine.spec.models import NetEngineSpec +def pytest_addoption(parser): + parser.addoption( + "--run-e2e", + action="store_true", + default=False, + help="Run e2e tests that require a live Docker daemon and pull real images", + ) + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--run-e2e"): + skip_e2e = pytest.mark.skip(reason="pass --run-e2e to run live Docker tests") + for item in items: + if item.get_closest_marker("e2e"): + item.add_marker(skip_e2e) + + def pytest_configure(config): """Keep Starlette's TestClient compatible with newer httpx releases.""" import inspect diff --git a/tests/fixtures/e2e-spec.yaml b/tests/fixtures/e2e-spec.yaml new file mode 100644 index 0000000..f3c7673 --- /dev/null +++ b/tests/fixtures/e2e-spec.yaml @@ -0,0 +1,141 @@ +# NetEngine Declarative Specification — E2E test fixture +# Mirrors minimal.yaml but uses 172.20.0.x for the core network so the +# subnet does not conflict with cloud CI runner host routes (10.0.0.0/8). + +metadata: + name: e2e-test + version: "1.0" + lifecycle: ephemeral + +substrate: + orchestrator: swarm + ntp: + enabled: true + servers: + - pool.ntp.org + networks: + platform: + type: bridge + subnet: 172.28.0.0/16 + description: "Platform management network" + core: + type: bridge + subnet: 172.20.0.0/24 + description: "In-world core network" + gateway: + platform_ip: 172.28.0.1 + core_ip: 172.20.0.1 + description: "Gateway stub" + +dns: + root: + enabled: true + type: authoritative + server: coredns + listen_ip: 172.20.0.2 + soa_primary_ns: root.internal + soa_email: admin.internal + serial_policy: timestamp + platform_zone: + name: platform.internal + type: authoritative + listen_ip: 172.20.0.3 + tlds: + - name: internal + description: "Default in-world TLD" + type: authoritative + listen_ip: 172.20.0.4 + +pki: + root_ca: + cn: "NetEngines Root CA" + o: "E2E Test" + c: "US" + key_storage_mode: ephemeral + cert_lifetime_days: 3650 + acme: + enabled: true + listen_ip: 172.20.0.6 + canonical_name: ca.platform.internal + dnssec_enabled: true + dnssec_ksk_lifetime_days: 365 + dnssec_zsk_lifetime_days: 30 + +identity_platform: + oidc_provider: keycloak + listen_ip: 172.20.0.7 + canonical_name: auth.platform.internal + realm_name: platform + admin_user: + username: admin + email: admin@platform.internal + scopes: + - "netengines:read" + - "netengines:write" + - "netengines:admin" + +world_registry: + enabled: true + listen_ip: 172.20.0.8 + canonical_name: registry.platform.internal + organizations: [] + operators: [] + whois: + enabled: true + listen_ip: 172.20.0.9 + port: 43 + +domain_registry: + enabled: true + listen_ip: 172.20.0.10 + canonical_name: domainreg.platform.internal + tld_delegations: [] + address_space: [] + registrar: + enabled: true + listen_ip: 172.20.0.11 + canonical_name: registrar.platform.internal + +identity_inworld: + oidc_provider: keycloak + listen_ip: 172.20.0.12 + canonical_name: auth.internal + realm_name: inworld + org_users: [] + scopes: + - profile + - email + - openid + +ands: + profiles: {} + instances: [] + +world_services: + mail: + enabled: false + storage: + enabled: false + +org_apps: + enabled: true + catalog: [] + deployments: [] + +gateway_portal: + enabled: true + real_internet: + mode: isolated + cross_world: + mode: none + +operator: + api: + enabled: true + listen_ip: 172.28.0.11 + port: 8080 + canonical_name: api.platform.internal + auth: + provider: oidc + issuer: "https://auth.platform.internal/realms/platform" + required_scope: "netengines:read" diff --git a/tests/integration/test_e2e_federation.py b/tests/integration/test_e2e_federation.py new file mode 100644 index 0000000..7febcbb --- /dev/null +++ b/tests/integration/test_e2e_federation.py @@ -0,0 +1,254 @@ +"""Cross-world federation end-to-end test: real CoreDNS + peer DNS stub. + +Run with: + pytest tests/integration/test_e2e_federation.py --run-e2e + +Requires: + - Docker daemon accessible on the host + - The 'core' and 'platform' Docker networks must not already exist + +What gets validated: + - GatewayPortalHandler runs without error in PEERED mode + - The peer's TLD forwarding stub is appended to the CoreDNS Corefile + - CoreDNS reloads the new config (SIGHUP sent) without crashing + - Runtime state records the peer federation output + +What is intentionally NOT tested here (requires dedicated infrastructure): + - nftables routing (needs a gateway container provisioned by a separate phase) + - Trust anchor cert (tested via unit tests in test_gateway_portal.py) + - Actual cross-world DNS resolution (needs a real peer world running) +""" + +from __future__ import annotations + +import asyncio +import io +import tarfile +from pathlib import Path + +import pytest + +from netengine.core.state import RuntimeState +from netengine.handlers.context import PhaseContext +from netengine.handlers.dns import DNSHandler +from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.gateway_portal_handler import GatewayPortalHandler +from netengine.handlers.substrate import SubstrateHandler +from netengine.logging import get_logger +from netengine.spec.loader import load_spec +from netengine.spec.models import ( + CrossWorldConfig, + CrossWorldPeer, + GatewayPortal, + RealInternetConfig, +) +from netengine.spec.types import GatewayCrossWorldMode, GatewayRealInternetMode + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +# ───────────────────────────────────────────── +# Helpers (shared with test_e2e_fullstack) +# ───────────────────────────────────────────── + + +def _docker_client(): + try: + import docker + + client = docker.from_env() + client.ping() + return client + except Exception: + return None + + +def _cleanup_docker(client) -> None: + for c in client.containers.list(all=True): + if c.name.startswith(("netengine_", "netengines_")): + try: + c.stop(timeout=5) + c.remove(force=True) + except Exception: + pass + for n in client.networks.list(): + if n.name in ("core", "platform"): + try: + n.remove() + except Exception: + pass + + +def _read_corefile_from_container(client, container_name: str) -> str: + """Return the current contents of /etc/coredns/Corefile inside the container. + + Uses Docker's get_archive API rather than exec+cat so it works with minimal + CoreDNS images that have no shell utilities in their PATH. + """ + container = client.containers.get(container_name) + bits, _ = container.get_archive("/etc/coredns/Corefile") + buf = io.BytesIO() + for chunk in bits: + buf.write(chunk) + buf.seek(0) + with tarfile.open(fileobj=buf) as tar: + members = tar.getmembers() + if not members: + return "" + f = tar.extractfile(members[0]) + return f.read().decode("utf-8", errors="replace") if f else "" + + +# ───────────────────────────────────────────── +# Federation test +# ───────────────────────────────────────────── + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_e2e_cross_world_federation(tmp_path, monkeypatch): + """PEERED mode: verify DNS stub for peer TLD is written to CoreDNS Corefile. + + The test boots phases 0-2 (substrate + CoreDNS) then runs + GatewayPortalHandler with a PEERED cross-world spec that references a + simulated peer world at 192.0.2.1 (TEST-NET — not routable). + + After the handler runs, the CoreDNS Corefile must contain a forwarding + stub for `world-b.internal` pointing at the peer's DNS resolver. + """ + monkeypatch.setenv("NETENGINE_ZONE_DIR", str(tmp_path / "coredns")) + + client = _docker_client() + if client is None: + pytest.skip("Docker daemon not available") + + spec = load_spec(FIXTURES_DIR / "e2e-spec.yaml") + docker_handler = DockerHandler() + state = RuntimeState() + ctx = PhaseContext( + spec=spec, + runtime_state=state, + logger=get_logger("e2e.federation"), + docker_client=docker_handler, + mock_mode=False, + zone_dir=str(tmp_path / "coredns"), + ) + + try: + # ── Bootstrap phases 0-2 ──────────────────────────────────────────── + await SubstrateHandler().execute(ctx) + await DNSHandler().execute(ctx) + + coredns = client.containers.get("netengine_coredns") + assert coredns.status == "running" + + # ── Build gateway portal spec with PEERED cross-world mode ────────── + peer = CrossWorldPeer( + name="world-b", + endpoint="192.0.2.1", # TEST-NET — safe, not routable + mode=GatewayCrossWorldMode.PEERED, + trust_anchor_cert=None, # Skip trust anchor install + ) + portal_spec = GatewayPortal( + enabled=True, + real_internet=RealInternetConfig( + mode=GatewayRealInternetMode.CUSTOM # CUSTOM is a no-op; avoids gateway container + ), + cross_world=CrossWorldConfig( + mode=GatewayCrossWorldMode.PEERED, + peers=[peer], + ), + ) + + # Patch the spec's gateway_portal for this test + original_portal = ctx.spec.gateway_portal + object.__setattr__(ctx.spec, "gateway_portal", portal_spec) + + try: + # ── Run gateway portal handler ─────────────────────────────────── + await GatewayPortalHandler().execute(ctx) + finally: + object.__setattr__(ctx.spec, "gateway_portal", original_portal) + + # ── Assertions ─────────────────────────────────────────────────────── + gp_output = ctx.runtime_state.gateway_portal_output + assert gp_output is not None + assert gp_output["enabled"] is True + assert gp_output["cross_world_mode"] == GatewayCrossWorldMode.PEERED.value + + peers_out = gp_output.get("federation", {}).get("peers", []) + assert len(peers_out) == 1, f"Expected 1 peer in output, got: {peers_out}" + + peer_out = peers_out[0] + assert peer_out["name"] == "world-b" + assert peer_out["dns_forwarding_configured"] is True, ( + "DNS stub for world-b.internal was not written to CoreDNS Corefile. " + f"Peer output: {peer_out}" + ) + + # ── Verify Corefile content ─────────────────────────────────────────── + # Brief pause for SIGHUP to propagate + await asyncio.sleep(1) + + corefile = _read_corefile_from_container(client, "netengine_coredns") + assert "world-b.internal" in corefile, ( + f"Expected 'world-b.internal' stub in CoreDNS Corefile but not found.\n" + f"Corefile:\n{corefile}" + ) + assert ( + "192.0.2.1" in corefile + ), f"Expected peer IP '192.0.2.1' in CoreDNS Corefile.\nCorefile:\n{corefile}" + + # CoreDNS should still be running (SIGHUP did not crash it) + coredns.reload() + assert ( + coredns.status == "running" + ), f"CoreDNS crashed after Corefile update (status={coredns.status})" + + finally: + _cleanup_docker(client) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_e2e_federation_none_mode_no_changes(tmp_path, monkeypatch): + """NONE mode: GatewayPortalHandler skips peer setup when cross_world is NONE.""" + monkeypatch.setenv("NETENGINE_ZONE_DIR", str(tmp_path / "coredns")) + + client = _docker_client() + if client is None: + pytest.skip("Docker daemon not available") + + spec = load_spec(FIXTURES_DIR / "e2e-spec.yaml") + docker_handler = DockerHandler() + state = RuntimeState() + ctx = PhaseContext( + spec=spec, + runtime_state=state, + logger=get_logger("e2e.federation.none"), + docker_client=docker_handler, + mock_mode=False, + zone_dir=str(tmp_path / "coredns"), + ) + + try: + await SubstrateHandler().execute(ctx) + await DNSHandler().execute(ctx) + + corefile_before = _read_corefile_from_container(client, "netengine_coredns") + + # minimal.yaml has cross_world.mode: none — run portal handler as-is + await GatewayPortalHandler().execute(ctx) + + gp_output = ctx.runtime_state.gateway_portal_output + assert gp_output is not None + assert gp_output["cross_world_mode"] == GatewayCrossWorldMode.NONE.value + + corefile_after = _read_corefile_from_container(client, "netengine_coredns") + # Corefile must not have changed — no stubs should have been added + assert ( + corefile_before == corefile_after + ), "Corefile was modified even though cross_world mode is NONE" + + finally: + _cleanup_docker(client) diff --git a/tests/integration/test_e2e_fullstack.py b/tests/integration/test_e2e_fullstack.py new file mode 100644 index 0000000..483f75e --- /dev/null +++ b/tests/integration/test_e2e_fullstack.py @@ -0,0 +1,297 @@ +"""End-to-end integration test: real Docker, live DNS, ACME, optional OIDC. + +Run with: + pytest tests/integration/test_e2e_fullstack.py --run-e2e + +Requires: + - Docker daemon accessible on the host + - Enough privileges to create bridge networks and containers + - About 1-2 min for phases 0-2 (CoreDNS image ~30 MB) + - About 5-8 min for phase 3 (step-ca image ~200 MB + CA generation) + - NETENGINE_KEYCLOAK_URL env var to enable the OIDC test + +What gets validated: + Phase 0 — real Docker networks (`core`, `platform`) created via Docker API + Phase 1-2 — CoreDNS container up, live SOA UDP query returns a valid response + Phase 3 — step-ca container up, ACME directory endpoint returns JSON + OIDC — Keycloak token endpoint issues a bearer token (optional) +""" + +from __future__ import annotations + +import asyncio +import json +import os +import socket +import ssl +import struct +import urllib.request +from pathlib import Path + +import pytest + +from netengine.core.state import RuntimeState +from netengine.handlers.context import PhaseContext +from netengine.handlers.dns import DNSHandler +from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.phase_pki import PKIPhaseHandler +from netengine.handlers.substrate import SubstrateHandler +from netengine.logging import get_logger +from netengine.spec.loader import load_spec + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +# ───────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────── + + +def _docker_client(): + """Return a docker SDK client, or None if Docker is unavailable.""" + try: + import docker + + client = docker.from_env() + client.ping() + return client + except Exception: + return None + + +def _get_container_ip(container, network_name: str) -> str: + """Return the container's IP on the named Docker network.""" + container.reload() + networks = container.attrs["NetworkSettings"]["Networks"] + return networks.get(network_name, {}).get("IPAddress", "") + + +def _cleanup_docker(client) -> None: + """Remove all netengine containers and the core/platform networks.""" + for c in client.containers.list(all=True): + if c.name.startswith(("netengine_", "netengines_")): + try: + c.stop(timeout=5) + c.remove(force=True) + except Exception: + pass + for n in client.networks.list(): + if n.name in ("core", "platform"): + try: + n.remove() + except Exception: + pass + + +def _send_dns_soa(server_ip: str, zone: str, timeout: float = 5.0) -> bool: + """Send a raw DNS SOA query over UDP and return True on a valid response. + + Does not depend on any third-party DNS library so the test stays + self-contained. The transaction ID (0x1234) and QR bit in the flags + are the only fields checked — we just need to confirm the server is + responding correctly. + """ + header = struct.pack(">HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0) + qname = b"" + for label in zone.rstrip(".").split("."): + enc = label.encode() + qname += bytes([len(enc)]) + enc + qname += b"\x00" + question = qname + struct.pack(">HH", 6, 1) # SOA + IN + query = header + question + + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.settimeout(timeout) + s.sendto(query, (server_ip, 53)) + data, _ = s.recvfrom(512) + resp_id, flags = struct.unpack(">HH", data[:4]) + return resp_id == 0x1234 and bool(flags & 0x8000) + except Exception: + return False + + +def _build_context(spec, zone_dir: str, state: RuntimeState | None = None) -> PhaseContext: + docker_handler = DockerHandler() + return PhaseContext( + spec=spec, + runtime_state=state or RuntimeState(), + logger=get_logger("e2e"), + docker_client=docker_handler, + mock_mode=False, + zone_dir=zone_dir, + ) + + +# ───────────────────────────────────────────── +# Phase 0 + 1-2: Substrate and DNS +# ───────────────────────────────────────────── + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_e2e_substrate_and_dns(tmp_path, monkeypatch): + """Phases 0-2: real Docker networks + CoreDNS + live SOA query. + + Validates: + - Docker networks 'core' and 'platform' are created by Phase 0 + - CoreDNS container is running after Phases 1-2 + - A raw UDP DNS SOA query to root.internal resolves at CoreDNS's IP + """ + monkeypatch.setenv("NETENGINE_ZONE_DIR", str(tmp_path / "coredns")) + + client = _docker_client() + if client is None: + pytest.skip("Docker daemon not available") + + spec = load_spec(FIXTURES_DIR / "e2e-spec.yaml") + ctx = _build_context(spec, str(tmp_path / "coredns")) + + try: + # ── Phase 0: Substrate ────────────────────────────────────────────── + await SubstrateHandler().execute(ctx) + + assert ctx.runtime_state.substrate_output is not None + assert "networks" in ctx.runtime_state.substrate_output + + net_names = {n.name for n in client.networks.list()} + assert "core" in net_names, "Docker network 'core' was not created by Phase 0" + assert "platform" in net_names, "Docker network 'platform' was not created by Phase 0" + + # ── Phases 1-2: DNS ───────────────────────────────────────────────── + await DNSHandler().execute(ctx) + + assert ctx.runtime_state.dns_output is not None + assert ctx.runtime_state.dns_output.get("healthy") is True + assert ctx.runtime_state.phase_completed.get("1") is True + assert ctx.runtime_state.phase_completed.get("2") is True + + # CoreDNS container must be running + coredns = client.containers.get("netengine_coredns") + assert coredns.status == "running", f"CoreDNS not running (status={coredns.status})" + + # Verify the declared listen IP was assigned on the core network + container_ip = _get_container_ip(coredns, "core") + expected_ip = spec.dns.root.listen_ip + assert ( + container_ip == expected_ip + ), f"CoreDNS IP on 'core' network is {container_ip!r}, expected {expected_ip!r}" + + # ── Live DNS validation ────────────────────────────────────────────── + # Brief pause for CoreDNS to finish binding + await asyncio.sleep(2) + + ok = _send_dns_soa(container_ip, "root.internal") + assert ok, ( + f"Live DNS SOA query to root.internal at {container_ip}:53 failed. " + "CoreDNS may not have started correctly." + ) + + finally: + _cleanup_docker(client) + + +# ───────────────────────────────────────────── +# Phase 3: PKI / ACME +# ───────────────────────────────────────────── + + +@pytest.mark.e2e +@pytest.mark.slow +@pytest.mark.asyncio +async def test_e2e_pki_acme_directory(tmp_path, monkeypatch): + """Phase 3: step-ca starts and ACME directory returns valid JSON. + + This test is marked @slow because it pulls the step-ca image (~200 MB) and + runs 'step ca init' which takes 30-60 seconds on first run. + + Validates: + - step-ca container is running after Phase 3 + - HTTPS GET to /acme/acme/directory returns JSON with 'newNonce' key + """ + monkeypatch.setenv("NETENGINE_ZONE_DIR", str(tmp_path / "coredns")) + + client = _docker_client() + if client is None: + pytest.skip("Docker daemon not available") + + spec = load_spec(FIXTURES_DIR / "e2e-spec.yaml") + ctx = _build_context(spec, str(tmp_path / "coredns")) + + try: + # Phases 0-2 + await SubstrateHandler().execute(ctx) + await DNSHandler().execute(ctx) + + # Phase 3: PKI + await PKIPhaseHandler().execute(ctx) + + assert ctx.runtime_state.pki_bootstrapped is True + assert ctx.runtime_state.ca_cert_pem is not None + + step_ca = client.containers.get("netengines_step_ca") + assert step_ca.status == "running", f"step-ca not running (status={step_ca.status})" + + # Verify ACME directory is reachable (self-signed cert → skip verify) + ca_ip = spec.pki.acme.listen_ip + ssl_ctx = ssl.create_default_context() + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + + acme_url = f"https://{ca_ip}/acme/acme/directory" + req = urllib.request.Request(acme_url) + with urllib.request.urlopen(req, context=ssl_ctx, timeout=10) as resp: + body = json.loads(resp.read().decode()) + + assert "newNonce" in body, f"ACME directory missing 'newNonce': {body}" + + finally: + _cleanup_docker(client) + + +# ───────────────────────────────────────────── +# Phase 4: OIDC login (optional — requires Keycloak) +# ───────────────────────────────────────────── + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_e2e_oidc_token(tmp_path, monkeypatch): + """OIDC token endpoint issues a bearer token. + + Requires a running Keycloak instance. Set NETENGINE_KEYCLOAK_URL to enable, + e.g.: + NETENGINE_KEYCLOAK_URL=http://localhost:8180 pytest --run-e2e + + The test creates a temporary realm, issues a token, and validates the JWT + structure. It does NOT run Phase 4 (too slow for default CI) — it uses + the pre-running Keycloak directly. + """ + keycloak_url = os.environ.get("NETENGINE_KEYCLOAK_URL", "").rstrip("/") + if not keycloak_url: + pytest.skip("NETENGINE_KEYCLOAK_URL not set — skipping OIDC test") + + import urllib.error + + # Check Keycloak health + try: + req = urllib.request.Request(f"{keycloak_url}/health/ready") + with urllib.request.urlopen(req, timeout=5) as resp: + assert resp.status == 200, "Keycloak health endpoint not ready" + except urllib.error.URLError as exc: + pytest.skip(f"Keycloak at {keycloak_url} not reachable: {exc}") + + # Obtain a token from the master realm using admin credentials + admin_password = os.environ.get("NETENGINE_KEYCLOAK_ADMIN_PASSWORD", "admin_dev_password") + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + payload = ( + f"client_id=admin-cli&username=admin&password={admin_password}&grant_type=password" + ).encode() + req = urllib.request.Request( + token_url, data=payload, headers={"Content-Type": "application/x-www-form-urlencoded"} + ) + with urllib.request.urlopen(req, timeout=10) as resp: + token_data = json.loads(resp.read().decode()) + + assert "access_token" in token_data, f"No access_token in response: {token_data}" + assert token_data.get("token_type", "").lower() == "bearer"