Implement DNS service deployment with CoreDNS and real verification

.env.example

@@ -1,2 +1,20 @@
+# ── Cloud Supabase (optional — used when NETENGINE_DB_URL is not set) ──
 SUPABASE_URL=https://xxxxx.supabase.co
-SUPABASE_SERVICE_KEY=eyJ...
+SUPABASE_SERVICE_KEY=eyJ...
+
+# ── Local Postgres (docker-compose.yml) ──
+NETENGINE_DB_URL=postgresql://netengine:dev_password@localhost:5432/netengine
+POSTGRES_PASSWORD=dev_password
+
+# ── Keycloak admin credentials ──
+KEYCLOAK_ADMIN_PASSWORD=admin_dev_password
+
+# ── Runtime behaviour ──
+# Set to true to skip all real Docker/DNS/PKI calls (unit-test mode)
+NETENGINE_MOCK=false
+
+# Where the DNS handler writes Corefile + zone files (CoreDNS bind-mounts this)
+NETENGINE_ZONE_DIR=./data/coredns
+
+# Where the RuntimeState JSON file is persisted
+NETENGINES_STATE_FILE=netengines_state.json

.gitignore

@@ -224,3 +224,4 @@ __marimo__/
 
 # NetEngine local runtime state
 netengines_state.json
+data/

docker-compose.yml

@@ -0,0 +1,97 @@
+version: "3.8"
+
+# Full NetEngine local bootstrap stack.
+# Services here start BEFORE `netengine up` runs.
+# CoreDNS and step-ca are deployed by phase handlers at runtime.
+#
+# Usage:
+#   docker compose up -d
+#   netengine up examples/minimal.yaml
+#
+# To run in mock mode (no real infra calls):
+#   NETENGINE_MOCK=true netengine up examples/minimal.yaml --mock
+
+services:
+  # ────────────────────────────────────────────
+  # Persistence layer
+  # ────────────────────────────────────────────
+  postgres:
+    image: postgres:15
+    container_name: netengine_postgres
+    environment:
+      POSTGRES_USER: netengine
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev_password}
+      POSTGRES_DB: netengine
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      # pgmq extension install script — runs at first startup
+      - ./scripts/install_pgmq.sh:/docker-entrypoint-initdb.d/install_pgmq.sh:ro
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U netengine -d netengine"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  # ────────────────────────────────────────────
+  # Platform identity (Keycloak)
+  # Deployed here so it starts independently of NetEngine phases.
+  # Phase 4 bootstraps realms and admin users via the admin API.
+  # ────────────────────────────────────────────
+  keycloak:
+    image: quay.io/keycloak/keycloak:24.0
+    container_name: netengine_keycloak
+    command: start-dev
+    environment:
+      KC_DB: postgres
+      KC_DB_URL: jdbc:postgresql://postgres:5432/netengine
+      KC_DB_USERNAME: netengine
+      KC_DB_PASSWORD: ${POSTGRES_PASSWORD:-dev_password}
+      KC_DB_SCHEMA: keycloak
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin_dev_password}
+      KC_HTTP_PORT: 8180
+      KC_HOSTNAME_STRICT: "false"
+      KC_HTTP_ENABLED: "true"
+    ports:
+      - "8180:8180"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:8180/health/ready || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 20
+      start_period: 60s
+
+  # ────────────────────────────────────────────
+  # NetEngine operator API
+  # Starts after postgres; mounts state file and zone dir.
+  # ────────────────────────────────────────────
+  netengine_api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: netengine_api
+    environment:
+      NETENGINE_DB_URL: postgresql://netengine:${POSTGRES_PASSWORD:-dev_password}@postgres:5432/netengine
+      NETENGINE_STATE_FILE: /data/netengines_state.json
+      NETENGINE_ZONE_DIR: /data/coredns
+      NETENGINE_MOCK: ${NETENGINE_MOCK:-false}
+    ports:
+      - "8080:8080"
+    volumes:
+      - netengine_data:/data
+    depends_on:
+      postgres:
+        condition: service_healthy
+    profiles:
+      # Only start the API container when explicitly requested.
+      # For local dev, run `netengine up` directly from the CLI.
+      - api
+
+volumes:
+  postgres_data:
+  netengine_data:

netengine/cli/main.py

@@ -1,14 +1,40 @@
 # netengine/cli/main.py
 import asyncio
 import logging
+import os
+from pathlib import Path
+
 import click
+
 from netengine.core.orchestrator import Orchestrator
 from netengine.core.state import RuntimeState
 from netengine.spec.loader import load_spec
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+MIGRATIONS_DIR = Path(__file__).parent.parent.parent / "migrations"
+
+
+async def _run_migrations(db_url: str) -> None:
+    """Run all SQL migration files in order against the given Postgres URL."""
+    import asyncpg  # type: ignore[import]
+
+    migration_files = sorted(MIGRATIONS_DIR.glob("*.sql"))
+    if not migration_files:
+        logger.info("No migration files found")
+        return
+
+    conn = await asyncpg.connect(db_url)
+    try:
+        for migration_path in migration_files:
+            sql = migration_path.read_text()
+            logger.info(f"Running migration: {migration_path.name}")
+            await conn.execute(sql)
+        logger.info(f"Applied {len(migration_files)} migration(s)")
+    finally:
+        await conn.close()
+
 
 @click.group()
 def cli():
@@ -17,15 +43,44 @@ def cli():
 
 @cli.command()
 @click.argument("spec_file", type=click.Path(exists=True))
-def up(spec_file):
+@click.option(
+    "--mock",
+    is_flag=True,
+    default=False,
+    envvar="NETENGINE_MOCK",
+    help="Run in mock mode (no real Docker/DNS calls).",
+)
+@click.option(
+    "--skip-migrations",
+    is_flag=True,
+    default=False,
+    help="Skip running database migrations on startup.",
+)
+def up(spec_file: str, mock: bool, skip_migrations: bool) -> None:
     """Boot a world from the given spec YAML."""
+    asyncio.run(_up(spec_file, mock, skip_migrations))
+
+
+async def _up(spec_file: str, mock: bool, skip_migrations: bool) -> None:
     spec = load_spec(spec_file)
-    orchestrator = Orchestrator(spec)
-    asyncio.run(orchestrator.execute_phases())
+
+    # Run migrations if a local Postgres URL is configured
+    if not skip_migrations and not mock:
+        db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL")
+        if db_url:
+            try:
+                await _run_migrations(db_url)
+            except Exception as exc:
+                logger.warning(f"Migrations failed (continuing anyway): {exc}")
+        else:
+            logger.debug("No NETENGINE_DB_URL set — skipping migrations")
+
+    orchestrator = Orchestrator(spec, mock_mode=mock)
+    await orchestrator.execute_phases()
 
 
 @cli.command()
-def status():
+def status() -> None:
     """Show current world state."""
     state = RuntimeState.load()
     phase_labels = {
@@ -45,13 +100,14 @@ def status():
         marker = "✓" if completed else "·"
         click.echo(f"  {marker} Phase {phase}: {label}")
     click.echo(f"CA certificate present: {bool(state.ca_cert_pem)}")
-    click.echo(f"step‑ca container ID: {state.step_ca_container_id}")
+    click.echo(f"step-ca container ID: {state.step_ca_container_id}")
+    if state.last_error:
+        click.echo(f"Last error: {state.last_error}")
 
 
 @cli.command()
-def down():
+def down() -> None:
     """Tear down the world (kill containers, remove volumes)."""
-    # Not fully implemented for M2 – will be done in M8.
     click.echo("Teardown not yet implemented.")
 
 

netengine/core/orchestrator.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Any, List, Type
+import os
+from typing import Any, List, Optional, Type
 
 from pydantic import ValidationError
 
@@ -41,18 +42,40 @@ class Orchestrator:
         (8, ServicesPhaseHandler),
     ]
 
-    def __init__(self, spec: NetEngineSpec | dict[str, Any]):
+    def __init__(self, spec: NetEngineSpec | dict[str, Any], mock_mode: Optional[bool] = None):
         """Initialize orchestrator with a validated NetEngine spec.
 
         Args:
             spec: Validated NetEngineSpec or raw YAML specification dictionary
+            mock_mode: Override for mock mode. When None, reads NETENGINE_MOCK env var.
         """
         self.spec = self._normalize_spec(spec)
         self.runtime_state = RuntimeState.load()
+
+        # Resolve mock_mode: explicit arg wins, then env var
+        effective_mock = (
+            mock_mode
+            if mock_mode is not None
+            else os.environ.get("NETENGINE_MOCK", "").lower() in ("1", "true", "yes")
+        )
+
+        # Initialise Docker client only when running for real
+        docker_client = None
+        if not effective_mock:
+            try:
+                from netengine.handlers.docker_handler import DockerHandler
+
+                docker_client = DockerHandler()
+            except Exception as exc:
+                logger.warning(f"Docker unavailable, falling back to mock mode: {exc}")
+                effective_mock = True
+
         self.context = PhaseContext(
             spec=self.spec,
             runtime_state=self.runtime_state,
             logger=logger,
+            docker_client=docker_client,
+            mock_mode=effective_mock,
         )
 
     @staticmethod
@@ -109,7 +132,7 @@ async def execute_phases(self, up_to_phase: int = 8) -> None:
 
             except Exception as e:
                 logger.error(f"Phase {phase_num} failed: {e}")
-                self.runtime_state.error = str(e)
+                self.runtime_state.last_error = str(e)
                 self.runtime_state.save()
                 raise
 

netengine/handlers/context.py

@@ -1,12 +1,18 @@
 """Phase execution context and runtime state."""
 
 import logging
-from dataclasses import dataclass
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Any, Optional
 
 from netengine.core.state import RuntimeState
 from netengine.spec.models import NetEngineSpec
 
+# Default directory for CoreDNS Corefile and zone files.
+# Overridden by NETENGINE_ZONE_DIR env var.
+DEFAULT_ZONE_DIR = str(Path.cwd() / "data" / "coredns")
+
 
 @dataclass
 class PhaseContext:
@@ -29,3 +35,15 @@ class PhaseContext:
     # Phase-specific config
     phase_name: Optional[str] = None
     phase_config: Optional[dict[str, Any]] = None
+
+    # When True, handlers skip real infrastructure calls (Docker, DNS queries, etc.)
+    # and return stub outputs. Set via NETENGINE_MOCK=true env var.
+    mock_mode: bool = field(
+        default_factory=lambda: os.environ.get("NETENGINE_MOCK", "").lower() in ("1", "true", "yes")
+    )
+
+    # Directory where the DNS handler writes Corefile + zone files.
+    # CoreDNS container bind-mounts this directory to /etc/coredns.
+    zone_dir: str = field(
+        default_factory=lambda: os.environ.get("NETENGINE_ZONE_DIR", DEFAULT_ZONE_DIR)
+    )