Fix operator API health leak, preserve bootstrap break‑glass, and fix structured logging

netengine/api/auth.py

@@ -29,6 +29,7 @@
 INSECURE_TLS_ENV = "NETENGINE_INSECURE_TLS"
 CA_BUNDLE_ENV = "NETENGINE_CA_BUNDLE"
 ADMIN_ROLES = {"admin", "netengine-admin", "operator-admin"}
+POST_PHASE4_BOOTSTRAP_ENV = "NETENGINES_BOOTSTRAP_SECRET_AFTER_PHASE4"
 
 _bearer = HTTPBearer(auto_error=False)
 
@@ -90,9 +91,9 @@ async def require_auth(
     state = RuntimeState.load()
     phase4_done = state.phase_completed.get("4", False)
 
+    secret = request.headers.get("X-Bootstrap-Secret", "")
     if not phase4_done:
-        # Bootstrap phase: accept secret in X-Bootstrap-Secret header
-        secret = request.headers.get("X-Bootstrap-Secret", "")
+        # Bootstrap phase: accept secret in X-Bootstrap-Secret header.
         if bootstrap_secret and secret == bootstrap_secret:
             return {"sub": "bootstrap", "roles": ["admin"]}
         # Also allow an unauthenticated health check
@@ -102,6 +103,17 @@ async def require_auth(
             status_code=401, detail="Bootstrap secret required (Phase 4 not yet complete)"
         )
 
+    # Post-Phase 4, OIDC is the default authority. The bootstrap secret can still
+    # be enabled as an explicit local break-glass credential for automation, but
+    # it must be separately opted in so deployments do not accidentally retain a
+    # static admin credential after identity bootstrap.
+    if (
+        _is_truthy(os.environ.get(POST_PHASE4_BOOTSTRAP_ENV))
+        and bootstrap_secret
+        and secret == bootstrap_secret
+    ):
+        return {"sub": "bootstrap", "roles": ["admin"]}
+
     if not credentials:
         raise HTTPException(status_code=401, detail="Bearer token required")
 

netengine/api/routes.py

@@ -114,7 +114,6 @@ async def health() -> dict[str, Any]:
         "status": overall,
         "phases": phases,
         "events": events,
-        "last_error": state.last_error,
         "last_error_present": bool(state.last_error),
     }
 

netengine/logs/middleware.py

@@ -302,8 +302,8 @@ async def __call__(self, scope, receive, send):
         request_logger = self.logger.bind(**context)
 
         # Log request
-        request_logger.info(
-            f"{scope['method']} {scope['path']}", extra={"event": "http.request.start"}
+        request_logger.bind(event="http.request.start").info(
+            f"{scope['method']} {scope['path']}"
         )
 
         start_time = time.time()
@@ -319,38 +319,27 @@ async def send_wrapper(message):
             await self.app(scope, receive, send_wrapper)
         except Exception as e:
             duration_ms = (time.time() - start_time) * 1000
-            request_logger.error(
-                f"Request failed: {type(e).__name__}",
-                extra={
-                    "event": "http.error",
-                    "status": 500,
-                    "duration_ms": duration_ms,
-                },
-            )
+            request_logger.bind(
+                event="http.error", status=500, duration_ms=duration_ms
+            ).error(f"Request failed: {type(e).__name__}")
             raise
         else:
             duration_ms = (time.time() - start_time) * 1000
 
             # Check for slow request
             if duration_ms > self.slow_request_threshold_ms:
-                request_logger.warning(
-                    f"Slow request: {scope['method']} {scope['path']}",
-                    extra={
-                        "event": "http.slow_request",
-                        "duration_ms": duration_ms,
-                        "threshold_ms": self.slow_request_threshold_ms,
-                        "status": status_code,
-                    },
-                )
+                request_logger.bind(
+                    event="http.slow_request",
+                    duration_ms=duration_ms,
+                    threshold_ms=self.slow_request_threshold_ms,
+                    status=status_code,
+                ).warning(f"Slow request: {scope['method']} {scope['path']}")
             else:
-                request_logger.info(
-                    f"{scope['method']} {scope['path']} {status_code}",
-                    extra={
-                        "event": "http.response.complete",
-                        "status": status_code,
-                        "duration_ms": duration_ms,
-                    },
-                )
+                request_logger.bind(
+                    event="http.response.complete",
+                    status=status_code,
+                    duration_ms=duration_ms,
+                ).info(f"{scope['method']} {scope['path']} {status_code}")
 
         finally:
             TraceContextManager.clear_context()

tests/integration/test_e2e_bootstrap.py

@@ -469,6 +469,7 @@ class TestFullMVPLifecycle:
     async def test_full_mvp_lifecycle(self, tmp_path, monkeypatch):
         monkeypatch.setenv("NETENGINE_STATE_FILE", str(tmp_path / "state.json"))
         monkeypatch.setenv("NETENGINES_BOOTSTRAP_SECRET", "mvp-secret")
+        monkeypatch.setenv("NETENGINES_BOOTSTRAP_SECRET_AFTER_PHASE4", "true")
 
         spec = load_spec(EXAMPLES_DIR / "minimal.yaml")
 

tests/test_api_auth.py

@@ -168,3 +168,41 @@ def fake_create_default_context(*, cafile=None):
     assert isinstance(ssl_option, ssl.SSLContext)
     assert captured_cafile is not None
     assert captured_cafile.endswith(".pem")
+
+
+@pytest.mark.asyncio
+async def test_require_auth_rejects_bootstrap_secret_after_phase4_without_opt_in(monkeypatch):
+    from fastapi import HTTPException
+
+    state = _phase4_state()
+    request = SimpleNamespace(
+        method="POST",
+        headers={"X-Bootstrap-Secret": "bootstrap-secret"},
+        url=SimpleNamespace(path="/api/v1/reload"),
+    )
+
+    monkeypatch.setenv("NETENGINES_BOOTSTRAP_SECRET", "bootstrap-secret")
+    monkeypatch.delenv(auth.POST_PHASE4_BOOTSTRAP_ENV, raising=False)
+    with patch("netengine.api.auth.RuntimeState.load", return_value=state):
+        with pytest.raises(HTTPException) as exc_info:
+            await auth.require_auth(request, None)
+
+    assert exc_info.value.status_code == 401
+    assert exc_info.value.detail == "Bearer token required"
+
+
+@pytest.mark.asyncio
+async def test_require_auth_allows_bootstrap_secret_after_phase4_with_opt_in(monkeypatch):
+    state = _phase4_state()
+    request = SimpleNamespace(
+        method="POST",
+        headers={"X-Bootstrap-Secret": "bootstrap-secret"},
+        url=SimpleNamespace(path="/api/v1/reload"),
+    )
+
+    monkeypatch.setenv("NETENGINES_BOOTSTRAP_SECRET", "bootstrap-secret")
+    monkeypatch.setenv(auth.POST_PHASE4_BOOTSTRAP_ENV, "true")
+    with patch("netengine.api.auth.RuntimeState.load", return_value=state):
+        result = await auth.require_auth(request, None)
+
+    assert result == {"sub": "bootstrap", "roles": ["admin"]}

tests/test_cli.py

@@ -485,8 +485,8 @@ def test_validate_json_reports_active_feature_states(tmp_path: Path) -> None:
     ]
 
 
-def test_validate_json_unsupported_active_feature_exits_nonzero(tmp_path: Path) -> None:
-    """Unsupported active fields should be machine-readable and fail CI."""
+def test_validate_json_experimental_active_feature_exits_zero(tmp_path: Path) -> None:
+    """Experimental active fields should be machine-readable and keep CI green."""
     spec_file = _write_cli_validate_spec(
         tmp_path, gateway_portal__real_internet__upstream_resolver_enabled=True
     )
@@ -495,12 +495,12 @@ def test_validate_json_unsupported_active_feature_exits_nonzero(tmp_path: Path)
 
     assert result.exit_code == 0, result.output
     payload = json.loads(result.output)
-    assert payload["ok"] is False
+    assert payload["ok"] is True
     assert (
         payload["feature_states"][0]["path"]
         == "gateway_portal.real_internet.upstream_resolver_enabled"
     )
-    assert payload["feature_states"][0]["state"] == "unsupported"
+    assert payload["feature_states"][0]["state"] == "experimental"
     assert payload["feature_states"][0]["current_value"] is True
     assert payload["feature_states"][0]["default_value"] is False
 

tests/test_gateway_federation.py

@@ -79,7 +79,17 @@ def test_real_internet_mode_mirrored_loads(
         self, tmp_path: Path, caplog: pytest.LogCaptureFixture
     ) -> None:
         spec_file = _write_spec(
-            tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}}
+            tmp_path,
+            {
+                "gateway_portal": {
+                    "real_internet": {
+                        "mode": "mirrored",
+                        "service_mirrors": [
+                            {"real_hostname": "api.example.com", "in_world_service": "10.1.2.3"}
+                        ],
+                    }
+                }
+            },
         )
         with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"):
             spec = load_spec(spec_file)

tests/test_spec_parsing.py

@@ -212,14 +212,6 @@ def test_mirrored_gateway_requires_service_mirrors(self, tmp_path) -> None:
             tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}}
         )
 
-        with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"):
-            spec = load_spec(spec_file)
-
-        assert spec.gateway_portal.real_internet.mode.value == "mirrored"
-        assert any(
-            "gateway_portal.real_internet.mode is experimental in alpha" in r.message
-            for r in caplog.records
-        )
         with pytest.raises(SpecLoadError, match="at least one service mirror is required"):
             load_spec(spec_file)