From 553c25e1ce597bd0ac2249180b5112c0ddfe59d6 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Fri, 26 Jun 2026 23:24:46 +0100 Subject: [PATCH 01/34] Refactor API token introspection credentials --- netengine/api/auth.py | 17 ++-- netengine/core/state.py | 2 + netengine/handlers/oidc_handler.py | 20 +++-- netengine/phases/phase_platform_identity.py | 7 +- tests/integration/test_phase4_platform.py | 24 +++++- tests/test_api_auth.py | 87 +++++++++++++++++++++ 6 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 tests/test_api_auth.py diff --git a/netengine/api/auth.py b/netengine/api/auth.py index 254c7b6..98f9434 100644 --- a/netengine/api/auth.py +++ b/netengine/api/auth.py @@ -47,18 +47,23 @@ async def require_auth( raise HTTPException(status_code=401, detail="Bearer token required") token = credentials.credentials - admin_password = getattr(state, "bootstrap_admin_password", None) or os.environ.get( - "KEYCLOAK_ADMIN_PASSWORD", "" + client_id = ( + getattr(state, "platform_client_auth_id", None) + or os.environ.get("KEYCLOAK_PLATFORM_CLIENT_ID") + or "platform-api" ) - if not admin_password: - raise HTTPException(status_code=500, detail="Keycloak admin credentials not configured") + client_secret = getattr(state, "platform_client_secret", None) or os.environ.get( + "KEYCLOAK_PLATFORM_CLIENT_SECRET", "" + ) + if not client_secret: + raise HTTPException(status_code=500, detail="Platform API client secret not configured") try: async with aiohttp.ClientSession() as session: async with session.post( f"{KEYCLOAK_ISSUER}/protocol/openid-connect/token/introspect", - data={"token": token}, - auth=aiohttp.BasicAuth("admin-cli", admin_password), + data={"token": token, "client_id": client_id}, + auth=aiohttp.BasicAuth(client_id, client_secret), ssl=False, ) as resp: if resp.status != 200: diff --git a/netengine/core/state.py b/netengine/core/state.py index 186fa0a..9619f7d 100644 --- a/netengine/core/state.py +++ b/netengine/core/state.py @@ -58,6 +58,8 @@ class RuntimeState: world_spec: Optional[Dict[str, Any]] = None bootstrap_admin_password: Optional[str] = None platform_client_id: Optional[str] = None + platform_client_auth_id: Optional[str] = None + platform_client_secret: Optional[str] = None # Drift detection and self-healing drift_history: list[Dict[str, Any]] = field(default_factory=list) diff --git a/netengine/handlers/oidc_handler.py b/netengine/handlers/oidc_handler.py index 36c9f4b..f67a334 100644 --- a/netengine/handlers/oidc_handler.py +++ b/netengine/handlers/oidc_handler.py @@ -98,8 +98,14 @@ async def create_client( name: str, redirect_uris: list[str] = None, public: bool = False, - ) -> str: - """Create an OIDC client in the given realm.""" + return_secret: bool = False, + ) -> str | tuple[str, str | None]: + """Create an OIDC client in the given realm. + + Returns the Keycloak client UUID by default. When ``return_secret`` is + true, also returns the confidential client secret generated for the + client. + """ payload = { "clientId": client_id, "name": name, @@ -113,13 +119,17 @@ async def create_client( "serviceAccountsEnabled": False, "protocol": "openid-connect", } - # If not public, generate a secret (we'll store it) + client_secret = None if not public: - payload["secret"] = self._generate_secret() + client_secret = self._generate_secret() + payload["secret"] = client_secret await self._admin_request("POST", f"realms/{realm}/clients", json=payload) # Get the client UUID clients = await self._admin_request("GET", f"realms/{realm}/clients?clientId={client_id}") - return clients[0]["id"] + keycloak_client_id = clients[0]["id"] + if return_secret: + return keycloak_client_id, client_secret + return keycloak_client_id async def create_user( self, diff --git a/netengine/phases/phase_platform_identity.py b/netengine/phases/phase_platform_identity.py index 7367fab..e7f4aed 100644 --- a/netengine/phases/phase_platform_identity.py +++ b/netengine/phases/phase_platform_identity.py @@ -102,12 +102,13 @@ async def execute(self, context: PhaseContext) -> None: ) # Create platform client for API authentication - client_id = await oidc.create_client( + client_id, client_secret = await oidc.create_client( realm="platform", client_id="platform-api", name="Platform API", redirect_uris=["https://api.platform.internal/callback"], public=False, + return_secret=True, ) # Add token mapper to include org claim in JWT @@ -129,11 +130,15 @@ async def execute(self, context: PhaseContext) -> None: context.runtime_state.platform_realm_id = realm_id context.runtime_state.admin_user_id = user_id context.runtime_state.platform_client_id = client_id + context.runtime_state.platform_client_auth_id = "platform-api" + context.runtime_state.platform_client_secret = client_secret context.runtime_state.identity_platform_output = { "keycloak_container_id": container_id, "platform_realm_id": realm_id, "admin_user_id": user_id, "platform_client_id": client_id, + "platform_client_auth_id": "platform-api", + "platform_client_secret": client_secret, "deployed_at": datetime.utcnow().isoformat(), } context.runtime_state.phase_completed["4"] = True diff --git a/tests/integration/test_phase4_platform.py b/tests/integration/test_phase4_platform.py index 721721c..f3d17c1 100644 --- a/tests/integration/test_phase4_platform.py +++ b/tests/integration/test_phase4_platform.py @@ -26,7 +26,9 @@ def patched_platform_deps(phase_context): mock_oidc = MagicMock() mock_oidc.create_platform_realm = AsyncMock(return_value="platform-realm-id") mock_oidc.create_admin_user = AsyncMock(return_value="admin-user-id") - mock_oidc.create_client = AsyncMock(return_value="platform-api-client-id") + mock_oidc.create_client = AsyncMock( + return_value=("platform-api-client-id", "platform-api-secret") + ) mock_oidc.add_token_mapper = AsyncMock() mock_dns = MagicMock() @@ -93,6 +95,8 @@ async def test_execute_output_has_required_keys(self, patched_platform_deps): "platform_realm_id", "admin_user_id", "platform_client_id", + "platform_client_auth_id", + "platform_client_secret", "deployed_at", ): assert key in output, f"Missing required key '{key}' in identity_platform_output" @@ -165,6 +169,22 @@ async def test_execute_creates_platform_realm(self, patched_platform_deps): mock_oidc.create_platform_realm.assert_awaited_once_with("platform") @pytest.mark.asyncio + async def test_execute_creates_confidential_platform_api_client(self, patched_platform_deps): + """Phase 4 should request the platform API client's generated secret.""" + ctx = patched_platform_deps["context"] + mock_oidc = patched_platform_deps["oidc"] + + await PlatformIdentityPhaseHandler().execute(ctx) + + mock_oidc.create_client.assert_awaited_once_with( + realm="platform", + client_id="platform-api", + name="Platform API", + redirect_uris=["https://api.platform.internal/callback"], + public=False, + return_secret=True, + ) + async def test_execute_persists_realm_and_user_ids(self, patched_platform_deps): """Phase 4 should write realm and user IDs to runtime_state.""" ctx = patched_platform_deps["context"] @@ -173,3 +193,5 @@ async def test_execute_persists_realm_and_user_ids(self, patched_platform_deps): assert ctx.runtime_state.platform_realm_id == "platform-realm-id" assert ctx.runtime_state.admin_user_id == "admin-user-id" assert ctx.runtime_state.platform_client_id == "platform-api-client-id" + assert ctx.runtime_state.platform_client_auth_id == "platform-api" + assert ctx.runtime_state.platform_client_secret == "platform-api-secret" diff --git a/tests/test_api_auth.py b/tests/test_api_auth.py new file mode 100644 index 0000000..7441282 --- /dev/null +++ b/tests/test_api_auth.py @@ -0,0 +1,87 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from fastapi import HTTPException +from fastapi.security import HTTPAuthorizationCredentials + +from netengine.api.auth import require_auth +from netengine.core.state import RuntimeState + + +class _PostContext: + def __init__(self, status=200, body=None): + self.status = status + self._body = body or {"active": True, "sub": "user-1"} + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def json(self): + return self._body + + +class _ClientSession: + def __init__(self, capture): + self.capture = capture + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def post(self, url, **kwargs): + self.capture["url"] = url + self.capture.update(kwargs) + return _PostContext() + + +@pytest.mark.asyncio +async def test_require_auth_introspection_uses_platform_api_client_secret(monkeypatch): + state = RuntimeState( + phase_completed={"4": True}, + identity_platform_output={"platform_client_id": "uuid"}, + platform_client_id="uuid", + platform_client_auth_id="platform-api", + platform_client_secret="stored-secret", + ) + capture = {} + request = SimpleNamespace(headers={}, url=SimpleNamespace(path="/api/v1/world")) + credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials="bearer-token") + + monkeypatch.setenv("KEYCLOAK_PLATFORM_CLIENT_ID", "ignored-env-client") + with ( + patch("netengine.api.auth.RuntimeState.load", return_value=state), + patch("netengine.api.auth.aiohttp.ClientSession", return_value=_ClientSession(capture)), + ): + data = await require_auth(request, credentials) + + assert data == {"active": True, "sub": "user-1"} + assert capture["data"] == {"token": "bearer-token", "client_id": "platform-api"} + assert capture["auth"].login == "platform-api" + assert capture["auth"].password == "stored-secret" + + +@pytest.mark.asyncio +async def test_require_auth_fails_when_platform_client_secret_missing(monkeypatch): + state = RuntimeState( + phase_completed={"4": True}, + identity_platform_output={"platform_client_id": "uuid"}, + platform_client_id="uuid", + platform_client_auth_id="platform-api", + platform_client_secret=None, + ) + request = SimpleNamespace(headers={}, url=SimpleNamespace(path="/api/v1/world")) + credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials="bearer-token") + + monkeypatch.delenv("KEYCLOAK_PLATFORM_CLIENT_SECRET", raising=False) + with patch("netengine.api.auth.RuntimeState.load", return_value=state): + with pytest.raises(HTTPException) as exc_info: + await require_auth(request, credentials) + + assert exc_info.value.status_code == 500 + assert exc_info.value.detail == "Platform API client secret not configured" From a5733628b67433a8744d3a11b6e8504c016c0069 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Fri, 26 Jun 2026 23:25:58 +0100 Subject: [PATCH 02/34] Harden import snapshot validation --- netengine/api/routes.py | 136 +++++++++++++++++++--- tests/integration/test_m8_operator_api.py | 95 ++++++++++++++- 2 files changed, 211 insertions(+), 20 deletions(-) diff --git a/netengine/api/routes.py b/netengine/api/routes.py index 3bb29ec..cfb94ff 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -13,7 +13,7 @@ import yaml from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import BaseModel +from pydantic import BaseModel, Field from netengine.api.auth import require_auth from netengine.core.reload import ReloadResult, apply_reload, check_immutability, compute_diff @@ -30,6 +30,63 @@ # Health # ───────────────────────────────────────────── +IMPORT_SCHEMA_VERSION = "netengine.import.v1" + +PHASE_REQUIRED_OUTPUTS = { + "0": ("substrate_output",), + "1": ("dns_output",), + "2": ("dns_output",), + "3": ("pki_bootstrapped",), + "4": ("identity_platform_output",), + "5": ("world_registry_output", "domain_registry_output"), + "6": ("identity_inworld_output",), + "7": ("ands_output",), + "8": ("world_services_output",), + "9": ("org_apps_output",), +} + + +def _validate_import_phase_state(state: RuntimeState) -> None: + """Reject import snapshots with invalid phase completion state.""" + unknown = sorted(set(state.phase_completed) - set(PHASE_REQUIRED_OUTPUTS)) + if unknown: + raise HTTPException( + status_code=422, detail=f"Unknown phase ID(s): {', '.join(unknown)}" + ) + + completed = { + phase for phase, is_completed in state.phase_completed.items() if is_completed + } + for phase in completed: + required_outputs = PHASE_REQUIRED_OUTPUTS[phase] + missing = [ + field for field in required_outputs if not getattr(state, field, None) + ] + if missing: + missing_str = ", ".join(missing) + raise HTTPException( + status_code=422, + detail=( + f"Phase {phase} is completed but missing required " + f"output(s): {missing_str}" + ), + ) + + completed_ints = sorted(int(phase) for phase in completed) + if completed_ints: + expected = set(range(completed_ints[-1] + 1)) + missing_prereqs = sorted(expected - set(completed_ints)) + if missing_prereqs: + missing_str = ", ".join(str(phase) for phase in missing_prereqs) + raise HTTPException( + status_code=422, + detail=( + "Impossible phase combination; missing prerequisite " + f"phase(s): {missing_str}" + ), + ) + + PHASE_LABELS = { "0": "Substrate", "1": "DNS root + platform zones", @@ -148,7 +205,9 @@ async def teardown_world( """ state = RuntimeState.load() if state.world_spec: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") + raw_lifecycle = (state.world_spec.get("metadata") or {}).get( + "lifecycle", "ephemeral" + ) if raw_lifecycle == "persistent" and not body.confirm: raise HTTPException( status_code=409, @@ -510,50 +569,89 @@ async def export_world(user: dict = Depends(require_auth)) -> dict[str, Any]: import datetime as _dt return { + "schema_version": IMPORT_SCHEMA_VERSION, "exported_at": _dt.datetime.utcnow().isoformat(), "spec": state.world_spec, "phase_completed": state.phase_completed, "ca_cert_pem": state.ca_cert_pem, + "substrate_output": state.substrate_output, "pki_output": state.pki_output, "dns_output": state.dns_output, + "identity_platform_output": state.identity_platform_output, + "world_registry_output": state.world_registry_output, + "domain_registry_output": state.domain_registry_output, + "identity_inworld_output": state.identity_inworld_output, "ands_output": state.ands_output, "world_services_output": state.world_services_output, + "org_apps_output": state.org_apps_output, } class ImportRequest(BaseModel): + schema_version: str = Field( + ..., description="Import snapshot schema/version identifier" + ) spec: dict[str, Any] - phase_completed: dict[str, bool] = {} + phase_completed: dict[str, bool] = Field(default_factory=dict) ca_cert_pem: str | None = None + substrate_output: dict[str, Any] | None = None pki_output: dict[str, Any] | None = None dns_output: dict[str, Any] | None = None + identity_platform_output: dict[str, Any] | None = None + world_registry_output: dict[str, Any] | None = None + domain_registry_output: dict[str, Any] | None = None + identity_inworld_output: dict[str, Any] | None = None ands_output: dict[str, Any] | None = None world_services_output: dict[str, Any] | None = None + org_apps_output: dict[str, Any] | None = None @router.post("/import") -async def import_world(body: ImportRequest, user: dict = Depends(require_auth)) -> dict[str, Any]: +async def import_world( + body: ImportRequest, user: dict = Depends(require_auth) +) -> dict[str, Any]: """Restore world state from an export snapshot (persistent mode only).""" state = RuntimeState.load() if state.world_spec: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") + raw_lifecycle = (state.world_spec.get("metadata") or {}).get( + "lifecycle", "ephemeral" + ) if raw_lifecycle == "ephemeral": raise HTTPException( status_code=409, detail="Import is only valid for persistent worlds" ) - phases_restored = list(body.phase_completed.keys()) - state.world_spec = body.spec - state.phase_completed = dict(body.phase_completed) - if body.ca_cert_pem: - state.ca_cert_pem = body.ca_cert_pem - if body.pki_output: - state.pki_output = body.pki_output - if body.dns_output: - state.dns_output = body.dns_output - if body.ands_output: - state.ands_output = body.ands_output - if body.world_services_output: - state.world_services_output = body.world_services_output - state.save() + if body.schema_version != IMPORT_SCHEMA_VERSION: + raise HTTPException( + status_code=422, + detail=f"Unsupported import schema_version: {body.schema_version}", + ) + + try: + spec = NetEngineSpec.model_validate(body.spec) + except Exception as exc: + raise HTTPException(status_code=422, detail=f"Spec parse error: {exc}") + + imported_state = RuntimeState( + world_spec=spec.model_dump(mode="json"), + phase_completed=dict(body.phase_completed), + ca_cert_pem=body.ca_cert_pem, + substrate_output=body.substrate_output, + pki_output=body.pki_output, + dns_output=body.dns_output, + identity_platform_output=body.identity_platform_output, + world_registry_output=body.world_registry_output, + domain_registry_output=body.domain_registry_output, + identity_inworld_output=body.identity_inworld_output, + ands_output=body.ands_output, + world_services_output=body.world_services_output, + org_apps_output=body.org_apps_output, + pki_bootstrapped=bool(body.ca_cert_pem or body.pki_output), + ) + _validate_import_phase_state(imported_state) + + phases_restored = [ + phase for phase, completed in body.phase_completed.items() if completed + ] + imported_state.save() return {"status": "imported", "phases_restored": phases_restored} diff --git a/tests/integration/test_m8_operator_api.py b/tests/integration/test_m8_operator_api.py index b61f9a3..c215477 100644 --- a/tests/integration/test_m8_operator_api.py +++ b/tests/integration/test_m8_operator_api.py @@ -409,6 +409,7 @@ def test_export_returns_spec_and_phase_data(self, tmp_path, monkeypatch): assert "spec" in data assert "phase_completed" in data assert "exported_at" in data + assert data["schema_version"] == "netengine.import.v1" def test_import_updates_state(self, tmp_path, monkeypatch): monkeypatch.setenv("NETENGINE_STATE_FILE", str(tmp_path / "state.json")) @@ -422,17 +423,109 @@ def test_import_updates_state(self, tmp_path, monkeypatch): from netengine.api.app import app client = TestClient(app) + spec = _load_example("minimal.yaml").model_dump(mode="json") + spec["metadata"]["lifecycle"] = "persistent" + resp = client.post( "/api/v1/import", json={ - "spec": {"metadata": {"name": "restored", "lifecycle": "persistent"}}, + "schema_version": "netengine.import.v1", + "spec": spec, "phase_completed": {"0": True, "1": True, "2": True}, + "substrate_output": {"networks": ["platform", "core"]}, + "dns_output": {"root_ip": "10.0.0.2"}, }, headers={"X-Bootstrap-Secret": "test-secret"}, ) assert resp.status_code == 200 assert "0" in resp.json()["phases_restored"] + def test_import_rejects_invalid_spec(self, tmp_path, monkeypatch): + client = _make_client(monkeypatch, tmp_path) + + state = RuntimeState() + state.world_spec = {"metadata": {"name": "x", "lifecycle": "persistent"}} + state.save() + + resp = client.post( + "/api/v1/import", + json={ + "schema_version": "netengine.import.v1", + "spec": {"metadata": {"name": "broken", "lifecycle": "persistent"}}, + "phase_completed": {}, + }, + headers={"X-Bootstrap-Secret": "test-secret"}, + ) + assert resp.status_code == 422 + assert "Spec parse error" in resp.json()["detail"] + + def test_import_rejects_unknown_phase(self, tmp_path, monkeypatch): + client = _make_client(monkeypatch, tmp_path) + + state = RuntimeState() + state.world_spec = {"metadata": {"name": "x", "lifecycle": "persistent"}} + state.save() + + spec = _load_example("minimal.yaml").model_dump(mode="json") + spec["metadata"]["lifecycle"] = "persistent" + resp = client.post( + "/api/v1/import", + json={ + "schema_version": "netengine.import.v1", + "spec": spec, + "phase_completed": {"99": True}, + }, + headers={"X-Bootstrap-Secret": "test-secret"}, + ) + assert resp.status_code == 422 + assert "Unknown phase ID" in resp.json()["detail"] + + def test_import_rejects_phase_completion_without_required_output( + self, tmp_path, monkeypatch + ): + client = _make_client(monkeypatch, tmp_path) + + state = RuntimeState() + state.world_spec = {"metadata": {"name": "x", "lifecycle": "persistent"}} + state.save() + + spec = _load_example("minimal.yaml").model_dump(mode="json") + spec["metadata"]["lifecycle"] = "persistent" + resp = client.post( + "/api/v1/import", + json={ + "schema_version": "netengine.import.v1", + "spec": spec, + "phase_completed": {"0": True}, + }, + headers={"X-Bootstrap-Secret": "test-secret"}, + ) + assert resp.status_code == 422 + assert "missing required output" in resp.json()["detail"] + + def test_import_rejects_skipped_prerequisite_phase(self, tmp_path, monkeypatch): + client = _make_client(monkeypatch, tmp_path) + + state = RuntimeState() + state.world_spec = {"metadata": {"name": "x", "lifecycle": "persistent"}} + state.save() + + spec = _load_example("minimal.yaml").model_dump(mode="json") + spec["metadata"]["lifecycle"] = "persistent" + resp = client.post( + "/api/v1/import", + json={ + "schema_version": "netengine.import.v1", + "spec": spec, + "phase_completed": {"0": True, "2": True}, + "substrate_output": {"networks": ["platform", "core"]}, + "dns_output": {"root_ip": "10.0.0.2"}, + }, + headers={"X-Bootstrap-Secret": "test-secret"}, + ) + assert resp.status_code == 422 + assert "Impossible phase combination" in resp.json()["detail"] + class TestQueuesRoute: def test_queues_returns_list(self, tmp_path, monkeypatch): From efae01dd0c0f5c9dde3e21078272413294124ad4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Jun 2026 02:25:58 +0000 Subject: [PATCH 03/34] Fix AND deduplication key, add 404 for missing AND, fix reheal loop filter - create_and: dedup check used i.get("name") but record stores "and_name", so every POST appended a duplicate; fixed key to "and_name" - remove_and: no 404 when AND doesn't exist; added existence check before delete - _reheal_dependent_phases: phase_key checked completion of the changed phase on every iteration instead of filtering to downstream candidate phases only; replaced with a phase_num > changed_phase_num guard on the candidate Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_013NEG7QhmpVoS4n6VLefFik --- netengine/api/routes.py | 5 ++++- netengine/core/drift_controller.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/netengine/api/routes.py b/netengine/api/routes.py index fb11031..047a383 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -394,7 +394,7 @@ async def create_and(body: ANDCreateRequest, user: dict = Depends(require_auth)) state = RuntimeState.load() ands_out = state.ands_output or {} instances: list[dict[str, Any]] = ands_out.get("instances", []) - if not any(i.get("name") == body.name for i in instances): + if not any(i.get("and_name") == body.name for i in instances): instances.append(record) ands_out["instances"] = instances state.ands_output = ands_out @@ -409,6 +409,9 @@ async def remove_and(and_name: str, user: dict = Depends(require_auth)) -> dict[ from netengine.core.supabase_client import get_db db = await get_db() + existing = await db.table("and_instances").select("and_name").eq("and_name", and_name).execute() + if not existing.data: + raise HTTPException(status_code=404, detail=f"AND {and_name} not found") await db.table("and_instances").delete().eq("and_name", and_name).execute() state = RuntimeState.load() diff --git a/netengine/core/drift_controller.py b/netengine/core/drift_controller.py index a57a52f..e4bfd3d 100644 --- a/netengine/core/drift_controller.py +++ b/netengine/core/drift_controller.py @@ -289,9 +289,12 @@ async def _reheal_dependent_phases( if phase_num in healed_phases: continue - phase_key = str(changed_phase_num) + # Only re-heal phases that come after the changed phase and are completed + if phase_num <= changed_phase_num: + continue - if not self.orchestrator.runtime_state.phase_completed.get(phase_key): + candidate_key = str(phase_num) + if not self.orchestrator.runtime_state.phase_completed.get(candidate_key): continue handler = handler_class() From e6aeb590e26ed46aab8c00abd8e476fb480ca6bf Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:25:42 +0100 Subject: [PATCH 04/34] Add phase event queues to initial migration --- migrations/001_initial.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql index 646cf57..54ccd4d 100644 --- a/migrations/001_initial.sql +++ b/migrations/001_initial.sql @@ -73,6 +73,8 @@ SELECT pgmq.create('world_health'); SELECT pgmq.create('world_health_dlq'); SELECT pgmq.create('gateway_portal_events'); SELECT pgmq.create('gateway_portal_events_dlq'); +SELECT pgmq.create('phase_events'); +SELECT pgmq.create('phase_events_dlq'); -- pgmq_send(queue_name, message) CREATE OR REPLACE FUNCTION pgmq_send(queue_name text, message text) From 27fb3b5a7a17d44a710a0018178563613afed5b1 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:27:07 +0100 Subject: [PATCH 05/34] Update event queue documentation --- README.md | 2 +- docs/COMPREHENSIVE_GAP_ANALYSIS.md | 77 ++++++++++++++++-------------- docs/SUPABASE_SETUP.md | 9 +++- docs/runbook.md | 5 ++ 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index a577032..e2d6515 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ poetry run netengine up spec.yaml --set metadata.name=my-world **Runtime state** is persisted to `netengines_state.json` after each phase so interrupted runs can resume where they left off. -**Events** flow phase-to-phase via pgmq queues: `dns_updates`, `oidc_provisioning`, `and_provisioning`, `inworld_admissions`, `services_admissions`. Each queue has a dead-letter queue (`*_dlq`) for failed messages. +**Events** flow phase-to-phase via pgmq queues defined by `netengine/events/queues.py::PRIMARY_QUEUES`. There are currently 11 primary queues (`dns_updates`, `oidc_provisioning`, `and_provisioning`, `inworld_admissions`, `services_admissions`, `and_admissions`, `pki_cert_rotation_events`, `drift_events`, `world_health`, `gateway_portal_events`, `phase_events`) plus 11 matching dead-letter queues (`*_dlq`) for failed messages. --- diff --git a/docs/COMPREHENSIVE_GAP_ANALYSIS.md b/docs/COMPREHENSIVE_GAP_ANALYSIS.md index 2c573cd..b59246c 100644 --- a/docs/COMPREHENSIVE_GAP_ANALYSIS.md +++ b/docs/COMPREHENSIVE_GAP_ANALYSIS.md @@ -150,44 +150,49 @@ ands: ## 2. 🟡 MEDIUM: EVENT INFRASTRUCTURE GAPS -### 2.1 Queue Registration Mismatch -**File**: `netengine/events/queues.py:27-33` (PRIMARY_QUEUES definition) +### 2.1 Queue Registration Baseline +**File**: `netengine/events/queues.py` (`PRIMARY_QUEUES` definition) -**Declared Queues in Code**: -```python -PRIMARY_QUEUES = { - "dns_updates", - "oidc_provisioning", - "and_provisioning", - "inworld_admissions", - "services_admissions", -} -``` +`netengine/events/queues.py::PRIMARY_QUEUES` is the source of truth for the +operator-facing pgmq queue set. The current baseline is 11 primary queues plus +11 matching dead-letter queues (`*_dlq`): -**Queues Referenced But Not in PRIMARY_QUEUES**: - -| Queue Name | Emitted By | Line | Status | -|---|---|---|---| -| `and_admissions` | phase_ands.py | 342 | ❌ Queue doesn't exist; will fail at runtime | -| `pki_cert_rotation_events` | pki_cert_rotation_worker.py | 16 | ❌ Queue undefined | -| `drift_events` | drift_controller.py | 315 | ❌ Queue undefined | -| `world_health` | monitoring/service.py | 75 | ❌ Queue undefined | - -**Impact**: -- When drift controller tries to emit: `await context.pgmq_client.send_to_queue("drift_events", ...)` → **fails because queue doesn't exist** -- Health monitoring events are lost -- Queue metrics endpoint (`/api/v1/queues`) doesn't report these queues -- Event replay CLI can't restore lost events from these queues - -**Evidence**: ```python -# netengine/phases/phase_ands.py:342 -await context.pgmq_client.send_to_queue( - "and_admissions", # This queue is never created! - EventEnvelope(...) +PRIMARY_QUEUES = ( + Queue.DNS_UPDATES, + Queue.OIDC_PROVISIONING, + Queue.AND_PROVISIONING, + Queue.INWORLD_ADMISSIONS, + Queue.SERVICES_ADMISSIONS, + Queue.AND_ADMISSIONS, + Queue.PKI_CERT_ROTATION_EVENTS, + Queue.DRIFT_EVENTS, + Queue.WORLD_HEALTH, + Queue.GATEWAY_PORTAL_EVENTS, + Queue.PHASE_EVENTS, ) ``` +**Registered Primary Queues**: + +| Queue Name | Purpose | +|---|---| +| `dns_updates` | DNS zone updates | +| `oidc_provisioning` | Identity setup | +| `and_provisioning` | Network isolation setup | +| `inworld_admissions` | In-world admission events | +| `services_admissions` | Service admission events | +| `and_admissions` | AND admission events | +| `pki_cert_rotation_events` | Certificate rotation events | +| `drift_events` | Drift detection and remediation events | +| `world_health` | Health check events | +| `gateway_portal_events` | Gateway portal lifecycle events | +| `phase_events` | Phase lifecycle events | + +**Operator note**: Queue metrics, replay tooling, and DLQ checks should derive +their queue inventory from `PRIMARY_QUEUES` rather than copying a hard-coded +count or list into runbooks. + --- ### 2.2 Missing Event Consumers @@ -397,9 +402,9 @@ old_spec = NetEngineSpec(**state.world_spec) # Uses stale snapshot - **Benefit**: Manages expectations ### 🟡 P1: High Value Gaps (3-5 Hours Each) -3. **Fix Queue Registration Mismatch** - - Add missing queues to PRIMARY_QUEUES - - Update queue creation logic in ConsumerSupervisor +3. **Maintain Queue Registration Baseline** + - Keep `PRIMARY_QUEUES` as the source of truth for the 11 primary queues + - Ensure ConsumerSupervisor creates each primary queue and its matching DLQ - **Files**: events/queues.py, core/consumer_supervisor.py - **Effort**: 2 hours @@ -457,7 +462,7 @@ old_spec = NetEngineSpec(**state.world_spec) # Uses stale snapshot | Category | Count | Most Critical | Easy Win | |----------|-------|---|---| | **Spec Fields (Declared, Not Implemented)** | 14+ | DNSSEC, CRL, OCSP, intermediate CA | Add warnings | -| **Event Infrastructure** | 4 | Queue mismatch | Fix PRIMARY_QUEUES | +| **Event Infrastructure** | 4 | Queue/DLQ inventory drift | Keep docs and tooling sourced from PRIMARY_QUEUES | | **API Gaps** | 4+ | Service toggle endpoint | Add PUT endpoints | | **Phase Logic** | 2 | Phase 9 prerequisites, Phase 2 explicit | Update graph, decompose handler | | **State Debt** | 7 | Container ID fields | Clean up deprecated fields | diff --git a/docs/SUPABASE_SETUP.md b/docs/SUPABASE_SETUP.md index e8f11fd..0d35992 100644 --- a/docs/SUPABASE_SETUP.md +++ b/docs/SUPABASE_SETUP.md @@ -367,12 +367,19 @@ NetEngine creates these tables: - `domain_records` — Domain registry - `operator_log` — API audit log -And these pgmq queues (if available): +And these pgmq queues (if available), sourced from `netengine/events/queues.py::PRIMARY_QUEUES`. NetEngine currently creates 11 primary queues plus 11 matching dead-letter queues (`*_dlq`): - `dns_updates` → DNS zone updates - `oidc_provisioning` → Identity setup - `and_provisioning` → Network isolation setup +- `inworld_admissions` → In-world admission events +- `services_admissions` → Service admission events +- `and_admissions` → AND admission events +- `pki_cert_rotation_events` → Certificate rotation events +- `drift_events` → Drift detection and remediation events - `world_health` → Health check events +- `gateway_portal_events` → Gateway portal lifecycle events +- `phase_events` → Phase lifecycle events --- diff --git a/docs/runbook.md b/docs/runbook.md index 98e2a90..6dc94a8 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -144,6 +144,11 @@ touching any live state. ### Inspecting event queues +NetEngine's pgmq queue inventory is defined in +`netengine/events/queues.py::PRIMARY_QUEUES`. There are currently 11 primary +queues, and each primary queue has one matching dead-letter queue (`*_dlq`) for +failed messages. + ```bash # Show all queue depths poetry run netengine events From 7ba9e397eac47429eae254e16d3b148a90809e00 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:28:38 +0100 Subject: [PATCH 06/34] Add explicit DLQ queue mapping helper --- netengine/api/routes.py | 6 +++--- netengine/cli/main.py | 6 +++--- netengine/core/pgmq_client.py | 3 ++- netengine/events/queues.py | 20 ++++++++++++++++++++ tests/test_event_queue_dlq_mapping.py | 22 ++++++++++++++++++++++ 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 tests/test_event_queue_dlq_mapping.py diff --git a/netengine/api/routes.py b/netengine/api/routes.py index 8fde7a8..72c4342 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -18,7 +18,7 @@ from netengine.api.auth import require_admin, require_auth from netengine.core.reload import ReloadResult, apply_reload, check_immutability, compute_diff from netengine.core.state import RuntimeState -from netengine.events.queues import PRIMARY_QUEUES +from netengine.events.queues import PRIMARY_QUEUES, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS from netengine.spec.loader import SpecLoadError, load_spec @@ -780,7 +780,7 @@ async def get_queue_state(user: dict = Depends(require_auth)) -> dict[str, Any]: metrics = {} try: - dlq_result = await db.rpc("pgmq_metrics", {"queue_name": f"{q}_dlq"}).execute() + dlq_result = await db.rpc("pgmq_metrics", {"queue_name": dlq_for(q)}).execute() dlq_metrics = dlq_result.data[0] if dlq_result.data else {} except Exception: dlq_metrics = {} @@ -790,7 +790,7 @@ async def get_queue_state(user: dict = Depends(require_auth)) -> dict[str, Any]: "queue": q, "depth": metrics.get("queue_length", 0), "oldest_msg_age_sec": metrics.get("oldest_msg_age_sec"), - "dlq": f"{q}_dlq", # DLQ name follows convention: {queue}_dlq + "dlq": dlq_for(q), "dlq_depth": dlq_metrics.get("queue_length", 0), } ) diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 14964b1..f788aee 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -11,7 +11,7 @@ from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState -from netengine.events.queues import PRIMARY_QUEUES, Queue +from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS from netengine.spec.loader import load_spec, load_spec_with_composition, load_spec_with_environment @@ -586,7 +586,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: if dlq: click.echo("\nDead-letter queue contents:\n") for q in queues_to_check: - dlq_name = f"{q}_dlq" + dlq_name = dlq_for(Queue(q)).value try: rows = await conn.fetch( "SELECT msg_id, message, enqueued_at, read_ct " @@ -621,7 +621,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: else: click.echo("\nEvent queue depths:\n") for q in queues_to_check: - dlq_name = f"{q}_dlq" + dlq_name = dlq_for(Queue(q)).value try: depth_row = await conn.fetchrow("SELECT count(*) AS depth FROM pgmq.q_$1", q) dlq_row = await conn.fetchrow( diff --git a/netengine/core/pgmq_client.py b/netengine/core/pgmq_client.py index aa9da7e..af555ce 100644 --- a/netengine/core/pgmq_client.py +++ b/netengine/core/pgmq_client.py @@ -1,6 +1,7 @@ import json from typing import Any, Dict, Optional +from netengine.events.queues import Queue, dlq_for from netengine.events.schema import EventEnvelope MAX_RETRIES = 3 @@ -71,7 +72,7 @@ async def archive_to_dlq(self, queue_name: str, msg_id: int, reason: str) -> Non parent_event_id=envelope.parent_event_id, retry_count=envelope.retry_count + 1, ) - await self.send(f"{queue_name}_dlq", dlq_envelope) + await self.send(dlq_for(Queue(queue_name)), dlq_envelope) else: requeue_envelope = EventEnvelope( event_id=envelope.event_id, diff --git a/netengine/events/queues.py b/netengine/events/queues.py index 34f6937..0bf5d65 100644 --- a/netengine/events/queues.py +++ b/netengine/events/queues.py @@ -51,6 +51,26 @@ class Queue(StrEnum): ) +DLQ_BY_PRIMARY: dict[Queue, Queue] = { + Queue.DNS_UPDATES: Queue.DNS_UPDATES_DLQ, + Queue.OIDC_PROVISIONING: Queue.OIDC_PROVISIONING_DLQ, + Queue.AND_PROVISIONING: Queue.AND_PROVISIONING_DLQ, + Queue.INWORLD_ADMISSIONS: Queue.INWORLD_ADMISSIONS_DLQ, + Queue.SERVICES_ADMISSIONS: Queue.SERVICES_ADMISSIONS_DLQ, + Queue.AND_ADMISSIONS: Queue.AND_ADMISSIONS_DLQ, + Queue.PKI_CERT_ROTATION_EVENTS: Queue.PKI_CERT_ROTATION_EVENTS_DLQ, + Queue.DRIFT_EVENTS: Queue.DRIFT_EVENTS_DLQ, + Queue.WORLD_HEALTH: Queue.WORLD_HEALTH_DLQ, + Queue.GATEWAY_PORTAL_EVENTS: Queue.GATEWAY_PORTAL_EVENTS_DLQ, + Queue.PHASE_EVENTS: Queue.PHASE_EVENTS_DLQ, +} + + +def dlq_for(queue: Queue) -> Queue: + """Return the dead-letter queue associated with a primary queue.""" + return DLQ_BY_PRIMARY[queue] + + def queue_for_event_type(event_type: str) -> Queue: """Return the PGMQ queue that should receive an emitted event type. diff --git a/tests/test_event_queue_dlq_mapping.py b/tests/test_event_queue_dlq_mapping.py new file mode 100644 index 0000000..5a2f7ff --- /dev/null +++ b/tests/test_event_queue_dlq_mapping.py @@ -0,0 +1,22 @@ +"""Regression tests for event queue DLQ mappings.""" + +from netengine.events.queues import DLQ_BY_PRIMARY, PRIMARY_QUEUES, Queue, dlq_for + + +def test_every_primary_queue_has_exactly_one_dlq_mapping() -> None: + """Keep the explicit DLQ map aligned with the primary queue registry.""" + assert set(DLQ_BY_PRIMARY) == set(PRIMARY_QUEUES) + assert len(DLQ_BY_PRIMARY) == len(PRIMARY_QUEUES) + assert len(set(DLQ_BY_PRIMARY.values())) == len(PRIMARY_QUEUES) + + +def test_every_mapped_dlq_exists_in_queue_registry() -> None: + """Each mapped DLQ must be a Queue enum member and not another primary queue.""" + queue_members = set(Queue) + primary_queues = set(PRIMARY_QUEUES) + + for primary, dlq in DLQ_BY_PRIMARY.items(): + assert primary in primary_queues + assert dlq in queue_members + assert dlq not in primary_queues + assert dlq_for(primary) is dlq From 4d15f0b799491491fb6e16d5fa5e15a896cba953 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:34:02 +0100 Subject: [PATCH 07/34] Add shared database migration service --- netengine/cli/main.py | 36 ++-- netengine/db/__init__.py | 0 netengine/db/migrations.py | 193 ++++++++++++++++++++ netengine/phases/phase_platform_identity.py | 4 +- netengine/utils/run_migrations.py | 70 ++----- tests/integration/test_phase4_platform.py | 2 +- 6 files changed, 228 insertions(+), 77 deletions(-) create mode 100644 netengine/db/__init__.py create mode 100644 netengine/db/migrations.py diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 14964b1..07aba8a 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -11,13 +11,13 @@ from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState +from netengine.db.migrations import MigrationRunResult, run_migrations from netengine.events.queues import PRIMARY_QUEUES, Queue from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS from netengine.spec.loader import load_spec, load_spec_with_composition, load_spec_with_environment logger = get_logger(__name__) -MIGRATIONS_DIR = Path(__file__).parent.parent.parent / "migrations" def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: @@ -53,24 +53,22 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: return overrides -async def _run_migrations(db_url: str) -> None: - """Run all SQL migration files in order against the given Postgres URL.""" - import asyncpg # type: ignore[import] - - migration_files = sorted(MIGRATIONS_DIR.glob("*.sql")) - if not migration_files: - logger.info("No migration files found") - return - - conn = await asyncpg.connect(db_url) - try: - for migration_path in migration_files: - sql = migration_path.read_text() - logger.info(f"Running migration: {migration_path.name}") - await conn.execute(sql) - logger.info(f"Applied {len(migration_files)} migration(s)") - finally: - await conn.close() +async def _run_migrations(db_url: str) -> MigrationRunResult: + """Run SQL migrations using the shared migration service.""" + result = await run_migrations(db_url) + for migration in result.results: + if migration.status == "applied": + logger.info( + f"Applied migration: {migration.filename} " + f"({migration.duration_seconds:.3f}s)" + ) + elif migration.status == "skipped": + logger.info(f"Skipped migration: {migration.filename} (already applied)") + logger.info( + f"Migrations complete: {result.applied_count} applied, " + f"{result.skipped_count} skipped, {result.failed_count} failed" + ) + return result @click.group() diff --git a/netengine/db/__init__.py b/netengine/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/netengine/db/migrations.py b/netengine/db/migrations.py new file mode 100644 index 0000000..080a316 --- /dev/null +++ b/netengine/db/migrations.py @@ -0,0 +1,193 @@ +"""Shared database migration service for NetEngine.""" + +from __future__ import annotations + +import hashlib +import os +import time +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import quote + +MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations" + + +@dataclass(frozen=True) +class MigrationResult: + """Outcome for a single migration file.""" + + filename: str + checksum: str + status: str + duration_seconds: float = 0.0 + applied_at: datetime | None = None + error: str | None = None + + +@dataclass(frozen=True) +class MigrationRunResult: + """Structured result for a migration run.""" + + migrations_dir: Path + results: tuple[MigrationResult, ...] = field(default_factory=tuple) + + @property + def applied_count(self) -> int: + return sum(1 for result in self.results if result.status == "applied") + + @property + def skipped_count(self) -> int: + return sum(1 for result in self.results if result.status == "skipped") + + @property + def failed_count(self) -> int: + return sum(1 for result in self.results if result.status == "failed") + + +class MigrationChecksumDriftError(RuntimeError): + """Raised when an already-applied migration file changes on disk.""" + + +_SCHEMA_MIGRATIONS_SQL = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + duration_seconds DOUBLE PRECISION NOT NULL, + success BOOLEAN NOT NULL, + error TEXT +) +""" + + +def discover_migrations(migrations_dir: Path | str = MIGRATIONS_DIR) -> list[Path]: + """Discover SQL migration files in lexical order.""" + return sorted(Path(migrations_dir).glob("*.sql"), key=lambda path: path.name) + + +def migration_checksum(sql: str) -> str: + """Return a stable checksum for a migration body.""" + return hashlib.sha256(sql.encode("utf-8")).hexdigest() + + +def database_url_from_environment() -> str | None: + """Build a database URL from NetEngine/Supabase-compatible environment variables.""" + db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + if db_url: + return db_url + + db_host = os.environ.get("SUPABASE_DB_HOST") + if not db_host: + return None + + db_port = os.environ.get("SUPABASE_DB_PORT", "5432") + db_user = os.environ.get("SUPABASE_DB_USER", "postgres") + db_password = os.environ.get("SUPABASE_DB_PASSWORD", "") + db_name = os.environ.get("SUPABASE_DB_NAME", "postgres") + auth = quote(db_user) + if db_password: + auth = f"{auth}:{quote(db_password)}" + return f"postgresql://{auth}@{db_host}:{db_port}/{quote(db_name)}" + + +async def run_migrations( + db_url: str | None = None, + migrations_dir: Path | str = MIGRATIONS_DIR, +) -> MigrationRunResult: + """Apply all pending SQL migrations and return structured outcomes.""" + import asyncpg # type: ignore[import] + + resolved_dir = Path(migrations_dir) + migration_files = discover_migrations(resolved_dir) + if db_url is None: + db_url = database_url_from_environment() + if not db_url: + raise RuntimeError("No database URL configured for migrations") + + conn = await asyncpg.connect(db_url) + results: list[MigrationResult] = [] + try: + await conn.execute(_SCHEMA_MIGRATIONS_SQL) + for migration_path in migration_files: + sql = migration_path.read_text(encoding="utf-8") + checksum = migration_checksum(sql) + filename = migration_path.name + existing = await conn.fetchrow( + "SELECT checksum, success FROM schema_migrations WHERE filename = $1", + filename, + ) + if existing and existing["success"] and existing["checksum"] == checksum: + results.append(MigrationResult(filename, checksum, "skipped")) + continue + if existing and existing["success"] and existing["checksum"] != checksum: + raise MigrationChecksumDriftError( + f"Checksum drift for migration {filename}: " + f"recorded {existing['checksum']}, current {checksum}" + ) + + start = time.perf_counter() + try: + async with conn.transaction(): + await conn.execute(sql) + duration = time.perf_counter() - start + await conn.execute( + """ + INSERT INTO schema_migrations + (filename, checksum, applied_at, duration_seconds, success, error) + VALUES ($1, $2, NOW(), $3, TRUE, NULL) + ON CONFLICT (filename) DO UPDATE SET + checksum = EXCLUDED.checksum, + applied_at = EXCLUDED.applied_at, + duration_seconds = EXCLUDED.duration_seconds, + success = TRUE, + error = NULL + """, + filename, + checksum, + duration, + ) + results.append( + MigrationResult( + filename, + checksum, + "applied", + duration_seconds=duration, + applied_at=datetime.now(UTC), + ) + ) + except Exception as exc: + duration = time.perf_counter() - start + error = str(exc) + await conn.execute( + """ + INSERT INTO schema_migrations + (filename, checksum, applied_at, duration_seconds, success, error) + VALUES ($1, $2, NOW(), $3, FALSE, $4) + ON CONFLICT (filename) DO UPDATE SET + checksum = EXCLUDED.checksum, + applied_at = EXCLUDED.applied_at, + duration_seconds = EXCLUDED.duration_seconds, + success = FALSE, + error = EXCLUDED.error + """, + filename, + checksum, + duration, + error, + ) + results.append( + MigrationResult( + filename, + checksum, + "failed", + duration_seconds=duration, + applied_at=datetime.now(UTC), + error=error, + ) + ) + raise RuntimeError(f"Migration {filename} failed: {error}") from exc + finally: + await conn.close() + + return MigrationRunResult(resolved_dir, tuple(results)) diff --git a/netengine/phases/phase_platform_identity.py b/netengine/phases/phase_platform_identity.py index 1fab0f0..57069ad 100644 --- a/netengine/phases/phase_platform_identity.py +++ b/netengine/phases/phase_platform_identity.py @@ -2,6 +2,7 @@ import secrets from datetime import UTC, datetime +from netengine.db.migrations import run_migrations from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext from netengine.handlers.dns import DNSHandler @@ -9,7 +10,6 @@ from netengine.handlers.oidc_handler import OIDCHandler from netengine.handlers.pki_handler import PKIHandler from netengine.logging import get_logger -from netengine.utils.run_migrations import apply_migrations logger = get_logger(__name__) @@ -23,7 +23,7 @@ async def execute(self, context: PhaseContext) -> None: # 1. Run Supabase migrations (idempotent) logger.info("Running Supabase migrations...") - await apply_migrations() + await run_migrations() # 2. Generate or retrieve bootstrap admin password for Keycloak admin_password = getattr(context.runtime_state, "bootstrap_admin_password", None) diff --git a/netengine/utils/run_migrations.py b/netengine/utils/run_migrations.py index 4c4603c..334ff63 100644 --- a/netengine/utils/run_migrations.py +++ b/netengine/utils/run_migrations.py @@ -1,62 +1,22 @@ -import asyncio -import os -from pathlib import Path -from urllib.parse import urlparse - +"""Compatibility wrapper for applying NetEngine database migrations.""" -async def apply_migrations() -> None: - """Apply SQL migrations to the local Postgres instance. +from __future__ import annotations - Reads NETENGINE_DB_URL (e.g. postgresql://user:pass@host:5432/db). - Falls back to SUPABASE_DB_* variables for backward compat with cloud setups. - """ - db_url = os.environ.get("NETENGINE_DB_URL") - - if db_url: - parsed = urlparse(db_url) - db_host = parsed.hostname or "localhost" - db_port = str(parsed.port or 5432) - db_user = parsed.username or "netengine" - db_password = parsed.password or "" - db_name = (parsed.path or "/netengine").lstrip("/") - else: - # Backward compat: Supabase cloud connection details - db_host = os.environ.get("SUPABASE_DB_HOST", "localhost") - db_port = os.environ.get("SUPABASE_DB_PORT", "5432") - db_user = os.environ.get("SUPABASE_DB_USER", "postgres") - db_password = os.environ.get("SUPABASE_DB_PASSWORD", "") - db_name = os.environ.get("SUPABASE_DB_NAME", "postgres") +import asyncio - sql_path = Path(__file__).parent.parent.parent / "migrations" / "001_initial.sql" - if not sql_path.exists(): - raise FileNotFoundError(f"Migration file not found: {sql_path}") +from netengine.db.migrations import MigrationRunResult, run_migrations - env = os.environ.copy() - if db_password: - env["PGPASSWORD"] = db_password - try: - process = await asyncio.create_subprocess_exec( - "psql", - "-h", - db_host, - "-p", - db_port, - "-U", - db_user, - "-d", - db_name, - "-f", - str(sql_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - stdout, stderr = await process.communicate() +async def apply_migrations(db_url: str | None = None) -> MigrationRunResult: + """Apply SQL migrations using the shared migration service.""" + return await run_migrations(db_url) - if process.returncode != 0: - error_msg = stderr.decode() if stderr else "Unknown error" - raise RuntimeError(f"Migration failed: {error_msg}") - except FileNotFoundError: - raise RuntimeError("psql command not found. Install PostgreSQL client tools.") +if __name__ == "__main__": + result = asyncio.run(apply_migrations()) + for migration in result.results: + print(f"{migration.status}: {migration.filename}") + print( + f"Migrations complete: {result.applied_count} applied, " + f"{result.skipped_count} skipped, {result.failed_count} failed" + ) diff --git a/tests/integration/test_phase4_platform.py b/tests/integration/test_phase4_platform.py index 7ad6ec6..5753611 100644 --- a/tests/integration/test_phase4_platform.py +++ b/tests/integration/test_phase4_platform.py @@ -33,7 +33,7 @@ def patched_platform_deps(phase_context): mock_dns.add_zone_record = AsyncMock() patches = [ - patch("netengine.phases.phase_platform_identity.apply_migrations", AsyncMock()), + patch("netengine.phases.phase_platform_identity.run_migrations", AsyncMock()), patch( "netengine.phases.phase_platform_identity.PKIHandler", MagicMock(return_value=mock_pki), From 5a92bc2f7be69717c68308b79c1dba5bad15df15 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:37:16 +0100 Subject: [PATCH 08/34] Use Queue enum for PGMQ call sites --- netengine/api/routes.py | 15 +++++++---- netengine/core/pgmq_client.py | 25 +++++++++++++------ netengine/events/queues.py | 23 +++++++++++++++++ netengine/handlers/domain_registry_handler.py | 3 ++- netengine/handlers/world_registry_handler.py | 13 +++++----- netengine/phases/phase_inworld_identity.py | 12 ++++----- netengine/phases/phase_services.py | 6 ++--- tests/test_migration_queues.py | 9 ++++++- tests/test_registry_handlers.py | 7 +++--- 9 files changed, 80 insertions(+), 33 deletions(-) diff --git a/netengine/api/routes.py b/netengine/api/routes.py index 8fde7a8..18b0549 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -18,7 +18,7 @@ from netengine.api.auth import require_admin, require_auth from netengine.core.reload import ReloadResult, apply_reload, check_immutability, compute_diff from netengine.core.state import RuntimeState -from netengine.events.queues import PRIMARY_QUEUES +from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS from netengine.spec.loader import SpecLoadError, load_spec @@ -780,7 +780,7 @@ async def get_queue_state(user: dict = Depends(require_auth)) -> dict[str, Any]: metrics = {} try: - dlq_result = await db.rpc("pgmq_metrics", {"queue_name": f"{q}_dlq"}).execute() + dlq_result = await db.rpc("pgmq_metrics", {"queue_name": dlq_for(q)}).execute() dlq_metrics = dlq_result.data[0] if dlq_result.data else {} except Exception: dlq_metrics = {} @@ -790,7 +790,7 @@ async def get_queue_state(user: dict = Depends(require_auth)) -> dict[str, Any]: "queue": q, "depth": metrics.get("queue_length", 0), "oldest_msg_age_sec": metrics.get("oldest_msg_age_sec"), - "dlq": f"{q}_dlq", # DLQ name follows convention: {queue}_dlq + "dlq": dlq_for(q), "dlq_depth": dlq_metrics.get("queue_length", 0), } ) @@ -805,8 +805,13 @@ async def replay_dlq(queue_name: str, user: dict = Depends(require_auth)) -> dic """Move all messages from a DLQ back to the main queue for retry.""" from netengine.core.pgmq_client import PGMQClient + try: + queue = Queue(queue_name) + dlq = dlq_for(queue) + except ValueError as exc: + raise HTTPException(status_code=404, detail=f"Unknown queue: {queue_name}") from exc + client = PGMQClient() - dlq = f"{queue_name}_dlq" replayed = 0 errors: list[str] = [] while True: @@ -821,7 +826,7 @@ async def replay_dlq(queue_name: str, user: dict = Depends(require_auth)) -> dic envelope = EventEnvelope(**_json.loads(msg["message"])) envelope.retry_count = 0 # reset retry counter - await client.send(queue_name, envelope) + await client.send(queue, envelope) await client.delete(dlq, msg["msg_id"]) replayed += 1 except Exception as exc: diff --git a/netengine/core/pgmq_client.py b/netengine/core/pgmq_client.py index aa9da7e..2c744a1 100644 --- a/netengine/core/pgmq_client.py +++ b/netengine/core/pgmq_client.py @@ -1,6 +1,7 @@ import json from typing import Any, Dict, Optional +from netengine.events.queues import Queue, dlq_for from netengine.events.schema import EventEnvelope MAX_RETRIES = 3 @@ -17,7 +18,7 @@ async def _get_db(self): self._db = await get_db() return self._db - async def send(self, queue_name: str, event: EventEnvelope) -> int: + async def send(self, queue_name: Queue, event: EventEnvelope) -> int: """Enqueue an event; returns message ID.""" db = await self._get_db() payload = event.to_dict() @@ -28,20 +29,28 @@ async def send(self, queue_name: str, event: EventEnvelope) -> int: raise RuntimeError(f"pgmq_send returned no data for queue '{queue_name}'") return result.data[0] - async def receive(self, queue_name: str, timeout: int = 5) -> Optional[Dict[str, Any]]: + async def receive( + self, queue_name: Queue, timeout: int = 5 + ) -> Optional[Dict[str, Any]]: """Pop a message from the queue.""" db = await self._get_db() - result = await db.rpc("pgmq_pop", {"queue_name": queue_name, "timeout": timeout}).execute() + result = await db.rpc( + "pgmq_pop", {"queue_name": queue_name, "timeout": timeout} + ).execute() if result.data: return result.data[0] return None - async def delete(self, queue_name: str, msg_id: int) -> None: + async def delete(self, queue_name: Queue, msg_id: int) -> None: """Acknowledge and delete a processed message.""" db = await self._get_db() - await db.rpc("pgmq_delete", {"queue_name": queue_name, "msg_id": msg_id}).execute() + await db.rpc( + "pgmq_delete", {"queue_name": queue_name, "msg_id": msg_id} + ).execute() - async def read_by_id(self, queue_name: str, msg_id: int) -> Optional[Dict[str, Any]]: + async def read_by_id( + self, queue_name: Queue, msg_id: int + ) -> Optional[Dict[str, Any]]: """Read a specific message by ID without consuming it.""" db = await self._get_db() result = await db.rpc( @@ -51,7 +60,7 @@ async def read_by_id(self, queue_name: str, msg_id: int) -> Optional[Dict[str, A return result.data[0] return None - async def archive_to_dlq(self, queue_name: str, msg_id: int, reason: str) -> None: + async def archive_to_dlq(self, queue_name: Queue, msg_id: int, reason: str) -> None: """Re-queue with incremented retry count, or move to DLQ after MAX_RETRIES.""" msg = await self.read_by_id(queue_name, msg_id) if not msg: @@ -71,7 +80,7 @@ async def archive_to_dlq(self, queue_name: str, msg_id: int, reason: str) -> Non parent_event_id=envelope.parent_event_id, retry_count=envelope.retry_count + 1, ) - await self.send(f"{queue_name}_dlq", dlq_envelope) + await self.send(dlq_for(queue_name), dlq_envelope) else: requeue_envelope = EventEnvelope( event_id=envelope.event_id, diff --git a/netengine/events/queues.py b/netengine/events/queues.py index 34f6937..3cb54a9 100644 --- a/netengine/events/queues.py +++ b/netengine/events/queues.py @@ -35,6 +35,29 @@ class Queue(StrEnum): PHASE_EVENTS_DLQ = "phase_events_dlq" +_DLQ_BY_PRIMARY: dict[Queue, Queue] = { + Queue.DNS_UPDATES: Queue.DNS_UPDATES_DLQ, + Queue.OIDC_PROVISIONING: Queue.OIDC_PROVISIONING_DLQ, + Queue.AND_PROVISIONING: Queue.AND_PROVISIONING_DLQ, + Queue.INWORLD_ADMISSIONS: Queue.INWORLD_ADMISSIONS_DLQ, + Queue.SERVICES_ADMISSIONS: Queue.SERVICES_ADMISSIONS_DLQ, + Queue.AND_ADMISSIONS: Queue.AND_ADMISSIONS_DLQ, + Queue.PKI_CERT_ROTATION_EVENTS: Queue.PKI_CERT_ROTATION_EVENTS_DLQ, + Queue.DRIFT_EVENTS: Queue.DRIFT_EVENTS_DLQ, + Queue.WORLD_HEALTH: Queue.WORLD_HEALTH_DLQ, + Queue.GATEWAY_PORTAL_EVENTS: Queue.GATEWAY_PORTAL_EVENTS_DLQ, + Queue.PHASE_EVENTS: Queue.PHASE_EVENTS_DLQ, +} + + +def dlq_for(queue: Queue) -> Queue: + """Return the dead-letter queue associated with a primary queue.""" + try: + return _DLQ_BY_PRIMARY[queue] + except KeyError as exc: + raise ValueError(f"Queue {queue!r} does not have a registered DLQ") from exc + + # Primary queues only — used for metrics/introspection endpoints PRIMARY_QUEUES: tuple[Queue, ...] = ( Queue.DNS_UPDATES, diff --git a/netengine/handlers/domain_registry_handler.py b/netengine/handlers/domain_registry_handler.py index be7ee1b..7f30ae7 100644 --- a/netengine/handlers/domain_registry_handler.py +++ b/netengine/handlers/domain_registry_handler.py @@ -3,6 +3,7 @@ from netengine.core.pgmq_client import PGMQClient from netengine.errors import RegistryError +from netengine.events.queues import Queue from netengine.events.schema import EventEnvelope @@ -48,4 +49,4 @@ async def register_domain(self, domain: str, org_name: str, ns_records: List[str emitted_by="domain_registry_handler", payload={"domain": domain, "org": org_name, "ns": ns_records}, ) - await self.pgmq.send("dns_updates", event) + await self.pgmq.send(Queue.DNS_UPDATES, event) diff --git a/netengine/handlers/world_registry_handler.py b/netengine/handlers/world_registry_handler.py index b5c28b8..e63a843 100644 --- a/netengine/handlers/world_registry_handler.py +++ b/netengine/handlers/world_registry_handler.py @@ -1,6 +1,7 @@ from typing import Any, List from netengine.core.pgmq_client import PGMQClient +from netengine.events.queues import Queue from netengine.events.schema import EventEnvelope @@ -36,8 +37,8 @@ async def admit_org(self, name: str, capabilities: List[str], and_profile: str) emitted_by="world_registry_handler", payload={"org_name": name, "capabilities": capabilities, "and_profile": and_profile}, ) - await self.pgmq.send("oidc_provisioning", event) - await self.pgmq.send("and_provisioning", event) + await self.pgmq.send(Queue.OIDC_PROVISIONING, event) + await self.pgmq.send(Queue.AND_PROVISIONING, event) async def list_orgs(self) -> List[Any]: """Return all orgs in the world registry.""" @@ -62,8 +63,8 @@ async def update_org(self, name: str, capabilities: List[str], and_profile: str) emitted_by="world_registry_handler", payload={"org_name": name, "capabilities": capabilities, "and_profile": and_profile}, ) - await self.pgmq.send("oidc_provisioning", event) - await self.pgmq.send("and_provisioning", event) + await self.pgmq.send(Queue.OIDC_PROVISIONING, event) + await self.pgmq.send(Queue.AND_PROVISIONING, event) async def remove_org(self, name: str) -> bool: """Remove an org from the world registry. Returns True if it existed.""" @@ -77,6 +78,6 @@ async def remove_org(self, name: str) -> bool: emitted_by="world_registry_handler", payload={"org_name": name}, ) - await self.pgmq.send("oidc_provisioning", event) - await self.pgmq.send("and_provisioning", event) + await self.pgmq.send(Queue.OIDC_PROVISIONING, event) + await self.pgmq.send(Queue.AND_PROVISIONING, event) return True diff --git a/netengine/phases/phase_inworld_identity.py b/netengine/phases/phase_inworld_identity.py index 3cf4b64..04f31c2 100644 --- a/netengine/phases/phase_inworld_identity.py +++ b/netengine/phases/phase_inworld_identity.py @@ -17,7 +17,7 @@ import aiohttp -from netengine.events.queues import queue_for_event_type +from netengine.events.queues import Queue, queue_for_event_type from netengine.events.schema import EventEnvelope from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext @@ -475,7 +475,7 @@ async def _consume_org_admission_events( while True: try: - msg = await context.pgmq_client.receive("inworld_admissions") + msg = await context.pgmq_client.receive(Queue.INWORLD_ADMISSIONS) if not msg: await asyncio.sleep(1) continue @@ -485,7 +485,7 @@ async def _consume_org_admission_events( if envelope.event_type != "org.admitted": # Skip non-admission events - await context.pgmq_client.delete("inworld_admissions", msg["msg_id"]) + await context.pgmq_client.delete(Queue.INWORLD_ADMISSIONS, msg["msg_id"]) continue payload = envelope.payload @@ -493,7 +493,7 @@ async def _consume_org_admission_events( if not org_name: logger.warning("org.admitted event missing org_name") - await context.pgmq_client.delete("inworld_admissions", msg["msg_id"]) + await context.pgmq_client.delete(Queue.INWORLD_ADMISSIONS, msg["msg_id"]) continue logger.info(f"Processing org admission: {org_name}") @@ -509,13 +509,13 @@ async def _consume_org_admission_events( logger.info(f"Provisioned in-world realm for org {org_name}") # Mark message as processed - await context.pgmq_client.delete("inworld_admissions", msg["msg_id"]) + await context.pgmq_client.delete(Queue.INWORLD_ADMISSIONS, msg["msg_id"]) except Exception as e: logger.error(f"Failed to process org admission event: {e}") # Archive to DLQ for manual review await context.pgmq_client.archive_to_dlq( - "inworld_admissions", msg["msg_id"], str(e) + Queue.INWORLD_ADMISSIONS, msg["msg_id"], str(e) ) except Exception as e: diff --git a/netengine/phases/phase_services.py b/netengine/phases/phase_services.py index e1accd9..006db20 100644 --- a/netengine/phases/phase_services.py +++ b/netengine/phases/phase_services.py @@ -12,7 +12,7 @@ from datetime import UTC, datetime from typing import Any -from netengine.events.queues import queue_for_event_type +from netengine.events.queues import Queue, queue_for_event_type from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext from netengine.handlers.dns import DNSHandler @@ -301,7 +301,7 @@ async def _consume_org_admission_events( while True: try: - msg = await context.pgmq_client.receive("services_admissions") + msg = await context.pgmq_client.receive(Queue.SERVICES_ADMISSIONS) if not msg: await asyncio.sleep(1) continue @@ -311,7 +311,7 @@ async def _consume_org_admission_events( logger.debug("Processing org.admitted event for services provisioning") # Mark message as processed - await context.pgmq_client.delete("services_admissions", msg["msg_id"]) + await context.pgmq_client.delete(Queue.SERVICES_ADMISSIONS, msg["msg_id"]) except Exception as e: logger.error(f"Org admission consumer error: {e}") diff --git a/tests/test_migration_queues.py b/tests/test_migration_queues.py index fbcdab0..2a215b4 100644 --- a/tests/test_migration_queues.py +++ b/tests/test_migration_queues.py @@ -3,7 +3,7 @@ from pathlib import Path import re -from netengine.events.queues import Queue +from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for REPO_ROOT = Path(__file__).resolve().parents[1] @@ -17,3 +17,10 @@ def test_initial_migration_declares_all_registered_queues() -> None: registered_queue_names = {queue.value for queue in Queue} assert migration_queue_names == registered_queue_names + + +def test_all_primary_queues_have_registered_dlqs() -> None: + """Keep DLQ lookups explicit rather than string-convention based.""" + for queue in PRIMARY_QUEUES: + assert dlq_for(queue) in Queue + assert dlq_for(queue).value in {registered.value for registered in Queue} diff --git a/tests/test_registry_handlers.py b/tests/test_registry_handlers.py index 2ed8523..d883964 100644 --- a/tests/test_registry_handlers.py +++ b/tests/test_registry_handlers.py @@ -5,6 +5,7 @@ import pytest from netengine.errors import RegistryError +from netengine.events.queues import Queue # ───────────────────────────────────────────────────────────────────────────── # Helpers @@ -68,8 +69,8 @@ async def test_admit_org_sends_two_pgmq_events(self, handler: MagicMock) -> None await handler.admit_org("acme", [], "residential") assert handler.pgmq.send.call_count == 2 queues = {call.args[0] for call in handler.pgmq.send.call_args_list} - assert "oidc_provisioning" in queues - assert "and_provisioning" in queues + assert Queue.OIDC_PROVISIONING in queues + assert Queue.AND_PROVISIONING in queues async def test_seed_from_spec_admits_each_org(self, handler: MagicMock) -> None: org1 = MagicMock() @@ -147,7 +148,7 @@ async def test_register_domain_upserts_record(self, handler: MagicMock, sb: Magi async def test_register_domain_sends_dns_update_event(self, handler: MagicMock) -> None: await handler.register_domain("acme.internal", "acme", []) handler.pgmq.send.assert_called_once() - assert handler.pgmq.send.call_args.args[0] == "dns_updates" + assert handler.pgmq.send.call_args.args[0] == Queue.DNS_UPDATES async def test_seed_address_pools_upserts_each_pool( self, handler: MagicMock, sb: MagicMock From 2d6a69a4bb95bbf1a2f60c2b47e5e02baf7902e9 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:37:37 +0100 Subject: [PATCH 09/34] Use initial migration for integration pgmq setup --- compose/compose.test-integration.yml | 9 +++------ migrations/001_initial.sql | 2 ++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/compose/compose.test-integration.yml b/compose/compose.test-integration.yml index c864d11..75c2516 100644 --- a/compose/compose.test-integration.yml +++ b/compose/compose.test-integration.yml @@ -38,13 +38,10 @@ services: PGPASSWORD: integration_test_password command: | sh -c " - psql -h postgres-integration -U netengine -d netengine_integration -c 'CREATE EXTENSION IF NOT EXISTS pgmq' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''dns_updates'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''oidc_provisioning'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''and_provisioning'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''inworld_admissions'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''services_admissions'');' + psql -h postgres-integration -U netengine -d netengine_integration -f /migrations/001_initial.sql " + volumes: + - ../migrations/001_initial.sql:/migrations/001_initial.sql:ro networks: - netengine-integration restart: on-failure diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql index 646cf57..54ccd4d 100644 --- a/migrations/001_initial.sql +++ b/migrations/001_initial.sql @@ -73,6 +73,8 @@ SELECT pgmq.create('world_health'); SELECT pgmq.create('world_health_dlq'); SELECT pgmq.create('gateway_portal_events'); SELECT pgmq.create('gateway_portal_events_dlq'); +SELECT pgmq.create('phase_events'); +SELECT pgmq.create('phase_events_dlq'); -- pgmq_send(queue_name, message) CREATE OR REPLACE FUNCTION pgmq_send(queue_name text, message text) From 646e08806976e8f4ee533fca7f7772a60f0f0f03 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:38:12 +0100 Subject: [PATCH 10/34] Add migration ledger tracking --- netengine/cli/main.py | 9 ++- netengine/utils/migrations.py | 100 ++++++++++++++++++++++++++++++ netengine/utils/run_migrations.py | 66 ++++++-------------- tests/test_migration_ledger.py | 83 +++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 51 deletions(-) create mode 100644 netengine/utils/migrations.py create mode 100644 tests/test_migration_ledger.py diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 14964b1..e6cb393 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -57,6 +57,8 @@ async def _run_migrations(db_url: str) -> None: """Run all SQL migration files in order against the given Postgres URL.""" import asyncpg # type: ignore[import] + from netengine.utils.migrations import apply_migration_files + migration_files = sorted(MIGRATIONS_DIR.glob("*.sql")) if not migration_files: logger.info("No migration files found") @@ -64,11 +66,8 @@ async def _run_migrations(db_url: str) -> None: conn = await asyncpg.connect(db_url) try: - for migration_path in migration_files: - sql = migration_path.read_text() - logger.info(f"Running migration: {migration_path.name}") - await conn.execute(sql) - logger.info(f"Applied {len(migration_files)} migration(s)") + applied_count = await apply_migration_files(conn, migration_files) + logger.info(f"Applied {applied_count} migration(s)") finally: await conn.close() diff --git a/netengine/utils/migrations.py b/netengine/utils/migrations.py new file mode 100644 index 0000000..faac3bd --- /dev/null +++ b/netengine/utils/migrations.py @@ -0,0 +1,100 @@ +"""Database migration helpers.""" + +from __future__ import annotations + +import hashlib +import time +from pathlib import Path +from types import TracebackType +from typing import Protocol + +from netengine.logging import get_logger + +logger = get_logger(__name__) + + +class AsyncTransaction(Protocol): + async def __aenter__(self) -> object: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: ... + + +class AsyncMigrationConnection(Protocol): + async def execute(self, query: str, *args: object) -> object: ... + + async def fetchval(self, query: str, *args: object) -> object: ... + + def transaction(self) -> AsyncTransaction: ... + + +MIGRATION_LEDGER_TABLE = "netengine_migration_ledger" +MIGRATION_LEDGER_SQL = f""" +CREATE TABLE IF NOT EXISTS {MIGRATION_LEDGER_TABLE} ( + filename TEXT PRIMARY KEY, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + duration_ms INTEGER, + success BOOLEAN NOT NULL, + error TEXT +); +""" + + +def migration_checksum(sql: str) -> str: + """Return the SHA-256 checksum for a migration body.""" + return hashlib.sha256(sql.encode("utf-8")).hexdigest() + + +async def apply_migration_files(conn: AsyncMigrationConnection, migration_files: list[Path]) -> int: + """Apply migration files through an asyncpg connection. + + Creates the bootstrap migration ledger before applying files. Already-applied + migrations are skipped when their checksums match. If an applied migration's + contents changed, abort before executing it. + """ + await conn.execute(MIGRATION_LEDGER_SQL) + + applied_count = 0 + for migration_path in migration_files: + sql = migration_path.read_text() + checksum = migration_checksum(sql) + filename = migration_path.name + existing_checksum = await conn.fetchval( + f"SELECT checksum FROM {MIGRATION_LEDGER_TABLE} WHERE filename = $1", + filename, + ) + + if existing_checksum is not None: + if existing_checksum != checksum: + raise RuntimeError( + "Applied migration changed: " + f"{filename} has checksum {checksum}, but the ledger contains " + f"{existing_checksum}. Create a new migration instead of editing " + "an applied migration." + ) + logger.info(f"Skipping already-applied migration: {filename}") + continue + + logger.info(f"Running migration: {filename}") + started = time.perf_counter() + async with conn.transaction(): + await conn.execute(sql) + duration_ms = round((time.perf_counter() - started) * 1000) + await conn.execute( + f""" + INSERT INTO {MIGRATION_LEDGER_TABLE} + (filename, checksum, duration_ms, success) + VALUES ($1, $2, $3, TRUE) + """, + filename, + checksum, + duration_ms, + ) + applied_count += 1 + + return applied_count diff --git a/netengine/utils/run_migrations.py b/netengine/utils/run_migrations.py index 4c4603c..f64a382 100644 --- a/netengine/utils/run_migrations.py +++ b/netengine/utils/run_migrations.py @@ -1,7 +1,7 @@ -import asyncio import os from pathlib import Path -from urllib.parse import urlparse + +from netengine.utils.migrations import apply_migration_files async def apply_migrations() -> None: @@ -10,53 +10,27 @@ async def apply_migrations() -> None: Reads NETENGINE_DB_URL (e.g. postgresql://user:pass@host:5432/db). Falls back to SUPABASE_DB_* variables for backward compat with cloud setups. """ + import asyncpg # type: ignore[import] + db_url = os.environ.get("NETENGINE_DB_URL") + migrations_dir = Path(__file__).parent.parent.parent / "migrations" + migration_files = sorted(migrations_dir.glob("*.sql")) + if not migration_files: + raise FileNotFoundError(f"No migration files found in: {migrations_dir}") if db_url: - parsed = urlparse(db_url) - db_host = parsed.hostname or "localhost" - db_port = str(parsed.port or 5432) - db_user = parsed.username or "netengine" - db_password = parsed.password or "" - db_name = (parsed.path or "/netengine").lstrip("/") + conn = await asyncpg.connect(db_url) else: - # Backward compat: Supabase cloud connection details - db_host = os.environ.get("SUPABASE_DB_HOST", "localhost") - db_port = os.environ.get("SUPABASE_DB_PORT", "5432") - db_user = os.environ.get("SUPABASE_DB_USER", "postgres") - db_password = os.environ.get("SUPABASE_DB_PASSWORD", "") - db_name = os.environ.get("SUPABASE_DB_NAME", "postgres") - - sql_path = Path(__file__).parent.parent.parent / "migrations" / "001_initial.sql" - if not sql_path.exists(): - raise FileNotFoundError(f"Migration file not found: {sql_path}") - - env = os.environ.copy() - if db_password: - env["PGPASSWORD"] = db_password - - try: - process = await asyncio.create_subprocess_exec( - "psql", - "-h", - db_host, - "-p", - db_port, - "-U", - db_user, - "-d", - db_name, - "-f", - str(sql_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, + parsed_port = int(os.environ.get("SUPABASE_DB_PORT", "5432")) + conn = await asyncpg.connect( + host=os.environ.get("SUPABASE_DB_HOST", "localhost"), + port=parsed_port, + user=os.environ.get("SUPABASE_DB_USER", "postgres"), + password=os.environ.get("SUPABASE_DB_PASSWORD", ""), + database=os.environ.get("SUPABASE_DB_NAME", "postgres"), ) - stdout, stderr = await process.communicate() - if process.returncode != 0: - error_msg = stderr.decode() if stderr else "Unknown error" - raise RuntimeError(f"Migration failed: {error_msg}") - - except FileNotFoundError: - raise RuntimeError("psql command not found. Install PostgreSQL client tools.") + try: + await apply_migration_files(conn, migration_files) + finally: + await conn.close() diff --git a/tests/test_migration_ledger.py b/tests/test_migration_ledger.py new file mode 100644 index 0000000..a4783aa --- /dev/null +++ b/tests/test_migration_ledger.py @@ -0,0 +1,83 @@ +"""Tests for migration ledger bookkeeping.""" + +from pathlib import Path +from types import TracebackType + +import pytest + +from netengine.utils.migrations import ( + MIGRATION_LEDGER_SQL, + apply_migration_files, + migration_checksum, +) + + +class FakeTransaction: + async def __aenter__(self) -> None: + return None + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + return False + + +class FakeConnection: + def __init__(self, checksums: dict[str, str] | None = None) -> None: + self.checksums = checksums or {} + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def execute(self, sql: str, *args: object) -> str: + self.executed.append((sql, args)) + if sql.strip().startswith("INSERT INTO netengine_migration_ledger"): + filename, checksum = args[:2] + self.checksums[str(filename)] = str(checksum) + return "OK" + + async def fetchval(self, sql: str, *args: object) -> str | None: + return self.checksums.get(str(args[0])) + + def transaction(self) -> FakeTransaction: + return FakeTransaction() + + +@pytest.mark.asyncio +async def test_apply_migration_files_records_success_after_running_sql(tmp_path: Path) -> None: + migration = tmp_path / "001_initial.sql" + migration.write_text("CREATE TABLE example (id int primary key);") + conn = FakeConnection() + + applied_count = await apply_migration_files(conn, [migration]) + + assert applied_count == 1 + assert conn.executed[0] == (MIGRATION_LEDGER_SQL, ()) + assert conn.executed[1] == (migration.read_text(), ()) + assert conn.executed[2][1][:2] == (migration.name, migration_checksum(migration.read_text())) + assert len(conn.executed[2][1]) == 3 + + +@pytest.mark.asyncio +async def test_apply_migration_files_skips_matching_ledger_entry(tmp_path: Path) -> None: + migration = tmp_path / "001_initial.sql" + migration.write_text("SELECT 1;") + conn = FakeConnection({migration.name: migration_checksum(migration.read_text())}) + + applied_count = await apply_migration_files(conn, [migration]) + + assert applied_count == 0 + assert [entry[0] for entry in conn.executed] == [MIGRATION_LEDGER_SQL] + + +@pytest.mark.asyncio +async def test_apply_migration_files_aborts_when_applied_migration_changed(tmp_path: Path) -> None: + migration = tmp_path / "001_initial.sql" + migration.write_text("SELECT 2;") + conn = FakeConnection({migration.name: migration_checksum("SELECT 1;")}) + + with pytest.raises(RuntimeError, match="Applied migration changed: 001_initial.sql"): + await apply_migration_files(conn, [migration]) + + assert [entry[0] for entry in conn.executed] == [MIGRATION_LEDGER_SQL] From 91e5d9713192ee300838bff913b24a83c67e0051 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:41:40 +0100 Subject: [PATCH 11/34] Make pgmq queue creation idempotent --- migrations/001_initial.sql | 58 ++++++++++++++++++++++------------ tests/test_migration_queues.py | 11 +++++-- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql index 646cf57..ef4b433 100644 --- a/migrations/001_initial.sql +++ b/migrations/001_initial.sql @@ -53,26 +53,44 @@ CREATE TABLE IF NOT EXISTS operator_log ( ); -- pgmq queues -SELECT pgmq.create('dns_updates'); -SELECT pgmq.create('dns_updates_dlq'); -SELECT pgmq.create('oidc_provisioning'); -SELECT pgmq.create('oidc_provisioning_dlq'); -SELECT pgmq.create('and_provisioning'); -SELECT pgmq.create('and_provisioning_dlq'); -SELECT pgmq.create('inworld_admissions'); -SELECT pgmq.create('inworld_admissions_dlq'); -SELECT pgmq.create('services_admissions'); -SELECT pgmq.create('services_admissions_dlq'); -SELECT pgmq.create('and_admissions'); -SELECT pgmq.create('and_admissions_dlq'); -SELECT pgmq.create('pki_cert_rotation_events'); -SELECT pgmq.create('pki_cert_rotation_events_dlq'); -SELECT pgmq.create('drift_events'); -SELECT pgmq.create('drift_events_dlq'); -SELECT pgmq.create('world_health'); -SELECT pgmq.create('world_health_dlq'); -SELECT pgmq.create('gateway_portal_events'); -SELECT pgmq.create('gateway_portal_events_dlq'); +DO $$ +DECLARE + queue_to_create text; +BEGIN + FOREACH queue_to_create IN ARRAY ARRAY[ + 'dns_updates', + 'oidc_provisioning', + 'and_provisioning', + 'inworld_admissions', + 'services_admissions', + 'and_admissions', + 'pki_cert_rotation_events', + 'drift_events', + 'world_health', + 'gateway_portal_events', + 'phase_events', + 'dns_updates_dlq', + 'oidc_provisioning_dlq', + 'and_provisioning_dlq', + 'inworld_admissions_dlq', + 'services_admissions_dlq', + 'and_admissions_dlq', + 'pki_cert_rotation_events_dlq', + 'drift_events_dlq', + 'world_health_dlq', + 'gateway_portal_events_dlq', + 'phase_events_dlq' + ] LOOP + IF NOT EXISTS ( + SELECT 1 + FROM pgmq.meta + WHERE meta.queue_name = queue_to_create + ) THEN + PERFORM pgmq.create(queue_to_create); + END IF; + END LOOP; +END; +$$; -- pgmq_send(queue_name, message) CREATE OR REPLACE FUNCTION pgmq_send(queue_name text, message text) diff --git a/tests/test_migration_queues.py b/tests/test_migration_queues.py index fbcdab0..937c228 100644 --- a/tests/test_migration_queues.py +++ b/tests/test_migration_queues.py @@ -8,12 +8,19 @@ REPO_ROOT = Path(__file__).resolve().parents[1] INITIAL_MIGRATION = REPO_ROOT / "migrations" / "001_initial.sql" -PGMQ_CREATE_RE = re.compile(r"SELECT\s+pgmq\.create\('([^']+)'\);", re.IGNORECASE) +PGMQ_QUEUE_ARRAY_RE = re.compile( + r"FOREACH\s+queue_to_create\s+IN\s+ARRAY\s+ARRAY\[(.*?)\]\s+LOOP", + re.IGNORECASE | re.DOTALL, +) +SQL_STRING_RE = re.compile(r"'([^']+)'") def test_initial_migration_declares_all_registered_queues() -> None: """Keep the initial pgmq migration aligned with the Queue registry.""" - migration_queue_names = set(PGMQ_CREATE_RE.findall(INITIAL_MIGRATION.read_text())) + migration = INITIAL_MIGRATION.read_text() + match = PGMQ_QUEUE_ARRAY_RE.search(migration) + assert match is not None + migration_queue_names = set(SQL_STRING_RE.findall(match.group(1))) registered_queue_names = {queue.value for queue in Queue} assert migration_queue_names == registered_queue_names From c81dc972c968dee31da6c0af1362a9d966c53165 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:41:52 +0100 Subject: [PATCH 12/34] Add migration CLI commands --- netengine/cli/main.py | 125 ++++++++++++++--- netengine/core/migrations.py | 221 ++++++++++++++++++++++++++++++ netengine/utils/run_migrations.py | 66 ++------- 3 files changed, 342 insertions(+), 70 deletions(-) create mode 100644 netengine/core/migrations.py diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 14964b1..f151abe 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -9,6 +9,7 @@ import click import yaml +from netengine.core.migrations import MigrationService, MigrationStatus from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState from netengine.events.queues import PRIMARY_QUEUES, Queue @@ -53,24 +54,59 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: return overrides +def _db_url_from_env() -> str | None: + """Return the database URL used by CLI migration operations.""" + return os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + + async def _run_migrations(db_url: str) -> None: - """Run all SQL migration files in order against the given Postgres URL.""" - import asyncpg # type: ignore[import] + """Apply pending SQL migrations against the given Postgres URL.""" + service = MigrationService(db_url, MIGRATIONS_DIR) + applied = await service.apply_pending() + if applied: + logger.info(f"Applied {len(applied)} migration(s)") + else: + logger.info("No pending migrations") - migration_files = sorted(MIGRATIONS_DIR.glob("*.sql")) - if not migration_files: - logger.info("No migration files found") - return - conn = await asyncpg.connect(db_url) - try: - for migration_path in migration_files: - sql = migration_path.read_text() - logger.info(f"Running migration: {migration_path.name}") - await conn.execute(sql) - logger.info(f"Applied {len(migration_files)} migration(s)") - finally: - await conn.close() +def _require_db_url() -> str: + db_url = _db_url_from_env() + if not db_url: + click.echo("Database URL is required: set NETENGINE_DB_URL or DATABASE_URL.", err=True) + sys.exit(2) + return db_url + + +def _print_migration_status(status: MigrationStatus) -> None: + click.echo("Migration status") + click.echo(f" Applied: {len(status.applied)}") + for record in status.applied: + applied_at = record.applied_at.isoformat() if record.applied_at else "unknown time" + click.echo(f" ✓ {record.version} {record.name} ({applied_at})") + + click.echo(f" Pending: {len(status.pending)}") + for migration in status.pending: + click.echo(f" • {migration.version} {migration.name} ({migration.path.name})") + + click.echo(f" Failed: {len(status.failed)}") + for record in status.failed: + detail = f": {record.error}" if record.error else "" + click.echo(f" ✗ {record.version} {record.name}{detail}") + + click.echo(f" Checksum drift: {len(status.checksum_drifted)}") + for migration, record in status.checksum_drifted: + click.echo( + f" ! {migration.version} {migration.name}: " + f"database={record.checksum or 'unknown'} file={migration.checksum}" + ) + + click.echo(" pgmq prerequisites:") + click.echo(f" Extension available: {'yes' if status.pgmq_available else 'no'}") + click.echo(f" Extension installed: {'yes' if status.pgmq_installed else 'no'}") + if status.missing_queues: + click.echo(f" Missing queues: {', '.join(status.missing_queues)}") + else: + click.echo(" Missing queues: none") @click.group() @@ -78,6 +114,63 @@ def cli() -> None: """NetEngine — spin up, reload, and tear down authority-autonomous worlds.""" +@cli.group() +def migrate() -> None: + """Manage NetEngine database migrations.""" + + +@migrate.command("up") +def migrate_up() -> None: + """Apply pending database migrations.""" + asyncio.run(_migrate_up()) + + +async def _migrate_up() -> None: + db_url = _require_db_url() + service = MigrationService(db_url, MIGRATIONS_DIR) + click.echo(f"Applying migrations from {MIGRATIONS_DIR}...") + try: + applied = await service.apply_pending() + except Exception as exc: + click.echo(f"Migration failed: {exc}", err=True) + sys.exit(1) + if applied: + click.echo(f"Applied {len(applied)} migration(s):") + for record in applied: + click.echo(f" ✓ {record.version} {record.name}") + else: + click.echo("No pending migrations.") + + +@migrate.command("status") +def migrate_status() -> None: + """Print applied, pending, failed, and drifted migrations.""" + asyncio.run(_migrate_status(exit_on_unhealthy=False)) + + +@migrate.command("check") +def migrate_check() -> None: + """Validate migrations and pgmq prerequisites for CI.""" + asyncio.run(_migrate_status(exit_on_unhealthy=True)) + + +async def _migrate_status(*, exit_on_unhealthy: bool) -> None: + db_url = _require_db_url() + service = MigrationService(db_url, MIGRATIONS_DIR) + try: + status = await service.status() + except Exception as exc: + click.echo(f"Unable to inspect migrations: {exc}", err=True) + sys.exit(1) + _print_migration_status(status) + if exit_on_unhealthy: + if status.ok: + click.echo("Migration check passed.") + else: + click.echo("Migration check failed.", err=True) + sys.exit(1) + + @cli.command() @click.argument("spec_file", type=click.Path(exists=True)) @click.option("--up-to", default=9, help="Stop after this phase number (0-9).") @@ -138,7 +231,7 @@ async def _up( click.echo("WARNING: running in mock mode — no real infrastructure will be created.") if not skip_migrations and not mock: - db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + db_url = _db_url_from_env() if db_url: try: await _run_migrations(db_url) diff --git a/netengine/core/migrations.py b/netengine/core/migrations.py new file mode 100644 index 0000000..36d4c2f --- /dev/null +++ b/netengine/core/migrations.py @@ -0,0 +1,221 @@ +"""Database migration service for NetEngine.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from netengine.events.queues import Queue + + +@dataclass(frozen=True) +class MigrationFile: + """A migration discovered on disk.""" + + version: str + name: str + path: Path + checksum: str + + +@dataclass(frozen=True) +class MigrationRecord: + """A migration record stored in the database.""" + + version: str + name: str + checksum: str | None + status: str + applied_at: datetime | None = None + error: str | None = None + + +@dataclass(frozen=True) +class MigrationStatus: + """Complete migration and pgmq prerequisite status.""" + + applied: list[MigrationRecord] + pending: list[MigrationFile] + failed: list[MigrationRecord] + checksum_drifted: list[tuple[MigrationFile, MigrationRecord]] + pgmq_available: bool + pgmq_installed: bool + missing_queues: list[str] + + @property + def ok(self) -> bool: + return ( + not self.pending + and not self.failed + and not self.checksum_drifted + and self.pgmq_available + and self.pgmq_installed + and not self.missing_queues + ) + + +class MigrationService: + """Apply and inspect SQL migrations against Postgres.""" + + def __init__(self, db_url: str, migrations_dir: Path) -> None: + self.db_url = db_url + self.migrations_dir = migrations_dir + + def discover(self) -> list[MigrationFile]: + migrations: list[MigrationFile] = [] + for path in sorted(self.migrations_dir.glob("*.sql")): + version, _, rest = path.stem.partition("_") + name = rest or path.stem + checksum = hashlib.sha256(path.read_bytes()).hexdigest() + migrations.append(MigrationFile(version=version, name=name, path=path, checksum=checksum)) + return migrations + + async def apply_pending(self) -> list[MigrationRecord]: + """Apply pending migrations and return records applied in this invocation.""" + import asyncpg # type: ignore[import] + + conn = await asyncpg.connect(self.db_url) + try: + await self._ensure_table(conn) + records = await self._records(conn) + by_version = {record.version: record for record in records} + applied: list[MigrationRecord] = [] + + for migration in self.discover(): + existing = by_version.get(migration.version) + if existing and existing.status == "applied": + continue + + sql = migration.path.read_text() + try: + await conn.execute(sql) + except Exception as exc: + error = str(exc) + await self._upsert_record(conn, migration, "failed", error) + raise RuntimeError(f"Migration {migration.path.name} failed: {error}") from exc + + await self._upsert_record(conn, migration, "applied", None) + record = MigrationRecord( + migration.version, + migration.name, + migration.checksum, + "applied", + ) + applied.append(record) + + return applied + finally: + await conn.close() + + async def status(self) -> MigrationStatus: + import asyncpg # type: ignore[import] + + conn = await asyncpg.connect(self.db_url) + try: + await self._ensure_table(conn) + records = await self._records(conn) + by_version = {record.version: record for record in records} + files = self.discover() + + pending = [m for m in files if by_version.get(m.version) is None] + failed = [record for record in records if record.status == "failed"] + drifted = [ + (migration, by_version[migration.version]) + for migration in files + if by_version.get(migration.version) + and by_version[migration.version].status == "applied" + and by_version[migration.version].checksum != migration.checksum + ] + + pgmq_available = await self._pgmq_available(conn) + pgmq_installed = await self._pgmq_installed(conn) + missing_queues = await self._missing_queues(conn) if pgmq_installed else [q.value for q in Queue] + + return MigrationStatus( + applied=[record for record in records if record.status == "applied"], + pending=pending, + failed=failed, + checksum_drifted=drifted, + pgmq_available=pgmq_available, + pgmq_installed=pgmq_installed, + missing_queues=missing_queues, + ) + finally: + await conn.close() + + async def _ensure_table(self, conn: Any) -> None: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS netengine_schema_migrations ( + version text PRIMARY KEY, + name text NOT NULL, + checksum text, + status text NOT NULL CHECK (status IN ('applied', 'failed')), + applied_at timestamptz, + error text + ) + """ + ) + + async def _records(self, conn: Any) -> list[MigrationRecord]: + rows = await conn.fetch( + """ + SELECT version, name, checksum, status, applied_at, error + FROM netengine_schema_migrations + ORDER BY version + """ + ) + return [ + MigrationRecord( + version=row["version"], + name=row["name"], + checksum=row["checksum"], + status=row["status"], + applied_at=row["applied_at"], + error=row["error"], + ) + for row in rows + ] + + async def _upsert_record( + self, conn: Any, migration: MigrationFile, status: str, error: str | None + ) -> None: + await conn.execute( + """ + INSERT INTO netengine_schema_migrations + (version, name, checksum, status, applied_at, error) + VALUES ($1, $2, $3, $4, CASE WHEN $4 = 'applied' THEN now() ELSE NULL END, $5) + ON CONFLICT (version) DO UPDATE SET + name = EXCLUDED.name, + checksum = EXCLUDED.checksum, + status = EXCLUDED.status, + applied_at = EXCLUDED.applied_at, + error = EXCLUDED.error + """, + migration.version, + migration.name, + migration.checksum, + status, + error, + ) + + async def _pgmq_available(self, conn: Any) -> bool: + return bool(await conn.fetchval("SELECT EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'pgmq')")) + + async def _pgmq_installed(self, conn: Any) -> bool: + return bool(await conn.fetchval("SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pgmq')")) + + async def _missing_queues(self, conn: Any) -> list[str]: + rows = await conn.fetch( + """ + SELECT q.queue_name + FROM unnest($1::text[]) AS q(queue_name) + WHERE to_regclass('pgmq.q_' || q.queue_name) IS NULL + ORDER BY q.queue_name + """, + [queue.value for queue in Queue], + ) + return [row["queue_name"] for row in rows] diff --git a/netengine/utils/run_migrations.py b/netengine/utils/run_migrations.py index 4c4603c..1d640b4 100644 --- a/netengine/utils/run_migrations.py +++ b/netengine/utils/run_migrations.py @@ -1,62 +1,20 @@ -import asyncio +"""Compatibility wrapper for applying NetEngine database migrations.""" + import os from pathlib import Path -from urllib.parse import urlparse + +from netengine.core.migrations import MigrationService async def apply_migrations() -> None: - """Apply SQL migrations to the local Postgres instance. + """Apply pending SQL migrations using the shared migration service. - Reads NETENGINE_DB_URL (e.g. postgresql://user:pass@host:5432/db). - Falls back to SUPABASE_DB_* variables for backward compat with cloud setups. + Reads NETENGINE_DB_URL first, then DATABASE_URL for consistency with the + CLI's startup and migration commands. """ - db_url = os.environ.get("NETENGINE_DB_URL") - - if db_url: - parsed = urlparse(db_url) - db_host = parsed.hostname or "localhost" - db_port = str(parsed.port or 5432) - db_user = parsed.username or "netengine" - db_password = parsed.password or "" - db_name = (parsed.path or "/netengine").lstrip("/") - else: - # Backward compat: Supabase cloud connection details - db_host = os.environ.get("SUPABASE_DB_HOST", "localhost") - db_port = os.environ.get("SUPABASE_DB_PORT", "5432") - db_user = os.environ.get("SUPABASE_DB_USER", "postgres") - db_password = os.environ.get("SUPABASE_DB_PASSWORD", "") - db_name = os.environ.get("SUPABASE_DB_NAME", "postgres") - - sql_path = Path(__file__).parent.parent.parent / "migrations" / "001_initial.sql" - if not sql_path.exists(): - raise FileNotFoundError(f"Migration file not found: {sql_path}") - - env = os.environ.copy() - if db_password: - env["PGPASSWORD"] = db_password - - try: - process = await asyncio.create_subprocess_exec( - "psql", - "-h", - db_host, - "-p", - db_port, - "-U", - db_user, - "-d", - db_name, - "-f", - str(sql_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - stdout, stderr = await process.communicate() - - if process.returncode != 0: - error_msg = stderr.decode() if stderr else "Unknown error" - raise RuntimeError(f"Migration failed: {error_msg}") + db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + if not db_url: + raise RuntimeError("Database URL is required: set NETENGINE_DB_URL or DATABASE_URL.") - except FileNotFoundError: - raise RuntimeError("psql command not found. Install PostgreSQL client tools.") + migrations_dir = Path(__file__).parent.parent.parent / "migrations" + await MigrationService(db_url, migrations_dir).apply_pending() From 0815a2536983db29c3a38489fc7e6440f35240d5 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:47:53 +0100 Subject: [PATCH 13/34] Fail CLI boot on migration errors --- netengine/cli/main.py | 28 +++++++++++++++++-- tests/test_cli.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 14964b1..8af97de 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -94,6 +94,12 @@ def cli() -> None: default=False, help="Skip running database migrations on startup.", ) +@click.option( + "--allow-migration-failure", + is_flag=True, + default=False, + help="Continue booting if database migrations fail (development escape hatch).", +) @click.option( "--env", "environment", help="Merge spec.{ENV}.yaml next to SPEC_FILE before booting." ) @@ -109,11 +115,22 @@ def up( up_to: int, mock: bool, skip_migrations: bool, + allow_migration_failure: bool, environment: str | None, set_values: tuple[str, ...], ) -> None: """Boot a world from SPEC_FILE.""" - asyncio.run(_up(spec_file, up_to, mock, skip_migrations, environment, set_values)) + asyncio.run( + _up( + spec_file, + up_to, + mock, + skip_migrations, + allow_migration_failure, + environment, + set_values, + ) + ) async def _up( @@ -121,6 +138,7 @@ async def _up( up_to: int, mock: bool, skip_migrations: bool, + allow_migration_failure: bool = False, environment: str | None = None, set_values: tuple[str, ...] = (), ) -> None: @@ -143,7 +161,13 @@ async def _up( try: await _run_migrations(db_url) except Exception as exc: - logger.warning(f"Migrations failed (continuing anyway): {exc}") + if allow_migration_failure: + logger.warning(f"Migrations failed (continuing anyway): {exc}") + else: + message = f"Migrations failed: {exc}" + logger.error(message) + click.echo(message, err=True) + sys.exit(1) orchestrator = Orchestrator(spec, mock_mode=mock) click.echo(f"Booting world from {spec_file} (phases 0–{up_to})…") diff --git a/tests/test_cli.py b/tests/test_cli.py index b0d989d..ab9c889 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -52,6 +52,70 @@ def test_up_invokes_execute_phases_with_example_spec(): mock_orchestrator.execute_phases.assert_awaited_once_with(up_to_phase=9) +def test_up_migration_failure_prevents_orchestrator_startup(): + """Migration failures should stop boot before the orchestrator is created.""" + spec_file = Path(__file__).parent.parent / "examples" / "minimal.yaml" + + with ( + patch.dict("os.environ", {"NETENGINE_DB_URL": "postgresql://example/db"}, clear=False), + patch("netengine.cli.main._run_migrations", new_callable=AsyncMock) as mock_migrations, + patch("netengine.cli.main.Orchestrator") as mock_orchestrator_class, + ): + mock_migrations.side_effect = RuntimeError("migration boom") + + result = CliRunner().invoke(cli_main.cli, ["up", str(spec_file)]) + + assert result.exit_code == 1 + assert "Migrations failed: migration boom" in result.output + mock_migrations.assert_awaited_once_with("postgresql://example/db") + mock_orchestrator_class.assert_not_called() + + +def test_up_allows_migration_failure_with_explicit_flag(): + """The development escape hatch should continue booting after migration failures.""" + spec_file = Path(__file__).parent.parent / "examples" / "minimal.yaml" + + with ( + patch.dict("os.environ", {"NETENGINE_DB_URL": "postgresql://example/db"}, clear=False), + patch("netengine.cli.main._run_migrations", new_callable=AsyncMock) as mock_migrations, + patch("netengine.cli.main.Orchestrator") as mock_orchestrator_class, + ): + mock_migrations.side_effect = RuntimeError("migration boom") + mock_orchestrator = mock_orchestrator_class.return_value + mock_orchestrator.execute_phases = AsyncMock() + mock_orchestrator.consumer_supervisor.consumers = {} + + result = CliRunner().invoke( + cli_main.cli, ["up", str(spec_file), "--allow-migration-failure"] + ) + + assert result.exit_code == 0, result.output + mock_migrations.assert_awaited_once_with("postgresql://example/db") + mock_orchestrator_class.assert_called_once() + mock_orchestrator.execute_phases.assert_awaited_once_with(up_to_phase=9) + + +def test_up_skip_migrations_bypasses_migration_execution(): + """--skip-migrations should remain the intentional migration bypass.""" + spec_file = Path(__file__).parent.parent / "examples" / "minimal.yaml" + + with ( + patch.dict("os.environ", {"NETENGINE_DB_URL": "postgresql://example/db"}, clear=False), + patch("netengine.cli.main._run_migrations", new_callable=AsyncMock) as mock_migrations, + patch("netengine.cli.main.Orchestrator") as mock_orchestrator_class, + ): + mock_orchestrator = mock_orchestrator_class.return_value + mock_orchestrator.execute_phases = AsyncMock() + mock_orchestrator.consumer_supervisor.consumers = {} + + result = CliRunner().invoke(cli_main.cli, ["up", str(spec_file), "--skip-migrations"]) + + assert result.exit_code == 0, result.output + mock_migrations.assert_not_awaited() + mock_orchestrator_class.assert_called_once() + mock_orchestrator.execute_phases.assert_awaited_once_with(up_to_phase=9) + + def test_status_output_includes_phase_9(): """The status command should show Phase 9 org applications.""" state = RuntimeState(phase_completed={"9": True}) From 3d4d8faad0b7e6374f68ea10d9d3483e1ead919b Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:48:46 +0100 Subject: [PATCH 14/34] Enhance migration failure semantics --- docs/migrations.md | 41 ++++ netengine/cli/main.py | 18 +- netengine/utils/migration_service.py | 274 +++++++++++++++++++++++++++ netengine/utils/run_migrations.py | 79 +++----- tests/test_migration_service.py | 38 ++++ 5 files changed, 383 insertions(+), 67 deletions(-) create mode 100644 docs/migrations.md create mode 100644 netengine/utils/migration_service.py create mode 100644 tests/test_migration_service.py diff --git a/docs/migrations.md b/docs/migrations.md new file mode 100644 index 0000000..642d431 --- /dev/null +++ b/docs/migrations.md @@ -0,0 +1,41 @@ +# Database migration semantics + +NetEngine applies SQL files from `migrations/` in filename order through the shared +`MigrationService`. + +## Partial-failure behavior + +- Each migration is recorded in `netengine_schema_migrations`. +- A migration is marked `applied` only after every SQL statement in that file completes. +- If a migration fails, the service records the failed filename, the current SQL statement + context, and the database error, then stops immediately. Later migration files are not + applied until the failed migration is corrected and rerun. +- Status output distinguishes: + - `applied`: the file completed successfully and its checksum still matches. + - `pending`: the file exists on disk and has no successful or failed record. + - `failed`: the previous run failed for that file. + - `checksum-drift`: the file was applied successfully, but its current checksum differs + from the recorded checksum. + +## Transaction boundaries + +When PostgreSQL allows it, each migration file runs inside a single transaction. If any +statement fails in a transactional migration, PostgreSQL rolls back statements from that +file and NetEngine records the failure after the rollback. + +Some PostgreSQL operations are not allowed inside transaction blocks and therefore cannot +be rolled back automatically. NetEngine detects common cases and executes those migration +files without wrapping them in a transaction. Avoid mixing non-transactional and ordinary +schema changes in the same file. + +Known non-transactional operations include: + +- `CREATE DATABASE` / `DROP DATABASE` +- `CREATE INDEX CONCURRENTLY` +- `REINDEX ... CONCURRENTLY` +- `VACUUM` +- `ALTER SYSTEM` +- `CREATE TABLESPACE` / `DROP TABLESPACE` + +If one of these migrations fails partway through, manually inspect the database, repair or +reverse any already-executed statements, and rerun migrations. diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 14964b1..30744ec 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -55,22 +55,10 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: async def _run_migrations(db_url: str) -> None: """Run all SQL migration files in order against the given Postgres URL.""" - import asyncpg # type: ignore[import] + from netengine.utils.migration_service import MigrationService - migration_files = sorted(MIGRATIONS_DIR.glob("*.sql")) - if not migration_files: - logger.info("No migration files found") - return - - conn = await asyncpg.connect(db_url) - try: - for migration_path in migration_files: - sql = migration_path.read_text() - logger.info(f"Running migration: {migration_path.name}") - await conn.execute(sql) - logger.info(f"Applied {len(migration_files)} migration(s)") - finally: - await conn.close() + service = MigrationService(db_url, MIGRATIONS_DIR, logger.info) + await service.apply() @click.group() diff --git a/netengine/utils/migration_service.py b/netengine/utils/migration_service.py new file mode 100644 index 0000000..83d47e0 --- /dev/null +++ b/netengine/utils/migration_service.py @@ -0,0 +1,274 @@ +"""Shared PostgreSQL migration service with explicit partial-failure semantics.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Iterable + +if TYPE_CHECKING: + import asyncpg # type: ignore[import] + +MIGRATION_TABLE = "netengine_schema_migrations" + + +class MigrationState(str, Enum): + """Operator-facing migration status values.""" + + APPLIED = "applied" + PENDING = "pending" + FAILED = "failed" + CHECKSUM_DRIFT = "checksum-drift" + + +@dataclass(frozen=True) +class MigrationStatus: + filename: str + checksum: str + state: MigrationState + applied_at: str | None = None + error: str | None = None + + +class MigrationApplyError(RuntimeError): + """Raised when a migration fails and later migrations must not run.""" + + def __init__(self, filename: str, statement_context: str | None, database_error: BaseException): + self.filename = filename + self.statement_context = statement_context + self.database_error = database_error + context = f" Statement context: {statement_context}" if statement_context else "" + super().__init__(f"Migration {filename} failed.{context} Database error: {database_error}") + + +NON_TRANSACTIONAL_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + re.compile(pattern, re.IGNORECASE | re.MULTILINE) + for pattern in ( + r"\bCREATE\s+DATABASE\b", + r"\bDROP\s+DATABASE\b", + r"\bCREATE\s+INDEX\s+CONCURRENTLY\b", + r"\bREINDEX\b.*\bCONCURRENTLY\b", + r"\bVACUUM\b", + r"\bALTER\s+SYSTEM\b", + r"\bCREATE\s+TABLESPACE\b", + r"\bDROP\s+TABLESPACE\b", + ) +) + + +def migration_checksum(sql: str) -> str: + return hashlib.sha256(sql.encode("utf-8")).hexdigest() + + +def postgres_allows_transaction(sql: str) -> bool: + """Return False for common PostgreSQL operations forbidden in transaction blocks.""" + + return not any(pattern.search(sql) for pattern in NON_TRANSACTIONAL_PATTERNS) + + +def split_sql_statements(sql: str) -> list[str]: + """Split SQL on semicolons while preserving quoted and dollar-quoted bodies.""" + + statements: list[str] = [] + start = 0 + i = 0 + quote: str | None = None + dollar_tag: str | None = None + line_comment = False + block_comment = False + + while i < len(sql): + ch = sql[i] + nxt = sql[i : i + 2] + + if line_comment: + if ch == "\n": + line_comment = False + i += 1 + continue + if block_comment: + if nxt == "*/": + block_comment = False + i += 2 + else: + i += 1 + continue + if quote: + if ch == quote: + if i + 1 < len(sql) and sql[i + 1] == quote: + i += 2 + else: + quote = None + i += 1 + else: + i += 1 + continue + if dollar_tag: + if sql.startswith(dollar_tag, i): + i += len(dollar_tag) + dollar_tag = None + else: + i += 1 + continue + + if nxt == "--": + line_comment = True + i += 2 + continue + if nxt == "/*": + block_comment = True + i += 2 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "$": + match = re.match(r"\$[A-Za-z_][A-Za-z0-9_]*\$|\$\$", sql[i:]) + if match: + dollar_tag = match.group(0) + i += len(dollar_tag) + continue + if ch == ";": + statement = sql[start : i + 1].strip() + if statement: + statements.append(statement) + start = i + 1 + i += 1 + + tail = sql[start:].strip() + if tail: + statements.append(tail) + return statements + + +class MigrationService: + def __init__(self, db_url: str, migrations_dir: Path, logger: Callable[[str], None] | None = None): + self.db_url = db_url + self.migrations_dir = migrations_dir + self.logger = logger or (lambda message: None) + + async def apply(self) -> list[MigrationStatus]: + asyncpg = _asyncpg() + conn = await asyncpg.connect(self.db_url) + try: + await self._ensure_table(conn) + statuses = await self.status(conn) + for status in statuses: + self.logger(f"Migration {status.filename}: {status.state.value}") + if status.state in {MigrationState.APPLIED, MigrationState.CHECKSUM_DRIFT}: + continue + await self._apply_one(conn, self.migrations_dir / status.filename, status.checksum) + return await self.status(conn) + finally: + await conn.close() + + async def status(self, conn: "asyncpg.Connection | None" = None) -> list[MigrationStatus]: + close_conn = conn is None + if conn is None: + asyncpg = _asyncpg() + conn = await asyncpg.connect(self.db_url) + try: + await self._ensure_table(conn) + rows = await conn.fetch(f"SELECT * FROM {MIGRATION_TABLE}") + by_name = {row["filename"]: row for row in rows} + statuses: list[MigrationStatus] = [] + for path in self._migration_files(): + sql = path.read_text() + checksum = migration_checksum(sql) + row = by_name.get(path.name) + if row is None: + state = MigrationState.PENDING + elif not row["success"]: + state = MigrationState.FAILED + elif row["checksum"] != checksum: + state = MigrationState.CHECKSUM_DRIFT + else: + state = MigrationState.APPLIED + statuses.append( + MigrationStatus( + filename=path.name, + checksum=checksum, + state=state, + applied_at=str(row["applied_at"]) if row and row["applied_at"] else None, + error=row["error"] if row else None, + ) + ) + return statuses + finally: + if close_conn: + await conn.close() + + async def _apply_one(self, conn: "asyncpg.Connection", path: Path, checksum: str) -> None: + sql = path.read_text() + statements = split_sql_statements(sql) + transactional = postgres_allows_transaction(sql) + context: str | None = None + try: + if transactional: + async with conn.transaction(): + for statement in statements: + context = _statement_context(statement) + await conn.execute(statement) + else: + self.logger(f"Migration {path.name}: pending (non-transactional operations detected)") + for statement in statements: + context = _statement_context(statement) + await conn.execute(statement) + await conn.execute( + f""" + INSERT INTO {MIGRATION_TABLE} (filename, checksum, success, error) + VALUES ($1, $2, TRUE, NULL) + ON CONFLICT (filename) DO UPDATE + SET checksum = EXCLUDED.checksum, success = TRUE, error = NULL, applied_at = now() + """, + path.name, + checksum, + ) + self.logger(f"Migration {path.name}: applied") + except Exception as exc: + await conn.execute( + f""" + INSERT INTO {MIGRATION_TABLE} (filename, checksum, success, error) + VALUES ($1, $2, FALSE, $3) + ON CONFLICT (filename) DO UPDATE + SET checksum = EXCLUDED.checksum, success = FALSE, error = EXCLUDED.error, applied_at = now() + """, + path.name, + checksum, + f"{context or 'unknown statement'}: {exc}", + ) + self.logger(f"Migration {path.name}: failed") + raise MigrationApplyError(path.name, context, exc) from exc + + async def _ensure_table(self, conn: "asyncpg.Connection") -> None: + await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {MIGRATION_TABLE} ( + filename TEXT PRIMARY KEY, + checksum TEXT NOT NULL, + success BOOLEAN NOT NULL, + error TEXT, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + + def _migration_files(self) -> Iterable[Path]: + return sorted(self.migrations_dir.glob("*.sql")) + + +def _statement_context(statement: str, max_length: int = 240) -> str: + normalized = " ".join(statement.split()) + return normalized[:max_length] + + +def _asyncpg() -> Any: + try: + import asyncpg # type: ignore[import] + except ModuleNotFoundError as exc: + raise RuntimeError("asyncpg is required to run database migrations") from exc + return asyncpg diff --git a/netengine/utils/run_migrations.py b/netengine/utils/run_migrations.py index 4c4603c..4a8e5a4 100644 --- a/netengine/utils/run_migrations.py +++ b/netengine/utils/run_migrations.py @@ -1,62 +1,37 @@ import asyncio import os from pathlib import Path -from urllib.parse import urlparse +from urllib.parse import quote + +from netengine.utils.migration_service import MigrationService + + +def _db_url_from_environment() -> str: + db_url = os.environ.get("NETENGINE_DB_URL") + if db_url: + return db_url + + db_host = os.environ.get("SUPABASE_DB_HOST", "localhost") + db_port = os.environ.get("SUPABASE_DB_PORT", "5432") + db_user = os.environ.get("SUPABASE_DB_USER", "postgres") + db_password = os.environ.get("SUPABASE_DB_PASSWORD", "") + db_name = os.environ.get("SUPABASE_DB_NAME", "postgres") + auth = quote(db_user, safe="") + if db_password: + auth = f"{auth}:{quote(db_password, safe='')}" + return f"postgresql://{auth}@{db_host}:{db_port}/{db_name}" async def apply_migrations() -> None: - """Apply SQL migrations to the local Postgres instance. + """Apply SQL migrations to Postgres with explicit partial-failure semantics. Reads NETENGINE_DB_URL (e.g. postgresql://user:pass@host:5432/db). - Falls back to SUPABASE_DB_* variables for backward compat with cloud setups. + Falls back to SUPABASE_DB_* variables for backward compatibility with cloud setups. """ - db_url = os.environ.get("NETENGINE_DB_URL") + migrations_dir = Path(__file__).parent.parent.parent / "migrations" + service = MigrationService(_db_url_from_environment(), migrations_dir, print) + await service.apply() - if db_url: - parsed = urlparse(db_url) - db_host = parsed.hostname or "localhost" - db_port = str(parsed.port or 5432) - db_user = parsed.username or "netengine" - db_password = parsed.password or "" - db_name = (parsed.path or "/netengine").lstrip("/") - else: - # Backward compat: Supabase cloud connection details - db_host = os.environ.get("SUPABASE_DB_HOST", "localhost") - db_port = os.environ.get("SUPABASE_DB_PORT", "5432") - db_user = os.environ.get("SUPABASE_DB_USER", "postgres") - db_password = os.environ.get("SUPABASE_DB_PASSWORD", "") - db_name = os.environ.get("SUPABASE_DB_NAME", "postgres") - - sql_path = Path(__file__).parent.parent.parent / "migrations" / "001_initial.sql" - if not sql_path.exists(): - raise FileNotFoundError(f"Migration file not found: {sql_path}") - - env = os.environ.copy() - if db_password: - env["PGPASSWORD"] = db_password - - try: - process = await asyncio.create_subprocess_exec( - "psql", - "-h", - db_host, - "-p", - db_port, - "-U", - db_user, - "-d", - db_name, - "-f", - str(sql_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - stdout, stderr = await process.communicate() - - if process.returncode != 0: - error_msg = stderr.decode() if stderr else "Unknown error" - raise RuntimeError(f"Migration failed: {error_msg}") - - except FileNotFoundError: - raise RuntimeError("psql command not found. Install PostgreSQL client tools.") + +if __name__ == "__main__": + asyncio.run(apply_migrations()) diff --git a/tests/test_migration_service.py b/tests/test_migration_service.py new file mode 100644 index 0000000..7ac24f1 --- /dev/null +++ b/tests/test_migration_service.py @@ -0,0 +1,38 @@ +from netengine.utils.migration_service import ( + MigrationApplyError, + postgres_allows_transaction, + split_sql_statements, +) + + +def test_split_sql_statements_preserves_plpgsql_function_body() -> None: + sql = """ + CREATE FUNCTION demo() RETURNS void LANGUAGE plpgsql AS $$ + BEGIN + RAISE NOTICE 'semi; inside'; + END; + $$; + CREATE TABLE demo_table(id int); + """ + + statements = split_sql_statements(sql) + + assert len(statements) == 2 + assert "RAISE NOTICE 'semi; inside';" in statements[0] + assert statements[1] == "CREATE TABLE demo_table(id int);" + + +def test_postgres_allows_transaction_detects_non_transactional_operations() -> None: + assert postgres_allows_transaction("CREATE TABLE demo(id int);") is True + assert postgres_allows_transaction("CREATE INDEX CONCURRENTLY idx ON demo(id);") is False + assert postgres_allows_transaction("VACUUM demo;") is False + + +def test_migration_apply_error_includes_filename_context_and_database_error() -> None: + error = MigrationApplyError("002_bad.sql", "CREATE TABLE bad", RuntimeError("boom")) + + message = str(error) + + assert "002_bad.sql" in message + assert "CREATE TABLE bad" in message + assert "boom" in message From 64bc96b89c0251f816a704f9ddb3d43f152058b8 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:48:48 +0100 Subject: [PATCH 15/34] docs: add migration rollback guidance --- README.md | 21 +++++++++++++++++++++ docs/runbook.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/README.md b/README.md index a577032..7bc32ab 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,27 @@ poetry run netengine up spec.yaml --set metadata.name=my-world --- + +## Operator migration guidance + +Alpha migrations are forward-only unless a migration file includes explicit manual rollback notes. Inspect applied migrations with: + +```bash +psql "$NETENGINE_DB_URL" -c "SELECT version, dirty FROM schema_migrations;" +psql "$NETENGINE_DB_URL" -c "SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1;" +``` + +The second command identifies the last applied migration. If a migration fails, stop writers, capture the error and current `schema_migrations` state, inspect partially-created objects, and follow that migration's manual recovery notes before retrying. Without explicit rollback notes, restore from backup or rebuild only if the database is disposable. Wiping and reapplying migrations is acceptable for local/dev/CI alpha databases with no durable data; it is unsafe for shared, persistent, staging, production, or customer-like environments without an approved backup/restore plan. + +pgmq queue additions should be treated as forward schema changes and should create both the queue and matching `*_dlq` queue. Prefer migrations over manual changes. If a queue must be created manually during alpha recovery, first inspect existing queues and then create both queues explicitly; remove queues only in disposable environments or after confirming no pending/audit messages are needed: + +```bash +psql "$NETENGINE_DB_URL" -c "SELECT queue_name FROM pgmq.list_queues() ORDER BY queue_name;" +psql "$NETENGINE_DB_URL" -c "SELECT pgmq.create('new_queue'); SELECT pgmq.create('new_queue_dlq');" +``` + +See `docs/runbook.md` for the full rollback and recovery procedure. + ## Architecture ``` diff --git a/docs/runbook.md b/docs/runbook.md index 98e2a90..8ba51ed 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -183,6 +183,56 @@ psql $NETENGINE_DB_URL -f migrations/001_initial.sql Then re-run `netengine up`. + +### Migration rollback and recovery (alpha) + +Alpha database migrations are **forward-only** unless the specific migration file includes explicit manual rollback notes. Do not assume a migration can be automatically reversed, and do not delete rows from `schema_migrations` as a rollback mechanism unless a migration's notes specifically instruct you to do so. + +Use `psql` to inspect applied migrations: + +```bash +psql "$NETENGINE_DB_URL" -c "SELECT version, dirty FROM schema_migrations;" +``` + +The last applied migration is the highest recorded migration version. If your local migration runner stores timestamps or filenames instead of integer versions, order by the recorded version/name exactly as stored by the runner: + +```bash +psql "$NETENGINE_DB_URL" -c "SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1;" +``` + +If a migration fails: + +1. Stop NetEngine processes that may write to the database. +2. Capture the failing migration version, `psql` output, and relevant application logs. +3. Inspect `schema_migrations` and the partially-created database objects before retrying. +4. If the database is marked dirty, fix the underlying SQL/object state first; then follow the migration's explicit manual recovery notes before re-running migrations. +5. If there are no rollback/recovery notes, restore from a known-good backup or rebuild an expendable environment rather than hand-editing production data. + +Wiping and reapplying migrations is acceptable only for disposable local/dev databases, CI databases, or alpha environments where you have confirmed that no durable world/operator data needs to be preserved. It is unsafe for shared, persistent, staging, production, or customer-like environments unless you have an approved backup/restore plan and explicit operator sign-off. + +For local-only rebuilds, a full schema wipe is the cleanest reset: + +```bash +psql "$NETENGINE_DB_URL" -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" +poetry run python -m netengine.utils.run_migrations +``` + +#### pgmq queue migration notes + +Queue additions are schema changes. Prefer adding new pgmq queues in a forward migration with both the primary queue and its dead-letter queue, and keep the migration queue list aligned with `netengine.events.queues.Queue`. Before adding a queue manually, verify whether the migration already created it: + +```bash +psql "$NETENGINE_DB_URL" -c "SELECT queue_name FROM pgmq.list_queues() ORDER BY queue_name;" +``` + +If an alpha migration fails because a pgmq queue is missing and the environment is otherwise recoverable, manually creating the queue can be an acceptable forward fix: + +```bash +psql "$NETENGINE_DB_URL" -c "SELECT pgmq.create('new_queue'); SELECT pgmq.create('new_queue_dlq');" +``` + +Manual removal is riskier: pgmq queue tables can contain pending, delayed, or archived operational messages. Only drop/remove queues in disposable environments, or after confirming the queue is unused and draining/archiving any messages required for audit or replay. Never remove a queue just to make `schema_migrations` look rolled back. + ### `SpecLoadError: Spec validation failed` The YAML spec has a field that failed Pydantic validation. The error message From 8b5781e56c2193687063450fe4ea4fbf7a43a582 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:48:57 +0100 Subject: [PATCH 16/34] Add pgmq migration integration tests --- compose/compose.test-integration.yml | 23 +- netengine/cli/main.py | 66 ++++- netengine/db/migrations.py | 84 +++++++ .../test_migrations_postgres_pgmq.py | 225 ++++++++++++++++++ 4 files changed, 373 insertions(+), 25 deletions(-) create mode 100644 tests/integration/test_migrations_postgres_pgmq.py diff --git a/compose/compose.test-integration.yml b/compose/compose.test-integration.yml index c864d11..37afa5d 100644 --- a/compose/compose.test-integration.yml +++ b/compose/compose.test-integration.yml @@ -6,7 +6,7 @@ version: '3.8' services: postgres-integration: - image: postgres:15 + image: ghcr.io/pgmq/pg15-pgmq:latest container_name: netengine_postgres_integration environment: POSTGRES_USER: netengine @@ -28,27 +28,6 @@ services: - netengine-integration restart: unless-stopped - pgmq-setup-integration: - image: postgres:15 - container_name: netengine_pgmq_setup_integration - depends_on: - postgres-integration: - condition: service_healthy - environment: - PGPASSWORD: integration_test_password - command: | - sh -c " - psql -h postgres-integration -U netengine -d netengine_integration -c 'CREATE EXTENSION IF NOT EXISTS pgmq' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''dns_updates'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''oidc_provisioning'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''and_provisioning'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''inworld_admissions'');' && - psql -h postgres-integration -U netengine -d netengine_integration -c 'SELECT pgmq.create_queue(''services_admissions'');' - " - networks: - - netengine-integration - restart: on-failure - keycloak-integration: image: quay.io/keycloak/keycloak:latest container_name: netengine_keycloak_integration diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 9df458f..b9a2ab7 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -11,7 +11,7 @@ from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState -from netengine.db.migrations import MigrationRunResult, run_migrations +from netengine.db.migrations import MigrationRunResult, migration_status, run_migrations from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS @@ -59,8 +59,7 @@ async def _run_migrations(db_url: str) -> MigrationRunResult: for migration in result.results: if migration.status == "applied": logger.info( - f"Applied migration: {migration.filename} " - f"({migration.duration_seconds:.3f}s)" + f"Applied migration: {migration.filename} " f"({migration.duration_seconds:.3f}s)" ) elif migration.status == "skipped": logger.info(f"Skipped migration: {migration.filename} (already applied)") @@ -167,6 +166,67 @@ async def _up( logger.info("Consumers stopped.") +@cli.group() +def migrate() -> None: + """Inspect and apply database migrations.""" + + +@migrate.command("run") +def migrate_run() -> None: + """Apply pending database migrations.""" + db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + if not db_url: + click.echo("No database URL configured for migrations", err=True) + sys.exit(2) + try: + asyncio.run(_run_migrations(db_url)) + except Exception as exc: + click.echo(f"Migrations failed: {exc}", err=True) + sys.exit(1) + + +@migrate.command("status") +def migrate_status() -> None: + """Show database migration status without applying migrations.""" + db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + if not db_url: + click.echo("No database URL configured for migrations", err=True) + sys.exit(2) + try: + report = asyncio.run(migration_status(db_url)) + except Exception as exc: + click.echo(f"Migration status failed: {exc}", err=True) + sys.exit(1) + for migration in report.results: + click.echo(f"{migration.status}: {migration.filename}") + click.echo( + f"Migrations status: {report.applied_count} applied, " + f"{report.pending_count} pending, {report.failed_count} failed, " + f"{report.drifted_count} drifted" + ) + + +@migrate.command("check") +def migrate_check() -> None: + """Exit non-zero when migrations are pending, failed, or drifted.""" + db_url = os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") + if not db_url: + click.echo("No database URL configured for migrations", err=True) + sys.exit(2) + try: + report = asyncio.run(migration_status(db_url)) + except Exception as exc: + click.echo(f"Migration check failed: {exc}", err=True) + sys.exit(1) + if report.pending_count or report.failed_count or report.drifted_count: + click.echo( + f"Migrations not current: {report.pending_count} pending, " + f"{report.failed_count} failed, {report.drifted_count} drifted" + ) + sys.exit(1) + click.echo("Migrations current.") + + @cli.command() @click.argument("spec_file", type=click.Path(exists=True)) def reload(spec_file: str) -> None: diff --git a/netengine/db/migrations.py b/netengine/db/migrations.py index 080a316..1b0b3a7 100644 --- a/netengine/db/migrations.py +++ b/netengine/db/migrations.py @@ -45,6 +45,41 @@ def failed_count(self) -> int: return sum(1 for result in self.results if result.status == "failed") +@dataclass(frozen=True) +class MigrationStatusResult: + """Current database status for a single migration file.""" + + filename: str + checksum: str + status: str + applied_at: datetime | None = None + error: str | None = None + + +@dataclass(frozen=True) +class MigrationStatusReport: + """Structured status report for migration files.""" + + migrations_dir: Path + results: tuple[MigrationStatusResult, ...] = field(default_factory=tuple) + + @property + def pending_count(self) -> int: + return sum(1 for result in self.results if result.status == "pending") + + @property + def applied_count(self) -> int: + return sum(1 for result in self.results if result.status == "applied") + + @property + def failed_count(self) -> int: + return sum(1 for result in self.results if result.status == "failed") + + @property + def drifted_count(self) -> int: + return sum(1 for result in self.results if result.status == "drifted") + + class MigrationChecksumDriftError(RuntimeError): """Raised when an already-applied migration file changes on disk.""" @@ -191,3 +226,52 @@ async def run_migrations( await conn.close() return MigrationRunResult(resolved_dir, tuple(results)) + + +async def migration_status( + db_url: str | None = None, + migrations_dir: Path | str = MIGRATIONS_DIR, +) -> MigrationStatusReport: + """Inspect migration state without applying pending migration files.""" + import asyncpg # type: ignore[import] + + resolved_dir = Path(migrations_dir) + migration_files = discover_migrations(resolved_dir) + if db_url is None: + db_url = database_url_from_environment() + if not db_url: + raise RuntimeError("No database URL configured for migrations") + + conn = await asyncpg.connect(db_url) + results: list[MigrationStatusResult] = [] + try: + await conn.execute(_SCHEMA_MIGRATIONS_SQL) + for migration_path in migration_files: + sql = migration_path.read_text(encoding="utf-8") + checksum = migration_checksum(sql) + filename = migration_path.name + existing = await conn.fetchrow( + "SELECT checksum, success, applied_at, error FROM schema_migrations WHERE filename = $1", + filename, + ) + if not existing: + status = "pending" + elif existing["success"] and existing["checksum"] == checksum: + status = "applied" + elif existing["success"]: + status = "drifted" + else: + status = "failed" + results.append( + MigrationStatusResult( + filename=filename, + checksum=checksum, + status=status, + applied_at=existing["applied_at"] if existing else None, + error=existing["error"] if existing else None, + ) + ) + finally: + await conn.close() + + return MigrationStatusReport(resolved_dir, tuple(results)) diff --git a/tests/integration/test_migrations_postgres_pgmq.py b/tests/integration/test_migrations_postgres_pgmq.py new file mode 100644 index 0000000..17988ae --- /dev/null +++ b/tests/integration/test_migrations_postgres_pgmq.py @@ -0,0 +1,225 @@ +"""Integration tests for the shared migration service against Postgres + pgmq.""" + +from __future__ import annotations + +import asyncio +import shutil +import subprocess +import time +from pathlib import Path +from uuid import uuid4 + +import pytest +from click.testing import CliRunner + +from netengine.cli.main import cli +from netengine.db.migrations import ( + MIGRATIONS_DIR, + discover_migrations, + migration_checksum, + run_migrations, +) +from netengine.events.queues import Queue + +asyncpg = pytest.importorskip("asyncpg") + +pytestmark = [pytest.mark.integration, pytest.mark.slow] + +POSTGRES_IMAGE = "ghcr.io/pgmq/pg15-pgmq:latest" +POSTGRES_PASSWORD = "integration_test_password" + + +@pytest.fixture(scope="session") +def pgmq_postgres_url() -> str: + """Start a fresh Postgres container with pgmq extension files installed.""" + if shutil.which("docker") is None: + pytest.skip("docker is required for pgmq migration integration tests") + + container_name = f"netengine-migration-pgmq-{uuid4().hex[:12]}" + run = subprocess.run( + [ + "docker", + "run", + "-d", + "--rm", + "--name", + container_name, + "-e", + f"POSTGRES_PASSWORD={POSTGRES_PASSWORD}", + "-p", + "127.0.0.1::5432", + POSTGRES_IMAGE, + ], + check=False, + capture_output=True, + text=True, + ) + if run.returncode != 0: + pytest.skip(f"could not start {POSTGRES_IMAGE}: {run.stderr.strip()}") + + try: + inspect = subprocess.run( + [ + "docker", + "inspect", + "-f", + '{{(index (index .NetworkSettings.Ports "5432/tcp") 0).HostPort}}', + container_name, + ], + check=True, + capture_output=True, + text=True, + ) + port = inspect.stdout.strip() + admin_url = f"postgresql://postgres:{POSTGRES_PASSWORD}@127.0.0.1:{port}/postgres" + deadline = time.monotonic() + 60 + while True: + try: + conn = asyncio.run(asyncpg.connect(admin_url)) + asyncio.run(conn.close()) + break + except Exception: + if time.monotonic() > deadline: + pytest.fail("timed out waiting for pgmq Postgres container") + time.sleep(1) + yield admin_url + finally: + subprocess.run(["docker", "rm", "-f", container_name], check=False, capture_output=True) + + +@pytest.fixture() +def fresh_database_url(pgmq_postgres_url: str) -> str: + """Create an isolated database for each migration test.""" + db_name = f"netengine_migration_{uuid4().hex}" + + async def create() -> str: + conn = await asyncpg.connect(pgmq_postgres_url) + try: + await conn.execute(f'CREATE DATABASE "{db_name}"') + finally: + await conn.close() + return pgmq_postgres_url.rsplit("/", 1)[0] + f"/{db_name}" + + url = asyncio.run(create()) + yield url + + async def drop() -> None: + conn = await asyncpg.connect(pgmq_postgres_url) + try: + await conn.execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = $1", db_name + ) + await conn.execute(f'DROP DATABASE IF EXISTS "{db_name}"') + finally: + await conn.close() + + asyncio.run(drop()) + + +async def _fetch(conn: object, query: str, *args: object) -> list[object]: + return list(await conn.fetch(query, *args)) + + +@pytest.mark.asyncio +async def test_fresh_database_applies_all_migrations_and_creates_expected_records_and_queues( + fresh_database_url: str, +) -> None: + result = await run_migrations(fresh_database_url) + + migration_files = discover_migrations(MIGRATIONS_DIR) + assert result.applied_count == len(migration_files) + assert result.failed_count == 0 + + conn = await asyncpg.connect(fresh_database_url) + try: + records = await _fetch( + conn, + """ + SELECT filename, checksum, success, error + FROM schema_migrations + ORDER BY filename + """, + ) + assert [record["filename"] for record in records] == [path.name for path in migration_files] + assert all(record["success"] is True for record in records) + assert all(record["error"] is None for record in records) + assert [record["checksum"] for record in records] == [ + migration_checksum(path.read_text(encoding="utf-8")) for path in migration_files + ] + + missing_queues = [] + for queue in Queue: + exists = await conn.fetchval( + "SELECT to_regclass($1) IS NOT NULL", f"pgmq.q_{queue.value}" + ) + if not exists: + missing_queues.append(queue.value) + assert missing_queues == [] + finally: + await conn.close() + + +@pytest.mark.asyncio +async def test_rerunning_migrations_skips_all_work(fresh_database_url: str) -> None: + first = await run_migrations(fresh_database_url) + second = await run_migrations(fresh_database_url) + + assert first.applied_count == len(discover_migrations(MIGRATIONS_DIR)) + assert second.applied_count == 0 + assert second.skipped_count == len(discover_migrations(MIGRATIONS_DIR)) + assert second.failed_count == 0 + + +@pytest.mark.asyncio +async def test_failed_migration_is_not_successful_and_stops_later_migrations( + fresh_database_url: str, tmp_path: Path +) -> None: + (tmp_path / "001_ok.sql").write_text("CREATE TABLE before_failure (id int);", encoding="utf-8") + (tmp_path / "002_failure.sql").write_text( + "SELECT definitely_not_a_function();", encoding="utf-8" + ) + (tmp_path / "003_later.sql").write_text( + "CREATE TABLE after_failure (id int);", encoding="utf-8" + ) + + with pytest.raises(RuntimeError, match="002_failure.sql"): + await run_migrations(fresh_database_url, tmp_path) + + conn = await asyncpg.connect(fresh_database_url) + try: + rows = await _fetch( + conn, + "SELECT filename, success FROM schema_migrations ORDER BY filename", + ) + assert [(row["filename"], row["success"]) for row in rows] == [ + ("001_ok.sql", True), + ("002_failure.sql", False), + ] + assert await conn.fetchval("SELECT to_regclass('public.after_failure')") is None + finally: + await conn.close() + + +def test_migrate_status_and_check_exit_codes(fresh_database_url: str) -> None: + runner = CliRunner() + env = {"NETENGINE_DB_URL": fresh_database_url} + + pending_status = runner.invoke(cli, ["migrate", "status"], env=env) + assert pending_status.exit_code == 0, pending_status.output + assert "pending" in pending_status.output + + pending_check = runner.invoke(cli, ["migrate", "check"], env=env) + assert pending_check.exit_code == 1, pending_check.output + assert "pending" in pending_check.output + + applied = asyncio.run(run_migrations(fresh_database_url)) + assert applied.failed_count == 0 + + current_status = runner.invoke(cli, ["migrate", "status"], env=env) + assert current_status.exit_code == 0, current_status.output + assert "applied" in current_status.output + assert "0 pending" in current_status.output + + current_check = runner.invoke(cli, ["migrate", "check"], env=env) + assert current_check.exit_code == 0, current_check.output + assert "Migrations current" in current_check.output From 5a96e5623e1f7c983303b1338ea973ca5929c9e7 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:53:17 +0100 Subject: [PATCH 17/34] Constrain cryptography below 49 --- poetry.lock | 99 ++++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0d6fd0a..f46316c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -737,58 +737,61 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "cryptography" -version = "49.0.0" +version = "48.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.9" groups = ["main"] files = [ - {file = "cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68"}, - {file = "cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9"}, - {file = "cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f"}, - {file = "cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459"}, - {file = "cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e"}, - {file = "cryptography-49.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866"}, - {file = "cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8"}, - {file = "cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3"}, - {file = "cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27"}, - {file = "cryptography-49.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61"}, - {file = "cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8"}, - {file = "cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36"}, - {file = "cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e"}, - {file = "cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b"}, - {file = "cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001"}, - {file = "cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b"}, - {file = "cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838"}, - {file = "cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5"}, - {file = "cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615"}, - {file = "cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6"}, - {file = "cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6"}, - {file = "cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493"}, + {file = "cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f"}, + {file = "cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f"}, + {file = "cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41"}, + {file = "cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6"}, + {file = "cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158"}, + {file = "cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24"}, + {file = "cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:e361afba8918070d376df76f408a4f67fec0ee9cff81a99e48fe9a233ef59e17"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d069066deead00ac7f090be101be875a06855908f7ec004c27b8fefb4acfb411"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:09f73a725d582cef64b91281a322cd798d14a33b2b6f2b7ad9531dc336d84c02"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:15254441469dd6bf027039453288e2072124f8b6603563f5d759e1c9b69273fa"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:8ace4507d1e6533c125f4fac754f8bb8b6a74c08e92179dabd7e16571a3efbf3"}, + {file = "cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c"}, + {file = "cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72"}, + {file = "cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9"}, + {file = "cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471"}, + {file = "cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2"}, + {file = "cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401"}, + {file = "cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b"}, + {file = "cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1"}, + {file = "cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475"}, + {file = "cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1"}, + {file = "cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac"}, + {file = "cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a"}, + {file = "cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd"}, + {file = "cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c"}, + {file = "cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9"}, + {file = "cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92"}, + {file = "cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a"}, + {file = "cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a"}, ] [package.dependencies] @@ -2741,4 +2744,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "f1760ce5973b88aefe4f13da1b7c1dec1f1903792f06518f90dbd01af5491179" +content-hash = "0038e142b340e054e7d38bceb41a65865cbcdcb8caf8038cc01d696e8b2ef58f" diff --git a/pyproject.toml b/pyproject.toml index 8f0fcbe..4fe6c5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ supabase = "^2.0" fastapi = ">=0.115" omegaconf = "^2.3" prometheus-client = "^0.25.0" -cryptography = "^49.0.0" +cryptography = ">=44,<49" [tool.poetry.group.dev.dependencies] pytest = "^7.4" From 6152adce3689010c37ad885410a40a5165c889f6 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 00:56:43 +0100 Subject: [PATCH 18/34] Fix CLI import after install --- netengine/cli/main.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/netengine/cli/main.py b/netengine/cli/main.py index e7f2c21..3ee3722 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -12,7 +12,7 @@ from netengine.core.migrations import MigrationService, MigrationStatus from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState -from netengine.db.migrations import MigrationRunResult, run_migrations +from netengine.db.migrations import MIGRATIONS_DIR from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS @@ -107,19 +107,6 @@ def _print_migration_status(status: MigrationStatus) -> None: click.echo(f" Missing queues: {', '.join(status.missing_queues)}") else: click.echo(" Missing queues: none") - from netengine.utils.migrations import apply_migration_files - - migration_files = sorted(MIGRATIONS_DIR.glob("*.sql")) - if not migration_files: - logger.info("No migration files found") - return - - conn = await asyncpg.connect(db_url) - try: - applied_count = await apply_migration_files(conn, migration_files) - logger.info(f"Applied {applied_count} migration(s)") - finally: - await conn.close() @click.group() From 1ce138c45db22b8fc3ed0323ecd087c822d7bfb0 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:00:05 +0100 Subject: [PATCH 19/34] Add feature state metadata for PKI spec fields --- netengine/spec/models.py | 78 ++++++++++++++++++++++++++++++++---- tests/test_feature_states.py | 44 ++++++++++++++++++++ 2 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 tests/test_feature_states.py diff --git a/netengine/spec/models.py b/netengine/spec/models.py index a7aaa12..6194406 100644 --- a/netengine/spec/models.py +++ b/netengine/spec/models.py @@ -23,6 +23,55 @@ class SpecModel(BaseModel): model_config = ConfigDict(frozen=True) +class FeatureState(str, Enum): + """Implementation maturity for spec fields. + + Values are serialized into JSON Schema so validators, tooling, and + future documentation generators can discover whether a field is stable, + experimental, reserved for future use, or currently unsupported. + """ + + STABLE = "stable" + EXPERIMENTAL = "experimental" + RESERVED = "reserved" + UNSUPPORTED = "unsupported" + + +FEATURE_STATE_JSON_SCHEMA_KEY = "feature_state" + + +def feature_state_extra(feature_state: FeatureState) -> dict[str, str]: + """Return JSON Schema metadata for a field's feature state.""" + + return {FEATURE_STATE_JSON_SCHEMA_KEY: feature_state.value} + + +def feature_state_field( + default: Any = ..., + *, + feature_state: FeatureState, + json_schema_extra: Optional[dict[str, Any]] = None, + **kwargs: Any, +) -> Any: + """Create a Pydantic field with discoverable feature-state metadata.""" + + extra: dict[str, Any] = feature_state_extra(feature_state) + if json_schema_extra: + extra.update(json_schema_extra) + return Field(default, json_schema_extra=extra, **kwargs) + + +PKI_FEATURE_STATES: dict[str, FeatureState] = { + "pki.intermediate_ca_enabled": FeatureState.RESERVED, + "pki.dnssec_enabled": FeatureState.UNSUPPORTED, + "pki.dnssec_ksk_lifetime_days": FeatureState.UNSUPPORTED, + "pki.dnssec_zsk_lifetime_days": FeatureState.UNSUPPORTED, + "pki.crl_enabled": FeatureState.UNSUPPORTED, + "pki.ocsp_enabled": FeatureState.UNSUPPORTED, + "pki.rotation_policy": FeatureState.EXPERIMENTAL, +} + + # ───────────────────────────────────────────── # METADATA # ───────────────────────────────────────────── @@ -219,13 +268,28 @@ class PKIPhase(SpecModel): root_ca: RootCAConfig = Field(default_factory=RootCAConfig) acme: ACMEConfig = Field(default_factory=ACMEConfig) - intermediate_ca_enabled: bool = Field(default=False) - dnssec_enabled: bool = Field(default=True) - dnssec_ksk_lifetime_days: int = Field(default=365) - dnssec_zsk_lifetime_days: int = Field(default=30) - crl_enabled: bool = Field(default=False) - ocsp_enabled: bool = Field(default=False) - rotation_policy: PKIRotationPolicy = Field(default_factory=PKIRotationPolicy) + intermediate_ca_enabled: bool = feature_state_field( + default=False, feature_state=PKI_FEATURE_STATES["pki.intermediate_ca_enabled"] + ) + dnssec_enabled: bool = feature_state_field( + default=True, feature_state=PKI_FEATURE_STATES["pki.dnssec_enabled"] + ) + dnssec_ksk_lifetime_days: int = feature_state_field( + default=365, feature_state=PKI_FEATURE_STATES["pki.dnssec_ksk_lifetime_days"] + ) + dnssec_zsk_lifetime_days: int = feature_state_field( + default=30, feature_state=PKI_FEATURE_STATES["pki.dnssec_zsk_lifetime_days"] + ) + crl_enabled: bool = feature_state_field( + default=False, feature_state=PKI_FEATURE_STATES["pki.crl_enabled"] + ) + ocsp_enabled: bool = feature_state_field( + default=False, feature_state=PKI_FEATURE_STATES["pki.ocsp_enabled"] + ) + rotation_policy: PKIRotationPolicy = feature_state_field( + default_factory=PKIRotationPolicy, + feature_state=PKI_FEATURE_STATES["pki.rotation_policy"], + ) # ───────────────────────────────────────────── diff --git a/tests/test_feature_states.py b/tests/test_feature_states.py new file mode 100644 index 0000000..9c5c4b0 --- /dev/null +++ b/tests/test_feature_states.py @@ -0,0 +1,44 @@ +"""Feature-state metadata tests for spec fields.""" + +from netengine.spec.models import ( + FEATURE_STATE_JSON_SCHEMA_KEY, + PKI_FEATURE_STATES, + FeatureState, + NetEngineSpec, + PKIPhase, +) + +PKI_FIELD_NAMES = { + "pki.intermediate_ca_enabled": "intermediate_ca_enabled", + "pki.dnssec_enabled": "dnssec_enabled", + "pki.dnssec_ksk_lifetime_days": "dnssec_ksk_lifetime_days", + "pki.dnssec_zsk_lifetime_days": "dnssec_zsk_lifetime_days", + "pki.crl_enabled": "crl_enabled", + "pki.ocsp_enabled": "ocsp_enabled", + "pki.rotation_policy": "rotation_policy", +} + + +def test_required_pki_fields_have_explicit_registry_feature_states() -> None: + """Every alpha-sensitive PKI field is explicitly tracked in the registry.""" + assert set(PKI_FEATURE_STATES) == set(PKI_FIELD_NAMES) + assert all(isinstance(state, FeatureState) for state in PKI_FEATURE_STATES.values()) + + +def test_required_pki_fields_expose_feature_state_on_model_fields() -> None: + """Validation tooling can discover feature states from Pydantic model fields.""" + for dotted_path, field_name in PKI_FIELD_NAMES.items(): + extra = PKIPhase.model_fields[field_name].json_schema_extra + assert extra is not None + assert extra[FEATURE_STATE_JSON_SCHEMA_KEY] == PKI_FEATURE_STATES[dotted_path].value + + +def test_required_pki_fields_expose_feature_state_in_json_schema() -> None: + """Documentation generators can discover feature states from JSON Schema.""" + pki_schema = NetEngineSpec.model_json_schema()["$defs"]["PKIPhase"]["properties"] + + for dotted_path, field_name in PKI_FIELD_NAMES.items(): + assert ( + pki_schema[field_name][FEATURE_STATE_JSON_SCHEMA_KEY] + == PKI_FEATURE_STATES[dotted_path].value + ) From 396b4462cd08e064665468a50fa6e19e70cf1c43 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:03:00 +0100 Subject: [PATCH 20/34] Validate feature-state gated spec fields --- examples/dev-sandbox.yaml | 9 +- examples/minimal.yaml | 2 +- examples/single-org.yaml | 4 +- netengine/cli/init_wizard.py | 2 +- netengine/spec/feature_state.py | 92 +++++++++++++++++++ netengine/spec/loader.py | 152 +++++++++++++++++++------------- netengine/spec/models.py | 2 +- tests/fixtures/e2e-spec.yaml | 2 +- tests/test_spec_parsing.py | 113 +++++++++++++----------- 9 files changed, 254 insertions(+), 124 deletions(-) create mode 100644 netengine/spec/feature_state.py diff --git a/examples/dev-sandbox.yaml b/examples/dev-sandbox.yaml index 95b9faf..e2f91c9 100644 --- a/examples/dev-sandbox.yaml +++ b/examples/dev-sandbox.yaml @@ -55,7 +55,7 @@ pki: enabled: true listen_ip: 10.0.0.6 canonical_name: ca.platform.internal - dnssec_enabled: true + dnssec_enabled: false identity_platform: oidc_provider: keycloak @@ -137,7 +137,7 @@ ands: dhcp: true nat: false inbound: allowed - reverse_dns: true + reverse_dns: false residential: dhcp: true nat: true @@ -196,10 +196,7 @@ org_apps: gateway_portal: enabled: true real_internet: - mode: mirrored - service_mirrors: - - real_hostname: github.com - in_world_service: gitea.acme.internal + mode: isolated cross_world: mode: none diff --git a/examples/minimal.yaml b/examples/minimal.yaml index dcbd92f..a6e4186 100644 --- a/examples/minimal.yaml +++ b/examples/minimal.yaml @@ -56,7 +56,7 @@ pki: enabled: true listen_ip: 10.0.0.6 canonical_name: ca.platform.internal - dnssec_enabled: true + dnssec_enabled: false dnssec_ksk_lifetime_days: 365 dnssec_zsk_lifetime_days: 30 diff --git a/examples/single-org.yaml b/examples/single-org.yaml index a87be97..1e326c2 100644 --- a/examples/single-org.yaml +++ b/examples/single-org.yaml @@ -60,7 +60,7 @@ pki: enabled: true listen_ip: 10.0.0.6 canonical_name: ca.platform.internal - dnssec_enabled: true + dnssec_enabled: false identity_platform: oidc_provider: keycloak @@ -134,7 +134,7 @@ ands: nat: false dynamic_ip: false inbound: allowed - reverse_dns: true + reverse_dns: false instances: - name: acme-corp-net org: acme-corp diff --git a/netengine/cli/init_wizard.py b/netengine/cli/init_wizard.py index 66cde1a..d23f391 100644 --- a/netengine/cli/init_wizard.py +++ b/netengine/cli/init_wizard.py @@ -631,7 +631,7 @@ def core(n: int) -> str: "canonical_name": "ca.platform.internal", }, "intermediate_ca_enabled": cfg.intermediate_ca, - "dnssec_enabled": True, + "dnssec_enabled": False, "dnssec_ksk_lifetime_days": 365, "dnssec_zsk_lifetime_days": 30, "crl_enabled": cfg.crl_enabled, diff --git a/netengine/spec/feature_state.py b/netengine/spec/feature_state.py new file mode 100644 index 0000000..e901e89 --- /dev/null +++ b/netengine/spec/feature_state.py @@ -0,0 +1,92 @@ +"""Feature-state metadata for spec fields that are not generally available.""" + +from dataclasses import dataclass +from typing import Literal + +FeatureState = Literal["unsupported", "experimental"] + + +@dataclass(frozen=True) +class FeatureStateEntry: + """Feature-state metadata for a spec field path.""" + + path: str + state: FeatureState + stage: str + reason: str + + +FEATURE_STATE_REGISTRY: tuple[FeatureStateEntry, ...] = ( + FeatureStateEntry( + path="pki.dnssec_enabled", + state="unsupported", + stage="alpha", + reason="DNSSEC key generation is not integrated with zone signing", + ), + FeatureStateEntry( + path="pki.crl_enabled", + state="unsupported", + stage="alpha", + reason="CRL publication and distribution points are not implemented", + ), + FeatureStateEntry( + path="pki.ocsp_enabled", + state="unsupported", + stage="alpha", + reason="OCSP responder deployment is not implemented", + ), + FeatureStateEntry( + path="gateway_portal.real_internet.mode", + state="unsupported", + stage="alpha", + reason="real-internet gateway policies are not implemented", + ), + FeatureStateEntry( + path="gateway_portal.real_internet.service_mirrors", + state="unsupported", + stage="alpha", + reason="service mirror provisioning is not implemented", + ), + FeatureStateEntry( + path="gateway_portal.real_internet.upstream_resolver_enabled", + state="unsupported", + stage="alpha", + reason="upstream resolver forwarding is not implemented", + ), + FeatureStateEntry( + path="gateway_portal.cross_world.mode", + state="unsupported", + stage="alpha", + reason="cross-world federation is not implemented", + ), + FeatureStateEntry( + path="gateway_portal.cross_world.peers", + state="unsupported", + stage="alpha", + reason="cross-world peer provisioning is not implemented", + ), + FeatureStateEntry( + path="ands.profiles.*.dynamic_ip", + state="unsupported", + stage="alpha", + reason="dynamic IP allocation is not implemented", + ), + FeatureStateEntry( + path="ands.profiles.*.reverse_dns", + state="unsupported", + stage="alpha", + reason="reverse DNS delegation is not implemented", + ), + FeatureStateEntry( + path="ands.profiles.*.bgp", + state="unsupported", + stage="alpha", + reason="BGP profile configuration is not implemented", + ), + FeatureStateEntry( + path="pki.intermediate_ca_enabled", + state="experimental", + stage="alpha", + reason="intermediate CA handling is available but still stabilizing", + ), +) diff --git a/netengine/spec/loader.py b/netengine/spec/loader.py index b545ef8..17f2030 100644 --- a/netengine/spec/loader.py +++ b/netengine/spec/loader.py @@ -2,6 +2,7 @@ import ipaddress import logging +from collections.abc import Iterator from pathlib import Path from typing import Any, Optional @@ -20,65 +21,98 @@ class SpecLoadError(Exception): pass -def _warn_unsupported(spec: NetEngineSpec) -> None: - """Emit warnings for spec fields that are declared but not yet implemented.""" - pki = spec.pki +def _resolve_feature_state_paths(spec: NetEngineSpec) -> Iterator[tuple[Any, str, Any, Any]]: + """Yield feature-state entries with concrete paths, values, and defaults.""" + from collections.abc import Mapping + + from pydantic.fields import PydanticUndefined + + from netengine.spec.feature_state import FEATURE_STATE_REGISTRY + + def _field_default(model: Any, field_name: str) -> Any: + field = model.__class__.model_fields[field_name] + if field.default is not PydanticUndefined: + return field.default + if field.default_factory is not None: + return field.default_factory() + return PydanticUndefined + + for entry in FEATURE_STATE_REGISTRY: + parts = entry.path.split(".") + nodes: list[tuple[str, Any, Any]] = [("", spec, PydanticUndefined)] + for part in parts: + next_nodes: list[tuple[str, Any, Any]] = [] + for prefix, node, default_node in nodes: + if part == "*": + if isinstance(node, Mapping): + for key, value in node.items(): + path = f"{prefix}.{key}" if prefix else str(key) + next_nodes.append((path, value, None)) + continue + + if isinstance(node, Mapping): + if part not in node: + continue + value = node[part] + default_value = ( + default_node.get(part) if isinstance(default_node, Mapping) else None + ) + else: + if not hasattr(node, part): + continue + value = getattr(node, part) + default_value = _field_default(node, part) + + path = f"{prefix}.{part}" if prefix else part + next_nodes.append((path, value, default_value)) + nodes = next_nodes + + for concrete_path, value, default_value in nodes: + yield entry, concrete_path, value, default_value + + +def _is_active_feature_value(value: Any, default_value: Any) -> bool: + """Return True when a gated field is enabled or set to active non-default data.""" + if hasattr(value, "value"): + comparable_value = value.value + else: + comparable_value = value + + if hasattr(default_value, "value"): + comparable_default = default_value.value + else: + comparable_default = default_value + + if isinstance(comparable_value, bool): + return comparable_value is True and comparable_value != comparable_default + if comparable_value in (None, "", [], {}, (), set()): + return False + return comparable_value != comparable_default + + +def _validate_feature_states(spec: NetEngineSpec) -> None: + """Validate unsupported spec fields and warn on experimental fields.""" + errors: list[str] = [] - if pki.dnssec_enabled: - logger.warning( - "pki.dnssec_enabled is set but DNSSEC is not yet implemented — field will be ignored" - ) - if pki.crl_enabled: - logger.warning( - "pki.crl_enabled is set but CRL is not yet implemented — field will be ignored" - ) - if pki.ocsp_enabled: - logger.warning( - "pki.ocsp_enabled is set but OCSP is not yet implemented — field will be ignored" - ) - gw = spec.gateway_portal - if gw.real_internet.mode.value != "isolated": - logger.warning( - f"gateway.real_internet.mode={gw.real_internet.mode.value!r} but real internet" - " mode is not yet implemented — gateway will remain isolated" - ) - if gw.real_internet.service_mirrors: - logger.warning( - "gateway.real_internet.service_mirrors is set but service mirrors are not yet" - " implemented — mirrors will be ignored" - ) - if gw.real_internet.upstream_resolver_enabled: - logger.warning( - "gateway.real_internet.upstream_resolver_enabled is set but upstream resolver" - " is not yet implemented — field will be ignored" - ) - if gw.cross_world.mode.value != "none": - logger.warning( - f"gateway.cross_world.mode={gw.cross_world.mode.value!r} but cross-world" - " federation is not yet implemented — gateway will remain isolated" - ) - if gw.cross_world.peers: - logger.warning( - "gateway.cross_world.peers is set but cross-world federation is not yet" - " implemented — peers will be ignored" + for entry, path, value, default_value in _resolve_feature_state_paths(spec): + if not _is_active_feature_value(value, default_value): + continue + message = f"{path} is {entry.state} in {entry.stage}: {entry.reason}" + if entry.state == "unsupported": + errors.append(message) + elif entry.state == "experimental": + logger.warning(message) + + if errors: + raise SpecLoadError( + "Unsupported spec features enabled:\n" + "\n".join(f" - {e}" for e in errors) ) - for profile_name, profile in spec.ands.profiles.items(): - if profile.dynamic_ip: - logger.warning( - f"ands.profiles.{profile_name}.dynamic_ip is set but dynamic IP allocation" - " is not yet implemented — field will be ignored" - ) - if profile.reverse_dns: - logger.warning( - f"ands.profiles.{profile_name}.reverse_dns is set but reverse DNS is not" - " yet implemented — field will be ignored" - ) - if profile.bgp is not None: - logger.warning( - f"ands.profiles.{profile_name}.bgp={profile.bgp!r} but BGP configuration" - " is not yet implemented — field will be ignored" - ) + +# Backwards-compatible name for callers/tests that imported the old helper. +def _warn_unsupported(spec: NetEngineSpec) -> None: + """Validate feature-state metadata and warn for experimental fields.""" + _validate_feature_states(spec) def _cross_validate(spec: NetEngineSpec) -> None: @@ -161,7 +195,7 @@ def load_spec(yaml_path: str | Path) -> NetEngineSpec: raise SpecLoadError(f"Spec validation failed: {e}") _cross_validate(spec) - _warn_unsupported(spec) + _validate_feature_states(spec) return spec @@ -221,7 +255,7 @@ def load_spec_with_composition( raise SpecLoadError(f"Spec validation failed: {e}") _cross_validate(spec) - _warn_unsupported(spec) + _validate_feature_states(spec) return spec @@ -279,5 +313,5 @@ def load_spec_with_environment( raise SpecLoadError(f"Spec validation failed: {e}") _cross_validate(spec) - _warn_unsupported(spec) + _validate_feature_states(spec) return spec diff --git a/netengine/spec/models.py b/netengine/spec/models.py index a7aaa12..678ca92 100644 --- a/netengine/spec/models.py +++ b/netengine/spec/models.py @@ -220,7 +220,7 @@ class PKIPhase(SpecModel): root_ca: RootCAConfig = Field(default_factory=RootCAConfig) acme: ACMEConfig = Field(default_factory=ACMEConfig) intermediate_ca_enabled: bool = Field(default=False) - dnssec_enabled: bool = Field(default=True) + dnssec_enabled: bool = Field(default=False) dnssec_ksk_lifetime_days: int = Field(default=365) dnssec_zsk_lifetime_days: int = Field(default=30) crl_enabled: bool = Field(default=False) diff --git a/tests/fixtures/e2e-spec.yaml b/tests/fixtures/e2e-spec.yaml index f3c7673..d1de5f5 100644 --- a/tests/fixtures/e2e-spec.yaml +++ b/tests/fixtures/e2e-spec.yaml @@ -57,7 +57,7 @@ pki: enabled: true listen_ip: 172.20.0.6 canonical_name: ca.platform.internal - dnssec_enabled: true + dnssec_enabled: false dnssec_ksk_lifetime_days: 365 dnssec_zsk_lifetime_days: 30 diff --git a/tests/test_spec_parsing.py b/tests/test_spec_parsing.py index cb2f28a..32ba3a1 100644 --- a/tests/test_spec_parsing.py +++ b/tests/test_spec_parsing.py @@ -139,21 +139,18 @@ def test_tld_defaults(self, single_org_spec: NetEngineSpec) -> None: assert tld.listen_ip is not None -class TestUnsupportedFieldWarnings: - """_warn_unsupported emits the right warnings for enabled-but-unimplemented fields.""" +class TestFeatureStateValidation: + """Feature-state validation rejects unsupported fields and warns on experiments.""" - def _make_spec(self, overrides: dict) -> NetEngineSpec: + def _write_spec(self, tmp_path, overrides: dict): from pathlib import Path import yaml - from netengine.spec.loader import _cross_validate - base = yaml.safe_load( (Path(__file__).parent.parent / "examples" / "minimal.yaml").read_text() ) - # deep-merge overrides def _merge(a, b): for k, v in b.items(): if isinstance(v, dict) and isinstance(a.get(k), dict): @@ -162,66 +159,76 @@ def _merge(a, b): a[k] = v _merge(base, overrides) - spec = NetEngineSpec(**base) - _cross_validate(spec) - return spec - - def test_dnssec_warns(self, caplog) -> None: - import logging - - spec = self._make_spec({"pki": {"dnssec_enabled": True}}) - with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): - from netengine.spec.loader import _warn_unsupported - - _warn_unsupported(spec) - assert any("dnssec_enabled" in r.message for r in caplog.records) - - def test_crl_warns(self, caplog) -> None: - import logging - - spec = self._make_spec({"pki": {"crl_enabled": True}}) - with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): - from netengine.spec.loader import _warn_unsupported + spec_file = tmp_path / "spec.yaml" + spec_file.write_text(yaml.safe_dump(base)) + return spec_file + + def test_unsupported_enabled_field_raises_spec_load_error(self, tmp_path) -> None: + spec_file = self._write_spec(tmp_path, {"pki": {"dnssec_enabled": True}}) + + with pytest.raises( + SpecLoadError, + match=( + "pki\\.dnssec_enabled is unsupported in alpha: " + "DNSSEC key generation is not integrated with zone signing" + ), + ): + load_spec(spec_file) - _warn_unsupported(spec) - assert any("crl_enabled" in r.message for r in caplog.records) + def test_unsupported_non_default_active_field_raises_spec_load_error(self, tmp_path) -> None: + spec_file = self._write_spec( + tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}} + ) - def test_ocsp_warns(self, caplog) -> None: - import logging + with pytest.raises( + SpecLoadError, + match="gateway_portal\\.real_internet\\.mode is unsupported in alpha", + ): + load_spec(spec_file) - spec = self._make_spec({"pki": {"ocsp_enabled": True}}) - with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): - from netengine.spec.loader import _warn_unsupported + def test_unsupported_profile_field_includes_dotted_path(self, tmp_path) -> None: + spec_file = self._write_spec( + tmp_path, + { + "ands": { + "profiles": { + "branch": { + "dhcp": True, + "nat": True, + "dynamic_ip": False, + "reverse_dns": True, + "inbound": "blocked", + } + } + } + }, + ) - _warn_unsupported(spec) - assert any("ocsp_enabled" in r.message for r in caplog.records) + with pytest.raises( + SpecLoadError, + match="ands\\.profiles\\.branch\\.reverse_dns is unsupported in alpha", + ): + load_spec(spec_file) - def test_real_internet_mode_warns(self, caplog) -> None: + def test_experimental_enabled_field_logs_warning(self, tmp_path, caplog) -> None: import logging - spec = self._make_spec({"gateway_portal": {"real_internet": {"mode": "mirrored"}}}) - with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): - from netengine.spec.loader import _warn_unsupported - - _warn_unsupported(spec) - assert any("real_internet.mode" in r.message for r in caplog.records) + spec_file = self._write_spec(tmp_path, {"pki": {"intermediate_ca_enabled": True}}) - def test_cross_world_mode_warns(self, caplog) -> None: - import logging - - spec = self._make_spec({"gateway_portal": {"cross_world": {"mode": "peered"}}}) with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): - from netengine.spec.loader import _warn_unsupported + spec = load_spec(spec_file) - _warn_unsupported(spec) - assert any("cross_world.mode" in r.message for r in caplog.records) + assert spec.pki.intermediate_ca_enabled is True + assert any( + "pki.intermediate_ca_enabled is experimental in alpha" in r.message + for r in caplog.records + ) def test_no_warnings_for_default_spec(self, caplog, minimal_spec) -> None: import logging with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): - from netengine.spec.loader import _warn_unsupported + from netengine.spec.loader import _validate_feature_states - _warn_unsupported(minimal_spec) - # dnssec_enabled defaults to True in the model, so one warning is expected for that - assert all("crl" not in r.message and "ocsp" not in r.message for r in caplog.records) + _validate_feature_states(minimal_spec) + assert not caplog.records From c3a600d5999a9795ef2ced271465657383c4350a Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:03:20 +0100 Subject: [PATCH 21/34] Add shared event emission failure diagnostics --- netengine/core/state.py | 3 + netengine/diagnostic/probes/events.py | 34 ++++++ netengine/diagnostic/runner.py | 13 ++- netengine/events/emitter.py | 116 +++++++++++++++++++ netengine/handlers/dns.py | 26 +---- netengine/handlers/gateway_portal_handler.py | 14 +-- netengine/handlers/phase_pki.py | 17 +-- netengine/handlers/substrate.py | 24 +--- netengine/phases/phase_ands.py | 25 +--- netengine/phases/phase_inworld_identity.py | 26 +---- netengine/phases/phase_services.py | 28 +---- tests/test_pgmq_emitters.py | 21 ++++ 12 files changed, 208 insertions(+), 139 deletions(-) create mode 100644 netengine/diagnostic/probes/events.py create mode 100644 netengine/events/emitter.py diff --git a/netengine/core/state.py b/netengine/core/state.py index 8735484..b12660f 100644 --- a/netengine/core/state.py +++ b/netengine/core/state.py @@ -80,6 +80,9 @@ class RuntimeState: # Gateway portal state gateway_portal_output: Optional[Dict[str, Any]] = None + # Recent structured failures from best-effort PGMQ event emission. + event_send_failures: list[Dict[str, Any]] = field(default_factory=list) + @classmethod def load(cls) -> "RuntimeState": state_file = get_state_file() diff --git a/netengine/diagnostic/probes/events.py b/netengine/diagnostic/probes/events.py new file mode 100644 index 0000000..375869d --- /dev/null +++ b/netengine/diagnostic/probes/events.py @@ -0,0 +1,34 @@ +"""Event emission diagnostics.""" + +from netengine.core.state import RuntimeState +from netengine.diagnostic.runner import ProbeResult, ProbeStatus +from netengine.spec.models import NetEngineSpec + +_PROBE_NAME = "Events" + + +async def probe(spec: NetEngineSpec) -> ProbeResult: + """Report recent PGMQ event-send failures recorded in runtime state.""" + del spec + state = RuntimeState.load() + failures = state.event_send_failures[-10:] + if not failures: + return ProbeResult( + name=_PROBE_NAME, + status=ProbeStatus.OK, + detail="No recent event-send failures recorded.", + ) + + latest = failures[-1] + detail = ( + f"{len(failures)} recent event-send failure(s). Latest: " + f"event_type={latest.get('event_type')}, queue={latest.get('queue')}, " + f"emitted_by={latest.get('emitted_by')}, event_id={latest.get('event_id')}, " + f"correlation_id={latest.get('correlation_id')}, exception={latest.get('exception')}" + ) + return ProbeResult( + name=_PROBE_NAME, + status=ProbeStatus.WARN, + detail=detail, + hint="Inspect runtime_state.event_send_failures and PGMQ connectivity.", + ) diff --git a/netengine/diagnostic/runner.py b/netengine/diagnostic/runner.py index 932af53..aa20f3f 100644 --- a/netengine/diagnostic/runner.py +++ b/netengine/diagnostic/runner.py @@ -60,7 +60,17 @@ async def _timed(self, fn: ProbeFunc) -> ProbeResult: def build_runner(spec: NetEngineSpec) -> DiagnosticRunner: """Build a DiagnosticRunner with all standard probes registered.""" - from netengine.diagnostic.probes import acme, dns, mail, network, oidc, pki, storage, whois + from netengine.diagnostic.probes import ( + acme, + dns, + events, + mail, + network, + oidc, + pki, + storage, + whois, + ) runner = DiagnosticRunner(spec) for probe_fn in [ @@ -72,6 +82,7 @@ def build_runner(spec: NetEngineSpec) -> DiagnosticRunner: mail.probe, storage.probe, whois.probe, + events.probe, ]: runner.register(probe_fn) return runner diff --git a/netengine/events/emitter.py b/netengine/events/emitter.py new file mode 100644 index 0000000..c1816ca --- /dev/null +++ b/netengine/events/emitter.py @@ -0,0 +1,116 @@ +"""Shared helpers for emitting NetEngine events to PGMQ.""" + +from datetime import UTC, datetime +from typing import Any + +from netengine.events.queues import Queue, queue_for_event_type +from netengine.events.schema import EventEnvelope +from netengine.handlers.context import PhaseContext + + +def _failure_record(event: EventEnvelope, queue: Queue | str, exc: Exception) -> dict[str, Any]: + """Build the structured runtime-state record for an event send failure.""" + return { + "event_type": event.event_type, + "queue": str(queue), + "emitted_by": event.emitted_by, + "exception": str(exc), + "event_id": event.event_id, + "correlation_id": event.correlation_id, + "parent_event_id": event.parent_event_id, + "failed_at": datetime.now(UTC).isoformat(), + } + + +def record_event_send_failure( + context: PhaseContext, + event: EventEnvelope, + queue: Queue | str, + exc: Exception, +) -> dict[str, Any]: + """Persist structured details about a failed event send in runtime state. + + The list is intentionally small and state-file friendly. It gives callers and + diagnostics a durable view of recent failures instead of relying only on log + messages. + """ + record = _failure_record(event, queue, exc) + failures = getattr(context.runtime_state, "event_send_failures", None) + if failures is None: + failures = [] + setattr(context.runtime_state, "event_send_failures", failures) + failures.append(record) + del failures[:-50] + + output_field = getattr(context, "phase_name", None) + phase_output_map = { + "substrate": "substrate_output", + "dns": "dns_output", + "pki": "pki_output", + "inworld_identity": "identity_inworld_output", + "ands": "ands_output", + "services": "world_services_output", + "gateway_portal": "gateway_portal_output", + } + if output_field in phase_output_map: + output_field = phase_output_map[output_field] + elif event.emitted_by == "substrate_handler": + output_field = "substrate_output" + elif event.emitted_by == "dns_handler": + output_field = "dns_output" + elif event.emitted_by == "pki_phase": + output_field = "pki_output" + elif event.emitted_by == "inworld_identity_handler": + output_field = "identity_inworld_output" + elif event.emitted_by == "ands_handler": + output_field = "ands_output" + elif event.emitted_by == "services_handler": + output_field = "world_services_output" + elif event.emitted_by == "gateway_portal_handler": + output_field = "gateway_portal_output" + + phase_output = getattr(context.runtime_state, output_field, None) if output_field else None + if isinstance(phase_output, dict): + phase_output.setdefault("event_send_failures", []).append(record) + + return record + + +async def emit_event( + context: PhaseContext, + *, + event_type: str, + emitted_by: str, + payload: dict[str, Any], + queue: Queue | None = None, +) -> EventEnvelope: + """Create, log, and best-effort enqueue an event. + + Send failures are captured on ``runtime_state.event_send_failures`` and, when + a phase output dict already exists, under that output's + ``event_send_failures`` key. + """ + event = EventEnvelope.create( + event_type=event_type, + emitted_by=emitted_by, + payload=payload, + correlation_id=getattr(context.runtime_state, "correlation_id", None), + parent_event_id=getattr(context.runtime_state, "parent_event_id", None), + ) + context.logger.info( + f"Event emitted: {event_type} " + f"(event_id={event.event_id}, correlation_id={event.correlation_id})" + ) + + if context.pgmq_client is None: + context.logger.debug("pgmq_client not available; event logged only") + return event + + target_queue = queue or queue_for_event_type(event_type) + try: + await context.pgmq_client.send(target_queue, event) + context.logger.debug(f"Event queued to pgmq: {event_type}") + except Exception as exc: + record = record_event_send_failure(context, event, target_queue, exc) + context.logger.warning(f"Failed to queue event to pgmq: {record}") + return event diff --git a/netengine/handlers/dns.py b/netengine/handlers/dns.py index 3017f6a..80bae27 100644 --- a/netengine/handlers/dns.py +++ b/netengine/handlers/dns.py @@ -17,8 +17,7 @@ from typing import Any from netengine.errors import DNSError -from netengine.events.queues import queue_for_event_type -from netengine.events.schema import EventEnvelope +from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext @@ -744,28 +743,7 @@ async def _emit_event( event_type: Type of event (e.g., "dns.zones_ready") payload: Event payload dict """ - event = EventEnvelope.create( - event_type=event_type, - emitted_by="dns_handler", - payload=payload, - correlation_id=context.runtime_state.correlation_id, - parent_event_id=context.runtime_state.parent_event_id, - ) - - context.logger.info( - f"Event emitted: {event_type} " - f"(event_id={event.event_id}, correlation_id={event.correlation_id})" - ) - - # Queue to pgmq for M4+ event processing - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(queue_for_event_type(event_type), event) - context.logger.debug(f"Event queued to pgmq: {event_type}") - except Exception as e: - context.logger.warning(f"Failed to queue event to pgmq: {e}") - else: - context.logger.debug("pgmq_client not available (M1-M3 testing); event logged only") + await emit_event(context, event_type=event_type, emitted_by="dns_handler", payload=payload) async def add_zone_record( self, diff --git a/netengine/handlers/gateway_portal_handler.py b/netengine/handlers/gateway_portal_handler.py index 66093b5..a0edcf1 100644 --- a/netengine/handlers/gateway_portal_handler.py +++ b/netengine/handlers/gateway_portal_handler.py @@ -16,7 +16,7 @@ from netengine.errors import GatewayError, PKIError from netengine.events.queues import Queue -from netengine.events.schema import EventEnvelope +from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext from netengine.handlers.docker_handler import DockerHandler @@ -307,16 +307,10 @@ async def _configure_peer_dns(self, context: PhaseContext, peer: CrossWorldPeer) async def _emit_event( self, context: PhaseContext, event_type: str, payload: dict[str, Any] ) -> None: - event = EventEnvelope.create( + await emit_event( + context, event_type=event_type, emitted_by="gateway_portal_handler", payload=payload, - correlation_id=getattr(context.runtime_state, "correlation_id", None), - parent_event_id=getattr(context.runtime_state, "parent_event_id", None), + queue=Queue.GATEWAY_PORTAL_EVENTS, ) - context.logger.info(f"Event emitted: {event_type}") - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(Queue.GATEWAY_PORTAL_EVENTS, event) - except Exception as exc: - context.logger.warning(f"Failed to queue gateway portal event: {exc}") diff --git a/netengine/handlers/phase_pki.py b/netengine/handlers/phase_pki.py index 4dde232..a540fd5 100644 --- a/netengine/handlers/phase_pki.py +++ b/netengine/handlers/phase_pki.py @@ -2,8 +2,7 @@ from datetime import UTC, datetime from typing import Any -from netengine.events.queues import queue_for_event_type -from netengine.events.schema import EventEnvelope +from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext from netengine.handlers.docker_handler import DockerHandler @@ -198,16 +197,4 @@ async def _prepare_app_cert_rotation(self, cn: str, cert_metadata: dict) -> None ) async def _emit_event(self, context, event_type, payload): - event = EventEnvelope.create( - event_type=event_type, - emitted_by="pki_phase", - payload=payload, - correlation_id=getattr(context.runtime_state, "correlation_id", None), - parent_event_id=getattr(context.runtime_state, "parent_event_id", None), - ) - context.logger.info(f"Event emitted: {event_type}") - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(queue_for_event_type(event_type), event) - except Exception as exc: - context.logger.warning(f"Failed to queue pki event: {exc}") + await emit_event(context, event_type=event_type, emitted_by="pki_phase", payload=payload) diff --git a/netengine/handlers/substrate.py b/netengine/handlers/substrate.py index 14c00c7..75fdc74 100644 --- a/netengine/handlers/substrate.py +++ b/netengine/handlers/substrate.py @@ -12,8 +12,7 @@ from typing import Any from netengine.errors import SubstrateError -from netengine.events.queues import queue_for_event_type -from netengine.events.schema import EventEnvelope +from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext @@ -477,23 +476,6 @@ async def _emit_event( event_type: Type of event (e.g., "substrate.initialized") payload: Event payload dict """ - event = EventEnvelope.create( - event_type=event_type, - emitted_by="substrate_handler", - payload=payload, - correlation_id=context.runtime_state.correlation_id, - parent_event_id=context.runtime_state.parent_event_id, + await emit_event( + context, event_type=event_type, emitted_by="substrate_handler", payload=payload ) - - context.logger.info( - f"Event emitted: {event_type} " - f"(event_id={event.event_id}, correlation_id={event.correlation_id})" - ) - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(queue_for_event_type(event_type), event) - context.logger.debug(f"Event queued to pgmq: {event_type}") - except Exception as e: - context.logger.warning(f"Failed to queue event to pgmq: {e}") - else: - context.logger.debug("pgmq_client not available; event logged only") diff --git a/netengine/phases/phase_ands.py b/netengine/phases/phase_ands.py index 15aa8cd..5470583 100644 --- a/netengine/phases/phase_ands.py +++ b/netengine/phases/phase_ands.py @@ -15,7 +15,8 @@ from datetime import UTC, datetime from typing import Any, Optional -from netengine.events.queues import Queue, queue_for_event_type +from netengine.events.queues import Queue +from netengine.events.emitter import emit_event from netengine.events.schema import EventEnvelope from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext @@ -410,24 +411,4 @@ async def _emit_event( event_type: str, payload: dict[str, Any], ) -> None: - event = EventEnvelope.create( - event_type=event_type, - emitted_by="ands_handler", - payload=payload, - correlation_id=context.runtime_state.correlation_id, - parent_event_id=context.runtime_state.parent_event_id, - ) - - context.logger.info( - f"Event emitted: {event_type} " - f"(event_id={event.event_id}, correlation_id={event.correlation_id})" - ) - - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(queue_for_event_type(event_type), event) - context.logger.debug(f"Event queued to pgmq: {event_type}") - except Exception as e: - context.logger.warning(f"Failed to queue event to pgmq: {e}") - else: - context.logger.debug("pgmq_client not available (M1-M6 testing); event logged only") + await emit_event(context, event_type=event_type, emitted_by="ands_handler", payload=payload) diff --git a/netengine/phases/phase_inworld_identity.py b/netengine/phases/phase_inworld_identity.py index 04f31c2..a0a2156 100644 --- a/netengine/phases/phase_inworld_identity.py +++ b/netengine/phases/phase_inworld_identity.py @@ -17,7 +17,8 @@ import aiohttp -from netengine.events.queues import Queue, queue_for_event_type +from netengine.events.queues import Queue +from netengine.events.emitter import emit_event from netengine.events.schema import EventEnvelope from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext @@ -542,25 +543,6 @@ async def _emit_event( event_type: Type of event (e.g., "inworld_identity.ready") payload: Event payload dict """ - event = EventEnvelope.create( - event_type=event_type, - emitted_by="inworld_identity_handler", - payload=payload, - correlation_id=context.runtime_state.correlation_id, - parent_event_id=context.runtime_state.parent_event_id, + await emit_event( + context, event_type=event_type, emitted_by="inworld_identity_handler", payload=payload ) - - context.logger.info( - f"Event emitted: {event_type} " - f"(event_id={event.event_id}, correlation_id={event.correlation_id})" - ) - - # Queue to pgmq for downstream processing (M7+) - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(queue_for_event_type(event_type), event) - context.logger.debug(f"Event queued to pgmq: {event_type}") - except Exception as e: - context.logger.warning(f"Failed to queue event to pgmq: {e}") - else: - context.logger.debug("pgmq_client not available (M1-M5 testing); event logged only") diff --git a/netengine/phases/phase_services.py b/netengine/phases/phase_services.py index 006db20..518222d 100644 --- a/netengine/phases/phase_services.py +++ b/netengine/phases/phase_services.py @@ -12,7 +12,8 @@ from datetime import UTC, datetime from typing import Any -from netengine.events.queues import Queue, queue_for_event_type +from netengine.events.queues import Queue +from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext from netengine.handlers.dns import DNSHandler @@ -330,27 +331,6 @@ async def _emit_event( event_type: Type of event (e.g., "services.ready") payload: Event payload dict """ - from netengine.events.schema import EventEnvelope - - event = EventEnvelope.create( - event_type=event_type, - emitted_by="services_handler", - payload=payload, - correlation_id=context.runtime_state.correlation_id, - parent_event_id=context.runtime_state.parent_event_id, + await emit_event( + context, event_type=event_type, emitted_by="services_handler", payload=payload ) - - context.logger.info( - f"Event emitted: {event_type} " - f"(event_id={event.event_id}, correlation_id={event.correlation_id})" - ) - - # Queue to pgmq for downstream processing - if context.pgmq_client is not None: - try: - await context.pgmq_client.send(queue_for_event_type(event_type), event) - context.logger.debug(f"Event queued to pgmq: {event_type}") - except Exception as e: - context.logger.warning(f"Failed to queue event to pgmq: {e}") - else: - context.logger.debug("pgmq_client not available (testing); event logged only") diff --git a/tests/test_pgmq_emitters.py b/tests/test_pgmq_emitters.py index 1827f06..b61f43f 100644 --- a/tests/test_pgmq_emitters.py +++ b/tests/test_pgmq_emitters.py @@ -51,3 +51,24 @@ async def test_emitters_send_queue_name_and_event( assert queue_name == expected_queue assert event.event_type == event_type assert event.payload == payload + + +async def test_emitters_record_structured_send_failure(context_with_pgmq: PhaseContext) -> None: + context_with_pgmq.runtime_state.dns_output = {"healthy": True} + context_with_pgmq.pgmq_client.send.side_effect = RuntimeError("pgmq down") + + await DNSHandler()._emit_event( + context_with_pgmq, + event_type="dns.zones_ready", + payload={"zones": ["root.internal"]}, + ) + + failures = context_with_pgmq.runtime_state.event_send_failures + assert len(failures) == 1 + assert failures[0]["event_type"] == "dns.zones_ready" + assert failures[0]["queue"] == Queue.DNS_UPDATES + assert failures[0]["emitted_by"] == "dns_handler" + assert failures[0]["exception"] == "pgmq down" + assert failures[0]["event_id"] + assert failures[0]["correlation_id"] == "correlation-123" + assert context_with_pgmq.runtime_state.dns_output["event_send_failures"] == failures From b79d5b7a8d390a7b92aaea0d136f6f71e990b9e7 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:03:46 +0100 Subject: [PATCH 22/34] Validate DLQ replay queue names --- netengine/api/routes.py | 11 ++--- netengine/events/queues.py | 23 ---------- tests/test_api_replay_dlq.py | 83 ++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 28 deletions(-) create mode 100644 tests/test_api_replay_dlq.py diff --git a/netengine/api/routes.py b/netengine/api/routes.py index 13bd972..1847f31 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -878,11 +878,12 @@ async def replay_dlq(queue_name: str, user: dict = Depends(require_auth)) -> dic """Move all messages from a DLQ back to the main queue for retry.""" from netengine.core.pgmq_client import PGMQClient - try: - queue = Queue(queue_name) - dlq = dlq_for(queue) - except ValueError as exc: - raise HTTPException(status_code=404, detail=f"Unknown queue: {queue_name}") from exc + primary_queue_names = {q.value for q in PRIMARY_QUEUES} + if queue_name not in primary_queue_names: + raise HTTPException(status_code=404, detail=f"Unknown queue: {queue_name}") + + queue = Queue(queue_name) + dlq = dlq_for(queue) client = PGMQClient() replayed = 0 diff --git a/netengine/events/queues.py b/netengine/events/queues.py index 2028752..0bf5d65 100644 --- a/netengine/events/queues.py +++ b/netengine/events/queues.py @@ -35,29 +35,6 @@ class Queue(StrEnum): PHASE_EVENTS_DLQ = "phase_events_dlq" -_DLQ_BY_PRIMARY: dict[Queue, Queue] = { - Queue.DNS_UPDATES: Queue.DNS_UPDATES_DLQ, - Queue.OIDC_PROVISIONING: Queue.OIDC_PROVISIONING_DLQ, - Queue.AND_PROVISIONING: Queue.AND_PROVISIONING_DLQ, - Queue.INWORLD_ADMISSIONS: Queue.INWORLD_ADMISSIONS_DLQ, - Queue.SERVICES_ADMISSIONS: Queue.SERVICES_ADMISSIONS_DLQ, - Queue.AND_ADMISSIONS: Queue.AND_ADMISSIONS_DLQ, - Queue.PKI_CERT_ROTATION_EVENTS: Queue.PKI_CERT_ROTATION_EVENTS_DLQ, - Queue.DRIFT_EVENTS: Queue.DRIFT_EVENTS_DLQ, - Queue.WORLD_HEALTH: Queue.WORLD_HEALTH_DLQ, - Queue.GATEWAY_PORTAL_EVENTS: Queue.GATEWAY_PORTAL_EVENTS_DLQ, - Queue.PHASE_EVENTS: Queue.PHASE_EVENTS_DLQ, -} - - -def dlq_for(queue: Queue) -> Queue: - """Return the dead-letter queue associated with a primary queue.""" - try: - return _DLQ_BY_PRIMARY[queue] - except KeyError as exc: - raise ValueError(f"Queue {queue!r} does not have a registered DLQ") from exc - - # Primary queues only — used for metrics/introspection endpoints PRIMARY_QUEUES: tuple[Queue, ...] = ( Queue.DNS_UPDATES, diff --git a/tests/test_api_replay_dlq.py b/tests/test_api_replay_dlq.py new file mode 100644 index 0000000..d1b97f7 --- /dev/null +++ b/tests/test_api_replay_dlq.py @@ -0,0 +1,83 @@ +"""API tests for DLQ replay route queue validation and mapping.""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +from netengine.api.app import app +from netengine.api.auth import require_auth +from netengine.events.queues import DLQ_BY_PRIMARY, Queue + + +class _FakePGMQClient: + instances: list["_FakePGMQClient"] = [] + + def __init__(self) -> None: + self.received_from: list[Queue] = [] + type(self).instances.append(self) + + async def receive(self, queue: Queue, timeout: int = 1): + self.received_from.append(queue) + return None + + async def send(self, queue: Queue, envelope) -> None: + raise AssertionError("send should not be called when the DLQ is empty") + + async def delete(self, queue: Queue, msg_id: int) -> None: + raise AssertionError("delete should not be called when the DLQ is empty") + + +def _client(monkeypatch) -> TestClient: + async def operator_user(): + return {"sub": "operator"} + + app.dependency_overrides[require_auth] = operator_user + monkeypatch.setattr("netengine.core.pgmq_client.PGMQClient", _FakePGMQClient) + _FakePGMQClient.instances.clear() + return TestClient(app) + + +def test_replay_accepts_known_primary_queue(monkeypatch) -> None: + client = _client(monkeypatch) + try: + response = client.post( + f"/api/v1/queues/{Queue.DNS_UPDATES.value}/dlq/replay", + headers={"Authorization": "Bearer token"}, + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + assert response.json() == {"replayed": 0, "errors": []} + assert len(_FakePGMQClient.instances) == 1 + + +def test_replay_rejects_unknown_queue(monkeypatch) -> None: + client = _client(monkeypatch) + try: + response = client.post( + "/api/v1/queues/not_a_primary_queue/dlq/replay", + headers={"Authorization": "Bearer token"}, + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 404 + assert response.json()["detail"] == "Unknown queue: not_a_primary_queue" + assert _FakePGMQClient.instances == [] + + +def test_replay_uses_declared_dlq_mapping(monkeypatch) -> None: + declared_dlq = Queue.PHASE_EVENTS_DLQ + monkeypatch.setitem(DLQ_BY_PRIMARY, Queue.DNS_UPDATES, declared_dlq) + client = _client(monkeypatch) + try: + response = client.post( + f"/api/v1/queues/{Queue.DNS_UPDATES.value}/dlq/replay", + headers={"Authorization": "Bearer token"}, + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + assert _FakePGMQClient.instances[0].received_from == [declared_dlq] From e7a0aa57e649a8b2a6684c8929c79adc6d041ccd Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:07:24 +0100 Subject: [PATCH 23/34] test: keep integration pgmq setup on migrations --- migrations/001_initial.sql | 23 ----------------------- tests/test_migration_queues.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql index d92ebee..b592961 100644 --- a/migrations/001_initial.sql +++ b/migrations/001_initial.sql @@ -91,29 +91,6 @@ BEGIN END LOOP; END; $$; -SELECT pgmq.create('dns_updates'); -SELECT pgmq.create('dns_updates_dlq'); -SELECT pgmq.create('oidc_provisioning'); -SELECT pgmq.create('oidc_provisioning_dlq'); -SELECT pgmq.create('and_provisioning'); -SELECT pgmq.create('and_provisioning_dlq'); -SELECT pgmq.create('inworld_admissions'); -SELECT pgmq.create('inworld_admissions_dlq'); -SELECT pgmq.create('services_admissions'); -SELECT pgmq.create('services_admissions_dlq'); -SELECT pgmq.create('and_admissions'); -SELECT pgmq.create('and_admissions_dlq'); -SELECT pgmq.create('pki_cert_rotation_events'); -SELECT pgmq.create('pki_cert_rotation_events_dlq'); -SELECT pgmq.create('drift_events'); -SELECT pgmq.create('drift_events_dlq'); -SELECT pgmq.create('world_health'); -SELECT pgmq.create('world_health_dlq'); -SELECT pgmq.create('gateway_portal_events'); -SELECT pgmq.create('gateway_portal_events_dlq'); -SELECT pgmq.create('phase_events'); -SELECT pgmq.create('phase_events_dlq'); - -- pgmq_send(queue_name, message) CREATE OR REPLACE FUNCTION pgmq_send(queue_name text, message text) RETURNS bigint diff --git a/tests/test_migration_queues.py b/tests/test_migration_queues.py index 7490e27..b2b9bf9 100644 --- a/tests/test_migration_queues.py +++ b/tests/test_migration_queues.py @@ -31,3 +31,15 @@ def test_all_primary_queues_have_registered_dlqs() -> None: for queue in PRIMARY_QUEUES: assert dlq_for(queue) in Queue assert dlq_for(queue).value in {registered.value for registered in Queue} + +COMPOSE_INTEGRATION = REPO_ROOT / "compose" / "compose.test-integration.yml" + + +def test_integration_compose_applies_initial_migration_for_pgmq_setup() -> None: + """Keep integration pgmq setup on the same migration path as the app.""" + compose = COMPOSE_INTEGRATION.read_text() + + assert "-f /migrations/001_initial.sql" in compose + assert "../migrations/001_initial.sql:/migrations/001_initial.sql:ro" in compose + assert "pgmq.create_queue" not in compose + assert "pgmq.create(" not in compose From 430122fe9e7c39c38ec6f79e57047e8ab3d83ec5 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:14:25 +0100 Subject: [PATCH 24/34] Document alpha feature support matrix --- docs/COMPREHENSIVE_GAP_ANALYSIS.md | 82 +++++++++++++++--------------- docs/GAP_SUMMARY.txt | 35 ++++++------- docs/spec-alpha-support.md | 55 ++++++++++++++++++++ 3 files changed, 111 insertions(+), 61 deletions(-) create mode 100644 docs/spec-alpha-support.md diff --git a/docs/COMPREHENSIVE_GAP_ANALYSIS.md b/docs/COMPREHENSIVE_GAP_ANALYSIS.md index b59246c..b7c3558 100644 --- a/docs/COMPREHENSIVE_GAP_ANALYSIS.md +++ b/docs/COMPREHENSIVE_GAP_ANALYSIS.md @@ -9,46 +9,45 @@ NetEngine exhibits **15+ categories of gaps** across the codebase, ranging from | Severity | Category | Count | Examples | |----------|----------|-------|----------| -| 🔴 **HIGH** | Unused Spec Fields (feature declared but never used) | 14+ | DNSSEC, CRL, OCSP, intermediate CA, service mirrors, cross-world peering | -| 🟡 **MEDIUM** | Missing Event Infrastructure | 4 | Event queues undefined, consumers not registered | +| 🔴 **HIGH** | Feature-State / Unsupported Spec Fields | 14+ | DNSSEC end-to-end signing, CRL publication, OCSP lifecycle, service mirrors, cross-world peering | +| 🟡 **MEDIUM** | Event Infrastructure Follow-up | 4 | Queue constants present; consumer coverage/registration still needs verification | | 🟡 **MEDIUM** | API Gaps | 4+ | No endpoint for AND profile updates, gateway config, service toggles | | 🟡 **MEDIUM** | Phase Prerequisites | 1 | Phase 9 missing prerequisite declaration | | 🟢 **LOW** | Test Coverage Gaps | 8+ | No tests for declared features | | 🟢 **LOW** | State/Schema Debt | 7 | Deprecated fields still accumulated | -**Total Impact**: ~30+ features partially or completely unimplemented, but declared in spec. Users can enable them in YAML configs with zero effect. +**Total Impact**: ~30+ features are partially implemented, gated by feature-state validation, or completely unimplemented. Some previously declared fields are now wired, but several remain alpha-unsupported or incomplete end-to-end. --- -## 1. 🔴 CRITICAL: UNUSED SPEC FIELDS (HIGH SEVERITY) +## 1. 🔴 CRITICAL: FEATURE-STATE / UNSUPPORTED SPEC FIELDS (HIGH SEVERITY) ### 1.1 PKI Configuration Gaps **File**: `netengine/spec/models.py:201-228` (PKIPhase class) -These fields are declared in the spec but **completely absent from handler logic**: +These fields are declared in the spec and should be read through the alpha support matrix before use. Verification shows several are now wired into handler logic, but some remain unsupported because the implementation is not complete end-to-end: | Field | Type | Default | Purpose | Files | Status | |-------|------|---------|---------|-------|--------| -| `intermediate_ca_enabled` | bool | False | Enable cert hierarchy | models.py:222 | ❌ Never read | -| `dnssec_enabled` | bool | True | DNSSEC support | models.py:223 | ❌ Never read | -| `dnssec_ksk_lifetime_days` | int | 365 | KSK rotation lifetime | models.py:224 | ❌ Never read | -| `dnssec_zsk_lifetime_days` | int | 30 | ZSK rotation lifetime | models.py:225 | ❌ Never read | -| `crl_enabled` | bool | False | Certificate revocation list | models.py:226 | ❌ Never read | -| `ocsp_enabled` | bool | False | Online cert status protocol | models.py:227 | ❌ Never read | +| `intermediate_ca_enabled` | bool | False | Enable cert hierarchy | models.py:271 | ⚠️ Read/wired; alpha stabilizing | +| `dnssec_enabled` | bool | True | DNSSEC support | models.py:274 | ⚠️ Read/wired for key generation; zone signing not integrated | +| `dnssec_ksk_lifetime_days` | int | 365 | KSK rotation lifetime | models.py:277 | ⚠️ Read as metadata; no automatic DNSSEC rotation | +| `dnssec_zsk_lifetime_days` | int | 30 | ZSK rotation lifetime | models.py:280 | ⚠️ Read as metadata; no automatic DNSSEC rotation | +| `crl_enabled` | bool | False | Certificate revocation list | models.py:283 | ⚠️ Read/wired for step-ca config; publication incomplete | +| `ocsp_enabled` | bool | False | Online cert status protocol | models.py:286 | ⚠️ Read/wired for step-ca config; responder lifecycle incomplete | +| `rotation_policy` | PKIRotationPolicy | enabled/defaults | Certificate rotation policy | models.py:289 | ⚠️ Read/wired; experimental | **Evidence:** ```bash -$ grep -r "dnssec_enabled\|crl_enabled\|ocsp_enabled\|intermediate_ca_enabled" netengine/handlers/ -# Returns: (nothing) -$ grep -r "dnssec_enabled\|crl_enabled\|ocsp_enabled\|intermediate_ca_enabled" netengine/phases/ -# Returns: (nothing) +$ rg -n "dnssec_enabled|crl_enabled|ocsp_enabled|intermediate_ca_enabled|rotation_policy" netengine/handlers netengine/workers netengine/api tests +# Returns handler, worker, API, and tests coverage for the wired PKI paths. ``` -**Handler Only Implements**: Root CA generation via `step ca init` (netengine/handlers/pki_handler.py:80-96) +**Handler Coverage**: `PKIHandler.bootstrap()` reads CRL, OCSP, and intermediate-CA flags; `PKIPhaseHandler.execute()` reads CRL, OCSP, intermediate CA, and DNSSEC fields; `_register_rotation_worker()` reads `rotation_policy`; the worker live-reloads `rotation_policy` from `world_spec`. -**Impact**: Users can set `pki: {dnssec_enabled: true, crl_enabled: true, intermediate_ca_enabled: true}` in their spec, but these settings have **zero effect**. The handlers don't check them, don't warn about them, don't implement them. +**Impact**: Users should consult `docs/spec-alpha-support.md`. Active unsupported feature values are validated by the spec loader, while experimental fields may work but remain subject to alpha changes. -**Root Cause**: step-ca (the underlying tool) supports these features, but NetEngine's Phase 3 handler never wires them up. +**Root Cause**: step-ca supports parts of these features, and NetEngine now wires several alpha paths. The remaining gap is end-to-end support: signed DNS zone publication/rotation, CRL distribution, OCSP lifecycle management, and stable trust-chain semantics are not complete. --- @@ -339,10 +338,10 @@ if queue_already_exists: | Cross-World Peering | GatewayPortal | (none) | 0% | | Service Mirrors | RealInternetConfig | (none) | 0% | | BGP Fabric | ANDsPhase | (none) | 0% | -| DNSSEC | PKIPhase | (none) | 0% | -| OCSP | PKIPhase | (none) | 0% | -| CRL | PKIPhase | (none) | 0% | -| Intermediate CA | PKIPhase | (none) | 0% | +| DNSSEC | PKIPhase | tests/test_pki_features.py | Partial: key generation and metadata | +| OCSP | PKIPhase | tests/test_pki_features.py | Partial: config injection | +| CRL | PKIPhase | tests/test_pki_features.py | Partial: config injection | +| Intermediate CA | PKIPhase | tests/test_pki_features.py | Partial: state/API/output coverage | | AND Dynamic IP | ANDProfileDef | (none) | 0% | | AND Reverse DNS | ANDProfileDef | (none) | 0% | | DMARC Policy | MailConfig | (none) | 0% | @@ -414,19 +413,18 @@ old_spec = NetEngineSpec(**state.world_spec) # Uses stale snapshot - **Files**: core/phase_graph.py, handlers/app_handler.py - **Effort**: 1 hour -5. **Implement Intermediate CA Support** - - Wire `intermediate_ca_enabled` to step-ca init - - Add tests - - **Files**: handlers/pki_handler.py, handlers/phase_pki.py, tests/ - - **Effort**: 4 hours - - **Benefit**: Enables proper cert hierarchy - -6. **Wire PKI Rotation Policy from Spec** - - Parse `spec.pki.rotation_policy` in phase_pki.py - - Pass cert-type configs to worker registration - - **Files**: handlers/phase_pki.py, spec/models.py +5. **Stabilize Intermediate CA Support** + - Keep `intermediate_ca_enabled` documented as alpha/stabilizing + - Verify trust-chain distribution semantics before marking stable + - **Files**: handlers/pki_handler.py, handlers/phase_pki.py, api/routes.py, tests/ + - **Effort**: 2-4 hours + - **Benefit**: Enables predictable cert hierarchy operations + +6. **Harden PKI Rotation Policy** + - Rotation policy is wired; add operational guarantees around cert-type coverage and graceful cutover + - **Files**: handlers/phase_pki.py, workers/pki_cert_rotation_worker.py, spec/models.py - **Effort**: 2-3 hours - - **Benefit**: Users control rotation via YAML + - **Benefit**: Users can rely on YAML/API-driven rotation during alpha 7. **Update world_spec on Reload** - Sync `state.world_spec` after successful reload @@ -483,8 +481,8 @@ old_spec = NetEngineSpec(**state.world_spec) # Uses stale snapshot 3. Document supported vs. planned features (P0) **If Feature-Driven (user needs):** -1. Implement intermediate CA (P1, high user value) -2. Wire PKI rotation policy (P1, improves ops) +1. Stabilize intermediate CA semantics (P1, high user value) +2. Harden PKI rotation policy behavior (P1, improves ops) 3. Add gateway config API endpoints (P1, enables hybrid worlds) --- @@ -497,8 +495,8 @@ old_spec = NetEngineSpec(**state.world_spec) # Uses stale snapshot | netengine/core/phase_graph.py | 39-46 | Add Phase 9 prerequisite | | netengine/spec/models.py | 201-228 | Add deprecation notes or implement | | netengine/api/routes.py | 1-800 | Add missing PUT endpoints | -| netengine/handlers/pki_handler.py | 62-96 | Wire intermediate_ca_enabled | -| netengine/handlers/phase_pki.py | 127-133 | Wire rotation_policy config | +| netengine/handlers/pki_handler.py | 62-120, 253-263 | Intermediate CA is read when enabled; stabilize semantics | +| netengine/handlers/phase_pki.py | 127-190 | Rotation policy is wired; harden operational behavior | | netengine/core/state.py | 51-72 | Remove deprecated fields | | tests/integration/ | Various | Add test cases for declared features | @@ -509,12 +507,12 @@ old_spec = NetEngineSpec(**state.world_spec) # Uses stale snapshot NetEngine has **30+ declared features that are partially or completely unimplemented**, with **four major infrastructure gaps**: 1. **Spec-Implementation Mismatch** (PKI, gateway, AND profiles) -2. **Event Infrastructure Inconsistency** (undefined queues, missing consumers) +2. **Event Infrastructure Follow-up** (queue constants present; consumer coverage/registration still needs verification) 3. **API Endpoint Gaps** (no service/gateway config endpoints) 4. **State/Phase Logic Debt** (deprecated fields, implicit Phase 2 handling) -**Most important fix**: Add warnings when users enable unsupported features (1-2 hours, prevents confusion). +**Most important fix**: Keep feature-state validation and `docs/spec-alpha-support.md` synchronized so unsupported and experimental features are visible before operators enable them. -**Highest ROI fix**: Implement intermediate CA support (4 hours, enables enterprise use cases). +**Highest ROI fix**: Stabilize intermediate CA support and PKI rotation policy semantics so wired alpha features can graduate safely. -**Biggest risk**: Queue registration mismatch could cause runtime failures in persistent mode (2 hours to fix, critical to do). +**Biggest risk**: Event consumer coverage and operational handling still need verification for persistent mode. diff --git a/docs/GAP_SUMMARY.txt b/docs/GAP_SUMMARY.txt index 60108a6..8d3054c 100644 --- a/docs/GAP_SUMMARY.txt +++ b/docs/GAP_SUMMARY.txt @@ -3,15 +3,15 @@ ╚═══════════════════════════════════════════════════════════════════════════╝ ┌─────────────────────────────────────────────────────────────────────────┐ -│ 🔴 CRITICAL GAPS (Spec Declared But Not Implemented) │ +│ 🔴 CRITICAL GAPS (Spec Gated, Partial, Or Not Implemented) │ └─────────────────────────────────────────────────────────────────────────┘ 1. PKI FEATURES (netengine/spec/models.py:201-228) - ❌ DNSSEC Support (dnssec_enabled, ksk/zsk lifetimes) - ❌ CRL (Certificate Revocation List) - crl_enabled - ❌ OCSP (Online Certificate Status Protocol) - ocsp_enabled - ❌ Intermediate CA - intermediate_ca_enabled - ⚠️ PKI Rotation Policy - declared but not wired from spec + ⚠️ DNSSEC Support (dnssec_enabled, ksk/zsk lifetimes) - key generation wired; zone signing/rotation incomplete + ⚠️ CRL (Certificate Revocation List) - crl_enabled read by handler; publication/distribution incomplete + ⚠️ OCSP (Online Certificate Status Protocol) - ocsp_enabled read by handler; responder lifecycle incomplete + ⚠️ Intermediate CA - intermediate_ca_enabled read and exposed; alpha semantics stabilizing + ⚠️ PKI Rotation Policy - wired from spec/API; experimental 2. GATEWAY FEATURES (netengine/spec/models.py:521-551) ❌ Real Internet Mode (service_mirrors, upstream_resolver) @@ -31,17 +31,14 @@ └─────────────────────────────────────────────────────────────────────────┘ 5. EVENT QUEUE MISMATCHES (netengine/events/queues.py:27-33) - ❌ Queue "and_admissions" referenced but not defined - ❌ Queue "pki_cert_rotation_events" referenced but not defined - ❌ Queue "drift_events" referenced but not defined - ❌ Queue "world_health" referenced but not defined - → RUNTIME RISK: Will fail when trying to emit to undefined queues + ✅ Queue registry now defines and_admissions, pki_cert_rotation_events, drift_events, and world_health + ⚠️ Remaining risk is consumer coverage/operational handling, not missing queue constants 6. MISSING API ENDPOINTS (netengine/api/routes.py) ❌ PUT /ands/{and_name}/profile (to update AND config) ❌ PUT /gateway (to modify gateway configuration) ❌ PUT /services/{name} (to enable/disable services) - ❌ PUT /pki/rotation-policy (to update rotation settings) + ✅ PUT /pki/rotation-policy (to update rotation settings) 7. INCOMPLETE PHASE LOGIC (netengine/core/) ❌ Phase 9 missing prerequisite for world_services_output @@ -68,10 +65,10 @@ 10. SILENT GRACEFUL DEGRADATION • pgmq unavailability → events disabled without warning • Queue creation deferral → potential runtime failures - • Handler field validation missing → unsupported configs allowed + • Feature-state validation exists; keep docs and registry synchronized 11. TEST COVERAGE GAPS - • No tests for: DNSSEC, OCSP, CRL, intermediate CA, real internet, + • Partial tests for PKI DNSSEC/OCSP/CRL/intermediate CA; still lacking real internet, cross-world, service mirrors, BGP, AND dynamic IP, AND reverse DNS ┌─────────────────────────────────────────────────────────────────────────┐ @@ -80,7 +77,7 @@ Total Gaps Identified: 30+ Declared But Unimplemented: 14+ fields -Missing Event Consumers: 4 queues +Event Follow-up Items: 4 queues Missing API Endpoints: 4+ endpoints Test Coverage Gaps: 8+ features @@ -88,10 +85,10 @@ Test Coverage Gaps: 8+ features │ ⚡ QUICK FIXES (High ROI) │ └─────────────────────────────────────────────────────────────────────────┘ -P0 (1-2 hours): Add warnings when users enable unsupported fields -P1 (2-4 hours): Fix queue registration mismatch -P1 (4 hours): Implement Intermediate CA support -P1 (2 hours): Wire PKI rotation policy from spec config +P0 (1-2 hours): Keep alpha support matrix synchronized with feature-state registry +P1 (2-4 hours): Verify event consumer coverage +P1 (2-4 hours): Stabilize Intermediate CA support +P1 (2 hours): Harden PKI rotation policy behavior P1 (1 hour): Add Phase 9 prerequisites P2 (3 hours): Clean up deprecated state fields P2 (4 hours): Add test coverage for declared features diff --git a/docs/spec-alpha-support.md b/docs/spec-alpha-support.md new file mode 100644 index 0000000..32498d1 --- /dev/null +++ b/docs/spec-alpha-support.md @@ -0,0 +1,55 @@ +# NetEngine Alpha Spec Support Matrix + +This matrix is the alpha support contract for spec fields that carry explicit +feature-state metadata. It combines the validation registry in +`netengine/spec/feature_state.py` with the PKI JSON-schema feature-state +metadata in `netengine/spec/models.py`. + +Feature states: + +- `stable`: supported for normal alpha use. No fields are explicitly marked + `stable` in the current registry; unlisted spec fields should not be inferred + to have this state. +- `experimental`: wired or partially wired, but APIs, state shape, or operator + behavior may change during alpha. +- `reserved`: accepted by the model as a forward-looking contract. Treat as + not generally available unless the caveat says otherwise. +- `unsupported`: rejected or considered unavailable when set to an active + non-default value during alpha validation. + +| Dotted field path | Feature state | Default value | Implementation owner/module | Alpha caveat | +|---|---:|---|---|---| +| `pki.intermediate_ca_enabled` | `reserved` / registry: `experimental` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki`, `netengine.api.routes` | step-ca's generated intermediate certificate can be read, exposed in Phase 3 output, and fetched through `GET /pki/intermediate-ca-cert`; behavior remains alpha/stabilizing and the model metadata is still conservative. | +| `pki.dnssec_enabled` | `unsupported` | `true` | `netengine.handlers.phase_pki`, `netengine.handlers.pki_handler`, spec loader validation | Active non-default enabling is unsupported in alpha validation. The handler can generate KSK/ZSK key material, but DNS zone signing and CoreDNS activation are not integrated, so this is not end-to-end DNSSEC support. | +| `pki.dnssec_ksk_lifetime_days` | `unsupported` | `365` | `netengine.handlers.pki_handler` | Lifetime is recorded in DNSSEC output metadata only; no automatic KSK rotation or signed-zone publication is implemented. | +| `pki.dnssec_zsk_lifetime_days` | `unsupported` | `30` | `netengine.handlers.pki_handler` | Lifetime is recorded in DNSSEC output metadata only; no automatic ZSK rotation or signed-zone publication is implemented. | +| `pki.crl_enabled` | `unsupported` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | Handler code can inject a step-ca CRL config and report a URL, but CRL publication/distribution-point plumbing is incomplete and active use is unsupported in alpha validation. | +| `pki.ocsp_enabled` | `unsupported` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | Handler code can inject OCSP-related step-ca config and report a URL, but responder deployment/verification is incomplete and active use is unsupported in alpha validation. | +| `pki.rotation_policy` | `experimental` | `{enabled: true, default_interval_hours: 24, default_warning_days: 30, cert_type_overrides: {}}` | `netengine.handlers.phase_pki`, `netengine.workers.pki_cert_rotation_worker`, `netengine.api.routes` | Wired from the spec into worker registration and live-reloaded from runtime state; policy shape and cert-type semantics may change during alpha. | +| `gateway_portal.real_internet.mode` | `unsupported` | `isolated` | `netengine.spec.loader` | Real-internet gateway policies are not implemented. | +| `gateway_portal.real_internet.service_mirrors` | `unsupported` | `[]` | `netengine.spec.loader` | Service mirror provisioning is not implemented. | +| `gateway_portal.real_internet.upstream_resolver_enabled` | `unsupported` | `false` | `netengine.spec.loader` | Upstream resolver forwarding is not implemented. | +| `gateway_portal.cross_world.mode` | `unsupported` | `none` | `netengine.spec.loader` | Cross-world federation is not implemented. | +| `gateway_portal.cross_world.peers` | `unsupported` | `[]` | `netengine.spec.loader` | Cross-world peer provisioning is not implemented. | +| `ands.profiles.*.dynamic_ip` | `unsupported` | profile default | `netengine.spec.loader` | Dynamic IP allocation in AND profiles is not implemented. | +| `ands.profiles.*.reverse_dns` | `unsupported` | profile default | `netengine.spec.loader` | Reverse DNS delegation from AND profiles is not implemented. | +| `ands.profiles.*.bgp` | `unsupported` | profile default | `netengine.spec.loader` | BGP profile configuration is not implemented. | + +## PKI alpha notes + +- **DNSSEC lifetimes**: `pki.dnssec_ksk_lifetime_days` and + `pki.dnssec_zsk_lifetime_days` are persisted as generated-key metadata by + `PKIHandler.setup_dnssec`, but they do not schedule rotation and do not wire + generated keys into signed CoreDNS zones. +- **CRL**: `pki.crl_enabled` reaches step-ca config injection code, but alpha + support does not provide complete CRL publication, distribution-point + integration, or client validation guarantees. +- **OCSP**: `pki.ocsp_enabled` reaches step-ca config injection code, but alpha + support does not provide a fully managed OCSP responder lifecycle. +- **Intermediate CA**: `pki.intermediate_ca_enabled` stores and exposes the + generated intermediate certificate when available, but remains stabilizing and + should not be treated as a stable trust-chain management interface. +- **Rotation policy**: `pki.rotation_policy` is wired into the PKI certificate + rotation worker and can be updated through the operator API. It remains + experimental because cert-type coverage and graceful cutover behavior are + still evolving. From 64f88357e2939df2c6b9bec29867ba8249ffa0cd Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:14:52 +0100 Subject: [PATCH 25/34] Disable DNSSEC by default in specs --- examples/dev-sandbox.yaml | 1 + examples/minimal.yaml | 3 +-- examples/single-org.yaml | 1 + netengine/spec/models.py | 2 +- tests/fixtures/e2e-spec.yaml | 3 +-- tests/test_config.py | 2 +- tests/test_spec_parsing.py | 2 ++ 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/dev-sandbox.yaml b/examples/dev-sandbox.yaml index e2f91c9..1136bc4 100644 --- a/examples/dev-sandbox.yaml +++ b/examples/dev-sandbox.yaml @@ -55,6 +55,7 @@ pki: enabled: true listen_ip: 10.0.0.6 canonical_name: ca.platform.internal + # DNSSEC is unsupported in alpha; leave disabled until key generation is integrated with zone signing. dnssec_enabled: false identity_platform: diff --git a/examples/minimal.yaml b/examples/minimal.yaml index a6e4186..658e8b2 100644 --- a/examples/minimal.yaml +++ b/examples/minimal.yaml @@ -56,9 +56,8 @@ pki: enabled: true listen_ip: 10.0.0.6 canonical_name: ca.platform.internal + # DNSSEC is unsupported in alpha; leave disabled until key generation is integrated with zone signing. dnssec_enabled: false - dnssec_ksk_lifetime_days: 365 - dnssec_zsk_lifetime_days: 30 identity_platform: oidc_provider: keycloak diff --git a/examples/single-org.yaml b/examples/single-org.yaml index 1e326c2..b9fefa4 100644 --- a/examples/single-org.yaml +++ b/examples/single-org.yaml @@ -60,6 +60,7 @@ pki: enabled: true listen_ip: 10.0.0.6 canonical_name: ca.platform.internal + # DNSSEC is unsupported in alpha; leave disabled until key generation is integrated with zone signing. dnssec_enabled: false identity_platform: diff --git a/netengine/spec/models.py b/netengine/spec/models.py index 6194406..627a2b5 100644 --- a/netengine/spec/models.py +++ b/netengine/spec/models.py @@ -272,7 +272,7 @@ class PKIPhase(SpecModel): default=False, feature_state=PKI_FEATURE_STATES["pki.intermediate_ca_enabled"] ) dnssec_enabled: bool = feature_state_field( - default=True, feature_state=PKI_FEATURE_STATES["pki.dnssec_enabled"] + default=False, feature_state=PKI_FEATURE_STATES["pki.dnssec_enabled"] ) dnssec_ksk_lifetime_days: int = feature_state_field( default=365, feature_state=PKI_FEATURE_STATES["pki.dnssec_ksk_lifetime_days"] diff --git a/tests/fixtures/e2e-spec.yaml b/tests/fixtures/e2e-spec.yaml index d1de5f5..4ae210d 100644 --- a/tests/fixtures/e2e-spec.yaml +++ b/tests/fixtures/e2e-spec.yaml @@ -57,9 +57,8 @@ pki: enabled: true listen_ip: 172.20.0.6 canonical_name: ca.platform.internal + # DNSSEC is unsupported in alpha; leave disabled until key generation is integrated with zone signing. dnssec_enabled: false - dnssec_ksk_lifetime_days: 365 - dnssec_zsk_lifetime_days: 30 identity_platform: oidc_provider: keycloak diff --git a/tests/test_config.py b/tests/test_config.py index 38fc858..fdf5aa0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -77,7 +77,7 @@ def _minimal_spec(name: str = "test-network") -> dict: "listen_ip": "10.0.0.6", "canonical_name": "ca.platform.internal", }, - "dnssec_enabled": True, + "dnssec_enabled": False, }, "identity_platform": { "oidc_provider": "keycloak", diff --git a/tests/test_spec_parsing.py b/tests/test_spec_parsing.py index 32ba3a1..d637dee 100644 --- a/tests/test_spec_parsing.py +++ b/tests/test_spec_parsing.py @@ -122,6 +122,7 @@ def test_minimal_spec_has_defaults(self, minimal_spec: NetEngineSpec) -> None: assert minimal_spec.substrate.orchestrator.value == "swarm" assert minimal_spec.dns.root.listen_ip == "10.0.0.2" assert minimal_spec.pki.root_ca.cert_lifetime_days == 3650 + assert minimal_spec.pki.dnssec_enabled is False assert minimal_spec.gateway_portal.enabled is True def test_networks_defaults(self, minimal_spec: NetEngineSpec) -> None: @@ -231,4 +232,5 @@ def test_no_warnings_for_default_spec(self, caplog, minimal_spec) -> None: from netengine.spec.loader import _validate_feature_states _validate_feature_states(minimal_spec) + assert minimal_spec.pki.dnssec_enabled is False assert not caplog.records From 260104931a14896bb25d1f315e758190ac588791 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:15:11 +0100 Subject: [PATCH 26/34] Add spec validation CLI command --- netengine/cli/main.py | 205 ++++++++++++++++++++++++++++++++++-------- tests/test_cli.py | 67 ++++++++++++++ 2 files changed, 233 insertions(+), 39 deletions(-) diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 4474083..3d23b24 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -12,11 +12,23 @@ from netengine.core.migrations import MigrationService, MigrationStatus from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState -from netengine.db.migrations import MIGRATIONS_DIR, MigrationRunResult, migration_status, run_migrations +from netengine.db.migrations import ( + MIGRATIONS_DIR, + MigrationRunResult, + migration_status, + run_migrations, +) from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS -from netengine.spec.loader import load_spec, load_spec_with_composition, load_spec_with_environment +from netengine.spec.loader import ( + SpecLoadError, + _is_active_feature_value, + _resolve_feature_state_paths, + load_spec, + load_spec_with_composition, + load_spec_with_environment, +) logger = get_logger(__name__) @@ -32,7 +44,9 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: parts = key.split(".") if any(part == "" for part in parts): - raise click.BadParameter("keys must be non-empty dotted paths", param_hint="--set") + raise click.BadParameter( + "keys must be non-empty dotted paths", param_hint="--set" + ) value = yaml.safe_load(raw_value) cursor = overrides @@ -54,13 +68,44 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: return overrides +def _load_spec_for_cli( + spec_file: str, + *, + environment: str | None = None, + set_values: tuple[str, ...] = (), +): + """Load a spec using the same composition semantics as ``up``.""" + overrides = _parse_set_overrides(set_values) + if environment: + return load_spec_with_environment( + spec_file, environment=environment, overrides=overrides or None + ) + if overrides: + return load_spec_with_composition(spec_file, overrides=overrides) + return load_spec(spec_file) + + +def _feature_state_explanations(spec: Any) -> list[str]: + """Return noteworthy active feature-state lines for ``validate --explain``.""" + lines: list[str] = [] + for entry, path, value, default_value in _resolve_feature_state_paths(spec): + if not _is_active_feature_value(value, default_value): + continue + value_text = getattr(value, "value", value) + lines.append( + f"{path}: {entry.state} ({entry.stage}) - {entry.reason}; value={value_text!r}" + ) + return lines + + async def _run_migrations(db_url: str) -> MigrationRunResult: """Run SQL migrations using the shared migration service.""" result = await run_migrations(db_url) for migration in result.results: if migration.status == "applied": logger.info( - f"Applied migration: {migration.filename} " f"({migration.duration_seconds:.3f}s)" + f"Applied migration: {migration.filename} " + f"({migration.duration_seconds:.3f}s)" ) elif migration.status == "skipped": logger.info(f"Skipped migration: {migration.filename} (already applied)") @@ -69,23 +114,27 @@ async def _run_migrations(db_url: str) -> MigrationRunResult: f"{result.skipped_count} skipped, {result.failed_count} failed" ) return result + + def _db_url_from_env() -> str | None: """Return the database URL used by CLI migration operations.""" return os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL") - - def _print_migration_status(status: MigrationStatus) -> None: click.echo("Migration status") click.echo(f" Applied: {len(status.applied)}") for record in status.applied: - applied_at = record.applied_at.isoformat() if record.applied_at else "unknown time" + applied_at = ( + record.applied_at.isoformat() if record.applied_at else "unknown time" + ) click.echo(f" ✓ {record.version} {record.name} ({applied_at})") click.echo(f" Pending: {len(status.pending)}") for migration in status.pending: - click.echo(f" • {migration.version} {migration.name} ({migration.path.name})") + click.echo( + f" • {migration.version} {migration.name} ({migration.path.name})" + ) click.echo(f" Failed: {len(status.failed)}") for record in status.failed: @@ -106,8 +155,6 @@ def _print_migration_status(status: MigrationStatus) -> None: click.echo(f" Missing queues: {', '.join(status.missing_queues)}") else: click.echo(" Missing queues: none") - service = MigrationService(db_url, MIGRATIONS_DIR, logger.info) - await service.apply() @click.group() @@ -172,6 +219,53 @@ async def _migrate_status(*, exit_on_unhealthy: bool) -> None: sys.exit(1) +@cli.command() +@click.argument("spec_file", type=click.Path(exists=True)) +@click.option( + "--explain", + is_flag=True, + default=False, + help="Print active experimental, reserved, unsupported, or otherwise noteworthy fields.", +) +@click.option( + "--env", + "environment", + help="Merge spec.{ENV}.yaml next to SPEC_FILE before validating.", +) +@click.option( + "--set", + "set_values", + multiple=True, + metavar="KEY=VALUE", + help="Override a spec value; repeat for multiple dotted keys.", +) +def validate( + spec_file: str, + explain: bool, + environment: str | None, + set_values: tuple[str, ...], +) -> None: + """Validate SPEC_FILE without booting a world.""" + try: + spec = _load_spec_for_cli( + spec_file, environment=environment, set_values=set_values + ) + except SpecLoadError as exc: + click.echo(f"Spec validation failed: {exc}", err=True) + sys.exit(1) + + click.echo(f"Spec validation succeeded: {spec.metadata.name}") + if explain: + explanations = _feature_state_explanations(spec) + if explanations: + click.echo("Feature-state details:") + for line in explanations: + prefix = "WARNING: " if ": experimental " in line else "" + click.echo(f" - {prefix}{line}") + else: + click.echo("Feature-state details: no active noteworthy fields.") + + @cli.command() @click.argument("spec_file", type=click.Path(exists=True)) @click.option("--up-to", default=9, help="Stop after this phase number (0-9).") @@ -195,7 +289,9 @@ async def _migrate_status(*, exit_on_unhealthy: bool) -> None: help="Continue booting if database migrations fail (development escape hatch).", ) @click.option( - "--env", "environment", help="Merge spec.{ENV}.yaml next to SPEC_FILE before booting." + "--env", + "environment", + help="Merge spec.{ENV}.yaml next to SPEC_FILE before booting.", ) @click.option( "--set", @@ -236,18 +332,12 @@ async def _up( environment: str | None = None, set_values: tuple[str, ...] = (), ) -> None: - overrides = _parse_set_overrides(set_values) - if environment: - spec = load_spec_with_environment( - spec_file, environment=environment, overrides=overrides or None - ) - elif overrides: - spec = load_spec_with_composition(spec_file, overrides=overrides) - else: - spec = load_spec(spec_file) + spec = _load_spec_for_cli(spec_file, environment=environment, set_values=set_values) if mock: - click.echo("WARNING: running in mock mode — no real infrastructure will be created.") + click.echo( + "WARNING: running in mock mode — no real infrastructure will be created." + ) if not skip_migrations and not mock: db_url = _db_url_from_env() @@ -369,7 +459,9 @@ def reload(spec_file: str) -> None: is_ephemeral = old_spec.metadata.lifecycle.value == "ephemeral" click.echo("Computing diff…") - result = asyncio.run(apply_reload(old_spec, new_spec, state, is_ephemeral=is_ephemeral)) + result = asyncio.run( + apply_reload(old_spec, new_spec, state, is_ephemeral=is_ephemeral) + ) if result.immutability_violations: click.echo("Reload REJECTED — immutable fields changed:", err=True) @@ -408,7 +500,9 @@ def status() -> None: @cli.command() @click.option("--yes", is_flag=True, help="Skip confirmation prompt.") -@click.option("--dry-run", is_flag=True, help="Show what would be removed without removing it.") +@click.option( + "--dry-run", is_flag=True, help="Show what would be removed without removing it." +) def down(yes: bool, dry_run: bool) -> None: """Tear down the running world (containers, networks, volumes).""" asyncio.run(_down(yes, dry_run)) @@ -417,7 +511,9 @@ def down(yes: bool, dry_run: bool) -> None: async def _down(yes: bool, dry_run: bool) -> None: state = RuntimeState.load() if state.world_spec and not dry_run: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") + raw_lifecycle = (state.world_spec.get("metadata") or {}).get( + "lifecycle", "ephemeral" + ) if raw_lifecycle == "persistent" and not yes: click.confirm( "This is a PERSISTENT world — all durable state will be destroyed. Continue?", @@ -469,7 +565,9 @@ async def _down(yes: bool, dry_run: bool) -> None: errors.append(f"{label}: {exc}") for network in client.networks.list(): - if network.name and any(network.name.startswith(p) for p in _CONTAINER_PREFIXES): + if network.name and any( + network.name.startswith(p) for p in _CONTAINER_PREFIXES + ): label = f"network:{network.name}" if dry_run: click.echo(f" would remove {label}") @@ -596,12 +694,16 @@ async def _diagnose(spec_file: str, as_json: bool) -> None: "status": r.status.value, "detail": r.detail, "hint": r.hint, - "elapsed_ms": round(r.elapsed_ms, 1) if r.elapsed_ms is not None else None, + "elapsed_ms": round(r.elapsed_ms, 1) + if r.elapsed_ms is not None + else None, } for r in results ] click.echo(_json.dumps(payload, indent=2)) - issues = sum(1 for r in results if r.status in (ProbeStatus.FAIL, ProbeStatus.WARN)) + issues = sum( + 1 for r in results if r.status in (ProbeStatus.FAIL, ProbeStatus.WARN) + ) if issues: sys.exit(1) return @@ -641,8 +743,12 @@ async def _diagnose(spec_file: str, as_json: bool) -> None: @cli.command() -@click.option("--interval", default=30, type=int, help="Poll interval in seconds (default 30).") -@click.option("--max-retries", default=3, type=int, help="Max self-heal retries per phase.") +@click.option( + "--interval", default=30, type=int, help="Poll interval in seconds (default 30)." +) +@click.option( + "--max-retries", default=3, type=int, help="Max self-heal retries per phase." +) @click.option("--no-auto-heal", is_flag=True, help="Detect drift but don't auto-heal.") def drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> None: """Watch running world for drift and optionally auto-heal (Ctrl+C to stop).""" @@ -655,7 +761,9 @@ async def _drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> N click.echo("No running world found — use `netengine up` first.", err=True) sys.exit(1) - click.echo(f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…") + click.echo( + f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…" + ) click.echo("Press Ctrl+C to stop.\n") orchestrator = Orchestrator(state.world_spec, mock_mode=False) @@ -775,7 +883,9 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: ) if rows: click.echo( - click.style(f" {dlq_name} ({len(rows)} message(s)):", bold=True) + click.style( + f" {dlq_name} ({len(rows)} message(s)):", bold=True + ) ) for row in rows: import json as _json @@ -785,14 +895,18 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: event_type = payload.get("event_type", "?") emitted_by = payload.get("emitted_by", "?") retry_count = payload.get("retry_count", 0) - dlq_reason = (payload.get("payload") or {}).get("dlq_reason", "") + dlq_reason = (payload.get("payload") or {}).get( + "dlq_reason", "" + ) click.echo( f" [{row['msg_id']}] {event_type} " f"from={emitted_by} retries={retry_count}" + (f" reason={dlq_reason}" if dlq_reason else "") ) except Exception: - click.echo(f" [{row['msg_id']}] (unparseable message)") + click.echo( + f" [{row['msg_id']}] (unparseable message)" + ) else: click.echo(f" {dlq_name}: empty") except Exception as exc: @@ -802,7 +916,9 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: for q in queues_to_check: dlq_name = dlq_for(Queue(q)).value try: - depth_row = await conn.fetchrow("SELECT count(*) AS depth FROM pgmq.q_$1", q) + depth_row = await conn.fetchrow( + "SELECT count(*) AS depth FROM pgmq.q_$1", q + ) dlq_row = await conn.fetchrow( "SELECT count(*) AS depth FROM pgmq.q_$1", dlq_name ) @@ -814,7 +930,9 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: else click.style("!", fg="yellow") ) dlq_status = ( - "" if dlq_depth == 0 else click.style(f" DLQ: {dlq_depth}", fg="red") + "" + if dlq_depth == 0 + else click.style(f" DLQ: {dlq_depth}", fg="red") ) click.echo(f" {status} {q:<30} depth={depth}{dlq_status}") except Exception as exc: @@ -866,7 +984,9 @@ def _print_status(state: RuntimeState) -> None: "dev-sandbox: two orgs, all services, dev apps." ), ) -@click.option("--output", "-o", default=None, help="Output file path (default: .yaml).") +@click.option( + "--output", "-o", default=None, help="Output file path (default: .yaml)." +) @click.option( "--yes", "-y", @@ -913,7 +1033,9 @@ def init( click.confirm(f"{early_path} already exists — overwrite?", abort=True) try: - cfg: WorldConfig = run_wizard(preset=preset, yes=yes, name=name, lifecycle=lifecycle) + cfg: WorldConfig = run_wizard( + preset=preset, yes=yes, name=name, lifecycle=lifecycle + ) except click.Abort: click.echo("\nAborted.", err=True) return @@ -939,7 +1061,10 @@ def init( import os as _os _os.unlink(tmp_path) - click.echo(f"\nSpec validation failed — please report this as a bug:\n {exc}", err=True) + click.echo( + f"\nSpec validation failed — please report this as a bug:\n {exc}", + err=True, + ) raise SystemExit(1) from exc import os as _os @@ -971,7 +1096,9 @@ def _print_init_summary(cfg: "Any", out_path: Path) -> None: click.echo(f"\n Organisations ({len(cfg.orgs)}):") for org in cfg.orgs: user_count = len(org.users) - click.echo(f" • {org.name:<20} profile={org.and_profile} users={user_count}") + click.echo( + f" • {org.name:<20} profile={org.and_profile} users={user_count}" + ) else: click.echo("\n Organisations: none (add later with `netengine reload`)") diff --git a/tests/test_cli.py b/tests/test_cli.py index ab9c889..8a4078a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -363,3 +363,70 @@ def test_up_supports_repeatable_set_overrides(): assert spec_arg.metadata.name == "my-world" assert spec_arg.world_services.mail.enabled is True mock_orchestrator.execute_phases.assert_awaited_once_with(up_to_phase=9) + + +def _write_cli_validate_spec(tmp_path: Path, **updates) -> Path: + """Write an example-derived spec for validate command tests.""" + import copy + import yaml + + source = Path(__file__).parent.parent / "examples" / "minimal.yaml" + data = yaml.safe_load(source.read_text()) + data = copy.deepcopy(data) + + for dotted_path, value in updates.items(): + cursor = data + parts = dotted_path.split("__") + for part in parts[:-1]: + cursor = cursor[part] + cursor[parts[-1]] = value + + spec_file = tmp_path / "spec.yaml" + spec_file.write_text(yaml.safe_dump(data, sort_keys=False)) + return spec_file + + +def test_validate_valid_spec_exits_zero() -> None: + """The validate command should accept a valid spec without booting.""" + spec_file = Path(__file__).parent.parent / "examples" / "minimal.yaml" + + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file)]) + + assert result.exit_code == 0, result.output + assert "Spec validation succeeded: minimal-example" in result.output + + +def test_validate_unsupported_enabled_feature_exits_nonzero(tmp_path: Path) -> None: + """Unsupported active feature gates should fail validation precisely.""" + spec_file = _write_cli_validate_spec(tmp_path, pki__crl_enabled=True) + + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file)]) + + assert result.exit_code == 1 + assert "Unsupported spec features enabled" in result.output + assert "pki.crl_enabled is unsupported" in result.output + + +def test_validate_experimental_enabled_feature_exits_zero_with_explanation(tmp_path: Path) -> None: + """Experimental active features should validate but be visible to operators.""" + spec_file = _write_cli_validate_spec(tmp_path, pki__intermediate_ca_enabled=True) + + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--explain"]) + + assert result.exit_code == 0, result.output + assert "Spec validation succeeded" in result.output + assert "WARNING:" in result.output + assert "pki.intermediate_ca_enabled: experimental" in result.output + + +def test_validate_explain_includes_dotted_field_paths_and_feature_states(tmp_path: Path) -> None: + """--explain should show concrete dotted paths and their feature states.""" + spec_file = _write_cli_validate_spec(tmp_path, pki__intermediate_ca_enabled=True) + + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--explain"]) + + assert result.exit_code == 0, result.output + assert "Feature-state details:" in result.output + assert "pki.intermediate_ca_enabled" in result.output + assert "experimental" in result.output + assert "alpha" in result.output From f0bc6ad74ad7a8c89919ee8f66f55960ea7c956a Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:20:17 +0100 Subject: [PATCH 27/34] Type PGMQ client results --- netengine/core/pgmq_client.py | 202 +++++++++++++++++++++++++--------- pyproject.toml | 1 - 2 files changed, 151 insertions(+), 52 deletions(-) diff --git a/netengine/core/pgmq_client.py b/netengine/core/pgmq_client.py index 8cb2286..2b0afa5 100644 --- a/netengine/core/pgmq_client.py +++ b/netengine/core/pgmq_client.py @@ -1,5 +1,7 @@ import json -from typing import Any, Dict, Optional +from datetime import datetime +from importlib import import_module +from typing import Any, NotRequired, Protocol, TypedDict, cast from netengine.events.queues import Queue, dlq_for from netengine.events.schema import EventEnvelope @@ -7,58 +9,79 @@ MAX_RETRIES = 3 +class PGMQMessage(TypedDict): + """Typed shape returned by PGMQ pop/read operations.""" + + msg_id: int + message: str + read_ct: NotRequired[int] + enqueued_at: NotRequired[str] + vt: NotRequired[str] + + +class _QueryResult(Protocol): + data: object + + +class _ExecutableQuery(Protocol): + async def execute(self) -> _QueryResult: ... + + +class _TableQuery(_ExecutableQuery, Protocol): + def select(self, cols: str = "*") -> "_TableQuery": ... + def insert(self, data: dict[str, Any]) -> "_TableQuery": ... + def upsert(self, data: dict[str, Any]) -> "_TableQuery": ... + def update(self, data: dict[str, Any]) -> "_TableQuery": ... + def delete(self) -> "_TableQuery": ... + def eq(self, col: str, val: Any) -> "_TableQuery": ... + def limit(self, n: int) -> "_TableQuery": ... + + +class _DatabaseClient(Protocol): + def rpc(self, func_name: str, params: dict[str, Any]) -> _ExecutableQuery: ... + def table(self, name: str) -> _TableQuery: ... + + +class _GetDb(Protocol): + async def __call__(self) -> _DatabaseClient: ... + + class PGMQClient: def __init__(self) -> None: - self._db = None + self._db: _DatabaseClient | None = None - async def _get_db(self): + async def _get_db(self) -> _DatabaseClient: if self._db is None: - from netengine.core.supabase_client import get_db - + module = import_module("netengine.core.supabase_client") + get_db = cast(_GetDb, module.get_db) self._db = await get_db() return self._db async def send(self, queue_name: Queue, event: EventEnvelope) -> int: """Enqueue an event; returns message ID.""" db = await self._get_db() - payload = event.to_dict() - result = await db.rpc( - "pgmq_send", {"queue_name": queue_name, "message": json.dumps(payload)} - ).execute() - if not result.data: - raise RuntimeError(f"pgmq_send returned no data for queue '{queue_name}'") - return result.data[0] + payload = self._encode_envelope(event) + result = await db.rpc("pgmq_send", {"queue_name": queue_name, "message": payload}).execute() + return self._message_id_from_result(result.data, queue_name) - async def receive( - self, queue_name: Queue, timeout: int = 5 - ) -> Optional[Dict[str, Any]]: + async def receive(self, queue_name: Queue, timeout: int = 5) -> PGMQMessage | None: """Pop a message from the queue.""" db = await self._get_db() - result = await db.rpc( - "pgmq_pop", {"queue_name": queue_name, "timeout": timeout} - ).execute() - if result.data: - return result.data[0] - return None + result = await db.rpc("pgmq_pop", {"queue_name": queue_name, "timeout": timeout}).execute() + return self._message_from_result(result.data, "pgmq_pop", queue_name) async def delete(self, queue_name: Queue, msg_id: int) -> None: """Acknowledge and delete a processed message.""" db = await self._get_db() - await db.rpc( - "pgmq_delete", {"queue_name": queue_name, "msg_id": msg_id} - ).execute() + await db.rpc("pgmq_delete", {"queue_name": queue_name, "msg_id": msg_id}).execute() - async def read_by_id( - self, queue_name: Queue, msg_id: int - ) -> Optional[Dict[str, Any]]: + async def read_by_id(self, queue_name: Queue, msg_id: int) -> PGMQMessage | None: """Read a specific message by ID without consuming it.""" db = await self._get_db() result = await db.rpc( "pgmq_read_by_id", {"queue_name": queue_name, "msg_id": msg_id} ).execute() - if result.data: - return result.data[0] - return None + return self._message_from_result(result.data, "pgmq_read_by_id", queue_name) async def archive_to_dlq(self, queue_name: Queue, msg_id: int, reason: str) -> None: """Re-queue with incremented retry count, or move to DLQ after MAX_RETRIES.""" @@ -66,30 +89,107 @@ async def archive_to_dlq(self, queue_name: Queue, msg_id: int, reason: str) -> N if not msg: return - envelope = EventEnvelope(**json.loads(msg["message"])) + envelope = self._decode_envelope(msg) await self.delete(queue_name, msg_id) if envelope.retry_count + 1 >= MAX_RETRIES: - dlq_envelope = EventEnvelope( - event_id=envelope.event_id, - correlation_id=envelope.correlation_id, - event_type=envelope.event_type, - emitted_by=envelope.emitted_by, - emitted_at=envelope.emitted_at, - payload={**envelope.payload, "dlq_reason": reason}, - parent_event_id=envelope.parent_event_id, - retry_count=envelope.retry_count + 1, + dlq_envelope = self._retry_envelope( + envelope, payload={**envelope.payload, "dlq_reason": reason} ) - await self.send(dlq_for(Queue(queue_name)), dlq_envelope) + await self.send(dlq_for(queue_name), dlq_envelope) else: - requeue_envelope = EventEnvelope( - event_id=envelope.event_id, - correlation_id=envelope.correlation_id, - event_type=envelope.event_type, - emitted_by=envelope.emitted_by, - emitted_at=envelope.emitted_at, - payload=envelope.payload, - parent_event_id=envelope.parent_event_id, - retry_count=envelope.retry_count + 1, - ) + requeue_envelope = self._retry_envelope(envelope, payload=envelope.payload) await self.send(queue_name, requeue_envelope) + + @staticmethod + def _encode_envelope(event: EventEnvelope) -> str: + return json.dumps(event.to_dict()) + + @staticmethod + def _decode_envelope(msg: PGMQMessage) -> EventEnvelope: + payload = json.loads(msg["message"]) + if not isinstance(payload, dict): + raise RuntimeError(f"PGMQ message {msg['msg_id']} did not contain a JSON object") + return PGMQClient._envelope_from_payload(cast(dict[str, Any], payload), msg["msg_id"]) + + @staticmethod + def _envelope_from_payload(payload: dict[str, Any], msg_id: int) -> EventEnvelope: + emitted_at = payload.get("emitted_at") + if isinstance(emitted_at, str): + payload = {**payload, "emitted_at": datetime.fromisoformat(emitted_at)} + elif not isinstance(emitted_at, datetime): + raise RuntimeError( + f"PGMQ message {msg_id} did not contain a valid emitted_at timestamp" + ) + return EventEnvelope(**payload) + + @staticmethod + def _retry_envelope(envelope: EventEnvelope, payload: dict[str, Any]) -> EventEnvelope: + return EventEnvelope( + event_id=envelope.event_id, + correlation_id=envelope.correlation_id, + event_type=envelope.event_type, + emitted_by=envelope.emitted_by, + emitted_at=envelope.emitted_at, + payload=payload, + parent_event_id=envelope.parent_event_id, + retry_count=envelope.retry_count + 1, + ) + + @staticmethod + def _message_id_from_result(data: object, queue_name: Queue) -> int: + first = PGMQClient._first_result_row(data, "pgmq_send", queue_name) + if isinstance(first, int): + return first + if isinstance(first, dict): + value = first.get("msg_id") + if isinstance(value, int): + return value + raise RuntimeError(f"pgmq_send returned invalid message ID for queue '{queue_name}'") + + @staticmethod + def _message_from_result(data: object, operation: str, queue_name: Queue) -> PGMQMessage | None: + first = PGMQClient._first_result_row(data, operation, queue_name, required=False) + if first is None: + return None + if not isinstance(first, dict): + raise RuntimeError(f"{operation} returned a non-object row for queue '{queue_name}'") + + msg_id = first.get("msg_id") + message = first.get("message") + if not isinstance(msg_id, int): + raise RuntimeError( + f"{operation} returned a row without integer msg_id for queue '{queue_name}'" + ) + if isinstance(message, dict): + message = json.dumps(message) + if not isinstance(message, str): + raise RuntimeError( + f"{operation} returned a row without string message for queue '{queue_name}'" + ) + + row: PGMQMessage = {"msg_id": msg_id, "message": message} + read_ct = first.get("read_ct") + if isinstance(read_ct, int): + row["read_ct"] = read_ct + for key in ("enqueued_at", "vt"): + value = first.get(key) + if isinstance(value, str): + row[key] = value + return row + + @staticmethod + def _first_result_row( + data: object, operation: str, queue_name: Queue, *, required: bool = True + ) -> object | None: + if data is None or data == []: + if required: + raise RuntimeError(f"{operation} returned no data for queue '{queue_name}'") + return None + if not isinstance(data, list): + raise RuntimeError(f"{operation} returned non-list data for queue '{queue_name}'") + if not data: + if required: + raise RuntimeError(f"{operation} returned no data for queue '{queue_name}'") + return None + return cast(object, data[0]) diff --git a/pyproject.toml b/pyproject.toml index 4fe6c5a..5890183 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ exclude = [ "netengine/api/", "netengine/cli/", "netengine/phases/", - "netengine/core/pgmq_client\\.py", "netengine/handlers/substrate\\.py", "netengine/handlers/dns\\.py", "netengine/handlers/pki_handler\\.py", From d1695e4bc439e0a09801f5de95552795b292243c Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:23:07 +0100 Subject: [PATCH 28/34] Add typed event payload factories --- netengine/core/drift_controller.py | 49 ++++--- netengine/events/emitter.py | 3 +- netengine/events/factory.py | 126 ++++++++++++++++++ netengine/events/payloads.py | 87 ++++++++++++ netengine/handlers/domain_registry_handler.py | 16 +-- netengine/handlers/world_registry_handler.py | 26 ++-- pyproject.toml | 2 - tests/test_event_factories.py | 39 ++++++ 8 files changed, 294 insertions(+), 54 deletions(-) create mode 100644 netengine/events/factory.py create mode 100644 netengine/events/payloads.py create mode 100644 tests/test_event_factories.py diff --git a/netengine/core/drift_controller.py b/netengine/core/drift_controller.py index 0c9e607..023587e 100644 --- a/netengine/core/drift_controller.py +++ b/netengine/core/drift_controller.py @@ -12,6 +12,7 @@ from typing import Any, Optional from netengine.core.orchestrator import Orchestrator +from netengine.events import factory as event_factory from netengine.events.queues import Queue from netengine.events.schema import EventEnvelope from netengine.handlers._base import BasePhaseHandler @@ -106,11 +107,11 @@ async def _run_one_iteration(self) -> None: phase_num, handler.__class__.__name__, "drift.detected", - { - "phase": phase_num, - "handler": handler.__class__.__name__, - "detected_at": datetime.now(UTC).isoformat(), - }, + event_factory.drift_detected( + phase=phase_num, + handler=handler.__class__.__name__, + detected_at=datetime.now(UTC).isoformat(), + ), ) if drift_this_round and self.auto_heal: @@ -126,10 +127,9 @@ async def _run_one_iteration(self) -> None: -1, "drift_controller", "drift.loop_error", - { - "error": str(e), - "error_at": datetime.now(UTC).isoformat(), - }, + event_factory.drift_loop_error( + error=str(e), error_at=datetime.now(UTC).isoformat() + ), ) async def _check_phase_health(self, phase_num: int, handler: BasePhaseHandler) -> bool: @@ -145,7 +145,7 @@ async def _check_phase_health(self, phase_num: int, handler: BasePhaseHandler) - try: is_healthy = await handler.healthcheck(self.orchestrator.context) self._update_drift_state(phase_num, handler.__class__.__name__, is_healthy) - return is_healthy + return bool(is_healthy) except Exception as e: logger.error(f"Phase {phase_num} healthcheck error: {e}") self._update_drift_state(phase_num, handler.__class__.__name__, False) @@ -241,10 +241,9 @@ async def _heal_phase(self, phase_num: int, handler: BasePhaseHandler) -> tuple[ phase_num, handler.__class__.__name__, "drift.self_healed", - { - "phase": phase_num, - "healed_at": datetime.now(UTC).isoformat(), - }, + event_factory.drift_self_healed( + phase=phase_num, healed_at=datetime.now(UTC).isoformat() + ), ) if drift_state: @@ -261,11 +260,9 @@ async def _heal_phase(self, phase_num: int, handler: BasePhaseHandler) -> tuple[ phase_num, handler.__class__.__name__, "drift.self_heal_failed", - { - "phase": phase_num, - "error": str(e), - "failed_at": datetime.now(UTC).isoformat(), - }, + event_factory.drift_self_heal_failed( + phase=phase_num, error=str(e), failed_at=datetime.now(UTC).isoformat() + ), ) if drift_state: @@ -314,7 +311,8 @@ async def _emit_drift_event( phase_num: int, handler_name: str, event_type: str, - payload: dict[str, Any], + event: EventEnvelope | None = None, + payload: dict[str, Any] | None = None, ) -> None: """Emit a drift event. @@ -329,11 +327,12 @@ async def _emit_drift_event( return try: - event = EventEnvelope.create( - event_type=event_type, - emitted_by="drift_controller", - payload=payload, - ) + if event is None: + event = event_factory.phase_event( + event_type=event_type, + emitted_by="drift_controller", + payload=payload or {}, + ) await self.orchestrator.context.pgmq_client.send(Queue.DRIFT_EVENTS, event) logger.debug(f"Drift event emitted: {event_type}") except Exception as e: diff --git a/netengine/events/emitter.py b/netengine/events/emitter.py index c1816ca..90cfa32 100644 --- a/netengine/events/emitter.py +++ b/netengine/events/emitter.py @@ -3,6 +3,7 @@ from datetime import UTC, datetime from typing import Any +from netengine.events import factory as event_factory from netengine.events.queues import Queue, queue_for_event_type from netengine.events.schema import EventEnvelope from netengine.handlers.context import PhaseContext @@ -90,7 +91,7 @@ async def emit_event( a phase output dict already exists, under that output's ``event_send_failures`` key. """ - event = EventEnvelope.create( + event = event_factory.phase_event( event_type=event_type, emitted_by=emitted_by, payload=payload, diff --git a/netengine/events/factory.py b/netengine/events/factory.py new file mode 100644 index 0000000..6cd9729 --- /dev/null +++ b/netengine/events/factory.py @@ -0,0 +1,126 @@ +"""Typed factory functions for creating event envelopes.""" + +from typing import Any, Mapping + +from netengine.events.payloads import ( + DomainRegisteredPayload, + DriftDetectedPayload, + DriftLoopErrorPayload, + DriftSelfHealedPayload, + DriftSelfHealFailedPayload, + GenericEventPayload, + OrgAdmittedPayload, + OrgRemovedPayload, + OrgUpdatedPayload, +) +from netengine.events.schema import EventEnvelope + + +def _create( + *, + event_type: str, + emitted_by: str, + payload: Mapping[str, Any], + correlation_id: str | None = None, + parent_event_id: str | None = None, +) -> EventEnvelope: + return EventEnvelope.create( + event_type=event_type, + emitted_by=emitted_by, + payload=dict(payload), + correlation_id=correlation_id, + parent_event_id=parent_event_id, + ) + + +def org_admitted( + *, + org_name: str, + capabilities: list[str], + and_profile: str, + correlation_id: str | None = None, + parent_event_id: str | None = None, +) -> EventEnvelope: + payload: OrgAdmittedPayload = { + "org_name": org_name, + "capabilities": capabilities, + "and_profile": and_profile, + } + return _create( + event_type="org.admitted", + emitted_by="world_registry_handler", + payload=payload, + correlation_id=correlation_id, + parent_event_id=parent_event_id, + ) + + +def org_updated(*, org_name: str, capabilities: list[str], and_profile: str) -> EventEnvelope: + payload: OrgUpdatedPayload = { + "org_name": org_name, + "capabilities": capabilities, + "and_profile": and_profile, + } + return _create(event_type="org.updated", emitted_by="world_registry_handler", payload=payload) + + +def org_removed(*, org_name: str) -> EventEnvelope: + payload: OrgRemovedPayload = {"org_name": org_name} + return _create(event_type="org.removed", emitted_by="world_registry_handler", payload=payload) + + +def domain_registered(*, domain: str, org_name: str, ns_records: list[str]) -> EventEnvelope: + payload: DomainRegisteredPayload = {"domain": domain, "org": org_name, "ns": ns_records} + return _create( + event_type="domain.registered", emitted_by="domain_registry_handler", payload=payload + ) + + +def drift_detected(*, phase: int, handler: str, detected_at: str) -> EventEnvelope: + payload: DriftDetectedPayload = {"phase": phase, "handler": handler, "detected_at": detected_at} + return _create(event_type="drift.detected", emitted_by="drift_controller", payload=payload) + + +def drift_loop_error(*, error: str, error_at: str) -> EventEnvelope: + payload: DriftLoopErrorPayload = {"error": error, "error_at": error_at} + return _create(event_type="drift.loop_error", emitted_by="drift_controller", payload=payload) + + +def drift_self_healed(*, phase: int, healed_at: str) -> EventEnvelope: + payload: DriftSelfHealedPayload = {"phase": phase, "healed_at": healed_at} + return _create(event_type="drift.self_healed", emitted_by="drift_controller", payload=payload) + + +def drift_self_heal_failed(*, phase: int, error: str, failed_at: str) -> EventEnvelope: + payload: DriftSelfHealFailedPayload = {"phase": phase, "error": error, "failed_at": failed_at} + return _create( + event_type="drift.self_heal_failed", emitted_by="drift_controller", payload=payload + ) + + +def extension_event(*, event_type: str, emitted_by: str, data: dict[str, Any]) -> EventEnvelope: + payload: GenericEventPayload = {"__extension_event__": True, "data": data} + return _create(event_type=event_type, emitted_by=emitted_by, payload=payload) + + +def phase_event( + *, + event_type: str, + emitted_by: str, + payload: dict[str, Any], + correlation_id: str | None = None, + parent_event_id: str | None = None, +) -> EventEnvelope: + """Create a typed phase event emitted through the shared emitter. + + Phase handlers still have heterogeneous payloads; routing through this factory + centralizes envelope creation while dedicated payload types are added + incrementally. + """ + return _create( + event_type=event_type, + emitted_by=emitted_by, + payload=payload, + correlation_id=correlation_id, + parent_event_id=parent_event_id, + ) diff --git a/netengine/events/payloads.py b/netengine/events/payloads.py new file mode 100644 index 0000000..16945b9 --- /dev/null +++ b/netengine/events/payloads.py @@ -0,0 +1,87 @@ +"""Typed payload definitions for known NetEngine events.""" + +from typing import Any, Literal, TypeAlias, TypedDict + + +class OrgAdmittedPayload(TypedDict): + org_name: str + capabilities: list[str] + and_profile: str + + +class OrgUpdatedPayload(TypedDict): + org_name: str + capabilities: list[str] + and_profile: str + + +class OrgRemovedPayload(TypedDict): + org_name: str + + +class DomainRegisteredPayload(TypedDict): + domain: str + org: str + ns: list[str] + + +class DriftDetectedPayload(TypedDict): + phase: int + handler: str + detected_at: str + + +class DriftLoopErrorPayload(TypedDict): + error: str + error_at: str + + +class DriftSelfHealedPayload(TypedDict): + phase: int + healed_at: str + + +class DriftSelfHealFailedPayload(TypedDict): + phase: int + error: str + failed_at: str + + +class GenericEventPayload(TypedDict, total=False): + """Fallback payload for explicitly unknown or extension events.""" + + __extension_event__: bool + data: dict[str, Any] + + +KnownEventType: TypeAlias = Literal[ + "org.admitted", + "org.updated", + "org.removed", + "domain.registered", + "drift.detected", + "drift.loop_error", + "drift.self_healed", + "drift.self_heal_failed", +] + +PhaseEventType: TypeAlias = Literal[ + "substrate.initialized", + "dns.zones_ready", + "pki.ready", + "inworld_identity.ready", +] + +EventType: TypeAlias = KnownEventType | PhaseEventType + +KnownEventPayload: TypeAlias = ( + OrgAdmittedPayload + | OrgUpdatedPayload + | OrgRemovedPayload + | DomainRegisteredPayload + | DriftDetectedPayload + | DriftLoopErrorPayload + | DriftSelfHealedPayload + | DriftSelfHealFailedPayload + | dict[str, Any] +) diff --git a/netengine/handlers/domain_registry_handler.py b/netengine/handlers/domain_registry_handler.py index 7f30ae7..f3423c6 100644 --- a/netengine/handlers/domain_registry_handler.py +++ b/netengine/handlers/domain_registry_handler.py @@ -1,18 +1,18 @@ # netengine/handlers/domain_registry_handler.py -from typing import Any, List +from typing import Any, List, cast from netengine.core.pgmq_client import PGMQClient from netengine.errors import RegistryError +from netengine.events import factory as event_factory from netengine.events.queues import Queue -from netengine.events.schema import EventEnvelope class DomainRegistryHandler: - def __init__(self): + def __init__(self) -> None: self._db = None self.pgmq = PGMQClient() - async def _get_db(self): + async def _get_db(self) -> Any: if self._db is None: from netengine.core.supabase_client import get_db @@ -36,7 +36,7 @@ async def allocate_address(self, and_name: str, profile: str) -> str: raise RegistryError(f"No address pool for profile {profile}") pool_cidr = result.data[0]["cidr"] await db.table("address_leases").upsert({"and_name": and_name, "cidr": pool_cidr}).execute() - return pool_cidr + return cast(str, pool_cidr) async def register_domain(self, domain: str, org_name: str, ns_records: List[str]) -> None: """Register a domain; emit DNS update event.""" @@ -44,9 +44,7 @@ async def register_domain(self, domain: str, org_name: str, ns_records: List[str await db.table("domain_records").upsert( {"domain": domain, "org_name": org_name, "ns_records": ns_records} ).execute() - event = EventEnvelope.create( - event_type="domain.registered", - emitted_by="domain_registry_handler", - payload={"domain": domain, "org": org_name, "ns": ns_records}, + event = event_factory.domain_registered( + domain=domain, org_name=org_name, ns_records=ns_records ) await self.pgmq.send(Queue.DNS_UPDATES, event) diff --git a/netengine/handlers/world_registry_handler.py b/netengine/handlers/world_registry_handler.py index e63a843..b86e63c 100644 --- a/netengine/handlers/world_registry_handler.py +++ b/netengine/handlers/world_registry_handler.py @@ -1,16 +1,16 @@ -from typing import Any, List +from typing import Any, List, cast from netengine.core.pgmq_client import PGMQClient +from netengine.events import factory as event_factory from netengine.events.queues import Queue -from netengine.events.schema import EventEnvelope class WorldRegistryHandler: - def __init__(self): + def __init__(self) -> None: self._db = None self.pgmq = PGMQClient() - async def _get_db(self): + async def _get_db(self) -> Any: if self._db is None: from netengine.core.supabase_client import get_db @@ -32,10 +32,8 @@ async def admit_org(self, name: str, capabilities: List[str], and_profile: str) db = await self._get_db() data = {"org_name": name, "capabilities": capabilities, "and_profile": and_profile} await db.table("world_registry").upsert(data).execute() - event = EventEnvelope.create( - event_type="org.admitted", - emitted_by="world_registry_handler", - payload={"org_name": name, "capabilities": capabilities, "and_profile": and_profile}, + event = event_factory.org_admitted( + org_name=name, capabilities=capabilities, and_profile=and_profile ) await self.pgmq.send(Queue.OIDC_PROVISIONING, event) await self.pgmq.send(Queue.AND_PROVISIONING, event) @@ -58,10 +56,8 @@ async def update_org(self, name: str, capabilities: List[str], and_profile: str) await db.table("world_registry").update( {"capabilities": capabilities, "and_profile": and_profile} ).eq("org_name", name).execute() - event = EventEnvelope.create( - event_type="org.updated", - emitted_by="world_registry_handler", - payload={"org_name": name, "capabilities": capabilities, "and_profile": and_profile}, + event = event_factory.org_updated( + org_name=name, capabilities=capabilities, and_profile=and_profile ) await self.pgmq.send(Queue.OIDC_PROVISIONING, event) await self.pgmq.send(Queue.AND_PROVISIONING, event) @@ -73,11 +69,7 @@ async def remove_org(self, name: str) -> bool: if not existing: return False await db.table("world_registry").delete().eq("org_name", name).execute() - event = EventEnvelope.create( - event_type="org.removed", - emitted_by="world_registry_handler", - payload={"org_name": name}, - ) + event = event_factory.org_removed(org_name=name) await self.pgmq.send(Queue.OIDC_PROVISIONING, event) await self.pgmq.send(Queue.AND_PROVISIONING, event) return True diff --git a/pyproject.toml b/pyproject.toml index 4fe6c5a..c291fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,12 +53,10 @@ exclude = [ "netengine/handlers/oidc_handler\\.py", "netengine/handlers/docker_handler\\.py", "netengine/handlers/and_handler\\.py", - "netengine/handlers/domain_registry_handler\\.py", "netengine/handlers/mail_handler\\.py", "netengine/handlers/minio_handler\\.py", "netengine/handlers/app_handler\\.py", "netengine/handlers/whois_server\\.py", - "netengine/handlers/world_registry_handler\\.py", "netengine/logging/middleware\\.py", "netengine/logging/sinks\\.py", ] diff --git a/tests/test_event_factories.py b/tests/test_event_factories.py new file mode 100644 index 0000000..3e55eb3 --- /dev/null +++ b/tests/test_event_factories.py @@ -0,0 +1,39 @@ +"""Tests for typed event factories.""" + +from netengine.events import factory as event_factory + + +def test_world_registry_factories_set_known_types_and_payloads() -> None: + admitted = event_factory.org_admitted( + org_name="acme", capabilities=["dns"], and_profile="residential" + ) + assert admitted.event_type == "org.admitted" + assert admitted.emitted_by == "world_registry_handler" + assert admitted.payload == { + "org_name": "acme", + "capabilities": ["dns"], + "and_profile": "residential", + } + + removed = event_factory.org_removed(org_name="acme") + assert removed.event_type == "org.removed" + assert removed.payload == {"org_name": "acme"} + + +def test_domain_and_drift_factories_set_known_types() -> None: + domain = event_factory.domain_registered( + domain="acme.internal", org_name="acme", ns_records=["ns1.internal"] + ) + assert domain.event_type == "domain.registered" + assert domain.payload == {"domain": "acme.internal", "org": "acme", "ns": ["ns1.internal"]} + + drift = event_factory.drift_detected(phase=3, handler="DNSHandler", detected_at="now") + assert drift.event_type == "drift.detected" + assert drift.payload == {"phase": 3, "handler": "DNSHandler", "detected_at": "now"} + + +def test_extension_factory_marks_explicit_unknown_payloads() -> None: + event = event_factory.extension_event( + event_type="vendor.custom", emitted_by="extension", data={"key": "value"} + ) + assert event.payload == {"__extension_event__": True, "data": {"key": "value"}} From d1bdc698d0ddf3462445e7e1c3b0e3886699d2b8 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:23:55 +0100 Subject: [PATCH 29/34] Add CLI doctor preflight command --- README.md | 13 +- docs/runbook.md | 4 +- netengine/cli/doctor.py | 411 ++++++++++++++++++++++++++++++++++++++++ netengine/cli/main.py | 4 + tests/test_doctor.py | 121 ++++++++++++ 5 files changed, 547 insertions(+), 6 deletions(-) create mode 100644 netengine/cli/doctor.py create mode 100644 tests/test_doctor.py diff --git a/README.md b/README.md index 52b4761..2c440f6 100644 --- a/README.md +++ b/README.md @@ -50,17 +50,20 @@ cd NetEngine # 2. Install dependencies poetry install -# 3. Start local Postgres + pgmq (includes pgmq extension pre-installed) -docker compose up -d db +# 3. Verify host prerequisites +poetry run netengine doctor -# 4. Apply migrations +# 4. Start local Postgres + pgmq (includes pgmq extension pre-installed) +docker compose up -d postgres + +# 5. Apply migrations poetry run python -m netengine.utils.run_migrations -# 5. Boot a minimal world +# 6. Boot a minimal world poetry run netengine up examples/minimal.yaml ``` -Check status at any time: +If you only want host/container checks before configuring Postgres, run `poetry run netengine doctor --skip-db`. Check status at any time: ```bash poetry run netengine status diff --git a/docs/runbook.md b/docs/runbook.md index f829046..811c257 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -44,14 +44,16 @@ This brings up the real persistence and identity layers. ### 1. Start backing services ```bash +poetry run netengine doctor --skip-db docker compose up -d +poetry run netengine doctor ``` This starts: - `netengine_postgres` on port 5432 (with pgmq extension) - Keycloak on port 8080 (platform identity, used in Phase 4) -Wait for postgres to be healthy: +The doctor command checks Python, Docker/Compose, required ports, writable runtime paths, database connectivity, pgmq, and stale Docker/state conflicts. Wait for postgres to be healthy: ```bash docker compose ps # all services should show "healthy" diff --git a/netengine/cli/doctor.py b/netengine/cli/doctor.py new file mode 100644 index 0000000..0e69551 --- /dev/null +++ b/netengine/cli/doctor.py @@ -0,0 +1,411 @@ +"""Host preflight checks for the NetEngine CLI.""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import tempfile +from dataclasses import asdict, dataclass +from enum import StrEnum +from pathlib import Path +from typing import Iterable +from urllib.parse import urlparse + +import click +import yaml + +from netengine.core.state import get_state_file + + +class DoctorStatus(StrEnum): + """Status values emitted by the doctor command.""" + + OK = "ok" + WARN = "warn" + FAIL = "fail" + SKIP = "skip" + + +@dataclass(frozen=True) +class DoctorCheckResult: + """Single preflight check result.""" + + name: str + status: DoctorStatus + detail: str + hint: str | None = None + group: str = "general" + required: bool = True + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _run(command: list[str], *, timeout: float = 8.0) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False) + + +def _compose_config() -> dict: + compose_file = _repo_root() / "docker-compose.yml" + try: + return yaml.safe_load(compose_file.read_text()) or {} + except Exception: + return {} + + +def _compose_ports_and_resources() -> tuple[set[tuple[int, str]], set[str], set[str], set[str]]: + config = _compose_config() + ports: set[tuple[int, str]] = set() + containers: set[str] = set() + volumes: set[str] = set((config.get("volumes") or {}).keys()) + networks: set[str] = set((config.get("networks") or {}).keys()) + for service_name, service in (config.get("services") or {}).items(): + containers.add(service.get("container_name") or f"netengine-{service_name}-1") + for volume in service.get("volumes") or []: + if isinstance(volume, str) and volume and not volume.startswith((".", "/", "~")): + volumes.add(volume.split(":", 1)[0]) + for port in service.get("ports") or []: + raw = str(port) + published = raw.split(":", 1)[0] if ":" in raw else raw + if published.isdigit(): + ports.add((int(published), "tcp")) + ports.update({(5432, "tcp"), (53, "tcp"), (53, "udp")}) + return ports, containers, volumes, networks + + +def _check_python() -> DoctorCheckResult: + ok = sys.version_info >= (3, 13) + return DoctorCheckResult( + "Python runtime", + DoctorStatus.OK if ok else DoctorStatus.FAIL, + f"Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}; pyproject requires ^3.13", + None if ok else "Install Python 3.13 or newer and recreate the virtualenv.", + "host", + ) + + +def _check_command(name: str, *, required: bool = True) -> DoctorCheckResult: + import shutil + + path = shutil.which(name) + return DoctorCheckResult( + f"command:{name}", + DoctorStatus.OK if path else (DoctorStatus.FAIL if required else DoctorStatus.SKIP), + path or "not found on PATH", + None if path else f"Install {name} or add it to PATH.", + "host", + required, + ) + + +def _check_docker_daemon() -> DoctorCheckResult: + try: + result = _run(["docker", "info"]) + except Exception as exc: + return DoctorCheckResult( + "Docker daemon", + DoctorStatus.FAIL, + str(exc), + "Start Docker and verify socket access.", + "docker", + ) + if result.returncode == 0: + return DoctorCheckResult( + "Docker daemon", DoctorStatus.OK, "docker info succeeded", group="docker" + ) + return DoctorCheckResult( + "Docker daemon", + DoctorStatus.FAIL, + (result.stderr or result.stdout).strip() or "docker info failed", + "Start Docker and verify the current user can access it.", + "docker", + ) + + +def _check_compose() -> DoctorCheckResult: + try: + result = _run(["docker", "compose", "version"]) + except Exception as exc: + return DoctorCheckResult( + "Docker Compose", + DoctorStatus.FAIL, + str(exc), + "Install the Docker Compose plugin.", + "docker", + ) + if result.returncode == 0: + return DoctorCheckResult( + "Docker Compose", + DoctorStatus.OK, + (result.stdout or "available").strip(), + group="docker", + ) + return DoctorCheckResult( + "Docker Compose", + DoctorStatus.FAIL, + (result.stderr or result.stdout).strip() or "docker compose version failed", + "Install or enable Docker Compose v2.", + "docker", + ) + + +def _parse_db_url(db_url: str | None) -> DoctorCheckResult: + if not db_url: + return DoctorCheckResult( + "Database URL", + DoctorStatus.FAIL, + "NETENGINE_DB_URL/DATABASE_URL is not set", + "Set NETENGINE_DB_URL=postgresql://netengine:dev_password@localhost:5432/netengine", + "database", + ) + parsed = urlparse(db_url) + ok = ( + parsed.scheme in {"postgres", "postgresql"} + and bool(parsed.hostname) + and bool(parsed.path.strip("/")) + ) + return DoctorCheckResult( + "Database URL", + DoctorStatus.OK if ok else DoctorStatus.FAIL, + "parseable PostgreSQL URL" if ok else "not a valid PostgreSQL URL", + None if ok else "Use a postgresql:// URL with host and database name.", + "database", + ) + + +def _check_psql(db_url: str, sql: str, name: str, *, hint: str) -> DoctorCheckResult: + try: + result = _run(["psql", db_url, "-tAc", sql]) + except Exception as exc: + return DoctorCheckResult(name, DoctorStatus.FAIL, str(exc), hint, "database") + if result.returncode == 0: + detail = (result.stdout or "ok").strip() + if name == "pgmq extension" and detail != "pgmq": + return DoctorCheckResult( + name, DoctorStatus.FAIL, "pgmq extension is not installed", hint, "database" + ) + return DoctorCheckResult(name, DoctorStatus.OK, detail, group="database") + return DoctorCheckResult( + name, + DoctorStatus.FAIL, + (result.stderr or result.stdout).strip() or "psql command failed", + hint, + "database", + ) + + +def _check_dir_writable(path: Path, name: str) -> DoctorCheckResult: + try: + path.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path, prefix=".netengine-doctor-", delete=True): + pass + return DoctorCheckResult(name, DoctorStatus.OK, f"writable: {path}", group="filesystem") + except Exception as exc: + return DoctorCheckResult( + name, + DoctorStatus.FAIL, + f"not writable: {path} ({exc})", + "Fix directory ownership or choose a writable path.", + "filesystem", + ) + + +def _check_state_file(path: Path) -> DoctorCheckResult: + if not path.exists(): + return DoctorCheckResult( + "State file", DoctorStatus.OK, f"no existing state at {path}", group="filesystem" + ) + try: + data = json.loads(path.read_text()) + phases = data.get("phase_completed", {}) if isinstance(data, dict) else {} + return DoctorCheckResult( + "State file", + DoctorStatus.WARN, + f"existing state file with {len(phases)} completed phase flag(s): {path}", + "Remove or back up the state file before bootstrapping a new world.", + "filesystem", + required=False, + ) + except Exception as exc: + return DoctorCheckResult( + "State file", + DoctorStatus.WARN, + f"existing unreadable/corrupt state file: {exc}", + "Inspect or remove the state file before continuing.", + "filesystem", + required=False, + ) + + +def _can_bind(port: int, proto: str) -> bool: + typ = socket.SOCK_DGRAM if proto == "udp" else socket.SOCK_STREAM + with socket.socket(socket.AF_INET, typ) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + return True + + +def _check_port(port: int, proto: str) -> DoctorCheckResult: + try: + _can_bind(port, proto) + return DoctorCheckResult( + f"port:{port}/{proto}", DoctorStatus.OK, "available on 127.0.0.1", group="ports" + ) + except PermissionError as exc: + return DoctorCheckResult( + f"port:{port}/{proto}", + DoctorStatus.WARN, + f"permission denied while probing: {exc}", + "Run with privileges or check listeners manually.", + "ports", + required=False, + ) + except OSError as exc: + return DoctorCheckResult( + f"port:{port}/{proto}", + DoctorStatus.FAIL, + f"unavailable on 127.0.0.1: {exc}", + f"Stop the process using {port}/{proto} or change the compose/spec port.", + "ports", + ) + + +def _docker_names(kind: str) -> set[str]: + try: + if kind == "container": + result = _run(["docker", "ps", "-a", "--format", "{{.Names}}"]) + elif kind == "volume": + result = _run(["docker", "volume", "ls", "--format", "{{.Name}}"]) + else: + result = _run(["docker", "network", "ls", "--format", "{{.Name}}"]) + except Exception: + return set() + return ( + {line.strip() for line in result.stdout.splitlines() if line.strip()} + if result.returncode == 0 + else set() + ) + + +def _check_docker_conflicts() -> list[DoctorCheckResult]: + _, containers, volumes, networks = _compose_ports_and_resources() + checks = [] + for kind, expected in (("container", containers), ("volume", volumes), ("network", networks)): + conflicts = sorted(expected & _docker_names(kind)) + checks.append( + DoctorCheckResult( + f"Docker {kind} names", + DoctorStatus.WARN if conflicts else DoctorStatus.OK, + ", ".join(conflicts) if conflicts else "no known name conflicts", + ( + "Run `netengine down` or remove stale Docker resources if these belong to an old run." + if conflicts + else None + ), + "docker", + required=False, + ) + ) + return checks + + +def run_checks( + db_url: str | None, state_file: Path, *, skip_db: bool = False +) -> list[DoctorCheckResult]: + checks = [ + _check_python(), + _check_command("docker"), + _check_command("psql"), + _check_command("step", required=False), + _check_docker_daemon(), + _check_compose(), + ] + if skip_db: + checks.append( + DoctorCheckResult( + "Database checks", + DoctorStatus.SKIP, + "--skip-db requested", + group="database", + required=False, + ) + ) + else: + parsed = _parse_db_url(db_url) + checks.append(parsed) + if db_url and parsed.status == DoctorStatus.OK: + checks.append( + _check_psql( + db_url, + "SELECT 1;", + "Postgres connectivity", + hint="Start Postgres or fix NETENGINE_DB_URL.", + ) + ) + checks.append( + _check_psql( + db_url, + "SELECT extname FROM pg_extension WHERE extname = 'pgmq';", + "pgmq extension", + hint="Install/enable pgmq in the configured database.", + ) + ) + ports, _, _, _ = _compose_ports_and_resources() + checks.extend(_check_port(port, proto) for port, proto in sorted(ports)) + checks.extend( + [ + _check_dir_writable(state_file.parent, "State directory"), + _check_dir_writable(Path.home() / ".netengine", "User runtime directory"), + _check_state_file(state_file), + ] + ) + checks.extend(_check_docker_conflicts()) + return checks + + +def _print_report(results: Iterable[DoctorCheckResult]) -> None: + by_group: dict[str, list[DoctorCheckResult]] = {} + for result in results: + by_group.setdefault(result.group, []).append(result) + symbols = { + DoctorStatus.OK: "✓", + DoctorStatus.WARN: "!", + DoctorStatus.FAIL: "✗", + DoctorStatus.SKIP: "-", + } + for group, group_results in by_group.items(): + click.echo(f"\n{group.title()}") + for result in group_results: + click.echo(f" {symbols[result.status]} {result.name}: {result.detail}") + if result.hint and result.status != DoctorStatus.OK: + click.echo(f" → {result.hint}") + + +@click.command("doctor") +@click.option( + "--db-url", + default=lambda: os.environ.get("NETENGINE_DB_URL") or os.environ.get("DATABASE_URL"), + help="PostgreSQL URL (defaults to NETENGINE_DB_URL or DATABASE_URL).", +) +@click.option( + "--state-file", + type=click.Path(path_type=Path), + default=get_state_file, + help="Runtime state file path.", +) +@click.option("--json", "as_json", is_flag=True, help="Output machine-readable JSON.") +@click.option("--skip-db", is_flag=True, help="Skip database connectivity and pgmq checks.") +def doctor(db_url: str | None, state_file: Path, as_json: bool, skip_db: bool) -> None: + """Run local host preflight checks before bootstrapping a world.""" + results = run_checks(db_url, state_file, skip_db=skip_db) + if as_json: + click.echo(json.dumps([asdict(r) for r in results], indent=2)) + else: + click.echo("NetEngine doctor preflight report") + _print_report(results) + if any(r.status == DoctorStatus.FAIL and r.required for r in results): + raise click.ClickException("required doctor checks failed") diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 3d23b24..77c5867 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -9,6 +9,7 @@ import click import yaml +from netengine.cli.doctor import doctor from netengine.core.migrations import MigrationService, MigrationStatus from netengine.core.orchestrator import Orchestrator from netengine.core.state import RuntimeState @@ -162,6 +163,9 @@ def cli() -> None: """NetEngine — spin up, reload, and tear down authority-autonomous worlds.""" +cli.add_command(doctor) + + @cli.group() def migrate() -> None: """Manage NetEngine database migrations.""" diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 0000000..d8f5838 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,121 @@ +"""Doctor command tests.""" + +from pathlib import Path +from types import SimpleNamespace + +from click.testing import CliRunner + +from netengine.cli import doctor as doctor_mod +from netengine.cli import main as cli_main +from netengine.cli.doctor import DoctorCheckResult, DoctorStatus + + +def test_doctor_appears_in_help() -> None: + result = CliRunner().invoke(cli_main.cli, ["--help"]) + + assert result.exit_code == 0, result.output + assert "doctor" in result.output + assert "Run local host preflight checks" in result.output + + +def test_doctor_json_uses_env_db_url_and_returns_nonzero_on_required_failure( + monkeypatch, tmp_path: Path +) -> None: + captured = {} + + def fake_run_checks(db_url: str | None, state_file: Path, *, skip_db: bool): + captured["db_url"] = db_url + captured["state_file"] = state_file + captured["skip_db"] = skip_db + return [ + DoctorCheckResult("Python runtime", DoctorStatus.OK, "ok", group="host"), + DoctorCheckResult("Docker daemon", DoctorStatus.FAIL, "not reachable", group="docker"), + ] + + state_file = tmp_path / "state.json" + monkeypatch.setenv("NETENGINE_DB_URL", "postgresql://netengine:dev@localhost:5432/netengine") + monkeypatch.setattr(doctor_mod, "run_checks", fake_run_checks) + + result = CliRunner().invoke(cli_main.cli, ["doctor", "--json", "--state-file", str(state_file)]) + + assert result.exit_code == 1 + assert '"status": "fail"' in result.output + assert captured == { + "db_url": "postgresql://netengine:dev@localhost:5432/netengine", + "state_file": state_file, + "skip_db": False, + } + + +def test_doctor_skip_db_succeeds_when_only_db_checks_are_skipped( + monkeypatch, tmp_path: Path +) -> None: + def fake_run_checks(db_url: str | None, state_file: Path, *, skip_db: bool): + assert skip_db is True + return [ + DoctorCheckResult( + "Database checks", + DoctorStatus.SKIP, + "--skip-db requested", + group="database", + required=False, + ) + ] + + monkeypatch.setattr(doctor_mod, "run_checks", fake_run_checks) + + result = CliRunner().invoke( + cli_main.cli, ["doctor", "--skip-db", "--state-file", str(tmp_path / "state.json")] + ) + + assert result.exit_code == 0, result.output + assert "Database checks" in result.output + + +def test_run_checks_invokes_psql_for_connectivity_and_pgmq(monkeypatch, tmp_path: Path) -> None: + commands = [] + + def fake_which(name: str) -> str | None: + return f"/usr/bin/{name}" + + def fake_run(command: list[str], *, timeout: float = 8.0): + commands.append(command) + stdout = "pgmq\n" if "pg_extension" in " ".join(command) else "1\n" + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + monkeypatch.setattr("shutil.which", fake_which) + monkeypatch.setattr(doctor_mod, "_run", fake_run) + monkeypatch.setattr(doctor_mod, "_can_bind", lambda port, proto: True) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + + results = doctor_mod.run_checks( + "postgresql://netengine:dev@localhost:5432/netengine", + tmp_path / "state.json", + skip_db=False, + ) + + assert any(r.name == "Postgres connectivity" and r.status == DoctorStatus.OK for r in results) + assert any(r.name == "pgmq extension" and r.status == DoctorStatus.OK for r in results) + assert [ + "psql", + "postgresql://netengine:dev@localhost:5432/netengine", + "-tAc", + "SELECT 1;", + ] in commands + assert [ + "psql", + "postgresql://netengine:dev@localhost:5432/netengine", + "-tAc", + "SELECT extname FROM pg_extension WHERE extname = 'pgmq';", + ] in commands + + +def test_state_file_conflict_is_warning(tmp_path: Path) -> None: + state_file = tmp_path / "state.json" + state_file.write_text('{"phase_completed": {"0": true}}') + + result = doctor_mod._check_state_file(state_file) + + assert result.status == DoctorStatus.WARN + assert result.required is False + assert "existing state file" in result.detail From fadf7fdd17896af10284da73fdc71512f6e00417 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:25:59 +0100 Subject: [PATCH 30/34] Type runtime phase outputs --- netengine/core/state.py | 202 +++++++++++++++++++++++++++----- netengine/events/emitter.py | 6 +- netengine/handlers/dns.py | 5 +- netengine/handlers/substrate.py | 5 +- 4 files changed, 183 insertions(+), 35 deletions(-) diff --git a/netengine/core/state.py b/netengine/core/state.py index b12660f..b5331f3 100644 --- a/netengine/core/state.py +++ b/netengine/core/state.py @@ -4,7 +4,7 @@ from dataclasses import asdict, dataclass, field from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional, TypedDict, cast from netengine.logging import get_logger @@ -16,6 +16,149 @@ DEFAULT_STATE_FILE = "netengines_state.json" +JsonPrimitive = str | int | float | bool | None +JsonValue = JsonPrimitive | list["JsonValue"] | dict[str, "JsonValue"] + + +class EventSendFailure(TypedDict): + event_type: str + queue: str + emitted_by: str + exception: str + event_id: str + correlation_id: str | None + parent_event_id: str | None + failed_at: str + + +class PhaseOutputBase(TypedDict, total=False): + deployed_at: str + event_send_failures: list[EventSendFailure] + + +class OrchestratorOutput(TypedDict, total=False): + type: str + status: str + healthy: bool + version: str + initialized_at: str + + +class NetworkOutput(TypedDict, total=False): + id: str + name: str + cidr: str + gateway: str + driver: str + + +class GatewayOutput(TypedDict, total=False): + networks: dict[str, str] + ip_addresses: dict[str, str] + healthy: bool + status: str + + +class NTPOutput(TypedDict, total=False): + enabled: bool + servers: list[str] + synchronized: bool + status: str + + +class SubstrateOutput(PhaseOutputBase, total=False): + orchestrator: OrchestratorOutput + networks: dict[str, NetworkOutput] + gateway: GatewayOutput + ntp: NTPOutput + + +class DNSZoneOutput(TypedDict, total=False): + name: str + soa: str + records: list[dict[str, JsonValue]] + listen_ip: str + healthy: bool + + +class TLDOutput(TypedDict, total=False): + name: str + listen_ip: str + records: list[dict[str, JsonValue]] + + +class DNSOutput(PhaseOutputBase, total=False): + root_zone: DNSZoneOutput + platform_zone: DNSZoneOutput + tlds: dict[str, TLDOutput] + zone_files: dict[str, str] + coredns_container_id: str + healthy: bool + + +class PKIOutput(PhaseOutputBase, total=False): + ca_ip: str + ca_dns: str + container_id: str | None + bootstrapped: bool + mock: bool + crl_url: str + crl_enabled: bool + ocsp_url: str + ocsp_enabled: bool + intermediate_ca_enabled: bool + intermediate_ca_cert_available: bool + intermediate_ca_cert: str + dnssec_enabled: bool + dnssec_zone: str + dnssec_ksk: str + dnssec_zsk: str + + +class IdentityPlatformOutput(PhaseOutputBase, total=False): + keycloak_container_id: str + platform_realm_id: str + admin_user_id: str + platform_client_id: str + platform_client_auth_id: str + platform_client_secret: str + + +class WorldRegistryOutput(PhaseOutputBase, total=False): + seeded: bool + + +class DomainRegistryOutput(PhaseOutputBase, total=False): + address_pools_seeded: bool + tld_delegations: list[dict[str, JsonValue]] + + +class GenericPhaseOutput(PhaseOutputBase, total=False): + status: str + healthy: bool + + +class DriftHistoryEvent(TypedDict): + phase_num: int + detected_at: str + healed_at: str | None + healing_failed: bool + error: str | None + + +class IssuedCertificateMetadata(TypedDict): + cert_type: str + issued_at: str | datetime + expires_at: str | datetime + sans: list[str] + rotated_at: str | datetime | None + version: int + + +class PKIRotationState(TypedDict, total=False): + last_check_by_type: dict[str, str | datetime] + + def get_state_file() -> Path: """Return the runtime state file path for the current environment.""" return Path(os.environ.get("NETENGINE_STATE_FILE", DEFAULT_STATE_FILE)) @@ -37,16 +180,16 @@ class RuntimeState: phase_completed: Dict[str, bool] = field(default_factory=dict) # Phase outputs (one dict per phase, populated on completion) - substrate_output: Optional[Dict[str, Any]] = None - dns_output: Optional[Dict[str, Any]] = None - pki_output: Optional[Dict[str, Any]] = None - identity_platform_output: Optional[Dict[str, Any]] = None - world_registry_output: Optional[Dict[str, Any]] = None - domain_registry_output: Optional[Dict[str, Any]] = None - identity_inworld_output: Optional[Dict[str, Any]] = None - ands_output: Optional[Dict[str, Any]] = None - world_services_output: Optional[Dict[str, Any]] = None - org_apps_output: Optional[Dict[str, Any]] = None + substrate_output: Optional[SubstrateOutput] = None + dns_output: Optional[DNSOutput] = None + pki_output: Optional[PKIOutput] = None + identity_platform_output: Optional[IdentityPlatformOutput] = None + world_registry_output: Optional[WorldRegistryOutput] = None + domain_registry_output: Optional[DomainRegistryOutput] = None + identity_inworld_output: Optional[GenericPhaseOutput] = None + ands_output: Optional[GenericPhaseOutput] = None + world_services_output: Optional[GenericPhaseOutput] = None + org_apps_output: Optional[GenericPhaseOutput] = None # Legacy container ID fields (kept for backward compat with existing handlers) gateway_container_id: Optional[str] = None @@ -66,12 +209,12 @@ class RuntimeState: platform_client_secret: Optional[str] = None # Drift detection and self-healing - drift_history: list[Dict[str, Any]] = field(default_factory=list) + drift_history: list[DriftHistoryEvent] = field(default_factory=list) last_drift_check_at: Optional[datetime] = None current_drift_phases: list[int] = field(default_factory=list) # PKI certificate rotation tracking - issued_certificates: Dict[str, Dict[str, Any]] = field(default_factory=dict) - pki_rotation_state: Dict[str, Any] = field(default_factory=dict) + issued_certificates: Dict[str, IssuedCertificateMetadata] = field(default_factory=dict) + pki_rotation_state: PKIRotationState = field(default_factory=lambda: cast(PKIRotationState, {})) # Extended PKI state intermediate_ca_cert: Optional[str] = None @@ -81,7 +224,7 @@ class RuntimeState: gateway_portal_output: Optional[Dict[str, Any]] = None # Recent structured failures from best-effort PGMQ event emission. - event_send_failures: list[Dict[str, Any]] = field(default_factory=list) + event_send_failures: list[EventSendFailure] = field(default_factory=list) @classmethod def load(cls) -> "RuntimeState": @@ -123,23 +266,24 @@ def load(cls) -> "RuntimeState": def _discard_completion_flags_without_outputs(self) -> None: """Ignore completion flags that do not have their matching phase output.""" phase_outputs = { - "0": ("substrate_output",), - "1": ("dns_output",), - "2": ("dns_output",), - "3": ("pki_bootstrapped",), - "4": ("identity_platform_output",), - "5": ("world_registry_output", "domain_registry_output"), - "6": ("identity_inworld_output",), - "7": ("ands_output",), - "8": ("world_services_output",), - "9": ("org_apps_output",), + "0": (self.substrate_output,), + "1": (self.dns_output,), + "2": (self.dns_output,), + "3": (self.pki_output,), + "4": (self.identity_platform_output,), + "5": (self.world_registry_output, self.domain_registry_output), + "6": (self.identity_inworld_output,), + "7": (self.ands_output,), + "8": (self.world_services_output,), + "9": (self.org_apps_output,), } - for phase, output_fields in phase_outputs.items(): - if self.phase_completed.get(phase) is True and not all( - getattr(self, output_field, None) for output_field in output_fields - ): + for phase, outputs in phase_outputs.items(): + if self.phase_completed.get(phase) is True and any(output is None for output in outputs): self.phase_completed.pop(phase, None) + if self.pki_output is None and self.phase_completed.get("3") is True: + self.phase_completed.pop("3", None) + def save(self) -> None: self._discard_completion_flags_without_outputs() data = asdict(self) diff --git a/netengine/events/emitter.py b/netengine/events/emitter.py index c1816ca..0e1bb52 100644 --- a/netengine/events/emitter.py +++ b/netengine/events/emitter.py @@ -3,12 +3,14 @@ from datetime import UTC, datetime from typing import Any +from netengine.core.state import EventSendFailure + from netengine.events.queues import Queue, queue_for_event_type from netengine.events.schema import EventEnvelope from netengine.handlers.context import PhaseContext -def _failure_record(event: EventEnvelope, queue: Queue | str, exc: Exception) -> dict[str, Any]: +def _failure_record(event: EventEnvelope, queue: Queue | str, exc: Exception) -> EventSendFailure: """Build the structured runtime-state record for an event send failure.""" return { "event_type": event.event_type, @@ -27,7 +29,7 @@ def record_event_send_failure( event: EventEnvelope, queue: Queue | str, exc: Exception, -) -> dict[str, Any]: +) -> EventSendFailure: """Persist structured details about a failed event send in runtime state. The list is intentionally small and state-file friendly. It gives callers and diff --git a/netengine/handlers/dns.py b/netengine/handlers/dns.py index 80bae27..cc4f15a 100644 --- a/netengine/handlers/dns.py +++ b/netengine/handlers/dns.py @@ -14,8 +14,9 @@ import asyncio from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, cast +from netengine.core.state import DNSOutput from netengine.errors import DNSError from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler @@ -118,7 +119,7 @@ async def execute(self, context: PhaseContext) -> None: dns_output["deployed_at"] = datetime.now(UTC).isoformat() - context.runtime_state.dns_output = dns_output + context.runtime_state.dns_output = cast(DNSOutput, dns_output) context.runtime_state.phase_completed["1"] = True context.runtime_state.phase_completed["2"] = True context.runtime_state.completed_at = datetime.now(UTC) diff --git a/netengine/handlers/substrate.py b/netengine/handlers/substrate.py index 75fdc74..265d079 100644 --- a/netengine/handlers/substrate.py +++ b/netengine/handlers/substrate.py @@ -9,10 +9,11 @@ """ from datetime import UTC, datetime -from typing import Any +from typing import Any, cast from netengine.errors import SubstrateError from netengine.events.emitter import emit_event +from netengine.core.state import SubstrateOutput from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext @@ -82,7 +83,7 @@ async def execute(self, context: PhaseContext) -> None: substrate_output["deployed_at"] = datetime.now(UTC).isoformat() - context.runtime_state.substrate_output = substrate_output + context.runtime_state.substrate_output = cast(SubstrateOutput, substrate_output) context.runtime_state.completed_at = datetime.now(UTC) logger.info("Phase 0: Substrate initialization complete") From 7445887718eaef275d40ba53ce7df6b2a653d6af Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:46:06 +0100 Subject: [PATCH 31/34] Add Docker adapter protocol for handlers --- netengine/handlers/and_handler.py | 4 +- netengine/handlers/app_handler.py | 4 +- netengine/handlers/context.py | 5 +- netengine/handlers/docker_handler.py | 98 +++++++++++++------- netengine/handlers/gateway_portal_handler.py | 14 ++- netengine/handlers/mail_handler.py | 4 +- netengine/handlers/minio_handler.py | 6 +- netengine/handlers/phase_pki.py | 13 ++- netengine/handlers/pki_handler.py | 4 +- netengine/handlers/protocols.py | 59 ++++++++++++ pyproject.toml | 1 - 11 files changed, 155 insertions(+), 57 deletions(-) create mode 100644 netengine/handlers/protocols.py diff --git a/netengine/handlers/and_handler.py b/netengine/handlers/and_handler.py index 8381e58..15beb22 100644 --- a/netengine/handlers/and_handler.py +++ b/netengine/handlers/and_handler.py @@ -7,14 +7,14 @@ from netengine.events.schema import EventEnvelope from netengine.handlers.context import PhaseContext from netengine.handlers.dns import DNSHandler -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol from netengine.handlers.domain_registry_handler import DomainRegistryHandler from netengine.handlers.gateway_handler import GatewayHandler from netengine.logging import get_logger class ANDHandler: - def __init__(self, docker: DockerHandler, state, context: PhaseContext | None = None): + def __init__(self, docker: DockerAdapterProtocol, state, context: PhaseContext | None = None): self.context = context or PhaseContext( spec={}, runtime_state=state, logger=get_logger(__name__) ) diff --git a/netengine/handlers/app_handler.py b/netengine/handlers/app_handler.py index bc7dc60..13c19d4 100644 --- a/netengine/handlers/app_handler.py +++ b/netengine/handlers/app_handler.py @@ -8,7 +8,7 @@ from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext from netengine.handlers.dns import DNSHandler -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol from netengine.handlers.oidc_handler import OIDCHandler from netengine.handlers.pki_handler import PKIHandler from netengine.logging import get_logger @@ -17,7 +17,7 @@ class AppHandler: def __init__( self, - docker: DockerHandler, + docker: DockerAdapterProtocol, dns: DNSHandler, pki: PKIHandler, oidc: OIDCHandler, diff --git a/netengine/handlers/context.py b/netengine/handlers/context.py index ceddb0b..f7f7c64 100644 --- a/netengine/handlers/context.py +++ b/netengine/handlers/context.py @@ -5,11 +5,12 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Optional +from netengine.handlers.protocols import DockerAdapterProtocol + from netengine.core.state import RuntimeState from netengine.spec.models import NetEngineSpec if TYPE_CHECKING: - import docker as docker_sdk from loguru import Logger from supabase import AsyncClient as SupabaseClient @@ -40,7 +41,7 @@ class PhaseContext: logger: "Logger" # Service clients (None until the relevant phase wires them up) - docker_client: Optional["docker_sdk.DockerClient"] = None + docker_client: Optional[DockerAdapterProtocol] = None kubernetes_client: Any = None supabase_client: Optional["SupabaseClient"] = None pgmq_client: Optional["PGMQClient"] = None diff --git a/netengine/handlers/docker_handler.py b/netengine/handlers/docker_handler.py index f6ac416..259dd99 100644 --- a/netengine/handlers/docker_handler.py +++ b/netengine/handlers/docker_handler.py @@ -1,21 +1,25 @@ # netengines/handlers/docker_handler.py import asyncio import os -from typing import Dict, List, Optional +from typing import Any import docker -from docker.types import IPAMConfig, IPAMPool class DockerHandler: - def __init__(self): + def __init__(self) -> None: self.client = docker.from_env() + @property + def containers(self) -> Any: + """Expose Docker SDK containers for legacy handlers during adapter migration.""" + return self.client.containers + async def ensure_volume(self, name: str) -> None: """Create a named volume if it doesn't exist.""" await asyncio.to_thread(self._ensure_volume_sync, name) - def _ensure_volume_sync(self, name: str): + def _ensure_volume_sync(self, name: str) -> None: try: self.client.volumes.get(name) except docker.errors.NotFound: @@ -23,8 +27,14 @@ def _ensure_volume_sync(self, name: str): # In DockerHandler async def run_container_one_off( - self, image, command, volumes, environment, working_dir=None, **kwargs - ): + self, + image: str, + command: list[str] | str, + volumes: dict[str, Any], + environment: dict[str, str] | None, + working_dir: str | None = None, + **kwargs: Any, + ) -> dict[str, Any]: return await asyncio.to_thread( self._run_container_one_off_sync, image, @@ -36,8 +46,14 @@ async def run_container_one_off( ) def _run_container_one_off_sync( - self, image, command, volumes, environment, working_dir, **kwargs - ): + self, + image: str, + command: list[str] | str, + volumes: dict[str, Any], + environment: dict[str, str] | None, + working_dir: str | None, + **kwargs: Any, + ) -> dict[str, Any]: container = self.client.containers.run( image=image, command=command, @@ -57,12 +73,12 @@ async def start_container( self, name: str, image: str, - command: List[str], - volumes: Dict[str, Dict[str, str]], - network: str, - ip: str, - environment: Dict[str, str], - **kwargs, + command: list[str] | str | None, + volumes: dict[str, Any], + network: str | None, + ip: str | None, + environment: dict[str, str] | None, + **kwargs: Any, ) -> str: """Start a long‑running container attached to a network with a fixed IP.""" return await asyncio.to_thread( @@ -78,10 +94,18 @@ async def start_container( ) def _start_container_sync( - self, name, image, command, volumes, network, ip, environment, **kwargs - ): + self, + name: str, + image: str, + command: list[str] | str | None, + volumes: dict[str, Any], + network: str | None, + ip: str | None, + environment: dict[str, str] | None, + **kwargs: Any, + ) -> str: # Ensure network exists (we assume it was created in Phase 0) - net = self.client.networks.get(network) + net = self.client.networks.get(network) if network is not None else None container = self.client.containers.run( image=image, command=command, @@ -93,14 +117,15 @@ def _start_container_sync( **kwargs, ) # Attach to network with specific IP - net.connect(container, ipv4_address=ip) - return container.id + if net is not None: + net.connect(container, ipv4_address=ip) + return str(container.id) - async def exec_command(self, container_id: str, cmd: List[str]) -> tuple[int, str]: + async def exec_command(self, container_id: str, cmd: list[str]) -> tuple[int, str]: """Execute a command inside a running container.""" return await asyncio.to_thread(self._exec_command_sync, container_id, cmd) - def _exec_command_sync(self, container_id, cmd): + def _exec_command_sync(self, container_id: str, cmd: list[str]) -> tuple[int, str]: container = self.client.containers.get(container_id) exec_result = container.exec_run(cmd, demux=False) output = exec_result.output or b"" @@ -109,18 +134,20 @@ def _exec_command_sync(self, container_id, cmd): async def stop_container(self, container_id: str) -> None: await asyncio.to_thread(self._stop_container_sync, container_id) - def _stop_container_sync(self, container_id): + def _stop_container_sync(self, container_id: str) -> None: container = self.client.containers.get(container_id) container.stop(timeout=10) container.remove() async def create_network( - self, name: str, driver: str = "bridge", subnet: str = None, internal: bool = False - ): + self, name: str, driver: str = "bridge", subnet: str | None = None, internal: bool = False + ) -> None: """Create a Docker network.""" await asyncio.to_thread(self._create_network_sync, name, driver, subnet, internal) - def _create_network_sync(self, name, driver, subnet, internal): + def _create_network_sync( + self, name: str, driver: str, subnet: str | None, internal: bool + ) -> None: ipam_pool = None if subnet: ipam_pool = docker.types.IPAMPool(subnet=subnet) @@ -129,24 +156,25 @@ def _create_network_sync(self, name, driver, subnet, internal): ipam_config = None self.client.networks.create(name=name, driver=driver, internal=internal, ipam=ipam_config) - async def connect_network(self, container: str, network: str, ip: str): + async def connect_network(self, container: str, network: str, ip: str | None) -> None: await asyncio.to_thread(self._connect_network_sync, container, network, ip) - def _connect_network_sync(self, container, network, ip): - net = self.client.networks.get(network) - net.connect(container, ipv4_address=ip) + def _connect_network_sync(self, container: str, network: str, ip: str | None) -> None: + net = self.client.networks.get(network) if network is not None else None + if net is not None: + net.connect(container, ipv4_address=ip) - async def disconnect_network(self, container: str, network: str): + async def disconnect_network(self, container: str, network: str) -> None: await asyncio.to_thread(self._disconnect_network_sync, container, network) - def _disconnect_network_sync(self, container, network): - net = self.client.networks.get(network) + def _disconnect_network_sync(self, container: str, network: str) -> None: + net = self.client.networks.get(network) if network is not None else None net.disconnect(container) - async def remove_network(self, name: str): + async def remove_network(self, name: str) -> None: await asyncio.to_thread(self._remove_network_sync, name) - def _remove_network_sync(self, name): + def _remove_network_sync(self, name: str) -> None: net = self.client.networks.get(name) net.remove() @@ -164,7 +192,7 @@ async def copy_to_container(self, container_id: str, src_path: str, dest_path: s tar_stream.seek(0) await asyncio.to_thread(self._copy_to_container_sync, container_id, tar_stream, dest_path) - def _copy_to_container_sync(self, container_id, tar_stream, dest_path): + def _copy_to_container_sync(self, container_id: str, tar_stream: Any, dest_path: str) -> None: container = self.client.containers.get(container_id) container.put_archive(os.path.dirname(dest_path), tar_stream) diff --git a/netengine/handlers/gateway_portal_handler.py b/netengine/handlers/gateway_portal_handler.py index a0edcf1..a130d57 100644 --- a/netengine/handlers/gateway_portal_handler.py +++ b/netengine/handlers/gateway_portal_handler.py @@ -19,7 +19,7 @@ from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol from netengine.handlers.gateway_handler import GatewayHandler from netengine.logging import get_logger from netengine.spec.models import CrossWorldPeer, GatewayPortal @@ -61,7 +61,11 @@ async def execute(self, context: PhaseContext) -> None: ) return - docker = context.docker_client or DockerHandler() # type: ignore[no-untyped-call] + if context.docker_client is None: + raise RuntimeError( + "Gateway portal requires context.docker_client when mock_mode is disabled" + ) + docker = context.docker_client gateway = GatewayHandler(docker) # ── Real Internet ─────────────────────────────────────────────────── @@ -162,7 +166,7 @@ async def _apply_cross_world( self, context: PhaseContext, gateway: GatewayHandler, - docker: DockerHandler, + docker: DockerAdapterProtocol, portal: GatewayPortal, ) -> dict[str, Any]: cross_world = portal.cross_world @@ -189,7 +193,7 @@ async def _setup_peer( self, context: PhaseContext, gateway: GatewayHandler, - docker: DockerHandler, + docker: DockerAdapterProtocol, peer: CrossWorldPeer, ) -> dict[str, Any]: """Wire up a single cross-world peer: trust anchor + routing + DNS.""" @@ -237,7 +241,7 @@ async def _setup_peer( async def _install_trust_anchor( self, context: PhaseContext, - docker: DockerHandler, + docker: DockerAdapterProtocol, peer_name: str, cert_pem: str, ) -> None: diff --git a/netengine/handlers/mail_handler.py b/netengine/handlers/mail_handler.py index bfc64fd..af89ba8 100644 --- a/netengine/handlers/mail_handler.py +++ b/netengine/handlers/mail_handler.py @@ -18,7 +18,7 @@ from netengine.handlers.context import PhaseContext from netengine.handlers.dns import DNSHandler -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol class MailHandler: @@ -27,7 +27,7 @@ class MailHandler: def __init__( self, context: PhaseContext, - docker: DockerHandler, + docker: DockerAdapterProtocol, dns: DNSHandler, ): self.context = context diff --git a/netengine/handlers/minio_handler.py b/netengine/handlers/minio_handler.py index cb56248..1cc9cd0 100644 --- a/netengine/handlers/minio_handler.py +++ b/netengine/handlers/minio_handler.py @@ -3,12 +3,14 @@ from datetime import UTC, datetime from netengine.handlers.dns import DNSHandler -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol from netengine.handlers.pki_handler import PKIHandler class StorageHandler: - def __init__(self, context, docker: DockerHandler, dns: DNSHandler, pki: PKIHandler, state): + def __init__( + self, context, docker: DockerAdapterProtocol, dns: DNSHandler, pki: PKIHandler, state + ): self.context = context self.docker = docker self.dns = dns diff --git a/netengine/handlers/phase_pki.py b/netengine/handlers/phase_pki.py index a540fd5..7e313e4 100644 --- a/netengine/handlers/phase_pki.py +++ b/netengine/handlers/phase_pki.py @@ -5,7 +5,7 @@ from netengine.events.emitter import emit_event from netengine.handlers._base import BasePhaseHandler from netengine.handlers.context import PhaseContext -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol from netengine.handlers.pki_handler import PKIHandler from netengine.logging import get_logger from netengine.workers.pki_cert_rotation_worker import CertTypeRotationConfig, PKICertRotationWorker @@ -43,8 +43,11 @@ async def execute(self, context: PhaseContext) -> None: ) return - # Use docker_client from context, falling back to a new DockerHandler - docker = context.docker_client if context.docker_client is not None else DockerHandler() + if context.docker_client is None: + raise RuntimeError( + "PKI phase requires context.docker_client when mock_mode is disabled" + ) + docker = context.docker_client pki = PKIHandler(docker, context.runtime_state, spec) # 1. Bootstrap CA (generate + start server) @@ -123,7 +126,9 @@ async def healthcheck(self, context: PhaseContext) -> bool: if context.mock_mode: return context.runtime_state.pki_output is not None try: - docker = context.docker_client if context.docker_client is not None else DockerHandler() + if context.docker_client is None: + return False + docker = context.docker_client pki = PKIHandler(docker, context.runtime_state, context.spec) return await pki.healthcheck() except Exception as exc: diff --git a/netengine/handlers/pki_handler.py b/netengine/handlers/pki_handler.py index 251081a..cc29374 100644 --- a/netengine/handlers/pki_handler.py +++ b/netengine/handlers/pki_handler.py @@ -12,14 +12,14 @@ from netengine.core.state import RuntimeState from netengine.errors import PKIError -from netengine.handlers.docker_handler import DockerHandler +from netengine.handlers.protocols import DockerAdapterProtocol from netengine.logging import get_logger logger = get_logger(__name__) class PKIHandler: - def __init__(self, docker: DockerHandler, state: RuntimeState, spec): + def __init__(self, docker: DockerAdapterProtocol, state: RuntimeState, spec): self.docker = docker self.state = state self.spec = spec diff --git a/netengine/handlers/protocols.py b/netengine/handlers/protocols.py new file mode 100644 index 0000000..7ed2a7c --- /dev/null +++ b/netengine/handlers/protocols.py @@ -0,0 +1,59 @@ +"""Shared protocols for handler dependencies.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class DockerAdapterProtocol(Protocol): + """Async Docker operations consumed by NetEngine handlers.""" + + client: Any + containers: Any + + async def ensure_volume(self, name: str) -> None: ... + + async def run_container_one_off( + self, + image: str, + command: list[str] | str, + volumes: dict[str, Any], + environment: dict[str, str] | None, + working_dir: str | None = None, + **kwargs: Any, + ) -> dict[str, Any]: ... + + async def start_container( + self, + name: str, + image: str, + command: list[str] | str | None, + volumes: dict[str, Any], + network: str | None, + ip: str | None, + environment: dict[str, str] | None, + **kwargs: Any, + ) -> str: ... + + async def exec_command(self, container_id: str, cmd: list[str]) -> tuple[int, str]: ... + + async def stop_container(self, container_id: str) -> None: ... + + async def create_network( + self, + name: str, + driver: str = "bridge", + subnet: str | None = None, + internal: bool = False, + ) -> None: ... + + async def connect_network(self, container: str, network: str, ip: str | None) -> None: ... + + async def disconnect_network(self, container: str, network: str) -> None: ... + + async def remove_network(self, name: str) -> None: ... + + async def copy_to_container(self, container_id: str, src_path: str, dest_path: str) -> None: ... + + async def signal_container(self, container_id: str, signal: str) -> None: ... diff --git a/pyproject.toml b/pyproject.toml index 4fe6c5a..9d0096c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ exclude = [ "netengine/handlers/pki_handler\\.py", "netengine/handlers/phase_pki\\.py", "netengine/handlers/oidc_handler\\.py", - "netengine/handlers/docker_handler\\.py", "netengine/handlers/and_handler\\.py", "netengine/handlers/domain_registry_handler\\.py", "netengine/handlers/mail_handler\\.py", From a874fd8974b90742e082dd2289417942dc8584b4 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:47:01 +0100 Subject: [PATCH 32/34] Add host-readiness preflight layer --- netengine/cli/doctor.py | 413 +++----------------------- netengine/diagnostic/__init__.py | 17 +- netengine/diagnostic/preflight.py | 463 ++++++++++++++++++++++++++++++ netengine/diagnostic/runner.py | 7 +- tests/test_doctor.py | 21 ++ 5 files changed, 553 insertions(+), 368 deletions(-) create mode 100644 netengine/diagnostic/preflight.py diff --git a/netengine/cli/doctor.py b/netengine/cli/doctor.py index 0e69551..d821b2b 100644 --- a/netengine/cli/doctor.py +++ b/netengine/cli/doctor.py @@ -1,388 +1,69 @@ -"""Host preflight checks for the NetEngine CLI.""" +"""CLI wrapper for NetEngine host-readiness checks. + +``netengine doctor`` validates local prerequisites before a spec is loaded or a +world is booted. Runtime diagnostics for an already configured/running world +live in :mod:`netengine.diagnostic.runner` and require ``NetEngineSpec``. +""" from __future__ import annotations import json import os -import socket -import subprocess -import sys -import tempfile -from dataclasses import asdict, dataclass -from enum import StrEnum +from dataclasses import asdict from pathlib import Path from typing import Iterable -from urllib.parse import urlparse import click -import yaml from netengine.core.state import get_state_file +from netengine.diagnostic import preflight as _preflight +from netengine.diagnostic.preflight import ( + DoctorCheckResult, + DoctorContext, + DoctorProbe, + DoctorStatus, + build_context, + run_preflight, +) - -class DoctorStatus(StrEnum): - """Status values emitted by the doctor command.""" - - OK = "ok" - WARN = "warn" - FAIL = "fail" - SKIP = "skip" - - -@dataclass(frozen=True) -class DoctorCheckResult: - """Single preflight check result.""" - - name: str - status: DoctorStatus - detail: str - hint: str | None = None - group: str = "general" - required: bool = True - - -def _repo_root() -> Path: - return Path(__file__).resolve().parents[2] - - -def _run(command: list[str], *, timeout: float = 8.0) -> subprocess.CompletedProcess[str]: - return subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False) - - -def _compose_config() -> dict: - compose_file = _repo_root() / "docker-compose.yml" - try: - return yaml.safe_load(compose_file.read_text()) or {} - except Exception: - return {} - - -def _compose_ports_and_resources() -> tuple[set[tuple[int, str]], set[str], set[str], set[str]]: - config = _compose_config() - ports: set[tuple[int, str]] = set() - containers: set[str] = set() - volumes: set[str] = set((config.get("volumes") or {}).keys()) - networks: set[str] = set((config.get("networks") or {}).keys()) - for service_name, service in (config.get("services") or {}).items(): - containers.add(service.get("container_name") or f"netengine-{service_name}-1") - for volume in service.get("volumes") or []: - if isinstance(volume, str) and volume and not volume.startswith((".", "/", "~")): - volumes.add(volume.split(":", 1)[0]) - for port in service.get("ports") or []: - raw = str(port) - published = raw.split(":", 1)[0] if ":" in raw else raw - if published.isdigit(): - ports.add((int(published), "tcp")) - ports.update({(5432, "tcp"), (53, "tcp"), (53, "udp")}) - return ports, containers, volumes, networks - - -def _check_python() -> DoctorCheckResult: - ok = sys.version_info >= (3, 13) - return DoctorCheckResult( - "Python runtime", - DoctorStatus.OK if ok else DoctorStatus.FAIL, - f"Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}; pyproject requires ^3.13", - None if ok else "Install Python 3.13 or newer and recreate the virtualenv.", - "host", - ) - - -def _check_command(name: str, *, required: bool = True) -> DoctorCheckResult: - import shutil - - path = shutil.which(name) - return DoctorCheckResult( - f"command:{name}", - DoctorStatus.OK if path else (DoctorStatus.FAIL if required else DoctorStatus.SKIP), - path or "not found on PATH", - None if path else f"Install {name} or add it to PATH.", - "host", - required, - ) - - -def _check_docker_daemon() -> DoctorCheckResult: - try: - result = _run(["docker", "info"]) - except Exception as exc: - return DoctorCheckResult( - "Docker daemon", - DoctorStatus.FAIL, - str(exc), - "Start Docker and verify socket access.", - "docker", - ) - if result.returncode == 0: - return DoctorCheckResult( - "Docker daemon", DoctorStatus.OK, "docker info succeeded", group="docker" - ) - return DoctorCheckResult( - "Docker daemon", - DoctorStatus.FAIL, - (result.stderr or result.stdout).strip() or "docker info failed", - "Start Docker and verify the current user can access it.", - "docker", - ) - - -def _check_compose() -> DoctorCheckResult: - try: - result = _run(["docker", "compose", "version"]) - except Exception as exc: - return DoctorCheckResult( - "Docker Compose", - DoctorStatus.FAIL, - str(exc), - "Install the Docker Compose plugin.", - "docker", - ) - if result.returncode == 0: - return DoctorCheckResult( - "Docker Compose", - DoctorStatus.OK, - (result.stdout or "available").strip(), - group="docker", - ) - return DoctorCheckResult( - "Docker Compose", - DoctorStatus.FAIL, - (result.stderr or result.stdout).strip() or "docker compose version failed", - "Install or enable Docker Compose v2.", - "docker", - ) - - -def _parse_db_url(db_url: str | None) -> DoctorCheckResult: - if not db_url: - return DoctorCheckResult( - "Database URL", - DoctorStatus.FAIL, - "NETENGINE_DB_URL/DATABASE_URL is not set", - "Set NETENGINE_DB_URL=postgresql://netengine:dev_password@localhost:5432/netengine", - "database", - ) - parsed = urlparse(db_url) - ok = ( - parsed.scheme in {"postgres", "postgresql"} - and bool(parsed.hostname) - and bool(parsed.path.strip("/")) - ) - return DoctorCheckResult( - "Database URL", - DoctorStatus.OK if ok else DoctorStatus.FAIL, - "parseable PostgreSQL URL" if ok else "not a valid PostgreSQL URL", - None if ok else "Use a postgresql:// URL with host and database name.", - "database", - ) - - -def _check_psql(db_url: str, sql: str, name: str, *, hint: str) -> DoctorCheckResult: - try: - result = _run(["psql", db_url, "-tAc", sql]) - except Exception as exc: - return DoctorCheckResult(name, DoctorStatus.FAIL, str(exc), hint, "database") - if result.returncode == 0: - detail = (result.stdout or "ok").strip() - if name == "pgmq extension" and detail != "pgmq": - return DoctorCheckResult( - name, DoctorStatus.FAIL, "pgmq extension is not installed", hint, "database" - ) - return DoctorCheckResult(name, DoctorStatus.OK, detail, group="database") - return DoctorCheckResult( - name, - DoctorStatus.FAIL, - (result.stderr or result.stdout).strip() or "psql command failed", - hint, - "database", - ) - - -def _check_dir_writable(path: Path, name: str) -> DoctorCheckResult: - try: - path.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile(dir=path, prefix=".netengine-doctor-", delete=True): - pass - return DoctorCheckResult(name, DoctorStatus.OK, f"writable: {path}", group="filesystem") - except Exception as exc: - return DoctorCheckResult( - name, - DoctorStatus.FAIL, - f"not writable: {path} ({exc})", - "Fix directory ownership or choose a writable path.", - "filesystem", - ) - - -def _check_state_file(path: Path) -> DoctorCheckResult: - if not path.exists(): - return DoctorCheckResult( - "State file", DoctorStatus.OK, f"no existing state at {path}", group="filesystem" - ) - try: - data = json.loads(path.read_text()) - phases = data.get("phase_completed", {}) if isinstance(data, dict) else {} - return DoctorCheckResult( - "State file", - DoctorStatus.WARN, - f"existing state file with {len(phases)} completed phase flag(s): {path}", - "Remove or back up the state file before bootstrapping a new world.", - "filesystem", - required=False, - ) - except Exception as exc: - return DoctorCheckResult( - "State file", - DoctorStatus.WARN, - f"existing unreadable/corrupt state file: {exc}", - "Inspect or remove the state file before continuing.", - "filesystem", - required=False, - ) - - -def _can_bind(port: int, proto: str) -> bool: - typ = socket.SOCK_DGRAM if proto == "udp" else socket.SOCK_STREAM - with socket.socket(socket.AF_INET, typ) as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", port)) - return True - - -def _check_port(port: int, proto: str) -> DoctorCheckResult: - try: - _can_bind(port, proto) - return DoctorCheckResult( - f"port:{port}/{proto}", DoctorStatus.OK, "available on 127.0.0.1", group="ports" - ) - except PermissionError as exc: - return DoctorCheckResult( - f"port:{port}/{proto}", - DoctorStatus.WARN, - f"permission denied while probing: {exc}", - "Run with privileges or check listeners manually.", - "ports", - required=False, - ) - except OSError as exc: - return DoctorCheckResult( - f"port:{port}/{proto}", - DoctorStatus.FAIL, - f"unavailable on 127.0.0.1: {exc}", - f"Stop the process using {port}/{proto} or change the compose/spec port.", - "ports", - ) - - -def _docker_names(kind: str) -> set[str]: - try: - if kind == "container": - result = _run(["docker", "ps", "-a", "--format", "{{.Names}}"]) - elif kind == "volume": - result = _run(["docker", "volume", "ls", "--format", "{{.Name}}"]) - else: - result = _run(["docker", "network", "ls", "--format", "{{.Name}}"]) - except Exception: - return set() - return ( - {line.strip() for line in result.stdout.splitlines() if line.strip()} - if result.returncode == 0 - else set() - ) - - -def _check_docker_conflicts() -> list[DoctorCheckResult]: - _, containers, volumes, networks = _compose_ports_and_resources() - checks = [] - for kind, expected in (("container", containers), ("volume", volumes), ("network", networks)): - conflicts = sorted(expected & _docker_names(kind)) - checks.append( - DoctorCheckResult( - f"Docker {kind} names", - DoctorStatus.WARN if conflicts else DoctorStatus.OK, - ", ".join(conflicts) if conflicts else "no known name conflicts", - ( - "Run `netengine down` or remove stale Docker resources if these belong to an old run." - if conflicts - else None - ), - "docker", - required=False, - ) - ) - return checks +__all__ = [ + "DoctorCheckResult", + "DoctorContext", + "DoctorProbe", + "DoctorStatus", + "build_context", + "doctor", + "run_checks", + "run_preflight", +] + +# Backwards-compatible test/extension hooks for the original CLI module surface. +_run = _preflight._run +_can_bind = _preflight._can_bind +_check_python = _preflight._check_python +_check_command = _preflight._check_command +_check_docker_daemon = _preflight._check_docker_daemon +_check_compose = _preflight._check_compose +_parse_db_url = _preflight._parse_db_url +_check_psql = _preflight._check_psql +_check_dir_writable = _preflight._check_dir_writable +_check_state_file = _preflight._check_state_file +_check_port = _preflight._check_port +_check_docker_conflicts = _preflight._check_docker_conflicts def run_checks( db_url: str | None, state_file: Path, *, skip_db: bool = False ) -> list[DoctorCheckResult]: - checks = [ - _check_python(), - _check_command("docker"), - _check_command("psql"), - _check_command("step", required=False), - _check_docker_daemon(), - _check_compose(), - ] - if skip_db: - checks.append( - DoctorCheckResult( - "Database checks", - DoctorStatus.SKIP, - "--skip-db requested", - group="database", - required=False, - ) - ) - else: - parsed = _parse_db_url(db_url) - checks.append(parsed) - if db_url and parsed.status == DoctorStatus.OK: - checks.append( - _check_psql( - db_url, - "SELECT 1;", - "Postgres connectivity", - hint="Start Postgres or fix NETENGINE_DB_URL.", - ) - ) - checks.append( - _check_psql( - db_url, - "SELECT extname FROM pg_extension WHERE extname = 'pgmq';", - "pgmq extension", - hint="Install/enable pgmq in the configured database.", - ) - ) - ports, _, _, _ = _compose_ports_and_resources() - checks.extend(_check_port(port, proto) for port, proto in sorted(ports)) - checks.extend( - [ - _check_dir_writable(state_file.parent, "State directory"), - _check_dir_writable(Path.home() / ".netengine", "User runtime directory"), - _check_state_file(state_file), - ] - ) - checks.extend(_check_docker_conflicts()) - return checks + """Run host-readiness checks without requiring a loaded NetEngine spec.""" + # Keep monkeypatches of the legacy CLI module effective for callers/tests. + _preflight._run = _run + _preflight._can_bind = _can_bind + return _preflight.run_checks(db_url, state_file, skip_db=skip_db) def _print_report(results: Iterable[DoctorCheckResult]) -> None: - by_group: dict[str, list[DoctorCheckResult]] = {} - for result in results: - by_group.setdefault(result.group, []).append(result) - symbols = { - DoctorStatus.OK: "✓", - DoctorStatus.WARN: "!", - DoctorStatus.FAIL: "✗", - DoctorStatus.SKIP: "-", - } - for group, group_results in by_group.items(): - click.echo(f"\n{group.title()}") - for result in group_results: - click.echo(f" {symbols[result.status]} {result.name}: {result.detail}") - if result.hint and result.status != DoctorStatus.OK: - click.echo(f" → {result.hint}") + _preflight._print_report(results) @click.command("doctor") @@ -400,7 +81,7 @@ def _print_report(results: Iterable[DoctorCheckResult]) -> None: @click.option("--json", "as_json", is_flag=True, help="Output machine-readable JSON.") @click.option("--skip-db", is_flag=True, help="Skip database connectivity and pgmq checks.") def doctor(db_url: str | None, state_file: Path, as_json: bool, skip_db: bool) -> None: - """Run local host preflight checks before bootstrapping a world.""" + """Run local host preflight checks before booting; use diagnose for world health.""" results = run_checks(db_url, state_file, skip_db=skip_db) if as_json: click.echo(json.dumps([asdict(r) for r in results], indent=2)) diff --git a/netengine/diagnostic/__init__.py b/netengine/diagnostic/__init__.py index e19bf4f..b70bcab 100644 --- a/netengine/diagnostic/__init__.py +++ b/netengine/diagnostic/__init__.py @@ -1,5 +1,20 @@ """Diagnostic probes for NetEngine world health checks.""" +from netengine.diagnostic.preflight import ( + DoctorCheckResult, + DoctorContext, + DoctorStatus, + run_preflight, +) from netengine.diagnostic.runner import DiagnosticRunner, ProbeResult, ProbeStatus, build_runner -__all__ = ["DiagnosticRunner", "ProbeResult", "ProbeStatus", "build_runner"] +__all__ = [ + "DiagnosticRunner", + "ProbeResult", + "ProbeStatus", + "build_runner", + "DoctorCheckResult", + "DoctorContext", + "DoctorStatus", + "run_preflight", +] diff --git a/netengine/diagnostic/preflight.py b/netengine/diagnostic/preflight.py new file mode 100644 index 0000000..751cf1e --- /dev/null +++ b/netengine/diagnostic/preflight.py @@ -0,0 +1,463 @@ +"""Host-readiness preflight probes for NetEngine. + +Doctor/preflight checks validate local prerequisites before a world is +bootstrapped: host commands, Docker availability, bindable ports, writable +runtime paths, and optional database prerequisites. These probes intentionally +operate on :class:`DoctorContext` instead of ``NetEngineSpec`` so ``netengine +doctor`` can run before any spec is loaded. + +By contrast, :mod:`netengine.diagnostic.runner` probes validate health of an +already configured/running world and require a loaded ``NetEngineSpec``. +""" + +from __future__ import annotations + +import json +import socket +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Callable, Iterable +from urllib.parse import urlparse + +import click +import yaml + + +class DoctorStatus(StrEnum): + """Status values emitted by the doctor command.""" + + OK = "ok" + WARN = "warn" + FAIL = "fail" + SKIP = "skip" + + +@dataclass(frozen=True) +class DoctorCheckResult: + """Single preflight check result.""" + + name: str + status: DoctorStatus + detail: str + hint: str | None = None + group: str = "general" + required: bool = True + + +@dataclass(frozen=True) +class DoctorContext: + """Input for host-readiness probes, independent of ``NetEngineSpec``.""" + + db_url: str | None + state_file: Path + project_root: Path + required_ports: tuple[tuple[int, str], ...] + required_commands: tuple[str, ...] = ("docker", "psql") + optional_commands: tuple[str, ...] = ("step",) + feature_flags: dict[str, bool] | None = None + + +DoctorProbe = Callable[[DoctorContext], DoctorCheckResult | Iterable[DoctorCheckResult]] + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _run(command: list[str], *, timeout: float = 8.0) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False) + + +def _compose_config(project_root: Path | None = None) -> dict: + compose_file = (project_root or _repo_root()) / "docker-compose.yml" + try: + return yaml.safe_load(compose_file.read_text()) or {} + except Exception: + return {} + + +def _compose_ports_and_resources( + project_root: Path | None = None, +) -> tuple[set[tuple[int, str]], set[str], set[str], set[str]]: + config = _compose_config(project_root) + ports: set[tuple[int, str]] = set() + containers: set[str] = set() + volumes: set[str] = set((config.get("volumes") or {}).keys()) + networks: set[str] = set((config.get("networks") or {}).keys()) + for service_name, service in (config.get("services") or {}).items(): + containers.add(service.get("container_name") or f"netengine-{service_name}-1") + for volume in service.get("volumes") or []: + if isinstance(volume, str) and volume and not volume.startswith((".", "/", "~")): + volumes.add(volume.split(":", 1)[0]) + for port in service.get("ports") or []: + raw = str(port) + published = raw.split(":", 1)[0] if ":" in raw else raw + if published.isdigit(): + ports.add((int(published), "tcp")) + ports.update({(5432, "tcp"), (53, "tcp"), (53, "udp")}) + return ports, containers, volumes, networks + + +def _check_python() -> DoctorCheckResult: + ok = sys.version_info >= (3, 13) + return DoctorCheckResult( + "Python runtime", + DoctorStatus.OK if ok else DoctorStatus.FAIL, + f"Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}; pyproject requires ^3.13", + None if ok else "Install Python 3.13 or newer and recreate the virtualenv.", + "host", + ) + + +def _check_command(name: str, *, required: bool = True) -> DoctorCheckResult: + import shutil + + path = shutil.which(name) + return DoctorCheckResult( + f"command:{name}", + DoctorStatus.OK if path else (DoctorStatus.FAIL if required else DoctorStatus.SKIP), + path or "not found on PATH", + None if path else f"Install {name} or add it to PATH.", + "host", + required, + ) + + +def _check_docker_daemon() -> DoctorCheckResult: + try: + result = _run(["docker", "info"]) + except Exception as exc: + return DoctorCheckResult( + "Docker daemon", + DoctorStatus.FAIL, + str(exc), + "Start Docker and verify socket access.", + "docker", + ) + if result.returncode == 0: + return DoctorCheckResult( + "Docker daemon", DoctorStatus.OK, "docker info succeeded", group="docker" + ) + return DoctorCheckResult( + "Docker daemon", + DoctorStatus.FAIL, + (result.stderr or result.stdout).strip() or "docker info failed", + "Start Docker and verify the current user can access it.", + "docker", + ) + + +def _check_compose() -> DoctorCheckResult: + try: + result = _run(["docker", "compose", "version"]) + except Exception as exc: + return DoctorCheckResult( + "Docker Compose", + DoctorStatus.FAIL, + str(exc), + "Install the Docker Compose plugin.", + "docker", + ) + if result.returncode == 0: + return DoctorCheckResult( + "Docker Compose", + DoctorStatus.OK, + (result.stdout or "available").strip(), + group="docker", + ) + return DoctorCheckResult( + "Docker Compose", + DoctorStatus.FAIL, + (result.stderr or result.stdout).strip() or "docker compose version failed", + "Install or enable Docker Compose v2.", + "docker", + ) + + +def _parse_db_url(db_url: str | None) -> DoctorCheckResult: + if not db_url: + return DoctorCheckResult( + "Database URL", + DoctorStatus.FAIL, + "NETENGINE_DB_URL/DATABASE_URL is not set", + "Set NETENGINE_DB_URL=postgresql://netengine:dev_password@localhost:5432/netengine", + "database", + ) + parsed = urlparse(db_url) + ok = ( + parsed.scheme in {"postgres", "postgresql"} + and bool(parsed.hostname) + and bool(parsed.path.strip("/")) + ) + return DoctorCheckResult( + "Database URL", + DoctorStatus.OK if ok else DoctorStatus.FAIL, + "parseable PostgreSQL URL" if ok else "not a valid PostgreSQL URL", + None if ok else "Use a postgresql:// URL with host and database name.", + "database", + ) + + +def _check_psql(db_url: str, sql: str, name: str, *, hint: str) -> DoctorCheckResult: + try: + result = _run(["psql", db_url, "-tAc", sql]) + except Exception as exc: + return DoctorCheckResult(name, DoctorStatus.FAIL, str(exc), hint, "database") + if result.returncode == 0: + detail = (result.stdout or "ok").strip() + if name == "pgmq extension" and detail != "pgmq": + return DoctorCheckResult( + name, DoctorStatus.FAIL, "pgmq extension is not installed", hint, "database" + ) + return DoctorCheckResult(name, DoctorStatus.OK, detail, group="database") + return DoctorCheckResult( + name, + DoctorStatus.FAIL, + (result.stderr or result.stdout).strip() or "psql command failed", + hint, + "database", + ) + + +def _check_dir_writable(path: Path, name: str) -> DoctorCheckResult: + try: + path.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path, prefix=".netengine-doctor-", delete=True): + pass + return DoctorCheckResult(name, DoctorStatus.OK, f"writable: {path}", group="filesystem") + except Exception as exc: + return DoctorCheckResult( + name, + DoctorStatus.FAIL, + f"not writable: {path} ({exc})", + "Fix directory ownership or choose a writable path.", + "filesystem", + ) + + +def _check_state_file(path: Path) -> DoctorCheckResult: + if not path.exists(): + return DoctorCheckResult( + "State file", DoctorStatus.OK, f"no existing state at {path}", group="filesystem" + ) + try: + data = json.loads(path.read_text()) + phases = data.get("phase_completed", {}) if isinstance(data, dict) else {} + return DoctorCheckResult( + "State file", + DoctorStatus.WARN, + f"existing state file with {len(phases)} completed phase flag(s): {path}", + "Remove or back up the state file before bootstrapping a new world.", + "filesystem", + required=False, + ) + except Exception as exc: + return DoctorCheckResult( + "State file", + DoctorStatus.WARN, + f"existing unreadable/corrupt state file: {exc}", + "Inspect or remove the state file before continuing.", + "filesystem", + required=False, + ) + + +def _can_bind(port: int, proto: str) -> bool: + typ = socket.SOCK_DGRAM if proto == "udp" else socket.SOCK_STREAM + with socket.socket(socket.AF_INET, typ) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + return True + + +def _check_port(port: int, proto: str) -> DoctorCheckResult: + try: + _can_bind(port, proto) + return DoctorCheckResult( + f"port:{port}/{proto}", DoctorStatus.OK, "available on 127.0.0.1", group="ports" + ) + except PermissionError as exc: + return DoctorCheckResult( + f"port:{port}/{proto}", + DoctorStatus.WARN, + f"permission denied while probing: {exc}", + "Run with privileges or check listeners manually.", + "ports", + required=False, + ) + except OSError as exc: + return DoctorCheckResult( + f"port:{port}/{proto}", + DoctorStatus.FAIL, + f"unavailable on 127.0.0.1: {exc}", + f"Stop the process using {port}/{proto} or change the compose/spec port.", + "ports", + ) + + +def _docker_names(kind: str) -> set[str]: + try: + if kind == "container": + result = _run(["docker", "ps", "-a", "--format", "{{.Names}}"]) + elif kind == "volume": + result = _run(["docker", "volume", "ls", "--format", "{{.Name}}"]) + else: + result = _run(["docker", "network", "ls", "--format", "{{.Name}}"]) + except Exception: + return set() + return ( + {line.strip() for line in result.stdout.splitlines() if line.strip()} + if result.returncode == 0 + else set() + ) + + +def _check_docker_conflicts(ctx: DoctorContext) -> list[DoctorCheckResult]: + _, containers, volumes, networks = _compose_ports_and_resources(ctx.project_root) + checks = [] + for kind, expected in (("container", containers), ("volume", volumes), ("network", networks)): + conflicts = sorted(expected & _docker_names(kind)) + checks.append( + DoctorCheckResult( + f"Docker {kind} names", + DoctorStatus.WARN if conflicts else DoctorStatus.OK, + ", ".join(conflicts) if conflicts else "no known name conflicts", + ( + "Run `netengine down` or remove stale Docker resources if these belong to an old run." + if conflicts + else None + ), + "docker", + required=False, + ) + ) + return checks + + +def build_context( + db_url: str | None, + state_file: Path, + *, + skip_db: bool = False, + project_root: Path | None = None, +) -> DoctorContext: + """Build the default doctor context from CLI inputs and compose metadata.""" + root = project_root or _repo_root() + ports, _, _, _ = _compose_ports_and_resources(root) + return DoctorContext( + db_url=db_url, + state_file=state_file, + project_root=root, + required_ports=tuple(sorted(ports)), + feature_flags={"skip_db": skip_db}, + ) + + +def _check_required_commands(ctx: DoctorContext) -> list[DoctorCheckResult]: + return [_check_command(name) for name in ctx.required_commands] + + +def _check_optional_commands(ctx: DoctorContext) -> list[DoctorCheckResult]: + return [_check_command(name, required=False) for name in ctx.optional_commands] + + +def _check_database(ctx: DoctorContext) -> list[DoctorCheckResult]: + if (ctx.feature_flags or {}).get("skip_db", False): + return [ + DoctorCheckResult( + "Database checks", + DoctorStatus.SKIP, + "--skip-db requested", + group="database", + required=False, + ) + ] + checks = [parsed := _parse_db_url(ctx.db_url)] + if ctx.db_url and parsed.status == DoctorStatus.OK: + checks.append( + _check_psql( + ctx.db_url, + "SELECT 1;", + "Postgres connectivity", + hint="Start Postgres or fix NETENGINE_DB_URL.", + ) + ) + checks.append( + _check_psql( + ctx.db_url, + "SELECT extname FROM pg_extension WHERE extname = 'pgmq';", + "pgmq extension", + hint="Install/enable pgmq in the configured database.", + ) + ) + return checks + + +def _check_ports(ctx: DoctorContext) -> list[DoctorCheckResult]: + return [_check_port(port, proto) for port, proto in ctx.required_ports] + + +def _check_filesystem(ctx: DoctorContext) -> list[DoctorCheckResult]: + return [ + _check_dir_writable(ctx.state_file.parent, "State directory"), + _check_dir_writable(Path.home() / ".netengine", "User runtime directory"), + _check_state_file(ctx.state_file), + ] + + +def standard_probes() -> tuple[DoctorProbe, ...]: + """Return host-readiness probes; each accepts only ``DoctorContext``.""" + return ( + lambda ctx: _check_python(), + _check_required_commands, + _check_optional_commands, + lambda ctx: _check_docker_daemon(), + lambda ctx: _check_compose(), + _check_database, + _check_ports, + _check_filesystem, + _check_docker_conflicts, + ) + + +def run_preflight( + ctx: DoctorContext, probes: Iterable[DoctorProbe] | None = None +) -> list[DoctorCheckResult]: + """Run host-readiness probes for a doctor context.""" + results: list[DoctorCheckResult] = [] + for probe in probes or standard_probes(): + result = probe(ctx) + if isinstance(result, DoctorCheckResult): + results.append(result) + else: + results.extend(result) + return results + + +def run_checks( + db_url: str | None, state_file: Path, *, skip_db: bool = False +) -> list[DoctorCheckResult]: + """Compatibility wrapper for callers that do not build ``DoctorContext``.""" + return run_preflight(build_context(db_url, state_file, skip_db=skip_db)) + + +def _print_report(results: Iterable[DoctorCheckResult]) -> None: + by_group: dict[str, list[DoctorCheckResult]] = {} + for result in results: + by_group.setdefault(result.group, []).append(result) + symbols = { + DoctorStatus.OK: "✓", + DoctorStatus.WARN: "!", + DoctorStatus.FAIL: "✗", + DoctorStatus.SKIP: "-", + } + for group, group_results in by_group.items(): + click.echo(f"\n{group.title()}") + for result in group_results: + click.echo(f" {symbols[result.status]} {result.name}: {result.detail}") + if result.hint and result.status != DoctorStatus.OK: + click.echo(f" → {result.hint}") diff --git a/netengine/diagnostic/runner.py b/netengine/diagnostic/runner.py index aa20f3f..f4ef2f0 100644 --- a/netengine/diagnostic/runner.py +++ b/netengine/diagnostic/runner.py @@ -1,4 +1,9 @@ -"""Diagnostic runner — orchestrates all probes concurrently.""" +"""Post-boot diagnostic runner — orchestrates world-health probes concurrently. + +These diagnostics require a loaded ``NetEngineSpec`` and validate a configured +or running world. Host-readiness checks for ``netengine doctor`` live in +:mod:`netengine.diagnostic.preflight` and run before any spec is loaded. +""" import asyncio from dataclasses import dataclass diff --git a/tests/test_doctor.py b/tests/test_doctor.py index d8f5838..7945aa9 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -8,6 +8,7 @@ from netengine.cli import doctor as doctor_mod from netengine.cli import main as cli_main from netengine.cli.doctor import DoctorCheckResult, DoctorStatus +from netengine.diagnostic.preflight import DoctorContext, run_preflight def test_doctor_appears_in_help() -> None: @@ -119,3 +120,23 @@ def test_state_file_conflict_is_warning(tmp_path: Path) -> None: assert result.status == DoctorStatus.WARN assert result.required is False assert "existing state file" in result.detail + + +def test_preflight_probe_accepts_doctor_context_without_spec(tmp_path: Path) -> None: + ctx = DoctorContext( + db_url=None, + state_file=tmp_path / "state.json", + project_root=tmp_path, + required_ports=(), + required_commands=(), + optional_commands=(), + feature_flags={}, + ) + + def probe(context: DoctorContext) -> DoctorCheckResult: + assert context.project_root == tmp_path + return DoctorCheckResult("custom", DoctorStatus.OK, "ready") + + results = run_preflight(ctx, probes=[probe]) + + assert results == [DoctorCheckResult("custom", DoctorStatus.OK, "ready")] From fe298cedf1cccc6c4784752eca9bb76bb8626293 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:55:42 +0100 Subject: [PATCH 33/34] docs: add alpha golden paths --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index 52b4761..f5a5d91 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,60 @@ Worlds are defined in YAML. See `examples/` for reference: | `examples/single-org.yaml` | One organisation with residential AND | | `examples/dev-sandbox.yaml` | Full dev setup with orgs, ANDs, mail, storage | +### Alpha golden paths + +These are the official alpha operator paths. Run them after the Quickstart setup to validate the supported lifecycle. + +#### Path A — Minimal smoke world + +```bash +poetry run netengine up examples/minimal.yaml +poetry run netengine up examples/minimal.yaml +poetry run netengine status +poetry run netengine diagnose examples/minimal.yaml +poetry run netengine down --dry-run +poetry run netengine down --yes +``` + +The second `up` proves idempotency. + +#### Path B — Single-org world + +Uses `examples/single-org.yaml` to prove org identity, DNS delegation, AND profile basics, and registry records. + +```bash +poetry run netengine up examples/single-org.yaml +poetry run netengine up examples/single-org.yaml +poetry run netengine status +poetry run netengine diagnose examples/single-org.yaml +poetry run netengine down --dry-run +poetry run netengine down --yes +``` + +#### Path C — Dev sandbox + +Uses `examples/dev-sandbox.yaml` as the feature-rich alpha demo. It is more experimental than Paths A/B if some integrations are still stabilizing. + +```bash +poetry run netengine up examples/dev-sandbox.yaml +poetry run netengine up examples/dev-sandbox.yaml +poetry run netengine status +poetry run netengine diagnose examples/dev-sandbox.yaml +poetry run netengine down --dry-run +poetry run netengine down --yes +``` + +#### Acceptance checklist + +- Fresh install works +- Boot completes +- Re-running `up` is idempotent +- `status` is accurate +- `diagnose` explains failures +- `reload` rejects immutable changes +- `down --dry-run` lists resources +- `down --yes` leaves no project-owned Docker resources behind + ### Spec composition Large specs can be split across files: From 6e99a9dbc2d6e1015e8bc6f6ade4c926fd8aaca6 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 01:55:44 +0100 Subject: [PATCH 34/34] Harden alpha state compatibility --- README.md | 8 +- netengine/api/routes.py | 79 +++++++-------- netengine/cli/main.py | 205 +++++++++++++++++++++++++-------------- netengine/core/state.py | 34 +++++++ netengine/spec/loader.py | 38 +++++++- netengine/spec/models.py | 20 +++- 6 files changed, 263 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 52b4761..a5e8465 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,13 @@ See `docs/runbook.md` for the full rollback and recovery procedure. └─────────────────────────────────────────────────┘ ``` -**Runtime state** is persisted to `netengines_state.json` after each phase so interrupted runs can resume where they left off. +**Runtime state** is persisted to `netengines_state.json` after each phase so interrupted runs can resume where they left off. The state file carries `schema_version: netengine.runtime_state.v1`; pre-version alpha.1 state files are detected as v1-compatible and stamped on the next save, while unknown future/foreign versions fail closed with instructions to export using the older release or migrate through a compatible release. The file is written atomically with `0600` permissions because it can contain runtime secrets (bootstrap admin password, OIDC client secrets, generated in-world admin passwords, and other phase outputs). Do not commit it, attach it to issues, or copy it into shared logs; use a sanitized support bundle instead. + +**Spec compatibility** is tracked in `metadata.schema_version` (default `netengine.spec.v1`). The loader accepts missing schema versions for existing alpha specs, rejects unsupported versions before boot/import, and includes the spec schema in support bundles so alpha.2+ can decide whether an alpha.1 world is safe to restore. + +**Support bundles** are produced with `netengine export --out netengine-support-bundle.json` or `GET /api/v1/export`. Bundles include schema metadata, the world spec, phase completion, public CA material, and sanitized phase outputs with secret-looking fields/private PEMs removed. Restore with `netengine import ` or `POST /api/v1/import`; import validates the bundle schema, spec schema compatibility, spec parseability, known phases, phase prerequisites, and required outputs before replacing local runtime state. + +**Persistent teardown safety** requires typed confirmation. CLI teardown of a persistent world must pass `netengine down --confirm `; the operator API requires both `confirm=true` and `confirmation` equal to the world name. `--yes` is intentionally not enough for persistent destructive operations. **Events** flow phase-to-phase via pgmq queues defined by `netengine/events/queues.py::PRIMARY_QUEUES`. There are currently 11 primary queues (`dns_updates`, `oidc_provisioning`, `and_provisioning`, `inworld_admissions`, `services_admissions`, `and_admissions`, `pki_cert_rotation_events`, `drift_events`, `world_health`, `gateway_portal_events`, `phase_events`) plus 11 matching dead-letter queues (`*_dlq`) for failed messages. diff --git a/netengine/api/routes.py b/netengine/api/routes.py index 1847f31..9fa3c6f 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -17,12 +17,12 @@ from netengine.api.auth import require_admin, require_auth from netengine.core.reload import ReloadResult, apply_reload, check_immutability, compute_diff -from netengine.core.state import RuntimeState +from netengine.core.state import RUNTIME_STATE_SCHEMA_VERSION, RuntimeState from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS from netengine.spec.loader import SpecLoadError, load_spec -from netengine.spec.models import NetEngineSpec +from netengine.spec.models import NetEngineSpec, SPEC_SCHEMA_VERSION logger = get_logger(__name__) router = APIRouter(prefix="/api/v1") @@ -33,6 +33,8 @@ # ───────────────────────────────────────────── IMPORT_SCHEMA_VERSION = "netengine.import.v1" +SUPPORT_BUNDLE_SCHEMA_VERSION = "netengine.support_bundle.v1" +SUPPORTED_IMPORT_SCHEMA_VERSIONS = {IMPORT_SCHEMA_VERSION, SUPPORT_BUNDLE_SCHEMA_VERSION} PHASE_REQUIRED_OUTPUTS = { "0": ("substrate_output",), @@ -52,25 +54,18 @@ def _validate_import_phase_state(state: RuntimeState) -> None: """Reject import snapshots with invalid phase completion state.""" unknown = sorted(set(state.phase_completed) - set(PHASE_REQUIRED_OUTPUTS)) if unknown: - raise HTTPException( - status_code=422, detail=f"Unknown phase ID(s): {', '.join(unknown)}" - ) + raise HTTPException(status_code=422, detail=f"Unknown phase ID(s): {', '.join(unknown)}") - completed = { - phase for phase, is_completed in state.phase_completed.items() if is_completed - } + completed = {phase for phase, is_completed in state.phase_completed.items() if is_completed} for phase in completed: required_outputs = PHASE_REQUIRED_OUTPUTS[phase] - missing = [ - field for field in required_outputs if not getattr(state, field, None) - ] + missing = [field for field in required_outputs if not getattr(state, field, None)] if missing: missing_str = ", ".join(missing) raise HTTPException( status_code=422, detail=( - f"Phase {phase} is completed but missing required " - f"output(s): {missing_str}" + f"Phase {phase} is completed but missing required " f"output(s): {missing_str}" ), ) @@ -83,24 +78,11 @@ def _validate_import_phase_state(state: RuntimeState) -> None: raise HTTPException( status_code=422, detail=( - "Impossible phase combination; missing prerequisite " - f"phase(s): {missing_str}" + "Impossible phase combination; missing prerequisite " f"phase(s): {missing_str}" ), ) -PHASE_LABELS = { - "0": "Substrate", - "1": "DNS root + platform zones", - "2": "DNS TLD hierarchy", - "3": "PKI + ACME", - "4": "Platform identity", - "5": "Registries", - "6": "In-world identity", - "7": "ANDs", - "8": "Services", -} - @router.get("/health") async def health() -> dict[str, Any]: """Per-phase healthcheck status.""" @@ -193,6 +175,9 @@ async def reload_world(body: ReloadRequest, user: dict = Depends(require_auth)) class WorldTeardownRequest(BaseModel): confirm: bool = False + confirmation: str | None = Field( + default=None, description="For persistent worlds, type the world name exactly." + ) @router.delete("/world") @@ -202,17 +187,19 @@ async def teardown_world( """Tear down the running world. Ephemeral: proceeds immediately. - Persistent: requires confirm=true in the request body. + Persistent: requires confirm=true and confirmation= in the request body. """ state = RuntimeState.load() if state.world_spec: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get( - "lifecycle", "ephemeral" - ) - if raw_lifecycle == "persistent" and not body.confirm: + raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") + world_name = (state.world_spec.get("metadata") or {}).get("name", "") + if raw_lifecycle == "persistent" and (not body.confirm or body.confirmation != world_name): raise HTTPException( status_code=409, - detail="Persistent world teardown requires confirm=true in request body", + detail=( + "Persistent world teardown requires confirm=true and " + f"confirmation={world_name!r} in request body" + ), ) removed: list[str] = [] @@ -982,7 +969,14 @@ async def export_world(user: dict = Depends(require_admin)) -> dict[str, Any]: return { "schema_version": IMPORT_SCHEMA_VERSION, + "support_bundle_schema_version": SUPPORT_BUNDLE_SCHEMA_VERSION, "exported_at": _dt.datetime.utcnow().isoformat(), + "runtime_state_schema_version": state.schema_version, + "supported_runtime_state_schema_version": RUNTIME_STATE_SCHEMA_VERSION, + "spec_schema_version": ((state.world_spec or {}).get("metadata") or {}).get( + "schema_version", SPEC_SCHEMA_VERSION + ), + "bundle_kind": "netengine.support", "spec": state.world_spec, "phase_completed": state.phase_completed, "ca_cert_pem": state.ca_cert_pem, @@ -998,9 +992,7 @@ async def export_world(user: dict = Depends(require_admin)) -> dict[str, Any]: class ImportRequest(BaseModel): - schema_version: str = Field( - ..., description="Import snapshot schema/version identifier" - ) + schema_version: str = Field(..., description="Import snapshot schema/version identifier") spec: dict[str, Any] phase_completed: dict[str, bool] = Field(default_factory=dict) ca_cert_pem: str | None = None @@ -1021,18 +1013,19 @@ async def import_world(body: ImportRequest, user: dict = Depends(require_admin)) """Restore world state from an export snapshot (persistent mode only).""" state = RuntimeState.load() if state.world_spec: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get( - "lifecycle", "ephemeral" - ) + raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") if raw_lifecycle == "ephemeral": raise HTTPException( status_code=409, detail="Import is only valid for persistent worlds" ) - if body.schema_version != IMPORT_SCHEMA_VERSION: + if body.schema_version not in SUPPORTED_IMPORT_SCHEMA_VERSIONS: raise HTTPException( status_code=422, - detail=f"Unsupported import schema_version: {body.schema_version}", + detail=( + f"Unsupported import schema_version: {body.schema_version}. " + f"Supported versions: {sorted(SUPPORTED_IMPORT_SCHEMA_VERSIONS)}" + ), ) try: @@ -1058,9 +1051,7 @@ async def import_world(body: ImportRequest, user: dict = Depends(require_admin)) ) _validate_import_phase_state(imported_state) - phases_restored = [ - phase for phase, completed in body.phase_completed.items() if completed - ] + phases_restored = [phase for phase, completed in body.phase_completed.items() if completed] imported_state.save() phases_restored = list(body.phase_completed.keys()) state.world_spec = body.spec diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 3d23b24..2d8cea5 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -44,9 +44,7 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: parts = key.split(".") if any(part == "" for part in parts): - raise click.BadParameter( - "keys must be non-empty dotted paths", param_hint="--set" - ) + raise click.BadParameter("keys must be non-empty dotted paths", param_hint="--set") value = yaml.safe_load(raw_value) cursor = overrides @@ -104,8 +102,7 @@ async def _run_migrations(db_url: str) -> MigrationRunResult: for migration in result.results: if migration.status == "applied": logger.info( - f"Applied migration: {migration.filename} " - f"({migration.duration_seconds:.3f}s)" + f"Applied migration: {migration.filename} " f"({migration.duration_seconds:.3f}s)" ) elif migration.status == "skipped": logger.info(f"Skipped migration: {migration.filename} (already applied)") @@ -125,16 +122,12 @@ def _print_migration_status(status: MigrationStatus) -> None: click.echo("Migration status") click.echo(f" Applied: {len(status.applied)}") for record in status.applied: - applied_at = ( - record.applied_at.isoformat() if record.applied_at else "unknown time" - ) + applied_at = record.applied_at.isoformat() if record.applied_at else "unknown time" click.echo(f" ✓ {record.version} {record.name} ({applied_at})") click.echo(f" Pending: {len(status.pending)}") for migration in status.pending: - click.echo( - f" • {migration.version} {migration.name} ({migration.path.name})" - ) + click.echo(f" • {migration.version} {migration.name} ({migration.path.name})") click.echo(f" Failed: {len(status.failed)}") for record in status.failed: @@ -247,9 +240,7 @@ def validate( ) -> None: """Validate SPEC_FILE without booting a world.""" try: - spec = _load_spec_for_cli( - spec_file, environment=environment, set_values=set_values - ) + spec = _load_spec_for_cli(spec_file, environment=environment, set_values=set_values) except SpecLoadError as exc: click.echo(f"Spec validation failed: {exc}", err=True) sys.exit(1) @@ -335,9 +326,7 @@ async def _up( spec = _load_spec_for_cli(spec_file, environment=environment, set_values=set_values) if mock: - click.echo( - "WARNING: running in mock mode — no real infrastructure will be created." - ) + click.echo("WARNING: running in mock mode — no real infrastructure will be created.") if not skip_migrations and not mock: db_url = _db_url_from_env() @@ -459,9 +448,7 @@ def reload(spec_file: str) -> None: is_ephemeral = old_spec.metadata.lifecycle.value == "ephemeral" click.echo("Computing diff…") - result = asyncio.run( - apply_reload(old_spec, new_spec, state, is_ephemeral=is_ephemeral) - ) + result = asyncio.run(apply_reload(old_spec, new_spec, state, is_ephemeral=is_ephemeral)) if result.immutability_violations: click.echo("Reload REJECTED — immutable fields changed:", err=True) @@ -499,24 +486,27 @@ def status() -> None: @cli.command() -@click.option("--yes", is_flag=True, help="Skip confirmation prompt.") -@click.option( - "--dry-run", is_flag=True, help="Show what would be removed without removing it." -) -def down(yes: bool, dry_run: bool) -> None: +@click.option("--yes", is_flag=True, help="Skip confirmation prompt for ephemeral worlds.") +@click.option("--confirm", default=None, help="For persistent worlds, type the world name exactly.") +@click.option("--dry-run", is_flag=True, help="Show what would be removed without removing it.") +def down(yes: bool, confirm: str | None, dry_run: bool) -> None: """Tear down the running world (containers, networks, volumes).""" - asyncio.run(_down(yes, dry_run)) + asyncio.run(_down(yes, confirm, dry_run)) -async def _down(yes: bool, dry_run: bool) -> None: +async def _down(yes: bool, confirm: str | None, dry_run: bool) -> None: state = RuntimeState.load() if state.world_spec and not dry_run: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get( - "lifecycle", "ephemeral" - ) - if raw_lifecycle == "persistent" and not yes: + raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") + world_name = (state.world_spec.get("metadata") or {}).get("name", "") + if raw_lifecycle == "persistent" and confirm != world_name: + raise click.ClickException( + "Persistent world teardown requires typed confirmation. " + f"Re-run with --confirm {world_name!r}." + ) + if raw_lifecycle != "persistent" and not yes: click.confirm( - "This is a PERSISTENT world — all durable state will be destroyed. Continue?", + "This world will be destroyed. Continue?", abort=True, ) @@ -565,9 +555,7 @@ async def _down(yes: bool, dry_run: bool) -> None: errors.append(f"{label}: {exc}") for network in client.networks.list(): - if network.name and any( - network.name.startswith(p) for p in _CONTAINER_PREFIXES - ): + if network.name and any(network.name.startswith(p) for p in _CONTAINER_PREFIXES): label = f"network:{network.name}" if dry_run: click.echo(f" would remove {label}") @@ -694,16 +682,12 @@ async def _diagnose(spec_file: str, as_json: bool) -> None: "status": r.status.value, "detail": r.detail, "hint": r.hint, - "elapsed_ms": round(r.elapsed_ms, 1) - if r.elapsed_ms is not None - else None, + "elapsed_ms": round(r.elapsed_ms, 1) if r.elapsed_ms is not None else None, } for r in results ] click.echo(_json.dumps(payload, indent=2)) - issues = sum( - 1 for r in results if r.status in (ProbeStatus.FAIL, ProbeStatus.WARN) - ) + issues = sum(1 for r in results if r.status in (ProbeStatus.FAIL, ProbeStatus.WARN)) if issues: sys.exit(1) return @@ -743,12 +727,8 @@ async def _diagnose(spec_file: str, as_json: bool) -> None: @cli.command() -@click.option( - "--interval", default=30, type=int, help="Poll interval in seconds (default 30)." -) -@click.option( - "--max-retries", default=3, type=int, help="Max self-heal retries per phase." -) +@click.option("--interval", default=30, type=int, help="Poll interval in seconds (default 30).") +@click.option("--max-retries", default=3, type=int, help="Max self-heal retries per phase.") @click.option("--no-auto-heal", is_flag=True, help="Detect drift but don't auto-heal.") def drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> None: """Watch running world for drift and optionally auto-heal (Ctrl+C to stop).""" @@ -761,9 +741,7 @@ async def _drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> N click.echo("No running world found — use `netengine up` first.", err=True) sys.exit(1) - click.echo( - f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…" - ) + click.echo(f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…") click.echo("Press Ctrl+C to stop.\n") orchestrator = Orchestrator(state.world_spec, mock_mode=False) @@ -837,6 +815,101 @@ def drift_status() -> None: click.echo("\nDrift history: (no events)") +@cli.command("export") +@click.option( + "--out", + "out_path", + type=click.Path(dir_okay=False), + default="netengine-support-bundle.json", + show_default=True, +) +def export_support_bundle(out_path: str) -> None: + """Write a sanitized support bundle for backup/support/import.""" + import datetime as _dt + import json as _json + + from netengine.api.routes import ( + IMPORT_SCHEMA_VERSION, + SUPPORT_BUNDLE_SCHEMA_VERSION, + _sanitize_export_value, + ) + from netengine.core.state import RUNTIME_STATE_SCHEMA_VERSION, RuntimeState + from netengine.spec.models import SPEC_SCHEMA_VERSION + + state = RuntimeState.load() + bundle = { + "schema_version": IMPORT_SCHEMA_VERSION, + "support_bundle_schema_version": SUPPORT_BUNDLE_SCHEMA_VERSION, + "bundle_kind": "netengine.support", + "exported_at": _dt.datetime.utcnow().isoformat(), + "runtime_state_schema_version": state.schema_version, + "supported_runtime_state_schema_version": RUNTIME_STATE_SCHEMA_VERSION, + "spec_schema_version": ((state.world_spec or {}).get("metadata") or {}).get( + "schema_version", SPEC_SCHEMA_VERSION + ), + "spec": state.world_spec, + "phase_completed": state.phase_completed, + "ca_cert_pem": state.ca_cert_pem, + "substrate_output": _sanitize_export_value(state.substrate_output), + "pki_output": _sanitize_export_value(state.pki_output), + "dns_output": _sanitize_export_value(state.dns_output), + "identity_platform_output": _sanitize_export_value(state.identity_platform_output), + "world_registry_output": _sanitize_export_value(state.world_registry_output), + "domain_registry_output": _sanitize_export_value(state.domain_registry_output), + "identity_inworld_output": _sanitize_export_value(state.identity_inworld_output), + "ands_output": _sanitize_export_value(state.ands_output), + "world_services_output": _sanitize_export_value(state.world_services_output), + "org_apps_output": _sanitize_export_value(state.org_apps_output), + } + path = Path(out_path) + path.write_text(_json.dumps(bundle, indent=2) + "\n") + os.chmod(path, 0o600) + click.echo(f"Wrote support bundle: {path}") + + +@cli.command("import") +@click.argument("bundle_file", type=click.Path(exists=True, dir_okay=False)) +def import_support_bundle(bundle_file: str) -> None: + """Validate and restore a compatible support bundle.""" + import json as _json + + from netengine.api.routes import SUPPORTED_IMPORT_SCHEMA_VERSIONS, _validate_import_phase_state + from netengine.core.state import RuntimeState + from netengine.spec.models import NetEngineSpec, SUPPORTED_SPEC_SCHEMA_VERSIONS + + body = _json.loads(Path(bundle_file).read_text()) + if body.get("schema_version") not in SUPPORTED_IMPORT_SCHEMA_VERSIONS: + raise click.ClickException( + f"Unsupported bundle schema_version: {body.get('schema_version')!r}" + ) + spec_data = body.get("spec") + if not isinstance(spec_data, dict): + raise click.ClickException("Support bundle is missing a spec object") + spec_schema = (spec_data.get("metadata") or {}).get("schema_version") + if spec_schema is not None and spec_schema not in SUPPORTED_SPEC_SCHEMA_VERSIONS: + raise click.ClickException(f"Unsupported spec metadata.schema_version: {spec_schema!r}") + spec = NetEngineSpec.model_validate(spec_data) + imported_state = RuntimeState( + world_spec=spec.model_dump(mode="json"), + phase_completed=dict(body.get("phase_completed") or {}), + ca_cert_pem=body.get("ca_cert_pem"), + substrate_output=body.get("substrate_output"), + pki_output=body.get("pki_output"), + dns_output=body.get("dns_output"), + identity_platform_output=body.get("identity_platform_output"), + world_registry_output=body.get("world_registry_output"), + domain_registry_output=body.get("domain_registry_output"), + identity_inworld_output=body.get("identity_inworld_output"), + ands_output=body.get("ands_output"), + world_services_output=body.get("world_services_output"), + org_apps_output=body.get("org_apps_output"), + pki_bootstrapped=bool(body.get("ca_cert_pem") or body.get("pki_output")), + ) + _validate_import_phase_state(imported_state) + imported_state.save() + click.echo(f"Imported support bundle for world: {spec.metadata.name}") + + @cli.command() @click.option( "--queue", @@ -883,9 +956,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: ) if rows: click.echo( - click.style( - f" {dlq_name} ({len(rows)} message(s)):", bold=True - ) + click.style(f" {dlq_name} ({len(rows)} message(s)):", bold=True) ) for row in rows: import json as _json @@ -895,18 +966,14 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: event_type = payload.get("event_type", "?") emitted_by = payload.get("emitted_by", "?") retry_count = payload.get("retry_count", 0) - dlq_reason = (payload.get("payload") or {}).get( - "dlq_reason", "" - ) + dlq_reason = (payload.get("payload") or {}).get("dlq_reason", "") click.echo( f" [{row['msg_id']}] {event_type} " f"from={emitted_by} retries={retry_count}" + (f" reason={dlq_reason}" if dlq_reason else "") ) except Exception: - click.echo( - f" [{row['msg_id']}] (unparseable message)" - ) + click.echo(f" [{row['msg_id']}] (unparseable message)") else: click.echo(f" {dlq_name}: empty") except Exception as exc: @@ -916,9 +983,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: for q in queues_to_check: dlq_name = dlq_for(Queue(q)).value try: - depth_row = await conn.fetchrow( - "SELECT count(*) AS depth FROM pgmq.q_$1", q - ) + depth_row = await conn.fetchrow("SELECT count(*) AS depth FROM pgmq.q_$1", q) dlq_row = await conn.fetchrow( "SELECT count(*) AS depth FROM pgmq.q_$1", dlq_name ) @@ -930,9 +995,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: else click.style("!", fg="yellow") ) dlq_status = ( - "" - if dlq_depth == 0 - else click.style(f" DLQ: {dlq_depth}", fg="red") + "" if dlq_depth == 0 else click.style(f" DLQ: {dlq_depth}", fg="red") ) click.echo(f" {status} {q:<30} depth={depth}{dlq_status}") except Exception as exc: @@ -984,9 +1047,7 @@ def _print_status(state: RuntimeState) -> None: "dev-sandbox: two orgs, all services, dev apps." ), ) -@click.option( - "--output", "-o", default=None, help="Output file path (default: .yaml)." -) +@click.option("--output", "-o", default=None, help="Output file path (default: .yaml).") @click.option( "--yes", "-y", @@ -1033,9 +1094,7 @@ def init( click.confirm(f"{early_path} already exists — overwrite?", abort=True) try: - cfg: WorldConfig = run_wizard( - preset=preset, yes=yes, name=name, lifecycle=lifecycle - ) + cfg: WorldConfig = run_wizard(preset=preset, yes=yes, name=name, lifecycle=lifecycle) except click.Abort: click.echo("\nAborted.", err=True) return @@ -1096,9 +1155,7 @@ def _print_init_summary(cfg: "Any", out_path: Path) -> None: click.echo(f"\n Organisations ({len(cfg.orgs)}):") for org in cfg.orgs: user_count = len(org.users) - click.echo( - f" • {org.name:<20} profile={org.and_profile} users={user_count}" - ) + click.echo(f" • {org.name:<20} profile={org.and_profile} users={user_count}") else: click.echo("\n Organisations: none (add later with `netengine reload`)") diff --git a/netengine/core/state.py b/netengine/core/state.py index b12660f..497f26f 100644 --- a/netengine/core/state.py +++ b/netengine/core/state.py @@ -14,6 +14,37 @@ logger = get_logger(__name__) DEFAULT_STATE_FILE = "netengines_state.json" +RUNTIME_STATE_SCHEMA_VERSION = "netengine.runtime_state.v1" +SUPPORTED_RUNTIME_STATE_SCHEMA_VERSIONS = {RUNTIME_STATE_SCHEMA_VERSION} + + +class RuntimeStateSchemaError(RuntimeError): + """Raised when a runtime state file cannot be loaded safely.""" + + +def _detect_or_migrate_state_schema(data: Dict[str, Any], state_file: Path) -> Dict[str, Any]: + """Return state data with a supported schema version or raise clear guidance.""" + schema_version = data.get("schema_version") + if schema_version is None: + # Alpha.1 state files predate explicit runtime state versioning. Treat them + # as v1-compatible and stamp the in-memory data so the next save persists it. + data["schema_version"] = RUNTIME_STATE_SCHEMA_VERSION + logger.warning( + "Runtime state file %s has no schema_version; assuming alpha.1-compatible %s. " + "Run `netengine export --out support-bundle.json` before upgrading further.", + state_file, + RUNTIME_STATE_SCHEMA_VERSION, + ) + return data + if schema_version not in SUPPORTED_RUNTIME_STATE_SCHEMA_VERSIONS: + supported = ", ".join(sorted(SUPPORTED_RUNTIME_STATE_SCHEMA_VERSIONS)) + raise RuntimeStateSchemaError( + f"Unsupported runtime state schema_version {schema_version!r} in {state_file}. " + f"This NetEngine build supports: {supported}. Export the world with the " + "older NetEngine version, upgrade through a release that provides a state " + "migration, or restore from a compatible support bundle." + ) + return data def get_state_file() -> Path: @@ -25,6 +56,8 @@ def get_state_file() -> Path: class RuntimeState: """Mutable runtime state, persisted to a local JSON file between phases.""" + schema_version: str = RUNTIME_STATE_SCHEMA_VERSION + # Execution trace correlation_id: Optional[str] = None parent_event_id: Optional[str] = None @@ -89,6 +122,7 @@ def load(cls) -> "RuntimeState": if state_file.exists(): with open(state_file, "r") as f: data: Dict[str, Any] = json.load(f) + data = _detect_or_migrate_state_schema(data, state_file) # datetime fields are stored as ISO strings for dt_field in ("started_at", "completed_at", "last_error_at", "last_drift_check_at"): if data.get(dt_field): diff --git a/netengine/spec/loader.py b/netengine/spec/loader.py index 17f2030..5f10a2e 100644 --- a/netengine/spec/loader.py +++ b/netengine/spec/loader.py @@ -10,7 +10,7 @@ from pydantic import ValidationError from netengine.config.loader import ConfigLoader -from netengine.spec.models import NetEngineSpec +from netengine.spec.models import NetEngineSpec, SUPPORTED_SPEC_SCHEMA_VERSIONS logger = logging.getLogger(__name__) @@ -189,6 +189,18 @@ def load_spec(yaml_path: str | Path) -> NetEngineSpec: if not isinstance(data, dict): raise SpecLoadError("Spec must be a YAML object (dict)") + metadata = data.get("metadata") + if not isinstance(metadata, dict): + raise SpecLoadError("Spec metadata must be a YAML object (dict)") + schema_version = metadata.get("schema_version") + if schema_version is not None and schema_version not in SUPPORTED_SPEC_SCHEMA_VERSIONS: + supported = ", ".join(sorted(SUPPORTED_SPEC_SCHEMA_VERSIONS)) + raise SpecLoadError( + f"Unsupported spec metadata.schema_version {schema_version!r}; " + f"supported versions: {supported}. Export with the older NetEngine version " + "or migrate the spec before booting this release." + ) + try: spec = NetEngineSpec(**data) except ValidationError as e: @@ -249,6 +261,18 @@ def load_spec_with_composition( if not isinstance(data, dict): raise SpecLoadError("Spec must be a YAML object (dict)") + metadata = data.get("metadata") + if not isinstance(metadata, dict): + raise SpecLoadError("Spec metadata must be a YAML object (dict)") + schema_version = metadata.get("schema_version") + if schema_version is not None and schema_version not in SUPPORTED_SPEC_SCHEMA_VERSIONS: + supported = ", ".join(sorted(SUPPORTED_SPEC_SCHEMA_VERSIONS)) + raise SpecLoadError( + f"Unsupported spec metadata.schema_version {schema_version!r}; " + f"supported versions: {supported}. Export with the older NetEngine version " + "or migrate the spec before booting this release." + ) + try: spec = NetEngineSpec(**data) except ValidationError as e: @@ -307,6 +331,18 @@ def load_spec_with_environment( if not isinstance(data, dict): raise SpecLoadError("Spec must be a YAML object (dict)") + metadata = data.get("metadata") + if not isinstance(metadata, dict): + raise SpecLoadError("Spec metadata must be a YAML object (dict)") + schema_version = metadata.get("schema_version") + if schema_version is not None and schema_version not in SUPPORTED_SPEC_SCHEMA_VERSIONS: + supported = ", ".join(sorted(SUPPORTED_SPEC_SCHEMA_VERSIONS)) + raise SpecLoadError( + f"Unsupported spec metadata.schema_version {schema_version!r}; " + f"supported versions: {supported}. Export with the older NetEngine version " + "or migrate the spec before booting this release." + ) + try: spec = NetEngineSpec(**data) except ValidationError as e: diff --git a/netengine/spec/models.py b/netengine/spec/models.py index 627a2b5..702cf01 100644 --- a/netengine/spec/models.py +++ b/netengine/spec/models.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Dict, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from netengine.spec.types import ( ANDProfile, @@ -38,6 +38,8 @@ class FeatureState(str, Enum): FEATURE_STATE_JSON_SCHEMA_KEY = "feature_state" +SPEC_SCHEMA_VERSION = "netengine.spec.v1" +SUPPORTED_SPEC_SCHEMA_VERSIONS = {SPEC_SCHEMA_VERSION} def feature_state_extra(feature_state: FeatureState) -> dict[str, str]: @@ -80,6 +82,22 @@ def feature_state_field( class SpecMetadata(SpecModel): """Top-level spec metadata.""" + schema_version: str = Field( + default=SPEC_SCHEMA_VERSION, + description="NetEngine spec schema/version identifier used for compatibility checks", + ) + + @field_validator("schema_version") + @classmethod + def validate_schema_version(cls, value: str) -> str: + """Reject unsupported spec schemas before boot/import.""" + if value not in SUPPORTED_SPEC_SCHEMA_VERSIONS: + supported = ", ".join(sorted(SUPPORTED_SPEC_SCHEMA_VERSIONS)) + raise ValueError( + f"unsupported spec schema_version {value!r}; supported versions: {supported}" + ) + return value + name: str = Field(..., description="World name") version: str = Field(default="1.0", description="Spec version") lifecycle: Lifecycle = Field(