diff --git a/README.md b/README.md index 28cc7d2..dcee97c 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,9 @@ The active development roadmap lives in the [GitHub project](https://github.com/ - [x] End-to-end integration test (real Docker, live DNS query, cert issuance, OIDC login) - [x] Complete operator API (org CRUD, AND management, domain management) -- [x] Cross-world federation +- [~] Cross-world federation — DNS-stub peering is prototyped and e2e-tested; full + nftables routing, trust-anchor wiring, and live cross-world resolution remain + gated `unsupported` (see [`docs/spec-alpha-support.md`](docs/spec-alpha-support.md)) - [x] `persistent` lifecycle mode (import/export, lifecycle guards, teardown confirmation) - [x] `netengine down --dry-run` diff --git a/docs/COMPREHENSIVE_GAP_ANALYSIS.md b/docs/COMPREHENSIVE_GAP_ANALYSIS.md index b7c3558..e1c4853 100644 --- a/docs/COMPREHENSIVE_GAP_ANALYSIS.md +++ b/docs/COMPREHENSIVE_GAP_ANALYSIS.md @@ -3,6 +3,23 @@ --- +> **⚠️ Status correction (2026-06-29).** A code re-audit found several items in +> this report are **stale / already resolved**: all four "missing" operator API +> endpoints now exist (`PUT /services/{name}`, `PUT /ands/{and_name}/profile`, +> `PUT /gateway`, `PUT /pki/rotation-policy`); mail SPF/DMARC are wired and the +> DKIM record now publishes the real key (no more `` placeholder); +> `/api/v1/reload` persists the updated spec; `PHASE_PREREQUISITES[9]` already +> requires `world_services_output` (Phase 2 is an intentional combined handler); +> and pgmq-disabled admission consumers now log at **WARNING** with an `events` +> block surfaced in `/health`. **Genuinely still open** (gated in +> `netengine/spec/feature_state.py`): PKI DNSSEC signing/rotation, CRL, OCSP, +> intermediate-CA stabilization; AND `dynamic_ip`/`reverse_dns`/`bgp`; and +> gateway real-internet + cross-world federation routing/resolution. The live +> contract is `docs/spec-alpha-support.md` + `feature_state.py`; the inventory +> below is preserved as the original audit. + +--- + ## Executive Summary NetEngine exhibits **15+ categories of gaps** across the codebase, ranging from unused spec fields and incomplete event infrastructure to missing API endpoints and test coverage. These fall into three severity tiers: diff --git a/docs/GAP_SUMMARY.txt b/docs/GAP_SUMMARY.txt index 8d3054c..cfc83dd 100644 --- a/docs/GAP_SUMMARY.txt +++ b/docs/GAP_SUMMARY.txt @@ -2,6 +2,40 @@ ║ NETENGINE: MAJOR GAPS DISCOVERED (2026-06-27) ║ ╚═══════════════════════════════════════════════════════════════════════════╝ +┌─────────────────────────────────────────────────────────────────────────┐ +│ ⚠️ STATUS CORRECTION (2026-06-29) — read before acting on this report │ +└─────────────────────────────────────────────────────────────────────────┘ + +A code re-audit on 2026-06-29 found several items below are STALE / already +resolved. The live alpha contract is docs/spec-alpha-support.md + +netengine/spec/feature_state.py, not this historical forensic snapshot. + +RESOLVED since this report was written: + • #6 Missing API endpoints — all present now: + PUT /services/{name} netengine/api/routes.py + PUT /ands/{and_name}/profile netengine/api/routes.py + PUT /gateway netengine/api/routes.py + PUT /pki/rotation-policy netengine/api/routes.py + • Mail SPF/DMARC — wired from spec (mail_handler.py:_inject_dns_records). + • Mail DKIM record — previously a "" placeholder; now publishes + the real generated key (mail_handler._dkim_txt_value). + • #8 /api/v1/reload stale world_spec — fixed; reload persists the new spec + (core/reload.py). + • #7 Phase 9 prerequisite — present: PHASE_PREREQUISITES[9] = ["world_services_output"] + (core/phase_graph.py). Phase 2 is intentionally combined into Phase 1 + (DNSHandler) — documented design, not a gap. + • #10 Silent pgmq degradation — admission-event consumers now log at WARNING + and /health surfaces an "events" block (enabled/detail). + +STILL OPEN (genuine v1 work — gated by feature_state.py): + • PKI DNSSEC zone signing/rotation, CRL publication, OCSP responder lifecycle, + intermediate-CA trust-chain stabilization. + • AND profile dynamic_ip / reverse_dns / bgp (only wizard defaults today). + • Gateway real-internet + cross-world federation routing/resolution (handler + wired but unproven; e2e covers DNS-stub peering only). + +Everything below is preserved as the original 2026-06-27 audit for history. + ┌─────────────────────────────────────────────────────────────────────────┐ │ 🔴 CRITICAL GAPS (Spec Gated, Partial, Or Not Implemented) │ └─────────────────────────────────────────────────────────────────────────┘ diff --git a/netengine/api/routes.py b/netengine/api/routes.py index 787a79f..0d7d25a 100644 --- a/netengine/api/routes.py +++ b/netengine/api/routes.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, Field from netengine.api.auth import _extract_roles, require_admin, require_auth +from netengine.core.db_client import pgmq_available from netengine.core.reload import ReloadResult, apply_reload, check_immutability, compute_diff from netengine.core.state import RUNTIME_STATE_SCHEMA_VERSION, RuntimeState from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for @@ -95,9 +96,24 @@ async def health() -> dict[str, Any]: completed = state.phase_completed.get(phase_id, False) phases[phase_id] = {"label": label, "completed": completed} overall = "ok" if all(p["completed"] for p in phases.values()) else "degraded" + + events_ok, events_reason = await pgmq_available() + events = { + "enabled": events_ok, + "backend": "pgmq", + "detail": ( + "event-driven provisioning active" + if events_ok + else f"events DISABLED — {events_reason}; " + "ANDs and per-org realms are not auto-provisioned" + ), + } + return { "status": overall, "phases": phases, + "events": events, + "last_error": state.last_error, "last_error_present": bool(state.last_error), } diff --git a/netengine/core/db_client.py b/netengine/core/db_client.py index 7a0d0c0..5a86213 100644 --- a/netengine/core/db_client.py +++ b/netengine/core/db_client.py @@ -218,3 +218,28 @@ async def close_pool() -> None: if _pool is not None: await _pool.close() _pool = None + + +async def pgmq_available() -> tuple[bool, Optional[str]]: + """Best-effort probe for whether the pgmq event backend is reachable. + + Used by the operator API ``/health`` endpoint and ``netengine doctor`` to + surface event-driven operation being disabled (e.g. ANDs and per-org realms + are not auto-provisioned without it). Never raises; on any failure it + returns ``(False, reason)`` so callers can report the degraded state. + + Returns: + ``(True, None)`` when the pgmq extension is installed and reachable, + otherwise ``(False, reason)``. + """ + try: + db = await get_local_db() + async with db._pool.acquire() as conn: + installed = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pgmq')" + ) + if installed: + return True, None + return False, "pgmq extension not installed" + except Exception as exc: # pragma: no cover - environment dependent + return False, f"database unreachable: {exc}" diff --git a/netengine/handlers/mail_handler.py b/netengine/handlers/mail_handler.py index af89ba8..fb7459d 100644 --- a/netengine/handlers/mail_handler.py +++ b/netengine/handlers/mail_handler.py @@ -69,7 +69,7 @@ async def deploy_postfix(self) -> dict[str, Any]: # 4. Inject DNS records (SPF, DKIM, DMARC) for each org self.logger.info("Injecting DNS records for mail infrastructure") - orgs_configured = await self._inject_dns_records() + orgs_configured = await self._inject_dns_records(dkim_public_pem) # 5. Provision virtual mailbox maps self.logger.info("Provisioning virtual mailbox maps") @@ -124,6 +124,27 @@ async def _generate_dkim_keys(self) -> tuple[str, str]: self.logger.info("DKIM RSA keys generated (2048-bit)") return private_pem, public_pem + @staticmethod + def _dkim_txt_value(public_pem: str) -> str: + """Build the DKIM TXT record value from a public key PEM. + + A DKIM ``p=`` tag carries the base64-encoded DER of the + SubjectPublicKeyInfo — i.e. the PEM body with the ``-----BEGIN/END + PUBLIC KEY-----`` armor and line breaks removed. + + Args: + public_pem: RSA public key in SubjectPublicKeyInfo PEM format. + + Returns: + DKIM TXT record value, e.g. ``v=DKIM1; k=rsa; p=MIIBIjANBg...``. + """ + body = "".join( + line.strip() + for line in public_pem.splitlines() + if line and not line.startswith("-----") + ) + return f"v=DKIM1; k=rsa; p={body}" + def _create_postfix_config(self, dkim_public_pem: str) -> str: """Create Postfix main.cf configuration. @@ -190,9 +211,14 @@ async def _start_postfix_container(self, config: str) -> str: return container_id - async def _inject_dns_records(self) -> list[str]: + async def _inject_dns_records(self, dkim_public_pem: str) -> list[str]: """Inject SPF, DKIM, DMARC DNS records for each org. + Args: + dkim_public_pem: Generated DKIM public key in PEM format, published + in the ``_dkim._domainkey`` TXT record so verifiers can validate + signatures. + Returns: List of org names configured """ @@ -223,7 +249,7 @@ async def _inject_dns_records(self) -> list[str]: # DKIM record if enabled if self.mail_config.dkim.enabled: dkim_name = f"_dkim._domainkey.{org_domain}" - dkim_value = "v=DKIM1; k=rsa; p=" # Simplified for MVP + dkim_value = self._dkim_txt_value(dkim_public_pem) await self.dns.add_zone_record( context=self.context, zone="internal", diff --git a/netengine/phases/phase_ands.py b/netengine/phases/phase_ands.py index d63efc6..d2088b5 100644 --- a/netengine/phases/phase_ands.py +++ b/netengine/phases/phase_ands.py @@ -341,7 +341,11 @@ async def _consume_org_admission_events( logger = context.logger if context.pgmq_client is None: - logger.info("pgmq_client not available; org admission events disabled") + logger.warning( + "pgmq_client not available; AND admission events are DISABLED — " + "ANDs will not be auto-provisioned from org.admitted events. " + "Provision pgmq (see docker-compose.yml) for event-driven operation." + ) return logger.info("Starting org admission event consumer") diff --git a/netengine/phases/phase_inworld_identity.py b/netengine/phases/phase_inworld_identity.py index f099d13..29bd1c5 100644 --- a/netengine/phases/phase_inworld_identity.py +++ b/netengine/phases/phase_inworld_identity.py @@ -469,7 +469,11 @@ async def _consume_org_admission_events( logger = context.logger if context.pgmq_client is None: - logger.info("pgmq_client not available; org admission events disabled") + logger.warning( + "pgmq_client not available; in-world admission events are DISABLED — " + "per-org realms will not be auto-provisioned from org.admitted events. " + "Provision pgmq (see docker-compose.yml) for event-driven operation." + ) return logger.info("Starting org admission event consumer") diff --git a/netengine/phases/phase_services.py b/netengine/phases/phase_services.py index daa28f3..74eab37 100644 --- a/netengine/phases/phase_services.py +++ b/netengine/phases/phase_services.py @@ -295,7 +295,11 @@ async def _consume_org_admission_events( logger = context.logger if context.pgmq_client is None: - logger.info("pgmq_client not available; org service provisioning disabled") + logger.warning( + "pgmq_client not available; org service provisioning events are DISABLED — " + "per-org services will not be auto-provisioned from org.admitted events. " + "Provision pgmq (see docker-compose.yml) for event-driven operation." + ) return logger.info("Starting org admission event consumer (services)") diff --git a/tests/integration/test_m8_mail.py b/tests/integration/test_m8_mail.py index 24b28e1..ee9a3cf 100644 --- a/tests/integration/test_m8_mail.py +++ b/tests/integration/test_m8_mail.py @@ -259,6 +259,27 @@ async def test_m8_disables_dkim_when_not_configured(self) -> None: # Verify DKIM is disabled in spec assert spec.world_services.mail.dkim.enabled is False + def test_dkim_txt_value_uses_real_public_key(self) -> None: + """The DKIM TXT value must carry the real key, not a placeholder.""" + from netengine.handlers.mail_handler import MailHandler + + public_pem = ( + "-----BEGIN PUBLIC KEY-----\n" + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\n" + "abcDEF1234567890\n" + "-----END PUBLIC KEY-----\n" + ) + + value = MailHandler._dkim_txt_value(public_pem) + + assert value.startswith("v=DKIM1; k=rsa; p=") + # The PEM armor and line breaks are stripped into a single base64 blob. + assert value == ( + "v=DKIM1; k=rsa; " "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAabcDEF1234567890" + ) + # Regression guard against the old "Simplified for MVP" placeholder. + assert "" not in value + class TestM8DNSRecords: """Tests that M8 injects DNS records (SPF, DKIM, DMARC, MX).""" diff --git a/tests/integration/test_m8_operator_api.py b/tests/integration/test_m8_operator_api.py index fc7d47b..edbbe4f 100644 --- a/tests/integration/test_m8_operator_api.py +++ b/tests/integration/test_m8_operator_api.py @@ -203,6 +203,12 @@ def test_health_returns_ok_when_no_state(self, tmp_path, monkeypatch): "9", } assert data["phases"]["9"] == {"label": "Org applications", "completed": False} + # Event backend status is always surfaced so operators can see when + # event-driven provisioning is disabled (pgmq unavailable). + assert "events" in data + assert isinstance(data["events"]["enabled"], bool) + assert data["events"]["backend"] == "pgmq" + assert data["events"]["detail"] def test_health_reports_degraded_when_phases_incomplete(self, tmp_path, monkeypatch): monkeypatch.setenv("NETENGINE_STATE_FILE", str(tmp_path / "state.json")) diff --git a/tests/test_feature_states.py b/tests/test_feature_states.py index 9c5c4b0..a5d051d 100644 --- a/tests/test_feature_states.py +++ b/tests/test_feature_states.py @@ -1,5 +1,8 @@ """Feature-state metadata tests for spec fields.""" +from pathlib import Path + +from netengine.spec.feature_state import FEATURE_STATE_REGISTRY from netengine.spec.models import ( FEATURE_STATE_JSON_SCHEMA_KEY, PKI_FEATURE_STATES, @@ -8,6 +11,8 @@ PKIPhase, ) +_SUPPORT_MATRIX = Path(__file__).resolve().parents[1] / "docs" / "spec-alpha-support.md" + PKI_FIELD_NAMES = { "pki.intermediate_ca_enabled": "intermediate_ca_enabled", "pki.dnssec_enabled": "dnssec_enabled", @@ -42,3 +47,19 @@ def test_required_pki_fields_expose_feature_state_in_json_schema() -> None: pki_schema[field_name][FEATURE_STATE_JSON_SCHEMA_KEY] == PKI_FEATURE_STATES[dotted_path].value ) + + +def test_every_registry_entry_is_documented_in_support_matrix() -> None: + """Each gated field must have a row in docs/spec-alpha-support.md. + + Keeps the alpha support contract in lock-step with the validation registry, + so removing a gate (for v1) forces a deliberate matrix update — mirroring + the supply-chain CI check that licenses stay documented. + """ + matrix_text = _SUPPORT_MATRIX.read_text(encoding="utf-8") + undocumented = [ + entry.path for entry in FEATURE_STATE_REGISTRY if f"`{entry.path}`" not in matrix_text + ] + assert not undocumented, ( + "feature_state registry entries missing from docs/spec-alpha-support.md: " f"{undocumented}" + )