Surface pgmq runtime state in doctor diagnostics

.github/workflows/ci.yaml

@@ -223,7 +223,7 @@ jobs:
           path: sbom.spdx.json
 
       - name: Vulnerability scan
-        uses: aquasecurity/trivy-action@0.30.0
+        uses: aquasecurity/trivy-action@v0.36.0
         with:
           scan-type: fs
           scan-ref: .

docs/licenses.md

@@ -13,28 +13,20 @@ Generated from installed Python package metadata. Re-run with `python scripts/ge
 | anyio | 4.14.0 | MIT | UNKNOWN |
 | asyncpg | 0.30.0 | Apache License, Version 2.0 | UNKNOWN |
 | attrs | 26.1.0 | MIT | UNKNOWN |
-| backports.zstd | 1.6.0 | PSF-2.0 | UNKNOWN |
 | black | 24.10.0 | MIT | UNKNOWN |
-| build | 1.5.0 | MIT | UNKNOWN |
-| CacheControl | 0.14.4 | Apache-2.0 | UNKNOWN |
 | certifi | 2026.6.17 | MPL-2.0 | https://github.com/certifi/python-certifi |
 | cffi | 2.0.0 | MIT | UNKNOWN |
 | cfgv | 3.5.0 | MIT | https://github.com/asottile/cfgv |
 | charset-normalizer | 3.4.7 | MIT | UNKNOWN |
-| cleo | 2.1.0 | MIT | https://github.com/python-poetry/cleo |
 | click | 8.4.1 | BSD-3-Clause | UNKNOWN |
 | coverage | 7.14.2 | Apache-2.0 | https://github.com/coveragepy/coveragepy |
-| crashtest | 0.4.1 | MIT | https://github.com/sdispater/crashtest |
 | cryptography | 48.0.1 | Apache-2.0 OR BSD-3-Clause | UNKNOWN |
 | deprecation | 2.1.0 | Apache 2 | http://deprecation.readthedocs.io/ |
 | distlib | 0.4.3 | PSF-2.0 | https://github.com/pypa/distlib |
 | dnspython | 2.8.0 | ISC | UNKNOWN |
 | docker | 7.1.0 | Apache-2.0 | UNKNOWN |
-| dulwich | 1.2.6 | Apache-2.0 OR GPL-2.0-or-later | UNKNOWN |
 | fastapi | 0.138.0 | MIT | UNKNOWN |
-| fastjsonschema | 2.21.2 | BSD | https://github.com/horejsek/python-fastjsonschema |
 | filelock | 3.29.4 | MIT | UNKNOWN |
-| findpython | 0.8.0 | MIT | UNKNOWN |
 | flake8 | 6.1.0 | MIT | https://github.com/pycqa/flake8 |
 | frozenlist | 1.8.0 | Apache-2.0 | https://github.com/aio-libs/frozenlist |
 | h11 | 0.16.0 | MIT | https://github.com/python-hyper/h11 |
@@ -46,18 +38,10 @@ Generated from installed Python package metadata. Re-run with `python scripts/ge
 | identify | 2.6.19 | MIT | https://github.com/pre-commit/identify |
 | idna | 3.18 | BSD-3-Clause | UNKNOWN |
 | iniconfig | 2.3.0 | MIT | UNKNOWN |
-| installer | 1.0.1 | MIT | UNKNOWN |
 | isort | 5.13.2 | MIT | https://pycqa.github.io/isort/ |
-| jaraco.classes | 3.4.0 | UNKNOWN | https://github.com/jaraco/jaraco.classes |
-| jaraco.context | 6.1.2 | MIT | UNKNOWN |
-| jaraco.functools | 4.5.0 | MIT | UNKNOWN |
-| jeepney | 0.9.0 | MIT | UNKNOWN |
-| keyring | 25.7.0 | MIT | UNKNOWN |
 | librt | 0.11.0 | MIT | UNKNOWN |
 | loguru | 0.7.3 | UNKNOWN | UNKNOWN |
 | mccabe | 0.7.0 | Expat license | https://github.com/pycqa/mccabe |
-| more-itertools | 11.1.0 | MIT | UNKNOWN |
-| msgpack | 1.2.1 | Apache-2.0 | UNKNOWN |
 | multidict | 6.7.1 | Apache License 2.0 | https://github.com/aio-libs/multidict |
 | mypy | 1.20.2 | MIT | UNKNOWN |
 | mypy_extensions | 1.1.0 | MIT | UNKNOWN |
@@ -66,13 +50,9 @@ Generated from installed Python package metadata. Re-run with `python scripts/ge
 | omegaconf | 2.3.1 | UNKNOWN | https://github.com/omry/omegaconf |
 | packaging | 26.2 | Apache-2.0 OR BSD-2-Clause | UNKNOWN |
 | pathspec | 1.1.1 | UNKNOWN | UNKNOWN |
-| pbs-installer | 2026.6.10 | MIT | UNKNOWN |
-| pip | 26.1 | MIT | UNKNOWN |
-| pkginfo | 1.12.1.2 | MIT | https://code.launchpad.net/~tseaver/pkginfo/trunk |
+| pip | 25.1.1 | MIT | UNKNOWN |
 | platformdirs | 4.10.0 | MIT | UNKNOWN |
 | pluggy | 1.6.0 | MIT | UNKNOWN |
-| poetry | 2.4.1 | MIT | UNKNOWN |
-| poetry-core | 2.4.0 | MIT | UNKNOWN |
 | postgrest | 2.31.0 | MIT | UNKNOWN |
 | pre-commit | 3.8.0 | MIT | https://github.com/pre-commit/pre-commit |
 | prometheus_client | 0.25.0 | Apache-2.0 AND BSD-2-Clause | UNKNOWN |
@@ -82,33 +62,22 @@ Generated from installed Python package metadata. Re-run with `python scripts/ge
 | pydantic | 2.13.4 | MIT | UNKNOWN |
 | pydantic_core | 2.46.4 | MIT | https://github.com/pydantic/pydantic |
 | pyflakes | 3.1.0 | MIT | https://github.com/PyCQA/pyflakes |
-| Pygments | 2.20.0 | BSD-2-Clause | UNKNOWN |
 | PyJWT | 2.13.0 | MIT | UNKNOWN |
-| pyproject_hooks | 1.2.0 | UNKNOWN | UNKNOWN |
-| pyright | 1.1.409 | MIT | https://github.com/RobertCraigie/pyright-python |
 | pytest | 7.4.4 | MIT | https://docs.pytest.org/en/latest/ |
 | pytest-asyncio | 0.21.2 | Apache 2.0 | https://github.com/pytest-dev/pytest-asyncio |
 | pytest-cov | 4.1.0 | MIT | https://github.com/pytest-dev/pytest-cov |
 | python-discovery | 1.4.2 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated document... | UNKNOWN |
 | python-dotenv | 1.2.2 | BSD-3-Clause | UNKNOWN |
-| pytokens | 0.4.1 | MIT License Copyright (c) 2024 Tushar Sadhwani Permission is hereby granted, free of charge, to any person obtaining... | UNKNOWN |
 | PyYAML | 6.0.3 | MIT | https://pyyaml.org/ |
-| RapidFuzz | 3.14.5 | MIT | UNKNOWN |
 | realtime | 2.31.0 | MIT | UNKNOWN |
 | requests | 2.34.2 | Apache-2.0 | UNKNOWN |
-| requests-toolbelt | 1.0.0 | Apache 2.0 | https://toolbelt.readthedocs.io/ |
-| ruff | 0.15.12 | MIT | https://docs.astral.sh/ruff |
-| SecretStorage | 3.5.0 | BSD-3-Clause | UNKNOWN |
-| shellingham | 1.5.4 | ISC License | https://github.com/sarugaku/shellingham |
 | starlette | 1.3.1 | BSD-3-Clause | UNKNOWN |
 | storage3 | 2.31.0 | MIT | UNKNOWN |
 | StrEnum | 0.4.15 | UNKNOWN | https://github.com/irgeek/StrEnum |
 | structlog | 24.4.0 | MIT OR Apache-2.0 | UNKNOWN |
 | supabase | 2.31.0 | MIT | UNKNOWN |
 | supabase-auth | 2.31.0 | MIT | UNKNOWN |
 | supabase-functions | 2.31.0 | MIT | UNKNOWN |
-| tomlkit | 0.15.0 | MIT | UNKNOWN |
-| trove-classifiers | 2026.6.1.19 | UNKNOWN | https://github.com/pypa/trove-classifiers |
 | types-PyYAML | 6.0.12.20260518 | Apache-2.0 | UNKNOWN |
 | typing-inspection | 0.4.2 | MIT | UNKNOWN |
 | typing_extensions | 4.15.0 | PSF-2.0 | UNKNOWN |

netengine/api/routes.py

@@ -23,7 +23,12 @@
 from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for
 from netengine.logging import get_logger
 from netengine.phase_labels import PHASE_LABELS
-from netengine.security.redaction import redact_for_api, redact_for_support_bundle
+from netengine.security.redaction import (
+    _contains_private_pem,
+    _is_secret_field,
+    redact_for_api,
+    redact_for_support_bundle,
+)
 from netengine.spec.loader import SpecLoadError, load_spec
 from netengine.spec.models import SPEC_SCHEMA_VERSION, NetEngineSpec
 

netengine/cli/main.py

@@ -1079,6 +1079,26 @@ def _print_status(state: RuntimeState) -> None:
     if state.step_ca_container_id:
         click.echo(f"step-ca container: {state.step_ca_container_id}")
 
+    failures = state.event_send_failures
+    if failures:
+        from netengine.events.emitter import _PGMQ_DISABLED_QUEUE
+
+        disabled_drops = [f for f in failures if f.get("queue") == _PGMQ_DISABLED_QUEUE]
+        send_failures = [f for f in failures if f.get("queue") != _PGMQ_DISABLED_QUEUE]
+        click.echo("")
+        if disabled_drops:
+            click.echo(
+                f"WARNING: pgmq client not wired — {len(disabled_drops)} event(s) silently "
+                "dropped in last run. Run `netengine doctor` for details."
+            )
+        if send_failures:
+            click.echo(f"Event send failures: {len(send_failures)}")
+            for failure in send_failures[-3:]:
+                click.echo(
+                    f"  ! [{failure.get('queue', '?')}] {failure.get('event_type', '?')}: "
+                    f"{failure.get('exception', '?')}"
+                )
+
 
 @cli.command()
 @click.option("--name", default=None, help="World name (pre-fills wizard prompt).")

netengine/diagnostic/db_doctor.py

@@ -3,10 +3,13 @@
 from __future__ import annotations
 
 import asyncio
+import json
 from collections.abc import Iterable
+from pathlib import Path
 from urllib.parse import urlparse
 
 from netengine.diagnostic.preflight import DoctorCheckResult, DoctorStatus
+from netengine.events.emitter import _PGMQ_DISABLED_QUEUE
 from netengine.events.queues import PRIMARY_QUEUES, dlq_for
 
 DB_URL_HINT = (
@@ -15,6 +18,10 @@
 )
 PGMQ_HINT = "Run database migrations or install pgmq before normal alpha boot."
 QUEUE_HINT = "Run `netengine migrate up`; migrations may create missing pgmq queues."
+PGMQ_RUNTIME_HINT = (
+    "pgmq_client was not wired during the last world run — events were silently dropped. "
+    "Ensure database migrations ran and pgmq is healthy, then re-run `netengine up`."
+)
 
 
 def _parse_db_url(db_url: str | None) -> DoctorCheckResult:
@@ -62,7 +69,7 @@ def _queue_name(row: object) -> str | None:
 
 async def _inspect_database(db_url: str, *, timeout: float) -> list[DoctorCheckResult]:
     try:
-        import asyncpg  # type: ignore[import]
+        import asyncpg  # type: ignore[import-untyped]
     except ImportError:
         return [
             DoctorCheckResult(
@@ -157,11 +164,62 @@ async def _inspect_database(db_url: str, *, timeout: float) -> list[DoctorCheckR
         await conn.close()
 
 
-def check_database(
-    db_url: str | None, *, timeout: float = 3.0
-) -> list[DoctorCheckResult]:
+def check_database(db_url: str | None, *, timeout: float = 3.0) -> list[DoctorCheckResult]:
     """Return actionable doctor checks for Postgres, pgmq, and event queues."""
     checks = [parsed := _parse_db_url(db_url)]
     if not db_url or parsed.status != DoctorStatus.OK:
         return checks
     return checks + asyncio.run(_inspect_database(db_url, timeout=timeout))
+
+
+def check_pgmq_runtime_state(state_file: Path) -> DoctorCheckResult:
+    """Check the runtime state file for events dropped because pgmq_client was not wired.
+
+    This surfaces the ephemeral pgmq-disabled condition that is otherwise only
+    visible at WARNING log level during a live run.
+    """
+    if not state_file.exists():
+        return DoctorCheckResult(
+            "pgmq runtime state",
+            DoctorStatus.SKIP,
+            "no runtime state file found (world not yet booted)",
+            group="database",
+            required=False,
+        )
+
+    try:
+        data = json.loads(state_file.read_text())
+    except Exception as exc:
+        return DoctorCheckResult(
+            "pgmq runtime state",
+            DoctorStatus.WARN,
+            f"could not read runtime state file: {exc}",
+            "Inspect the state file manually or re-run `netengine up`.",
+            group="database",
+            required=False,
+        )
+
+    failures: list[dict[str, object]] = data.get("event_send_failures") or []
+    disabled_drops = [f for f in failures if f.get("queue") == _PGMQ_DISABLED_QUEUE]
+    if disabled_drops:
+        return DoctorCheckResult(
+            "pgmq runtime state",
+            DoctorStatus.WARN,
+            (
+                f"{len(disabled_drops)} event(s) silently dropped in last run "
+                f"because pgmq_client was not wired "
+                f"(most recent: {disabled_drops[-1].get('event_type', 'unknown')} "
+                f"at {disabled_drops[-1].get('failed_at', 'unknown')})"
+            ),
+            PGMQ_RUNTIME_HINT,
+            group="database",
+            required=False,
+        )
+
+    return DoctorCheckResult(
+        "pgmq runtime state",
+        DoctorStatus.OK,
+        "no pgmq-disabled drops recorded in last run",
+        group="database",
+        required=False,
+    )

netengine/diagnostic/preflight.py

@@ -119,6 +119,8 @@ def _parse_compose_port(raw_port: object) -> tuple[int, str] | None:
     if isinstance(raw_port, dict):
         published = raw_port.get("published") or raw_port.get("target")
         proto = str(raw_port.get("protocol") or "tcp").lower()
+        if published is None:
+            return None
         try:
             return int(published), proto
         except (TypeError, ValueError):
@@ -330,9 +332,7 @@ def _can_bind(port: int, proto: str) -> bool:
 
 def _check_port(port: int, proto: str) -> DoctorCheckResult:
     name = f"port:{port}/{proto}"
-    label = next(
-        (p.label for p in KNOWN_LOCAL_PORTS if p.port == port and p.proto == proto), None
-    )
+    label = next((p.label for p in KNOWN_LOCAL_PORTS if p.port == port and p.proto == proto), None)
     detail_suffix = f" ({label})" if label else ""
     try:
         _can_bind(port, proto)
@@ -479,20 +479,20 @@ def _check_docker_conflicts(ctx: DoctorContext) -> list[DoctorCheckResult]:
                 (
                     DoctorStatus.FAIL
                     if kind == "container" and conflicts
-                    else DoctorStatus.WARN
-                    if conflicts
-                    else DoctorStatus.OK
+                    else DoctorStatus.WARN if conflicts else DoctorStatus.OK
                 ),
                 ", ".join(conflicts) if conflicts else "no known name conflicts",
                 (
                     "Stop/remove conflicting containers before startup."
                     if kind == "container" and conflicts
                     else (
-                        "Run `netengine down` or remove stale Docker resources if these "
-                        "belong to an old run."
+                        (
+                            "Run `netengine down` or remove stale Docker resources if these "
+                            "belong to an old run."
+                        )
+                        if conflicts
+                        else None
                     )
-                    if conflicts
-                    else None
                 ),
                 "docker",
                 required=kind == "container",
@@ -604,6 +604,12 @@ def _check_filesystem(ctx: DoctorContext) -> list[DoctorCheckResult]:
     ]
 
 
+def _check_pgmq_runtime_state(ctx: DoctorContext) -> DoctorCheckResult:
+    from netengine.diagnostic.db_doctor import check_pgmq_runtime_state
+
+    return check_pgmq_runtime_state(ctx.state_file)
+
+
 def standard_probes() -> tuple[DoctorProbe, ...]:
     """Return host-readiness probes; each accepts only ``DoctorContext``."""
     return (
@@ -614,6 +620,7 @@ def standard_probes() -> tuple[DoctorProbe, ...]:
         lambda ctx: _check_docker_daemon(),
         lambda ctx: _check_compose(),
         _check_database,
+        _check_pgmq_runtime_state,
         _check_ports,
         _check_filesystem,
         _check_docker_conflicts,

netengine/diagnostic/runner.py

@@ -60,14 +60,10 @@ def __init__(
         self.elapsed_ms = elapsed_ms
         self.remediation = remediation or hint
         self.related_phase = related_phase if related_phase is not None else phase
-        self.related_resource = (
-            related_resource if related_resource is not None else resource
-        )
+        self.related_resource = related_resource if related_resource is not None else resource
         log_commands = related_logs if related_logs is not None else logs
         self.related_logs = list(log_commands or [])
-        self.command_to_retry = (
-            command_to_retry if command_to_retry is not None else retry_command
-        )
+        self.command_to_retry = command_to_retry if command_to_retry is not None else retry_command
 
     @property
     def phase(self) -> int | None:

netengine/events/emitter.py

@@ -9,6 +9,8 @@
 from netengine.events.schema import EventEnvelope
 from netengine.handlers.context import PhaseContext
 
+_PGMQ_DISABLED_QUEUE = "<pgmq_disabled>"
+
 
 def _failure_record(event: EventEnvelope, queue: Queue | str, exc: Exception) -> EventSendFailure:
     """Build the structured runtime-state record for an event send failure."""
@@ -105,7 +107,16 @@ async def emit_event(
     )
 
     if context.pgmq_client is None:
-        context.logger.debug("pgmq_client not available; event logged only")
+        context.logger.warning(
+            "pgmq_client not wired; event dropped "
+            f"(event_type={event_type}, emitted_by={emitted_by})"
+        )
+        record_event_send_failure(
+            context,
+            event,
+            _PGMQ_DISABLED_QUEUE,
+            RuntimeError("pgmq_client not wired — event silently dropped"),
+        )
         return event
 
     target_queue = queue or queue_for_event_type(event_type)

netengine/handlers/and_handler.py

@@ -7,9 +7,9 @@
 from netengine.events.schema import EventEnvelope
 from netengine.handlers.context import PhaseContext
 from netengine.handlers.dns import DNSHandler
-from netengine.handlers.protocols import DockerAdapterProtocol, PGMQAdapterProtocol
 from netengine.handlers.domain_registry_handler import DomainRegistryHandler
 from netengine.handlers.gateway_handler import GatewayHandler
+from netengine.handlers.protocols import DockerAdapterProtocol, PGMQAdapterProtocol
 from netengine.logging import get_logger
 from netengine.spec.models import NetEngineSpec
 

netengine/handlers/app_handler.py

@@ -6,9 +6,9 @@
 from netengine.handlers._base import BasePhaseHandler
 from netengine.handlers.context import PhaseContext
 from netengine.handlers.dns import DNSHandler
-from netengine.handlers.protocols import DockerAdapterProtocol, PGMQAdapterProtocol
 from netengine.handlers.oidc_handler import OIDCHandler
 from netengine.handlers.pki_handler import PKIHandler
+from netengine.handlers.protocols import DockerAdapterProtocol, PGMQAdapterProtocol
 from netengine.logging import get_logger
 from netengine.spec.models import NetEngineSpec
 

netengine/handlers/context.py

@@ -5,9 +5,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from netengine.handlers.protocols import DockerAdapterProtocol, PGMQAdapterProtocol
-
 from netengine.core.state import RuntimeState
+from netengine.handlers.protocols import DockerAdapterProtocol, PGMQAdapterProtocol
 from netengine.spec.models import NetEngineSpec
 
 if TYPE_CHECKING: