Nowlert CE v3.1.0: Amber WebUI and QA fixes

.env.example

@@ -1,7 +1,7 @@
 # Nowlert production Compose defaults.
 # Copy this file to .env and adjust it on the deployment host.
 
-NOWLERT_IMAGE=theriark/nowlert-ce:3.0.0
+NOWLERT_IMAGE=theriark/nowlert-ce:3.1.0
 NOWLERT_UID=1000
 NOWLERT_GID=1000
 NOWLERT_SMTP_PORT=8025

.github/scripts/dokploy_release.py

@@ -184,27 +184,66 @@ def current_image(application_id: str) -> str:
     return image.strip()
 
 
-def wait_health(url: str, timeout_seconds: int) -> None:
+def wait_health(
+    url: str,
+    timeout_seconds: int,
+    *,
+    expected_version: str = "",
+) -> None:
     deadline = time.monotonic() + timeout_seconds
     last_error = "health endpoint did not respond"
+    consecutive_matches = 0
+
     while time.monotonic() < deadline:
-        request = urllib.request.Request(url, headers={"accept": "application/json"})
+        request = urllib.request.Request(
+            url,
+            headers={"accept": "application/json"},
+        )
         try:
             with urllib.request.urlopen(request, timeout=15) as response:
                 body = response.read().decode("utf-8", errors="replace")
                 status_code = response.status
             payload = json.loads(body)
-            if status_code == 200 and payload.get("status") == "ok":
-                print(
-                    f"PASS: {url} returned HTTP 200 status=ok "
-                    f"version={payload.get('version', 'unknown')}"
+            observed_version = str(payload.get("version") or "").strip()
+            healthy = status_code == 200 and payload.get("status") == "ok"
+
+            if not healthy:
+                consecutive_matches = 0
+                last_error = f"HTTP {status_code}: {body[:500]}"
+            elif expected_version and observed_version != expected_version:
+                consecutive_matches = 0
+                last_error = (
+                    f"healthy response is still version {observed_version or 'unknown'}; "
+                    f"waiting for {expected_version}"
                 )
-                return
-            last_error = f"HTTP {status_code}: {body[:500]}"
-        except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError) as exc:
+                print(f"WAIT: {url} {last_error}")
+            else:
+                consecutive_matches += 1
+                if consecutive_matches >= 2:
+                    print(
+                        f"PASS: {url} returned two consecutive HTTP 200 "
+                        f"status=ok responses version={observed_version or 'unknown'}"
+                    )
+                    return
+                last_error = (
+                    f"received first matching healthy response version="
+                    f"{observed_version or 'unknown'}; confirming"
+                )
+                print(f"WAIT: {url} {last_error}")
+        except (
+            urllib.error.URLError,
+            urllib.error.HTTPError,
+            json.JSONDecodeError,
+            TimeoutError,
+        ) as exc:
+            consecutive_matches = 0
             last_error = str(exc)
-        time.sleep(10)
-    raise DokployError(f"Health check timed out for {url}: {last_error}")
+        time.sleep(5)
+
+    expected = f" version={expected_version}" if expected_version else ""
+    raise DokployError(
+        f"Health check timed out for {url}{expected}: {last_error}"
+    )
 
 
 def deploy(args: argparse.Namespace) -> None:
@@ -232,7 +271,11 @@ def deploy(args: argparse.Namespace) -> None:
         )
         print(f"Deployment requested for {args.application_id}: {args.image}")
 
-    wait_health(args.health_url, args.timeout)
+    wait_health(
+        args.health_url,
+        args.timeout,
+        expected_version=args.expected_version,
+    )
     observed = current_image(args.application_id)
     if observed != args.image:
         raise DokployError(
@@ -251,6 +294,29 @@ def assert_image(args: argparse.Namespace) -> None:
     print(f"PASS: {args.application_id} runs {observed}")
 
 
+def promotion_smoke(args: argparse.Namespace) -> None:
+    """Passively verify a promoted image without submitting notification events."""
+    validate_image(args.image)
+    marker = args.success_marker.strip()
+    if not marker:
+        raise DokployError("Promotion smoke success marker must not be empty")
+
+    wait_health(
+        args.health_url,
+        args.timeout,
+        expected_version=args.expected_version,
+    )
+    observed = current_image(args.application_id)
+    if observed != args.image:
+        raise DokployError(
+            f"Dokploy application {args.application_id} reports {observed}, expected {args.image}"
+        )
+
+    print(f"PASS: passive promotion smoke confirmed exact image {observed}")
+    print("PASS: notification delivery tests disabled for this promotion smoke")
+    print(marker)
+
+
 def list_schedule_deployments(schedule_id: str) -> list[dict[str, Any]]:
     response = request_json(
         "GET",
@@ -353,6 +419,7 @@ def build_parser() -> argparse.ArgumentParser:
     deploy_parser.add_argument("--title", default="Immutable image deployment")
     deploy_parser.add_argument("--description", default="Managed by GitHub Actions")
     deploy_parser.add_argument("--timeout", type=int, default=600)
+    deploy_parser.add_argument("--expected-version", default="")
     deploy_parser.add_argument("--allow-mutable", action="store_true")
     deploy_parser.add_argument("--noop-ok", action="store_true")
     deploy_parser.set_defaults(func=deploy)
@@ -362,6 +429,15 @@ def build_parser() -> argparse.ArgumentParser:
     assert_parser.add_argument("--image", required=True)
     assert_parser.set_defaults(func=assert_image)
 
+    smoke_parser = subparsers.add_parser("promotion-smoke")
+    smoke_parser.add_argument("--application-id", required=True)
+    smoke_parser.add_argument("--image", required=True)
+    smoke_parser.add_argument("--health-url", required=True)
+    smoke_parser.add_argument("--expected-version", required=True)
+    smoke_parser.add_argument("--success-marker", required=True)
+    smoke_parser.add_argument("--timeout", type=int, default=300)
+    smoke_parser.set_defaults(func=promotion_smoke)
+
     schedule_parser = subparsers.add_parser("run-schedule")
     schedule_parser.add_argument("--schedule-id", required=True)
     schedule_parser.add_argument("--success-marker", required=True)

.github/scripts/finalize_release.py

@@ -28,13 +28,29 @@
     "stage": {"workflow_dispatch"},
     "production_reference": {"workflow_dispatch"},
 }
-STAGE_SCHEDULE_ID = "meHx1NPZfCxqASLe4RAYS"
-STAGE_SUCCESS_MARKER = "STAGE CE QA PASSED"
+STAGE_SCHEDULE_ID = "meHx1NPZfCxqASLe4RAYS"  # legacy v3.0 backfill only
+STAGE_SUCCESS_MARKER = "STAGE CE QA PASSED"  # legacy v3.0 backfill only
 PRODREF_APPLICATION_ID = "-Qb71PLUZmBHLJ_Iv68Oo"
-PRODREF_SCHEDULE_ID = "JVSEdTk4tt4vKPbG3cKFZ"
-PRODREF_QA_HOST = "vm-13"
-PRODREF_QA_ROOT = "/var/lib/nowlert-qa/evidence"
-PRODREF_SUCCESS_MARKER = "PRODUCTION REFERENCE CE POST-PROMOTION SMOKE PASSED"
+PRODREF_SCHEDULE_ID = "JVSEdTk4tt4vKPbG3cKFZ"  # legacy v3.0 backfill only
+PRODREF_QA_HOST = "vm-13"  # legacy v3.0 backfill only
+PRODREF_QA_ROOT = "/var/lib/nowlert-qa/evidence"  # legacy v3.0 backfill only
+PRODREF_SUCCESS_MARKER = "PRODUCTION REFERENCE CE POST-PROMOTION SMOKE PASSED"  # legacy
+STAGE_SILENT_SUCCESS_MARKER = "STAGE CE SILENT PROMOTION SMOKE PASSED"
+PRODREF_SILENT_SUCCESS_MARKER = (
+    "PRODUCTION REFERENCE CE SILENT PROMOTION SMOKE PASSED"
+)
+SILENT_DELIVERY_DISABLED_MARKER = (
+    "PASS: notification delivery tests disabled for this promotion smoke"
+)
+SILENT_PROMOTION_FORBIDDEN_LOG_FRAGMENTS = (
+    "===== DELIVERY =====",
+    "Expected deliveries:",
+    "Accepted deliveries:",
+    "Related firing run:",
+    "stage-ce-all-",
+    "stage-ce-zabbix-",
+    "stage-ce-portainer-",
+)
 
 
 def fail(message: str) -> None:
@@ -120,6 +136,25 @@ def require_in_logs(logs: str, value: str, description: str, run_id: int) -> Non
         fail(f"Run {run_id} logs do not contain {description}: {value}")
 
 
+def require_absent_from_logs(logs: str, value: str, description: str, run_id: int) -> None:
+    if value in logs:
+        fail(f"Run {run_id} unexpectedly contains {description}: {value}")
+
+
+def validate_silent_promotion_logs(logs: str, run_id: int, marker: str) -> None:
+    require_in_logs(logs, marker, "the notification-silent promotion marker", run_id)
+    require_in_logs(
+        logs,
+        SILENT_DELIVERY_DISABLED_MARKER,
+        "the explicit delivery-test-disabled marker",
+        run_id,
+    )
+    for value in SILENT_PROMOTION_FORBIDDEN_LOG_FRAGMENTS:
+        require_absent_from_logs(
+            logs, value, "notification delivery QA activity", run_id
+        )
+
+
 def validate_args(args: argparse.Namespace) -> None:
     if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?", args.version):
         fail("Version must be a semantic version beginning with v, for example v3.0.0")
@@ -129,7 +164,9 @@ def validate_args(args: argparse.Namespace) -> None:
         re.escape(EXPECTED_IMAGE_PREFIX) + r"[0-9a-f]{64}", args.final_image
     ):
         fail("Final image must be an immutable nowlert-ce GHCR digest")
-    if not re.fullmatch(r"[A-Za-z0-9_-]+", args.qa_schedule_deployment_id):
+    if args.qa_schedule_deployment_id and not re.fullmatch(
+        r"[A-Za-z0-9_-]+", args.qa_schedule_deployment_id
+    ):
         fail("QA schedule deployment ID contains unexpected characters")
     if not args.release_notes.strip():
         fail("Release notes must not be empty")
@@ -198,12 +235,20 @@ def validate_runs(args: argparse.Namespace) -> dict[str, dict[str, Any]]:
         "the immutable image digest",
         args.stage_run,
     )
-    require_in_logs(
-        stage_logs,
-        f"PASS: schedule {STAGE_SCHEDULE_ID} emitted marker: {STAGE_SUCCESS_MARKER}",
-        "the successful VM-12 Stage gate confirmation",
-        args.stage_run,
-    )
+
+    if args.qa_schedule_deployment_id:
+        # Backward-compatible validation for the immutable v3.0 ledger backfill.
+        require_in_logs(
+            stage_logs,
+            f"PASS: schedule {STAGE_SCHEDULE_ID} emitted marker: {STAGE_SUCCESS_MARKER}",
+            "the successful legacy VM-12 Stage gate confirmation",
+            args.stage_run,
+        )
+    else:
+        validate_silent_promotion_logs(
+            stage_logs, args.stage_run, STAGE_SILENT_SUCCESS_MARKER
+        )
+
     require_in_logs(
         prodref_logs,
         args.final_image,
@@ -216,33 +261,75 @@ def validate_runs(args: argparse.Namespace) -> dict[str, dict[str, Any]]:
         "the supplied Stage promotion run reference",
         args.production_reference_run,
     )
-    require_in_logs(
-        prodref_logs,
-        f"Execution host: {PRODREF_QA_HOST}",
-        "the VM-13 execution host",
-        args.production_reference_run,
-    )
-    require_in_logs(
-        prodref_logs,
-        (
-            f"Detected schedule deployment {args.qa_schedule_deployment_id} "
-            f"for {PRODREF_SCHEDULE_ID}"
-        ),
-        "the exact VM-13 QA schedule deployment",
-        args.production_reference_run,
-    )
-    require_in_logs(
-        prodref_logs,
-        f"PASS: schedule {PRODREF_SCHEDULE_ID} emitted marker: {PRODREF_SUCCESS_MARKER}",
-        "the successful VM-13 marker confirmation",
-        args.production_reference_run,
-    )
+
+    if args.qa_schedule_deployment_id:
+        require_in_logs(
+            prodref_logs,
+            f"Execution host: {PRODREF_QA_HOST}",
+            "the legacy VM-13 execution host",
+            args.production_reference_run,
+        )
+        require_in_logs(
+            prodref_logs,
+            (
+                f"Detected schedule deployment {args.qa_schedule_deployment_id} "
+                f"for {PRODREF_SCHEDULE_ID}"
+            ),
+            "the exact legacy VM-13 QA schedule deployment",
+            args.production_reference_run,
+        )
+        require_in_logs(
+            prodref_logs,
+            f"PASS: schedule {PRODREF_SCHEDULE_ID} emitted marker: {PRODREF_SUCCESS_MARKER}",
+            "the successful legacy VM-13 marker confirmation",
+            args.production_reference_run,
+        )
+    else:
+        validate_silent_promotion_logs(
+            prodref_logs,
+            args.production_reference_run,
+            PRODREF_SILENT_SUCCESS_MARKER,
+        )
 
     return runs
 
 
 def write_outputs(args: argparse.Namespace, runs: dict[str, dict[str, Any]]) -> None:
     created_at = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
+
+    if args.qa_schedule_deployment_id:
+        qa_evidence = {
+            "type": "legacy_schedule_promotion_chain",
+            "host": PRODREF_QA_HOST,
+            "root": PRODREF_QA_ROOT,
+            "schedule_id": PRODREF_SCHEDULE_ID,
+            "schedule_deployment_id": args.qa_schedule_deployment_id,
+            "success_marker": PRODREF_SUCCESS_MARKER,
+            "notification_delivery_tests": True,
+        }
+        evidence_summary = (
+            f"- Legacy VM-13 schedule deployment: `{args.qa_schedule_deployment_id}`\n"
+            f"- Legacy VM-13 success marker: `{PRODREF_SUCCESS_MARKER}`\n"
+        )
+    else:
+        qa_evidence = {
+            "type": "notification_silent_promotion_chain",
+            "notification_delivery_tests": False,
+            "stage": {
+                "promotion_run": str(args.stage_run),
+                "success_marker": STAGE_SILENT_SUCCESS_MARKER,
+            },
+            "production_reference": {
+                "promotion_run": str(args.production_reference_run),
+                "success_marker": PRODREF_SILENT_SUCCESS_MARKER,
+            },
+        }
+        evidence_summary = (
+            f"- Stage silent-smoke marker: `{STAGE_SILENT_SUCCESS_MARKER}`\n"
+            f"- Production Reference silent-smoke marker: `{PRODREF_SILENT_SUCCESS_MARKER}`\n"
+            "- Notification delivery tests during promotion: disabled\n"
+        )
+
     manifest = {
         "schema_version": 1,
         "edition": "ce",
@@ -253,13 +340,7 @@ def write_outputs(args: argparse.Namespace, runs: dict[str, dict[str, Any]]) ->
         "production_reference_run": str(args.production_reference_run),
         "final_image": args.final_image,
         "production_reference_application_id": PRODREF_APPLICATION_ID,
-        "qa_evidence": {
-            "host": PRODREF_QA_HOST,
-            "root": PRODREF_QA_ROOT,
-            "schedule_id": PRODREF_SCHEDULE_ID,
-            "schedule_deployment_id": args.qa_schedule_deployment_id,
-            "success_marker": PRODREF_SUCCESS_MARKER,
-        },
+        "qa_evidence": qa_evidence,
         "workflow_runs": {
             key: {
                 "id": str(run["id"]),
@@ -293,9 +374,7 @@ def write_outputs(args: argparse.Namespace, runs: dict[str, dict[str, Any]]) ->
 - Development run: `{args.development_run}`
 - Stage promotion run: `{args.stage_run}`
 - Production Reference run: `{args.production_reference_run}`
-- VM-13 schedule deployment: `{args.qa_schedule_deployment_id}`
-- VM-13 success marker: `{PRODREF_SUCCESS_MARKER}`
-- Rebuild during promotion: no
+{evidence_summary}- Rebuild during promotion: no
 - Deployment during release finalisation: no
 """
     Path(args.summary_path).write_text(summary, encoding="utf-8")
@@ -310,7 +389,11 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--development-run", required=True, type=int)
     parser.add_argument("--stage-run", required=True, type=int)
     parser.add_argument("--production-reference-run", required=True, type=int)
-    parser.add_argument("--qa-schedule-deployment-id", required=True)
+    parser.add_argument(
+        "--qa-schedule-deployment-id",
+        default="",
+        help="Deprecated: legacy v3.0 schedule evidence only",
+    )
     parser.add_argument("--release-notes", required=True)
     parser.add_argument("--manifest-path", default="release-manifest.json")
     parser.add_argument("--summary-path", default="release-summary.md")