test: reconcile ambiguous setup response

Author: misunders2d

scripts/ci/ordinary-server-upgrade-e2e.sh

@@ -66,6 +66,44 @@ report_failure() {
         --property=SubState --property=Result --property=ExecMainStatus >&2
       sudo journalctl -u "$unit" --no-pager -n 12 >&2 || true
     fi
+      if [[ "$unit" == "agentnet-c0-responder.service" ]] &&
+        sudo test -f /var/lib/agentnet-c0/config.json &&
+        sudo test -f /var/lib/agentnet/guided-join.key.pem &&
+        [[ -x "${PYTHON_0144:-}" ]]; then
+        sudo install -o agentnet-c0 -g agentnet-c0 -m 0400 \
+          /var/lib/agentnet/guided-join.key.pem "$WORK/c0-probe-key.pem"
+        sudo -u agentnet-c0 env \
+          HOME=/var/lib/agentnet-c0 \
+          NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
+          "$PYTHON_0144" - /var/lib/agentnet-c0/config.json "$WORK/c0-probe-key.pem" <<'PY' >&2 || true
+import json
+import sys
+from pathlib import Path
+
+from agentnet.supervisor.c0_responder import _client, load_c0_responder_config
+
+client, _core = _client(
+    load_c0_responder_config(Path(sys.argv[1])),
+    Path(sys.argv[2]),
+    transport=None,
+)
+try:
+    response = client.c0_pilot_readiness()
+    try:
+        body = response.json()
+    except ValueError:
+        body = {"message": "non-JSON response"}
+    safe_body = {
+        key: body[key]
+        for key in ("schema", "status", "code", "message")
+        if isinstance(body, dict) and key in body
+    }
+    print(json.dumps({"c0_probe_status": response.status_code, "body": safe_body}, sort_keys=True))
+finally:
+    client.close()
+PY
+        sudo rm -f "$WORK/c0-probe-key.pem"
+      fi
   done
   return "$status"
 }
@@ -244,17 +282,72 @@ plan_setup() {
     server-agent setup --request "$INPUTS/server-setup.json"
 }
 
+RELEASED_SETUP_START_RECONCILED=false
+
+released_setup_is_live() {
+  local prefix="$1"
+  local version
+  version="$(sudo jq -r '.version' "$prefix/lib/node_modules/@misunders2d/agentnet/package.json")"
+  local unit
+  for unit in agentnet-core.service agentnet-approval.service agentnet-c0-responder.service; do
+    [[ "$(sudo systemctl show "$unit" --property=ActiveState --value)" == "active" ]] || return 1
+    [[ "$(sudo systemctl show "$unit" --property=UnitFileState --value)" == "enabled" ]] || return 1
+  done
+  [[ "$(sudo systemctl show agentnet-credential-renew.timer --property=ActiveState --value)" == "active" ]] || return 1
+  [[ "$(sudo systemctl show agentnet-credential-renew.timer --property=UnitFileState --value)" == "enabled" ]] || return 1
+  sudo jq -e --arg version "$version" \
+    '.package_version == $version and .profile == "always_on_server_agent"' \
+    /var/lib/agentnet-setup/setup.json >/dev/null || return 1
+  env NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
+    curl --fail --silent --show-error https://approval.agentnet.test/healthz |
+    jq -e --arg version "$version" \
+      '.service == "agentnet-approval" and .version == $version and .status == "alive"' \
+      >/dev/null || return 1
+  env NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
+    curl --fail --silent --show-error https://core.agentnet.test/healthz |
+    jq -e --arg version "$version" \
+      '.service == "agentnet-core" and .version == $version and .status == "alive"' \
+      >/dev/null
+}
+
+wait_for_released_setup_live() {
+  local prefix="$1"
+  local attempt
+  for attempt in $(seq 1 120); do
+    if released_setup_is_live "$prefix"; then
+      return 0
+    fi
+    sleep 0.25
+  done
+  return 1
+}
+
 apply_setup() {
   local prefix="$1"
   local digest="$2"
   local output="$3"
+  local exit_code=0
   run_evidence "$output" sudo -- env \
     PATH="$prefix/bin:/usr/bin:/bin" \
     AGENTNET_UV="$prefix/bin/uv" \
     NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
     "$prefix/bin/node" "$(launcher "$prefix")" \
     server-agent setup --request "$INPUTS/server-setup.json" \
-    --expected-request-digest "$digest" --apply --start
+    --expected-request-digest "$digest" --apply --start || exit_code=$?
+  if [[ "$exit_code" -eq 0 ]]; then
+    return 0
+  fi
+  if [[ "$prefix" == "$PREFIX_0144" ]] &&
+    jq -e '.status == "blocked" and .blocker == "systemd_start"' "$output" >/dev/null &&
+    wait_for_released_setup_live "$prefix"; then
+    # Released 0.1.50 can lose the final systemctl response after the immutable
+    # marker, exact units, and exact health identities have all converged. Do
+    # not reapply while those units are live; retain the blocked response and
+    # record only the independently observed postcondition.
+    RELEASED_SETUP_START_RECONCILED=true
+    return 0
+  fi
+  return "$exit_code"
 }
 
 psql_agentnet() {
@@ -622,7 +715,11 @@ jq -e --arg harness "$HARNESS_ID" --arg credential "$CREDENTIAL_ID" \
 sudo install -d -o root -g root -m 0755 /var/lib/systemd/timers
 sudo touch /var/lib/systemd/timers/stamp-agentnet-credential-renew.timer
 apply_setup "$PREFIX_0144" "$DIGEST_0144" "$APPLY_BOUND_0144"
-jq -e '.status == "operational" and .identity_enrolled == true and .authority_granted == false' "$APPLY_BOUND_0144" >/dev/null
+if [[ "$RELEASED_SETUP_START_RECONCILED" == "true" ]]; then
+  jq -e '.status == "blocked" and .blocker == "systemd_start"' "$APPLY_BOUND_0144" >/dev/null
+else
+  jq -e '.status == "operational" and .identity_enrolled == true and .authority_granted == false' "$APPLY_BOUND_0144" >/dev/null
+fi
 sudo grep -Fxq 'OnUnitActiveSec=1h' /etc/systemd/system/agentnet-credential-renew.timer
 sudo grep -Fxq 'Persistent=true' /etc/systemd/system/agentnet-credential-renew.timer
 ! sudo grep -Fq 'OnUnitInactiveSec=' /etc/systemd/system/agentnet-credential-renew.timer