test: reconcile ambiguous setup response

Author: misunders2d

evidence/local/2026-08-09-v0.1.51/manifest.json

@@ -4,7 +4,7 @@
   "scope": "v0_1_51_canonical_owner_recovery_candidate",
   "verification_status": "PASS",
   "release_certified": false,
-  "verification_scope": "PASS applies only to the selected local source, focused recovery/communication tests, broad hermetic suite, reproducible package, and installed-byte commands recorded below; it is not release certification and does not include the exact v0.1.51 two-machine flow, fresh installed-host upgrade, dedicated PostgreSQL recovery, installed-harness pin gates, or any owner/privileged/external must-not-ship gate",
+  "verification_scope": "PASS applies only to the selected local source, focused recovery/communication tests, broad hermetic suite, reproducible package, and installed-byte commands recorded below; it is not release certification and does not include release-eligible two-machine proof, fresh installed-host upgrade, dedicated PostgreSQL recovery, installed-harness pin gates, or any owner/privileged/external must-not-ship gate",
   "source_revision_kind": "working_tree_candidate",
   "release_source_tree_sha256": "0906475b88bb57673fa2ab4ea793803d9931fad36e97a00519d52f7bb39685ea",
   "supersedes_package_evidence": "evidence/local/2026-08-07-v0.1.50/manifest.json",
@@ -35,8 +35,8 @@
       "result": "NOT RUN: exact same-commit GitHub workflow result awaits commit and push; no source-local result is substituted"
     },
     {
-      "command": "Two isolated Ubuntu 24.04 machines: one-command server setup; server-origin guided laptop join; complete browser OIDC and independent approval; send authenticated messages bidirectionally; prove offline reconnect, duplicate safety, and recipient acknowledgement",
-      "result": "NOT RUN: exact v0.1.51 cross-machine reliability proof awaits two isolated hosts; no historical run is inherited"
+      "command": "Ad-hoc live trial on two isolated Ubuntu 24.04.4 VMs with separate qcow2 disks and exact v0.1.51 wheel: send authenticated messages bidirectionally; stop and restart VM A Core; inspect recovered delivery, duplicate safety, exact attribution, and recipient acknowledgement",
+      "result": "OBSERVED PASS, NOT ACCEPTED AS RELEASE EVIDENCE: fresh A-to-B event f66de4d4-e993-505f-8fe3-d9969e85f83e recovered unchanged after a verified Core offline interval and reached recipient_committed; fresh B-to-A event 8f5f7fa8-dfb0-587a-9c91-784c716e66fe reached recipient_committed. The ad-hoc provisioning driver was not retained, so no must-not-ship gate or formal two-machine evidence status is promoted. Observation: evidence/local/2026-08-09-v0.1.51/two-vm-communication.json"
     },
     {
       "command": "Fresh Ubuntu 24.04 installed-host matrix: clean v0.1.51 setup and direct installed v0.1.45/v0.1.46/v0.1.47/v0.1.48/v0.1.49/v0.1.50 to v0.1.51 upgrades",
@@ -55,7 +55,7 @@
     "source": "AgentNet v0.1.51 canonical-owner recovery candidate on Linux x86_64 with locked CPython 3.13.13 dependencies",
     "callback_incident": "No external workflow result is included in this local candidate evidence",
     "packed_generations": "Local npm check validates the source package and two clean recursively packed and installed generations",
-    "two_vm_source_proof": "Not rerun for exact v0.1.51; cross-machine reliability trials await isolated hosts",
+    "two_vm_source_proof": "Observed exact packaged v0.1.51 behavior on two live isolated Ubuntu 24.04.4 VMs, but excluded from formal release evidence because the ad-hoc provisioning driver was not retained; production durability and must-not-ship gates remain unproven",
     "root_installed_external_host": "Not rerun for exact v0.1.51; fresh installed-host setup and direct upgrade evidence await GitHub CI"
   },
   "artifacts": [

evidence/local/2026-08-09-v0.1.51/two-vm-communication.json

@@ -0,0 +1,65 @@
+{
+  "schema": "agentnet.two-vm-communication-observation.v1",
+  "evaluated_at": "2026-08-09T12:08:14.219Z",
+  "trial_result": "PASS",
+  "release_evidence_eligible": false,
+  "release_certified": false,
+  "exclusion_reason": "Live commands and outputs were observed, but the ad-hoc provisioning driver was not retained as a repository script; this record cannot promote a release gate.",
+  "package": {
+    "version": "0.1.51",
+    "wheel_sha256": "8030d8cbd62ef05f6b3134994ccbd884f343356d16203d02484bbefde59af10e"
+  },
+  "hosts": [
+    {
+      "vm": "A",
+      "observed": [
+        "host=agentnet-vm-a",
+        "kernel=6.8.0-136-generic",
+        "os=Ubuntu 24.04.4 LTS",
+        "agentnet=agentnet 0.1.51"
+      ]
+    },
+    {
+      "vm": "B",
+      "observed": [
+        "host=agentnet-vm-b",
+        "kernel=6.8.0-136-generic",
+        "os=Ubuntu 24.04.4 LTS",
+        "agentnet=agentnet 0.1.51"
+      ]
+    }
+  ],
+  "launch_commands": {
+    "vm_a": "qemu-system-x86_64 -enable-kvm -machine q35 -cpu host -m 1536 -smp 2 -drive file=/tmp/agentnet-vm-e2e/vm-a.qcow2,format=qcow2,if=virtio -drive file=/tmp/agentnet-vm-e2e/seed-a.iso,format=raw,media=cdrom,readonly=on -netdev user,id=uplink,hostfwd=tcp:127.0.0.1:22221-:22 -device virtio-net-pci,netdev=uplink,mac=52:54:00:12:34:01 -netdev socket,id=agentnet,listen=127.0.0.1:24500 -device virtio-net-pci,netdev=agentnet,mac=52:54:00:12:50:01 -display none -serial file:/tmp/agentnet-vm-e2e/vm-a.serial.log -monitor none",
+    "vm_b": "qemu-system-x86_64 -enable-kvm -machine q35 -cpu host -m 1536 -smp 2 -drive file=/tmp/agentnet-vm-e2e/vm-b.qcow2,format=qcow2,if=virtio -drive file=/tmp/agentnet-vm-e2e/seed-b.iso,format=raw,media=cdrom,readonly=on -netdev user,id=uplink,hostfwd=tcp:127.0.0.1:22222-:22 -device virtio-net-pci,netdev=uplink,mac=52:54:00:12:34:02 -netdev socket,id=agentnet,connect=127.0.0.1:24500 -device virtio-net-pci,netdev=agentnet,mac=52:54:00:12:50:02 -display none -serial file:/tmp/agentnet-vm-e2e/vm-b.serial.log -monitor none"
+  },
+  "scope": {
+    "domain_id": "two-vm.example",
+    "collaboration_scope_id": "scope:two-vm-v0151",
+    "conversation_id": "conversation:two-vm-v0151"
+  },
+  "observations": {
+    "a_to_b": {
+      "status": 202,
+      "event_id": "f66de4d4-e993-505f-8fe3-d9969e85f83e",
+      "accepted_fact": "accepted_local",
+      "duplicate": true,
+      "offline_request_failed": true,
+      "recovered_after_restart": true,
+      "sender_harness_id": "synthetic-harness-6676033caa7c48e09cd10df1f8b8fc58",
+      "recipient_fact": "recipient_committed",
+      "acknowledgement_receipt_id": "0c38e37d-a80a-49b6-aebc-c49f1658da05",
+      "acknowledgement_duplicate": true
+    },
+    "b_to_a": {
+      "status": 202,
+      "event_id": "8f5f7fa8-dfb0-587a-9c91-784c716e66fe",
+      "accepted_fact": "accepted_local",
+      "duplicate": true,
+      "sender_harness_id": "synthetic-harness-ee8130787fe14f06921fff47ce01adfb",
+      "recipient_fact": "recipient_committed",
+      "acknowledgement_receipt_id": "24ed7807-ad08-4474-9464-f6b9226c3969"
+    }
+  },
+  "claim_boundary": "Observed exact packaged v0.1.51 bidirectional communication, duplicate safety, A-to-B recovery after Core offline interval, exact sender attribution, and recipient acknowledgement on two live local VMs. Does not prove production durability, multi-node HA, public-network operation, owner policy, privileged-host isolation, or any must-not-ship gate."
+}

scripts/ci/ordinary-server-upgrade-e2e.sh

@@ -143,6 +143,7 @@ trap 'report_failure "$LINENO" "$BASH_COMMAND"' ERR
 
 cleanup() {
   set +e
+  sudo systemctl unset-environment SSL_CERT_FILE >/dev/null 2>&1
   sudo systemctl start nginx >/dev/null 2>&1
   sudo systemctl stop agentnet-credential-renew-e2e.timer >/dev/null 2>&1
   sudo systemctl disable --now \
@@ -244,17 +245,72 @@ plan_setup() {
     server-agent setup --request "$INPUTS/server-setup.json"
 }
 
+RELEASED_SETUP_START_RECONCILED=false
+
+released_setup_is_live() {
+  local prefix="$1"
+  local version
+  version="$(sudo jq -r '.version' "$prefix/lib/node_modules/@misunders2d/agentnet/package.json")"
+  local unit
+  for unit in agentnet-core.service agentnet-approval.service agentnet-c0-responder.service; do
+    [[ "$(sudo systemctl show "$unit" --property=ActiveState --value)" == "active" ]] || return 1
+    [[ "$(sudo systemctl show "$unit" --property=UnitFileState --value)" == "enabled" ]] || return 1
+  done
+  [[ "$(sudo systemctl show agentnet-credential-renew.timer --property=ActiveState --value)" == "active" ]] || return 1
+  [[ "$(sudo systemctl show agentnet-credential-renew.timer --property=UnitFileState --value)" == "enabled" ]] || return 1
+  sudo jq -e --arg version "$version" \
+    '.package_version == $version and .profile == "always_on_server_agent"' \
+    /var/lib/agentnet-setup/setup.json >/dev/null || return 1
+  env NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
+    curl --fail --silent --show-error https://approval.agentnet.test/healthz |
+    jq -e --arg version "$version" \
+      '.service == "agentnet-approval" and .version == $version and .status == "alive"' \
+      >/dev/null || return 1
+  env NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
+    curl --fail --silent --show-error https://core.agentnet.test/healthz |
+    jq -e --arg version "$version" \
+      '.service == "agentnet-core" and .version == $version and .status == "alive"' \
+      >/dev/null
+}
+
+wait_for_released_setup_live() {
+  local prefix="$1"
+  local attempt
+  for attempt in $(seq 1 120); do
+    if released_setup_is_live "$prefix"; then
+      return 0
+    fi
+    sleep 0.25
+  done
+  return 1
+}
+
 apply_setup() {
   local prefix="$1"
   local digest="$2"
   local output="$3"
+  local exit_code=0
   run_evidence "$output" sudo -- env \
     PATH="$prefix/bin:/usr/bin:/bin" \
     AGENTNET_UV="$prefix/bin/uv" \
     NO_PROXY="$NO_PROXY_VALUE" no_proxy="$NO_PROXY_VALUE" \
     "$prefix/bin/node" "$(launcher "$prefix")" \
     server-agent setup --request "$INPUTS/server-setup.json" \
-    --expected-request-digest "$digest" --apply --start
+    --expected-request-digest "$digest" --apply --start || exit_code=$?
+  if [[ "$exit_code" -eq 0 ]]; then
+    return 0
+  fi
+  if [[ "$prefix" == "$PREFIX_0144" ]] &&
+    jq -e '.status == "blocked" and .blocker == "systemd_start"' "$output" >/dev/null &&
+    wait_for_released_setup_live "$prefix"; then
+    # Released 0.1.50 can lose the final systemctl response after the immutable
+    # marker, exact units, and exact health identities have all converged. Do
+    # not reapply while those units are live; retain the blocked response and
+    # record only the independently observed postcondition.
+    RELEASED_SETUP_START_RECONCILED=true
+    return 0
+  fi
+  return "$exit_code"
 }
 
 psql_agentnet() {
@@ -361,6 +417,7 @@ EOF
 sudo ln -s /etc/nginx/sites-available/agentnet-upgrade-e2e /etc/nginx/sites-enabled/agentnet-upgrade-e2e
 sudo nginx -t
 sudo systemctl restart nginx
+sudo systemctl set-environment SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
 
 # Disposable request inputs contain synthetic CI-only credentials and no
 # repository, organization, environment, or long-lived secret.
@@ -615,14 +672,17 @@ run_as_core_service "$AGENTNET_0144" server-agent activate \
 jq -e --arg harness "$HARNESS_ID" --arg credential "$CREDENTIAL_ID" \
   '.activated == true and .harness_id == $harness and .credential_id == $credential and .authority_granted == false' \
   "$WORK/activate-0.1.45.json" >/dev/null
-# Public 0.1.45 uses Persistent=true. Seed the fresh runner's standard systemd
-# timer stamp so enabling the historical timer does not fabricate a missed
-# activation from before this disposable host existed and race setup's exact
-# postcondition check.
+# Public 0.1.45 uses Persistent=true. Seed its timer stamp before first setup
+# so a fresh runner cannot renew the synthetic credential before the upgrade
+# snapshot and invalidate the concurrently starting C0 responder.
 sudo install -d -o root -g root -m 0755 /var/lib/systemd/timers
 sudo touch /var/lib/systemd/timers/stamp-agentnet-credential-renew.timer
 apply_setup "$PREFIX_0144" "$DIGEST_0144" "$APPLY_BOUND_0144"
-jq -e '.status == "operational" and .identity_enrolled == true and .authority_granted == false' "$APPLY_BOUND_0144" >/dev/null
+if [[ "$RELEASED_SETUP_START_RECONCILED" == "true" ]]; then
+  jq -e '.status == "blocked" and .blocker == "systemd_start"' "$APPLY_BOUND_0144" >/dev/null
+else
+  jq -e '.status == "operational" and .identity_enrolled == true and .authority_granted == false' "$APPLY_BOUND_0144" >/dev/null
+fi
 sudo grep -Fxq 'OnUnitActiveSec=1h' /etc/systemd/system/agentnet-credential-renew.timer
 sudo grep -Fxq 'Persistent=true' /etc/systemd/system/agentnet-credential-renew.timer
 ! sudo grep -Fq 'OnUnitInactiveSec=' /etc/systemd/system/agentnet-credential-renew.timer
@@ -641,6 +701,9 @@ OLD_CORE_PID="$(sudo systemctl show agentnet-core.service --property=MainPID --v
 OLD_APPROVAL_PID="$(sudo systemctl show agentnet-approval.service --property=MainPID --value)"
 [[ "$OLD_CORE_PID" =~ ^[1-9][0-9]*$ ]]
 [[ "$OLD_APPROVAL_PID" =~ ^[1-9][0-9]*$ ]]
+sudo systemctl stop agentnet-credential-renew.timer
+sudo systemctl reset-failed agentnet-credential-renew.service
+
 
 PLAN_0145="$WORK/plan-0.1.46.json"
 APPLY_0145="$WORK/apply-0.1.46.json"