From e22b2ab16b881db8e5126dd97ee6e787879c6a9a Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:14:35 -0400 Subject: [PATCH] Make an abnormal owner exit fail StatusLifecycleTest by name The durable-leg halt test failed on one PostgreSQL leg in each of the last two CI runs with only "(no repo frame)": the runner prints every exception banner it finds and found none, the shape of a link exit from the start_link-linked pipeline owner. It did not reproduce in 200 isolated runs or 12 full live-suite runs on the dev box (16 and 4 cores). The four tests where a halt stops the owner now unlink it, keep a monitor, and assert it stops :normal within 15s (the durable leg's 5s budget plus margin), following StartLinkTest's 1.6.1 precedent. A real abnormal exit still fails, now at a named assertion with its reason. Verification: the file passes 12 of 12; with the owner's pipeline-DOWN stop mutated to :boom, the test fails "pipeline owner exited abnormally: :boom" at status_lifecycle_test.exs:177; mutation restored. --- CHANGELOG.md | 10 ++++ test/ash_replicant/status_lifecycle_test.exs | 53 ++++++++++++++------ 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 878dd4e..49588e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- StatusLifecycleTest's four halt tests no longer let an abnormal pipeline + owner exit kill the test through its start_link link. The durable-leg test + died that way on one PostgreSQL leg in each of CI runs 40681408838 and the + run before it, printing only the formatter's `(no repo frame)` receipt. The + tests now unlink the owner, keep a monitor, and assert it stops `:normal` + within 15s, so the same exit fails at a named assertion with its reason in + the local output (proven red with the owner stopping `:boom`). + ### Security - Require Ash `>= 3.37.2 and < 4.0.0-0`. This excludes diff --git a/test/ash_replicant/status_lifecycle_test.exs b/test/ash_replicant/status_lifecycle_test.exs index 97c6a93..48d0911 100644 --- a/test/ash_replicant/status_lifecycle_test.exs +++ b/test/ash_replicant/status_lifecycle_test.exs @@ -64,6 +64,24 @@ defmodule AshReplicant.StatusLifecycleTest do defp tombstone, do: :persistent_term.get(@tombstone_key, nil) + # start_link links the owner to this test. An abnormal owner exit would kill + # the test with a link exit, which carries no stacktrace, so the battery could + # only print "(no repo frame)" (CI run 40681408838 and the PG17 run before it, + # neither able to name the cause). Unlinked with the monitor kept, the same + # exit FLUNKS at the assertion below with its reason in the local output. + defp start_owner!(opts) do + assert {:ok, owner} = AshReplicant.start_link(opts) + Process.unlink(owner) + {owner, Process.monitor(owner)} + end + + # A halted owner stops :normal after its node-local and durable legs; the + # durable leg's own budget is 5s, so 15s bounds the stop with margin. + defp assert_owner_stopped_normally({owner, ref}) do + assert_receive {:DOWN, ^ref, :process, ^owner, reason}, 15_000 + assert reason == :normal, "pipeline owner exited abnormally: #{inspect(reason)}" + end + setup do AshReplicant.stop_supervised(@slot) :persistent_term.erase(@entry) @@ -120,16 +138,16 @@ defmodule AshReplicant.StatusLifecycleTest do describe "degraded → halted (the census fault budget)" do test "census faults degrade, and the budget halt persists a value-free tombstone" do capture_log(fn -> - assert {:ok, _owner} = - AshReplicant.start_link( - start_opts( - census: [ - interval_ms: 50, - timeout_ms: 100, - max_consecutive_faults: 2 - ] - ) - ) + owner = + start_owner!( + start_opts( + census: [ + interval_ms: 50, + timeout_ms: 100, + max_consecutive_faults: 2 + ] + ) + ) # Below the budget the pipeline is live and degraded (public # catching_up — never healthy: census is the health authority). @@ -149,6 +167,7 @@ defmodule AshReplicant.StatusLifecycleTest do # then the erase) — poll the erase, never assert it immediately # (the O02-era flake class this file fights). eventually(fn -> :none == :persistent_term.get(@entry, :none) end) + assert_owner_stopped_normally(owner) end) end end @@ -156,8 +175,7 @@ defmodule AshReplicant.StatusLifecycleTest do describe "halted — an unexplained pipeline death" do test "a pipeline halt without a recorded cause writes the generic tombstone" do capture_log(fn -> - assert {:ok, _owner} = - AshReplicant.start_link(start_opts(census: [enabled?: false])) + owner = start_owner!(start_opts(census: [enabled?: false])) eventually(fn -> match?(%Generation{}, :persistent_term.get(@entry, :none)) end) @@ -169,6 +187,8 @@ defmodule AshReplicant.StatusLifecycleTest do assert %AshReplicant.Status.Tombstone{cause: :pipeline_terminated, class: :halt} = tombstone() + + assert_owner_stopped_normally(owner) end) end @@ -208,8 +228,7 @@ defmodule AshReplicant.StatusLifecycleTest do on_exit(fn -> :telemetry.detach({__MODULE__, :durable_failure}) end) capture_log(fn -> - assert {:ok, _owner} = - AshReplicant.start_link(start_opts(census: [enabled?: false])) + owner = start_owner!(start_opts(census: [enabled?: false])) eventually(fn -> match?(%Generation{}, :persistent_term.get(@entry, :none)) end) @@ -272,13 +291,14 @@ defmodule AshReplicant.StatusLifecycleTest do flunk("durable tombstone record neither fired nor lawfully skipped") end end + + assert_owner_stopped_normally(owner) end) end test "a precise recorded cause is not stomped by the generic writer" do capture_log(fn -> - assert {:ok, _owner} = - AshReplicant.start_link(start_opts(census: [enabled?: false])) + owner = start_owner!(start_opts(census: [enabled?: false])) eventually(fn -> match?(%Generation{}, :persistent_term.get(@entry, :none)) end) @@ -295,6 +315,7 @@ defmodule AshReplicant.StatusLifecycleTest do eventually(fn -> :none == :persistent_term.get(@entry, :none) end) assert %AshReplicant.Status.Tombstone{cause: :schema_change_destructive} = tombstone() + assert_owner_stopped_normally(owner) end) end end