Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed

- StatusLifecycleTest's four halt tests no longer let an abnormal pipeline
owner exit kill the test through its start_link link. The durable-leg test
died that way on one PostgreSQL leg in each of CI runs 40681408838 and the
run before it, printing only the formatter's `(no repo frame)` receipt. The
tests now unlink the owner, keep a monitor, and assert it stops `:normal`
within 15s, so the same exit fails at a named assertion with its reason in
the local output (proven red with the owner stopping `:boom`).

### Security

- Require Ash `>= 3.37.2 and < 4.0.0-0`. This excludes
Expand Down
53 changes: 37 additions & 16 deletions test/ash_replicant/status_lifecycle_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,24 @@ defmodule AshReplicant.StatusLifecycleTest do

defp tombstone, do: :persistent_term.get(@tombstone_key, nil)

# start_link links the owner to this test. An abnormal owner exit would kill
# the test with a link exit, which carries no stacktrace, so the battery could
# only print "(no repo frame)" (CI run 40681408838 and the PG17 run before it,
# neither able to name the cause). Unlinked with the monitor kept, the same
# exit FLUNKS at the assertion below with its reason in the local output.
defp start_owner!(opts) do
assert {:ok, owner} = AshReplicant.start_link(opts)
Process.unlink(owner)
{owner, Process.monitor(owner)}
end

# A halted owner stops :normal after its node-local and durable legs; the
# durable leg's own budget is 5s, so 15s bounds the stop with margin.
defp assert_owner_stopped_normally({owner, ref}) do
assert_receive {:DOWN, ^ref, :process, ^owner, reason}, 15_000
assert reason == :normal, "pipeline owner exited abnormally: #{inspect(reason)}"
end

setup do
AshReplicant.stop_supervised(@slot)
:persistent_term.erase(@entry)
Expand Down Expand Up @@ -120,16 +138,16 @@ defmodule AshReplicant.StatusLifecycleTest do
describe "degraded → halted (the census fault budget)" do
test "census faults degrade, and the budget halt persists a value-free tombstone" do
capture_log(fn ->
assert {:ok, _owner} =
AshReplicant.start_link(
start_opts(
census: [
interval_ms: 50,
timeout_ms: 100,
max_consecutive_faults: 2
]
)
)
owner =
start_owner!(
start_opts(
census: [
interval_ms: 50,
timeout_ms: 100,
max_consecutive_faults: 2
]
)
)

# Below the budget the pipeline is live and degraded (public
# catching_up — never healthy: census is the health authority).
Expand All @@ -149,15 +167,15 @@ defmodule AshReplicant.StatusLifecycleTest do
# then the erase) — poll the erase, never assert it immediately
# (the O02-era flake class this file fights).
eventually(fn -> :none == :persistent_term.get(@entry, :none) end)
assert_owner_stopped_normally(owner)
end)
end
end

describe "halted — an unexplained pipeline death" do
test "a pipeline halt without a recorded cause writes the generic tombstone" do
capture_log(fn ->
assert {:ok, _owner} =
AshReplicant.start_link(start_opts(census: [enabled?: false]))
owner = start_owner!(start_opts(census: [enabled?: false]))

eventually(fn -> match?(%Generation{}, :persistent_term.get(@entry, :none)) end)

Expand All @@ -169,6 +187,8 @@ defmodule AshReplicant.StatusLifecycleTest do

assert %AshReplicant.Status.Tombstone{cause: :pipeline_terminated, class: :halt} =
tombstone()

assert_owner_stopped_normally(owner)
end)
end

Expand Down Expand Up @@ -208,8 +228,7 @@ defmodule AshReplicant.StatusLifecycleTest do
on_exit(fn -> :telemetry.detach({__MODULE__, :durable_failure}) end)

capture_log(fn ->
assert {:ok, _owner} =
AshReplicant.start_link(start_opts(census: [enabled?: false]))
owner = start_owner!(start_opts(census: [enabled?: false]))

eventually(fn -> match?(%Generation{}, :persistent_term.get(@entry, :none)) end)

Expand Down Expand Up @@ -272,13 +291,14 @@ defmodule AshReplicant.StatusLifecycleTest do
flunk("durable tombstone record neither fired nor lawfully skipped")
end
end

assert_owner_stopped_normally(owner)
end)
end

test "a precise recorded cause is not stomped by the generic writer" do
capture_log(fn ->
assert {:ok, _owner} =
AshReplicant.start_link(start_opts(census: [enabled?: false]))
owner = start_owner!(start_opts(census: [enabled?: false]))

eventually(fn -> match?(%Generation{}, :persistent_term.get(@entry, :none)) end)

Expand All @@ -295,6 +315,7 @@ defmodule AshReplicant.StatusLifecycleTest do
eventually(fn -> :none == :persistent_term.get(@entry, :none) end)

assert %AshReplicant.Status.Tombstone{cause: :schema_change_destructive} = tombstone()
assert_owner_stopped_normally(owner)
end)
end
end
Expand Down
Loading