diff --git a/CLAUDE.md b/CLAUDE.md index 7b3d861..25b1a3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,9 +12,10 @@ Providers, in order: 2. **Hetzner Cloud** is second. 3. **AWS** comes later. It must remain possible without changes to the core. -**Status:** M0 Foundation (2026-09-27), M1 Vultr infrastructure (2026-09-28) and M2 Nomad bootstrap (2026-10-07) are -complete. v0.1.0, the first release that builds a Nomad cluster, was tagged on 2026-10-07: archives, `.deb` and `.rpm` -packages, a Homebrew cask (`ingvarch/tap/tent`), and the tent-node binaries that its nodes download. +**Status:** M0 Foundation (2026-09-27), M1 Vultr infrastructure (2026-09-28), M2 Nomad bootstrap (2026-10-07) and M3 +Day-2 operations (2026-10-14) are complete. v0.1.0, the first release that builds a Nomad cluster, was tagged on +2026-10-07: archives, `.deb` and `.rpm` packages, a Homebrew cask (`ingvarch/tap/tent`), and the tent-node binaries +that its nodes download. - The skeleton is in place: Go module, `tent version`, Makefile, lint rules, CI on Linux, macOS and Windows, and a GoReleaser release pipeline. The repository is public, the release secrets are set, and the archives and packages ship third-party licence notices (ADR-0020). Renovate updates the Go modules and GitHub Actions. @@ -86,14 +87,15 @@ packages, a Homebrew cask (`ingvarch/tap/tent`), and the tent-node binaries that Vultr on 2026-10-07 (spike v12, run `9pxbqn`); its one unexpected row was a fault of the script, which is fixed. - M2.9 added the E2E suite on Vultr (`test/e2e`, `test/e2e/vultrapi`, `test/e2e/janitor`, `hack/e2e-janitor`, ADR-0034): the scenario `smoke` builds a cluster per image (ubuntu-24.04 and ubuntu-26.04) and is built to check - the four exit criteria of M2, and a janitor deletes the objects of `e2e-` clusters that are older than 3 hours. + the four exit criteria of M2, and a janitor deletes the objects of `e2e-` clusters that are older than 4 hours. `make e2e` runs it from the maintainer's machine, not in CI. The labels `tent/e2e` and `tent/e2e-run` are removed. The runs of 2026-10-07 found four faults, all fixed: a node refuses a tent-node of another version than its tent's (`make e2e` builds both with one version); `update` asked for an intro token before Nomad's keyring had a key (the Nomad step now waits for it); the suite's `validate` lacked `--allow-single-server`; and the probe's one control site, `archive.ubuntu.com`, did not answer over IPv4 (it tries three now). The next three runs passed on both images (`r7l48w`, `58sglh`, `g57k85`): that is the exit of M2. -- M3 Day-2 operations is in progress, in parts M3.1 to M3.9 (`docs/roadmap.md`). +- M3 Day-2 operations was built in parts M3.1 to M3.9 (`docs/roadmap.md`) and is complete since 2026-10-14 (runs + `qkmdrs` and `hik34s`). - M3.1 added the rollout decisions (`internal/rollout`, ADR-0035; maintainer decisions 33 to 35): `Next(State, Mode)` returns one step of a rolling update (`Roll`) or of a removal of surplus nodes (`Shrink`) from what the cloud and Nomad report, and reaches nothing itself. A simulator with golden step sequences, invariants and a @@ -210,6 +212,20 @@ packages, a Homebrew cask (`ingvarch/tap/tent`), and the tent-node binaries that `latest` refuses when the earlier backups are of more than one rebuild. - `nomadops` has `Jobs`, `TokenAccessor` and the class `ErrDenied`. `delete cluster` deletes the backups and the record. The run on Vultr gave a rebuilt server its old private address (platform notes §3.16). + - M3.9 added the E2E scenarios `ha` and `upgrade` and a probe (`test/e2e` `ha_test.go`, `upgrade_test.go`, + `cluster_test.go`, `probe_test.go`, `scenario_text_test.go`; ADR-0043; maintainer decision 51): `make e2e` runs + `smoke`, then `ha` and `upgrade` in parallel (at most 10 machines), and `E2E_RUN` picks scenarios. + - `ha` changes the specs, rolls the servers and then the workers, restarts an old worker between the rolls so that + it finds the new servers through `refresh-join`, and backs up Nomad, deletes the servers, rebuilds them and + restores the backup. `upgrade` rolls a combined cluster from Nomad 2.0.6 (`E2E_UPGRADE_FROM`) to the channel's + version. + - A probe polls the servers for a leader and the service for a healthy allocation during each roll and fails the + step on a breach of its limits (ADR-0043). The timeout of `make e2e` is 180 minutes, and a step ends 20 minutes + before it so the cleanups still delete the clusters. + - Decision 51: no command takes an upgrade back once the roll's first new server has joined (ADR-0041). + - Runs on Vultr (platform notes §3.16): `w28exz` found two faults of the suite (a creation date without a zone, + and checks that cannot be read after a hard stop of a server), `ae5dhs` passed `ha` and `upgrade`, and + `qkmdrs` and `hik34s` passed the full run. The channel lists 2.0.6 as tested since `ae5dhs`. ## Read before changing anything diff --git a/Makefile b/Makefile index 83194ad..275cb29 100644 --- a/Makefile +++ b/Makefile @@ -36,9 +36,11 @@ test: # e2e runs the E2E suite on Vultr from this machine (test/e2e/README.md). It runs dev-upload, which needs the R2 keys # and TENT_DEV_S3_URL, so the nodes get the tent-node of the same build as bin/tent, and it needs VULTR_API_KEY. +# E2E_RUN, when set, is the -run pattern, for example make e2e E2E_RUN='TestHA|TestUpgrade'. e2e: @lines="$$(SHELL=/bin/sh $(MAKE) -s --no-print-directory dev-upload)" && eval "$$lines" && \ - E2E_TENT="$(CURDIR)/bin/tent" go test -tags e2e -count=1 -v -timeout 90m ./test/e2e + E2E_TENT="$(CURDIR)/bin/tent" go test -tags e2e -count=1 -v -timeout 180m \ + $(if $(E2E_RUN),-run "$(E2E_RUN)") ./test/e2e golangci-lint-version: @command -v golangci-lint >/dev/null 2>&1 || { \ diff --git a/docs/adr/0034-e2e-suite-on-vultr.md b/docs/adr/0034-e2e-suite-on-vultr.md index 4ddd8eb..f854a27 100644 --- a/docs/adr/0034-e2e-suite-on-vultr.md +++ b/docs/adr/0034-e2e-suite-on-vultr.md @@ -1,6 +1,8 @@ # ADR-0034: The E2E suite on Vultr -- **Status:** Accepted +- **Status:** Accepted; amended by [ADR-0043](0043-e2e-scenarios-ha-and-upgrade.md) (the scenarios `ha` and `upgrade`, a + probe of the leader and the service during a roll, `E2E_RUN`, the timeout of 180 minutes, the janitor's age of 4 + hours) - **Date:** 2026-10-07 - **Deciders:** ingvarch - **Related:** amends [ADR-0012](0012-testing-strategy.md), [ADR-0014](0014-vultr-first-provider-and-e2e.md), diff --git a/docs/adr/0041-upgrading-a-clusters-nomad-version.md b/docs/adr/0041-upgrading-a-clusters-nomad-version.md index bae2d2d..32522f8 100644 --- a/docs/adr/0041-upgrading-a-clusters-nomad-version.md +++ b/docs/adr/0041-upgrading-a-clusters-nomad-version.md @@ -95,7 +95,8 @@ A local lab on 2026-10-10 ran Nomad 2.0.6 and 2.0.7 together in both directions 14. **A combined group is the one exception.** A new combined node runs a client of the new version while the other servers are old, and its create is not held back, since a combined group has no other order. The guide advises against a client ahead of its servers because new client features need new servers. The local run covered a newer - client with older servers and not a combined agent. M3.9's `upgrade` scenario runs it on Vultr. + client with older servers and not a combined agent. M3.9's `upgrade` scenario ran it on Vultr + (`w28exz`, `ae5dhs`): a 2.0.7 combined node joined 2.0.6 ones and the roll ended. 15. **`rollout.CheckNewNode` holds the version rules of a new node** for `Next`, `serversHold` and `update`. A refusal says `the cluster's version is`, because the pin can differ from the version that a check holds. `validate cluster` takes the same phrase and warns that `update` has not applied the specs while the completed spec differs. @@ -126,19 +127,21 @@ A local lab on 2026-10-10 ran Nomad 2.0.6 and 2.0.7 together in both directions lowered joins later ahead of the pin, and `rolling-update` refuses until the version is raised again or `update` removes the machine. - **No command takes an upgrade back once the roll's first new server has joined** (Q14 of - [architecture §18](../architecture.md#18-open-questions), open). The way back is by hand: delete that machine in the - cloud, wait until its peer is gone, and run `tent update cluster`. + [architecture §18](../architecture.md#18-open-questions), answered (a) on 2026-10-13: decision 51, with no lab for + (b)). The way back is by hand: delete that machine in the cloud, wait until its peer is gone, and run `tent update + cluster`. - **`validate cluster` exits with 2 from `upgrade --yes` until the roll has ended,** since the nodes differ from the specs. -- **Not verified on a real cluster:** whether autopilot's report holds a version for every Raft peer at the read - before the first client of a first build. M3.9's run on Vultr shows it. +- **Autopilot's report held a version for every Raft peer** at the read before the first client of a first build, in + the create of the `ha` cluster on Vultr, in both runs `w28exz` and `ae5dhs` (2026-10-14). - **The tests know two Nomad versions, one patch apart.** A move to the next minor version is tested as a rule and has never run. ### Follow-ups -- **M3.9** runs the `upgrade` scenario on Vultr from 2.0.6 to 2.0.7 with a combined cluster, and may then add 2.0.6 to - the channel's `tested` list. +- **M3.9** ran the `upgrade` scenario on Vultr from 2.0.6 to 2.0.7 with a combined cluster (runs `w28exz` and + `ae5dhs`, [ADR-0043](0043-e2e-scenarios-ha-and-upgrade.md)), and 2.0.6 is in the channel's `tested` list since + `ae5dhs`. - A flag for the target version, a flow with a third version and an in-place upgrade of the binary ([ADR-0005](0005-immutable-nodes-and-nomad-aware-rollouts.md)) are left for later. diff --git a/docs/adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md b/docs/adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md index 91a7d3d..bf26582 100644 --- a/docs/adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md +++ b/docs/adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md @@ -140,8 +140,10 @@ ### Follow-ups -- M3.9's `ha` scenario runs a backup and a restore on Vultr. `tent backup delete`, `backup create --keep N` and a - restore inside `update` wait for issues. +- M3.9's `ha` scenario ran a backup and a restore on Vultr in run `ae5dhs` (2026-10-14) with three servers: Vultr gave + the rebuilt servers one of their old private addresses back, the workers came back on their own, `validate` passed + after the 6-minute wait and no forced roll was needed ([ADR-0043](0043-e2e-scenarios-ha-and-upgrade.md)). + `tent backup delete`, `backup create --keep N` and a restore inside `update` wait for issues. ## Alternatives considered diff --git a/docs/adr/0043-e2e-scenarios-ha-and-upgrade.md b/docs/adr/0043-e2e-scenarios-ha-and-upgrade.md new file mode 100644 index 0000000..7e1594d --- /dev/null +++ b/docs/adr/0043-e2e-scenarios-ha-and-upgrade.md @@ -0,0 +1,200 @@ +# ADR-0043: E2E scenarios ha and upgrade + +- **Status:** Accepted +- **Date:** 2026-10-14 +- **Deciders:** ingvarch +- **Related:** amends [ADR-0034](0034-e2e-suite-on-vultr.md) (see its Status line); builds on + [ADR-0038](0038-rolling-update-of-server-groups.md), [ADR-0039](0039-rolling-update-of-combined-groups.md), + [ADR-0041](0041-upgrading-a-clusters-nomad-version.md) and + [ADR-0042](0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md); + [architecture §15](../architecture.md#15-testing), [§18](../architecture.md#18-open-questions) (decisions 39 and + 51), [platform notes §3.16](../platform-notes.md#316-spike-runs) + +## Context + +The exit criterion of M3 in [the roadmap](../roadmap.md) is "E2E `ha` and `upgrade` pass on Vultr, with no quorum loss +and no job downtime beyond the configured migrations". The maintainer allows runs on a real cloud of at most ten +machines at once. Decision 39 puts `ha` on ubuntu-24.04 and `upgrade` on ubuntu-26.04 in one run, with the backup step +in `ha`. + +[ADR-0034](0034-e2e-suite-on-vultr.md) made the suite a black box: it drives `bin/tent`, reads Vultr with its own client +and Nomad with its own mTLS client, and imports nothing under `internal/`. The scenarios keep that rule. Two things in +them are new. A roll runs for tens of minutes, so the suite needs a way to see a quorum loss or a job outage during the +roll. And `go test` panics on its timeout and runs no cleanup, while the step timeouts of a +long scenario add up to more than any `-timeout`. + +The facts that the design rests on, from the source of Nomad 1.11.3 and from runs +([platform notes §1.6](../platform-notes.md#16-the-agent-on-a-node), [§3.16](../platform-notes.md#316-spike-runs)): + +- A read of the leader without `stale=true` is forwarded, and Nomad holds it for up to 5 s before it answers `No + cluster leader`. +- A read of an allocation's checks goes over the client's connection to a server. tent stops a server with a hard + power-off. In the runs, the checks could not be read for up to 37 s after some of those stops (20.1 s in `ae5dhs` and + 7.1 s in `qkmdrs`, both in `upgrade`), and other stops caused no such poll. By a reading of the source, a client keeps + that connection until its keepalive fails and its next heartbeat registers the node on another server. +- A client that restarts keeps its node ID, and Nomad's client logs `client: node registration complete` at INFO after + each registration. + +## Decision + +1. **Three top-level tests, in this order.** `TestSmoke` keeps its steps and is not parallel. `TestHA` and + `TestUpgrade` call `t.Parallel()` first. Go runs the tests that are not parallel first and releases the parallel + ones after them, so a full run builds smoke's 4 machines and then `ha` and `upgrade` together: `ha` has 3 servers + and 2 workers (6 during a roll or the forced roll), `upgrade` has 3 combined nodes (4 during its roll), so at most + 10 machines are up at once. +2. **Names and images.** The clusters are `e2e--ha` and `e2e--upgrade`; both fit the cluster name rule of + `api/v1alpha1/validation.go`, and the janitor needs no change. `ha` takes the first image of `E2E_IMAGES` and + `upgrade` the last: with the default list ubuntu-24.04 and ubuntu-26.04, with one image in the list both take it. +3. **Variables of the run.** `E2E_RUN`, when set, is the `-run` pattern that `make e2e` passes, for example + `make e2e E2E_RUN='TestHA|TestUpgrade'`. `E2E_UPGRADE_FROM`, of the form `N.N.N`, is the Nomad version that `upgrade` + builds on; it defaults to `2.0.6`. The suite does not know the channel: it reads the target version from the specs + after `tent upgrade cluster`, and checks it against the version that the command prints. +4. **Timeouts.** `make e2e` passes `-timeout 180m`. A step's context also ends 20 minutes before the test's deadline + (`cleanupReserve`: the delete of a cluster and a margin), so a step that runs long fails and the cleanups still + delete the clusters. A `-timeout` under 20 minutes ends every step at once. The cleanup also deletes a cluster when + the end of the step's context cut its delete step short. The janitor deletes a cluster once it is older than 4 hours, + above the 3 hours of a run, so a sweep cannot take the clusters of a run that is near its end, whatever the clock + skew between Vultr and the runner. +5. **The job.** `testdata/web-pair.nomad.hcl` is the job `e2e-web` with `count = 2` and a `migrate` block with Nomad's + defaults written out (`max_parallel = 1`, `health_check = "checks"`, `min_healthy_time = "10s"`, + `healthy_deadline = "5m"`), since the exit criterion speaks of the configured migrations. A drain stops one + allocation, waits until its replacement's checks have passed for 10 s and then stops the next, so one allocation is + healthy at every moment. Both scenarios submit it without a purge, so it stays until `delete cluster`. +6. **The probe.** While a roll runs, a probe polls every second. It lists the cluster's server machines (the combined + machines in `upgrade`) through Vultr every 10 s (in a goroutine of its own, so a slow Vultr API never holds up a + poll) and asks all of them at once: + - **The leader**, with `GET /v1/status/leader?stale=true` and a 3 s timeout. With `stale=true` each server answers + from its own Raft view at once. The outcome is `led` when any server names a leader, `leaderless` when at least one + server answered and none named a leader, and `unreached` when no server answered. An empty answer never makes a + poll `leaderless` while another server names a leader, since a server outside the quorum, such as a new one before + it joins, answers an empty leader. + - **The service**, through a server that named a leader or else one that answered, with a 7 s timeout (above Nomad's + 5 s hold of a forwarded read): the allocations of the job, the checks of each running allocation and the + registrations of the service. The outcome is `served` when one running allocation has a successful check of the + service and a registration. It is `unserved` when the reads answered and show no running allocation, a running + allocation without a registration, or no healthy allocation among the checks that answered. It is `unreached` when + no server answered the leader read, or when the allocations or the registrations could not be read. It is + `unchecked` when those answered, a running allocation has a registration, and every checks read failed. + - **The verdict** fails the step when: + - `leaderless` lasts more than 10 s (a transfer takes about 1 s, a lost quorum lasts until a person acts); + - `unserved` lasts more than 10 s (two check intervals of the job); + - `unreached` lasts more than 30 s, for the leader or the service; + - `unchecked` lasts more than 60 s (a stretch of 37.2 s after a hard stop of a server in run `w28exz`, with a + margin); + - two polls start more than 30 s apart (the probe itself stalled, so it saw nothing); + - the stretches of one bad outcome add up to more than twice its limit in one roll (`leaderless` and `unserved` + 20 s, `unreached` 60 s, `unchecked` 120 s), which catches a leader that is lost every other second; + - there are fewer than 10 polls, or the first and the last poll are not both `led` and `served` (the probe started + too late or stopped too early). + A stretch lasts from the first poll of an outcome to the start of the next poll with another outcome. The + summary of every verdict, passing or not, is logged with the count of each outcome, the longest stretch and the + total time of each, and the longest gap between polls. + - **The end of a roll.** tent exits 40 to 47 s after the last hard stop of a server, and the checks may stay + unreadable for up to 37 s after it. When tent exits 0, the probe goes on polling until a poll is `led` and + `served`, for at most the roll's longest stretch limit (60 s, or 30 s in the worker roll of `ha`), and the log + says how long it polled. A stretch that is still open then is judged against its own limit, and a roll whose + probe never got a good poll back fails on the last-poll rule. + - **The limits of each roll.** The server roll of `ha` and the roll of `upgrade`, which stops combined nodes, use + the limits above. The worker roll of `ha` stops no server, so an `unchecked` poll there means that no client that + runs an allocation answers while Nomad still lists them: its `unchecked` limit is the `unserved` one (10 s, 20 s + in all). + - **The first poll** uses the server addresses that the step listed and checked before it started the probe, so a + failed list at the start cannot fail the first poll. + - **The records** go to `results//-probe-.tsv` as they come, so a run that is cut short keeps + them. +7. **`ha`, step by step.** + 1. `create` with 3 servers and 2 workers; `validate`; `export`; `service` (the pair job, two allocations running and + the service healthy). + 2. `change`: `tent get clusters -o json`, then a document with `spec.nomad.extraConfig` added (a comment for the + servers and a `client { meta { e2e_roll = "" } }` block for the workers), `tent replace -f`. The change moves + the spec hash of every machine, and the new workers carry the run id in their node meta. + 3. `update`, then `rolling-update --exit-code` exits 2. + 4. `roll-servers`: `rolling-update --yes --nodegroups servers` (timeout 60 minutes) with the probe; afterwards Vultr + lists no old server, 3 servers and the old workers. + 5. `restart-client`: export again, then the old worker with the lowest label restarts over SSH + (`systemd-run --on-active=2 --unit=e2e-reboot systemctl reboot`) and must come back with a new boot ID, the same + node ID, the line + `client: node registration complete` in the journal of its new boot, `ready` and `eligible` in Nomad, and the + service served. The worker was built with the old servers, so its seed is dead and only what `refresh-join` wrote + leads it to the new ones; the record holds `05-join.hcl`. + 6. `roll-workers`: `rolling-update --yes` (timeout 45 minutes) with the probe; `rolling-update --exit-code` exits 0; + Vultr lists no old worker; every worker's node has `e2e_roll` set to the run id. + 7. `validate-again`. + 8. `backup`: `backup create`, `backup list` (one backup, of the servers that run now); the suite deletes the + cluster's server machines through the Vultr API (only machines tagged with the cluster and the role `server`, and + no empty result); `update` builds new servers; the backup is `earlier`; `backup restore`; the job `e2e-web` + answers; the suite waits 6 minutes, since Nomad lists the backup's nodes as ready for about 5 minutes whether + they answer or not; `validate cluster` runs, and only when it fails, `rolling-update --yes --force --nodegroups + workers` and `validate --wait 5m` run; finally two allocations run and the service is healthy. + 9. `delete` and the leak check, unless `E2E_KEEP` is set. + + The roll is in two runs for #106: a roll of the whole cluster replaces the workers too, and a worker built with the + new servers would not test a dead seed. +8. **`upgrade`, step by step.** `create` with `--combined --servers 3 --nomad-version ` (the warning + that the version is untested is logged and does not fail the step); `validate`; `export`; `service`; + `versions-before` (3 nodes on the old version); `upgrade` (`tent upgrade cluster --yes` with a timeout of 5 minutes, + since it downloads and verifies the release's signed checksums; the target read from the specs, which must differ + from the old version and be the version that the command prints; then `update` and `rolling-update --exit-code` + exits 2); `roll` with the probe (timeout 60 minutes), then exit code 0; `versions-after` (3 nodes on the target, + `ready` and `eligible`; 3 alive members named after the current machines with the target in their `build` tag); + `validate-again` (exit 0, and the output names the target); `delete` and the leak check. +9. **Records.** Each run writes these files into `results//` beside the logs of ADR-0034: + `-roll-change.json`, `-probe-servers.tsv` and `-probe-workers.tsv` (`ha`), + `-probe-nodes.tsv` (`upgrade`), `-restart.txt` and `-backup.txt` (`ha`). A probe file has + the columns `start`, `took_ms`, `leader`, `leader_address`, `service` and `detail`. + +## Consequences + +### Positive + +- The exit criterion is checked at every poll of two rolls and a restore, on two images, in one run. +- A run that fails leaves the probe's records, so the gap that failed it can be read. +- A step that runs past its time fails while the cleanups can still delete the clusters. + +### Negative / trade-offs + +- A run of `ha` and `upgrade` took 39 minutes (run `ae5dhs`): `ha` ran create 7 min 43 s, the + server roll 12 min 14 s, the worker roll 5 min 45 s and the backup step 9 min 36 s (6 minutes of it the wait); + `upgrade` ran create 4 min 2 s and the roll 14 min 46 s. The full run `qkmdrs`, with `smoke` first, took about 45 + minutes. Vultr bills each machine for + at least one hour; the plan's estimate was under 2 US dollars for a run. +- The probe cannot read a check for up to 37 s after a hard stop of a server. In those polls it knows that the + allocations run and are registered, and counts the poll `unchecked`, with a limit of 60 s (120 s in all) in the rolls + that stop a server. A job that fails its checks in that window would pass. +- The suite depends on Nomad's text `client: node registration complete`. A release that changes it fails the + `restart-client` step. +- The probe asks the servers' public addresses, which the run's `--api-access` opens to the runner. It cannot see a + fault that only shows on the service's own port. + +### Follow-ups + +- Runs: + - `w28exz` (2026-10-14, `ha` and `upgrade`) found two faults of the suite, both fixed: Vultr's firewall list gave a + creation date without a zone, which failed the leak check, and the probe counted polls without readable checks as + `unreached` (see Alternatives). + - `ae5dhs` (2026-10-14, `ha` and `upgrade`): both passed. The probe saw no `leaderless` and no `unserved` poll; the + longest `unchecked` stretch was 20.1 s and the longest `unreached` stretch 7.1 s. + - `qkmdrs` (2026-10-14, the full `make e2e`: `smoke`, `ha` and `upgrade`): every scenario passed. The probes saw no + `leaderless`, `unserved` or `unreached` poll; the longest `unchecked` stretch was 7.1 s. + - `hik34s` (2026-10-14, the full `make e2e` with the fixes of the final review): every scenario passed, and every + poll of every probe was `led` and `served`. +- The channel lists 2.0.6 as tested since `ae5dhs` ([ADR-0041](0041-upgrading-a-clusters-nomad-version.md)). +- Hetzner E2E and runs in CI wait as [architecture §15](../architecture.md#15-testing) says. + +## Alternatives considered + +- **A probe of the service's port from outside.** The cluster's firewall opens no job port to the runner, and opening + one would change what the cluster under test is. The probe reads the service through Nomad's API instead. +- **`rolling-update --force` in place of a change of the specs.** `--force` replaces machines whose specs did not + change, so no roll would be due from a changed spec, and no new worker would carry a value that the suite can read + back. The change of `extraConfig` gives both. +- **A reboot through Vultr's API.** The step tests that the agent finds the new servers after a boot of its own + machine. A reboot over SSH needs only the machine and the runner's SSH access, which the suite already has; Vultr's + API adds a call whose timing is not documented. +- **One roll of the whole cluster with a restart of a new worker.** A worker built with the new servers has the new + servers as its seed, so it would come back even if `refresh-join` never wrote anything. #106 asks for a worker whose + seed is dead, which needs the two rolls. +- **Counting a failed checks read as `unreached` with a limit of 30 s.** Run `w28exz` failed on it: after a hard stop of + a server, one allocations read through that server timed out (7.1 s), then every checks read failed for 30.2 s while + the allocations and registrations answered and both allocations ran, one `unreached` stretch of 37.2 s in all. The + outcome `unchecked` keeps a poll that cannot read the checks apart from a poll that reached nothing. diff --git a/docs/adr/README.md b/docs/adr/README.md index 8ff44c1..7ababbb 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -55,7 +55,7 @@ Nygard's format, lightly extended; see [template.md](template.md). | [0031](0031-bootstrap-in-update.md) | Bootstrap in `update` | Accepted; amended by 0032, 0033, 0034, 0036, 0038, 0039, 0040, 0041 and 0042 | | [0032](0032-joined-label-scrub-and-delete-guard.md) | The joined label, the scrub and the delete guard | Accepted; amended by 0033, 0037, 0039 and 0040 | | [0033](0033-operator-commands.md) | Operator commands: validate, export nomad and ui | Accepted; amended by 0040 and 0041 | -| [0034](0034-e2e-suite-on-vultr.md) | The E2E suite on Vultr | Accepted | +| [0034](0034-e2e-suite-on-vultr.md) | The E2E suite on Vultr | Accepted; amended by 0043 | | [0035](0035-rollout-decisions.md) | Rollout decisions | Accepted; the refusal of two voters and single servers is gone (built in M3.4); amended by 0036, 0037, 0038, 0039, 0040 and 0041 | | [0036](0036-nomad-calls-of-a-roll.md) | The Nomad calls of a roll | Accepted; amended by 0038 and 0042 | | [0037](0037-rolling-update-of-client-groups.md) | Rolling update of client groups | Accepted; amended by 0038, 0039, 0040 and 0041 | @@ -64,3 +64,4 @@ Nygard's format, lightly extended; see [template.md](template.md). | [0040](0040-scale-down-through-the-rollout.md) | Scale down through the rollout | Accepted; amended by 0041 | | [0041](0041-upgrading-a-clusters-nomad-version.md) | Upgrading a cluster's Nomad version | Accepted | | [0042](0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md) | Backups in the state store and the restore after a loss of the servers | Accepted | +| [0043](0043-e2e-scenarios-ha-and-upgrade.md) | E2E scenarios ha and upgrade | Accepted | diff --git a/docs/architecture.md b/docs/architecture.md index 8cf918b..315fe3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -502,8 +502,9 @@ github.com/ingvarch/tent │ ├── channels/ # embedded channel files: the Nomad versions allowed and tested, the CNI plugins │ ├── buildinfo/ # version, commit, date (ldflags); which release a version counts as │ └── buildconfig/ # tests only: CI, Makefile and release config agree; what tent-node links -├── test/e2e/ # black-box E2E suite against real clouds (Vultr first); only main_test.go, smoke_test.go -│ │ # and harness_test.go carry //go:build e2e; imports none of tent's internal packages (ADR-0034) +├── test/e2e/ # black-box E2E suite against real clouds (Vultr first); only main_test.go, harness_test.go, +│ │ # cluster_test.go and the scenarios (smoke, ha, upgrade) carry //go:build e2e; imports none of +│ │ # tent's internal packages (ADR-0034) │ ├── vultrapi/ # the suite's own Vultr API client over net/http │ └── janitor/ # finds and deletes the objects of E2E clusters that earlier runs left │ └── janitortest/ # tests only: a fake Vultr API for the janitor @@ -3490,7 +3491,8 @@ right order. No rule was added for the roll. - **The exception is a combined group.** Its new node runs a client of the new version while the other servers are old, and its create is not held back, since a combined group has no other order. Nomad's guide advises against it. A 2.0.7 client with an intro token joined three 2.0.6 servers on the local Nomad, and a combined agent itself was not - run; M3.9's `upgrade` scenario runs it on Vultr. + run; M3.9's `upgrade` scenario ran it on Vultr (a 2.0.7 combined node joined 2.0.6 ones and the roll ended, runs + `w28exz` and `ae5dhs`). **Servers**, one at a time. A server group never goes below its size: it creates the replacement first. @@ -3872,6 +3874,9 @@ cni: arm64: ``` +Since run `ae5dhs` (2026-10-14) the embedded file lists `tested: [2.0.6, 2.0.7]`; the file above stays as it was on +2026-09-28. + - A channel holds Nomad and the CNI plugins only. Images stay in the provider's table ([ADR-0023](adr/0023-vultr-inventory-dedupe-and-images.md)) and the API default (decision 4 of [18](#18-open-questions)). @@ -4756,7 +4761,8 @@ of M3.8 reaches the state store alone. The other commands come with later milest Details in [ADR-0012](adr/0012-testing-strategy.md). The E2E platform is chosen in [ADR-0014](adr/0014-vultr-first-provider-and-e2e.md) and its running is set in -[ADR-0034](adr/0034-e2e-suite-on-vultr.md). +[ADR-0034](adr/0034-e2e-suite-on-vultr.md) and extended by the scenarios of +[ADR-0043](adr/0043-e2e-scenarios-ha-and-upgrade.md). 1. **Unit tests:** - defaults and validation; @@ -5392,7 +5398,7 @@ Details in [ADR-0012](adr/0012-testing-strategy.md). The E2E platform is chosen the metadata endpoint, and `delete` leaves nothing behind. - **Marking and the janitor.** Clusters are named `e2e--`. The janitor (`hack/e2e-janitor`, and `TestMain` before every run) deletes the objects of a cluster whose name starts with `e2e-` and whose oldest object - is older than 3 hours: instances first (polled every 10 s, up to 5 minutes, until none is listed), then firewall + is older than 4 hours: instances first (polled every 10 s, up to 5 minutes, until none is listed), then firewall groups, VPCs and SSH keys. A refused VPC delete is repeated every 10 s, for up to 5 minutes per VPC. Without `--yes` it deletes nothing. - **Black box.** The suite imports none of tent's internal packages. It has its own Vultr API client and Nomad @@ -5400,8 +5406,14 @@ Details in [ADR-0012](adr/0012-testing-strategy.md). The E2E platform is chosen - Billing has a one-hour minimum per VM. A green run took 6 to 7 minutes on 2026-10-07: `create` 4 to 6.5 minutes, `delete` about 20 s, the leak check up to 2 minutes on the cluster whose check runs while the other is still being built, every other step seconds ([platform notes §3.16](platform-notes.md#316-spike-runs)). - - The scenarios `ha` and `upgrade` are not built yet; they are the exit of M3. The checks of ADR-0012's `security` - scenario run in `smoke`. + - **Scenarios `ha` and `upgrade`** (M3.9, [ADR-0043](adr/0043-e2e-scenarios-ha-and-upgrade.md)). They run after + `smoke`, in parallel with each other, so a run has at most 10 machines. `ha` has 3 servers and 2 workers on the + first image: it changes the specs, rolls the servers, restarts an old worker that must find the new servers on + its own, rolls the workers, backs up Nomad, deletes the servers, has `update` build new ones and restores the + backup. `upgrade` builds a combined cluster of 3 on Nomad 2.0.6 (`E2E_UPGRADE_FROM`), runs `tent upgrade cluster` + and rolls it, on the last image. A probe polls the servers for a leader and the service for a healthy allocation + during each roll and fails the step on a leaderless or unserved stretch longer than 10 s (limits for the other + outcomes are in the ADR). They are the exit of M3. The checks of ADR-0012's `security` scenario run in `smoke`. - Nightly runs and runs on a PR label wait until a runner can reach the account. - Hetzner E2E, including the `arm` scenario on CAX, is added in M4 once an account can create servers reliably. 6. **Platform spike.** [`hack/vultr-spike`](../hack/vultr-spike/README.md) verifies undocumented Vultr behaviour @@ -5502,7 +5514,7 @@ See [ADR-0013](adr/0013-technology-stack.md). Releases and CI follow | Vultr counted a deleted machine against the account's instance limit, so the create of a replacement was refused (seen 2026-10-06, run `rugw2m`) | the replacement of a client that never registered fails in an account at its limit | the Vultr provider sends a refused node create again for up to 2 minutes after a delete that it made ([11.3](#113-creating-a-node)); a limit that still holds then returns the limit error with its hint, and so does a rerun of tent right after a run that deleted, since the window does not outlive the process | | Hetzner capacity and account limits (5 servers, creation restrictions since June 2026) | Hetzner provider cannot be E2E-tested | Vultr first (ADR-0014); Hetzner E2E in M4 | | Nomad 2.x version skew rules not yet restated: the upgrade guide of 2026-10-10 states them for 1.x ([platform notes §1.3](platform-notes.md#13-upgrades-and-version-skew)) | broken upgrades | channels allow one major version from a minimum, and tent warns about versions they have not tested; a version moves up by at most one minor version at a time, against the pin and against the oldest node that Nomad reports; servers before clients; no node older than one that runs ([13.5](#135-tent-upgrade-cluster---yes), [ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)); 2.0.6 and 2.0.7 ran together on a local Nomad on 2026-10-10, and a move to the next minor version has never run | -| A new combined node runs a client of the new Nomad version while the other servers run the old one | the guide advises against a client ahead of its servers, since new client features need new servers | a combined group has no other order; a 2.0.7 client joined three 2.0.6 servers on a local Nomad (a combined agent was not run); M3.9's `upgrade` scenario runs it on Vultr ([13.3](#133-tent-rolling-update-cluster---yes)) | +| A new combined node runs a client of the new Nomad version while the other servers run the old one | the guide advises against a client ahead of its servers, since new client features need new servers | a combined group has no other order; a 2.0.7 client joined three 2.0.6 servers on a local Nomad (a combined agent was not run); M3.9's `upgrade` scenario ran it on Vultr, runs `w28exz` and `ae5dhs` ([13.3](#133-tent-rolling-update-cluster---yes)) | | HashiCorp's embedded release key expires on 2030-03-01, or is rotated or revoked | tent cannot verify Nomad downloads, or trusts a revoked key | a weekly CI job fails 180 days before the expiry; a tent release embeds the new key ([8.5](#85-artifacts-and-verification)) | | BUSL licence of Nomad | a paid managed offering would need a commercial licence | tent downloads official binaries and never redistributes them; stays free (not legal advice) | | Nomad reads a rendered value differently from the HCL1 that the tests use: Nomad parses with its fork `v1.0.1-nomad-1`, the tests with upstream v1.0.0 | a node does not start, or runs with another setting | strict quoting refuses what HCL1 cannot read back; tests parse every golden back; the weekly `online` job runs `nomad config validate` of the channel's minimum and recommended Nomad on the goldens ([ADR-0030](adr/0030-nomad-on-nodes.md)) | @@ -5755,20 +5767,16 @@ Decided on 2026-10-12: overrides the embedded file ([13.5](#135-tent-upgrade-cluster---yes), [ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)). -Open for the maintainer, from the final review of M3.7 (Q14): -- **Taking an upgrade back once the roll's first new server has joined.** After `upgrade --yes`, `update --yes` and - the first step of the roll, the cluster has one server on the new Nomad. `update` then refuses the old version in - the spec, and `rolling-update` refuses specs that `update` has not applied. When that server is unhealthy, the roll - does not go on either. The way back today is by hand: delete that machine in the cloud, wait until its peer is gone, - and run `tent update cluster` - ([ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)). - - (a) Keep it: the way is forward, and ADR-0041 names the way back by hand. - - (b) Let `update` take the upgrade back: the version check leaves out the servers and nodes whose machines the plan - removes, so the plan at the old version removes the new server as surplus. Nomad's guide says that downgrading - servers needs a new cluster, and whether old servers work on after a newer server led them is not verified, so (b) - needs a lab first. - - Recommendation: (a) for M3.7, and a lab for (b) before M3.9's `upgrade` scenario if the maintainer wants a way - back. +Decided on 2026-10-13: + +51. **No command takes an upgrade back once the roll's first new server has joined** (Q14 of the final review of + M3.7, answered (a)). After `upgrade --yes`, `update --yes` and the first step of the roll, the cluster has one + server on the new Nomad. `update` then refuses the old version in the spec, and `rolling-update` refuses specs that + `update` has not applied. When that server is unhealthy, the roll does not go on either. The way back is by hand: + delete that machine in the cloud, wait until its peer is gone, and run `tent update cluster` + ([ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)). The maintainer did not take (b), an `update` that + removes the new server as surplus, since Nomad's guide says that downgrading servers needs a new cluster, and + whether old servers work on after a newer server led them is not verified. There is no lab for (b). Decisions 18 to 20 are recorded in [ADR-0028](adr/0028-tent-node-agent-units-and-delivery.md), decision 21 in [ADR-0029](adr/0029-host-firewall-runtime-and-cni-on-nodes.md), decisions 22 to 25 in @@ -5777,7 +5785,7 @@ in [ADR-0032](adr/0032-joined-label-scrub-and-delete-guard.md), and decisions 29 [ADR-0033](adr/0033-operator-commands.md), decisions 33 to 35 in [ADR-0035](adr/0035-rollout-decisions.md), decisions 39 to 42 in [ADR-0037](adr/0037-rolling-update-of-client-groups.md), decisions 37, 38, 43, 44 and 48 in [ADR-0038](adr/0038-rolling-update-of-server-groups.md) (decisions 37 and 38 were first named in ADR-0035, item 15), and -decision 49 in [ADR-0039](adr/0039-rolling-update-of-combined-groups.md), and decisions 45 and 50 in +decision 49 in [ADR-0039](adr/0039-rolling-update-of-combined-groups.md), decisions 45, 50 and 51 in [ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md), and decisions 46 and 47 in [ADR-0042](adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md). diff --git a/docs/platform-notes.md b/docs/platform-notes.md index d6c9058..e6fe94a 100644 --- a/docs/platform-notes.md +++ b/docs/platform-notes.md @@ -927,6 +927,9 @@ through Playwright, with a throwaway `httputil.ReverseProxy`, and read in the so `downgrade`, `upgrade`, `newer version`, `older version`, `protocol version`, `minimum version`, `version skew`). - Not shown: a combined agent, a drain, Docker tasks, Linux, versions more than one patch apart, and tent-node's `refresh-join` (the clients learned the new servers from Nomad's heartbeats). + - **On Vultr (runs `w28exz` and `ae5dhs`, 2026-10-14):** a 2.0.7 combined node joined 2.0.6 ones and the roll of the + three nodes ended, as in the local lab, with the leader moving once and no `leaderless` poll + ([3.16](#316-spike-runs)). ### 1.4 Downloads and verification @@ -1229,6 +1232,21 @@ the VPC interface was up by `network-online.target` and the go-sockaddr template running Nomad stopped in 1.1 s, before Docker, without a drain; the client was ready and eligible after the reboot. Nomad did not reattach to its container after `systemctl restart docker` ([6.2](#62-firewalls-on-the-host)). +**After a hard stop of a server (run `w28exz`, 2026-10-14, Nomad 2.0.7).** At about 01:07:29 a server stopped (a +power-off). The probe's next read of the allocations through it timed out (7.1 s). Then for 30.2 s the reads of the +checks of both running allocations failed, while the reads of the allocations and the service's registrations answered +through other servers and listed both allocations running. A read of the checks goes over the connection of the +allocation's client to a server. By a reading of Nomad's source, the client keeps that connection until its keepalive +fails (5 s interval, about 10 s more for the ping) and its next heartbeat registers the node on another server. Run +`ae5dhs` showed one stretch of 20.1 s without readable checks (`upgrade`) and one poll of 7.1 s in which the read of the +registrations through a server that had just stopped failed (`ha`). Run `qkmdrs` showed one of 7.1 s (`upgrade`). The +server rolls of `ha` in both runs showed no stretch without readable checks. + +**A rebooted client (run `ae5dhs`, 2026-10-14).** A worker that was built with the old servers and restarted after all +three servers had been replaced came back with the same node ID. SSH answered from a new boot 2 min 17 s after the +restart, and 11 s later the journal of that boot held `client: node registration complete` and Nomad listed the node +ready and eligible. After the boot its `05-join.hcl` named the three new servers. + **A tent-node of another version** (the first E2E run, run `0oqujb`, and a debug cluster built the same way, `e2e-dbgaa1-2404`, 2026-10-07). tent-node's `preflight` refuses a tent-node whose version is not the one in the NodeConfig's tent-node asset. The suite had built tent with `go build`, so its version was `dev`, and the tent-node that @@ -1770,6 +1788,10 @@ Facts dated 2026-09-27 were read in the v3.33.0 source. and HTTP from another instance, so the VPC does not bind addresses to instances. Nothing reserves such an address, though, and Vultr may later assign it to a new instance. - **Traffic.** Ping between instances over the VPC works with the image's default host firewall (spike 2026-09-25). +- **A freed private address goes to a new machine (run `ae5dhs`, 2026-10-14).** After the rolls of `ha`, the new workers + had the addresses of servers that the roll had deleted. After the servers were deleted and rebuilt, the three new + servers got 10.64.0.3, .6 and .7, one of the three old addresses back (the servers before the delete had .3, .8 and + .9), while the workers were at .4 and .5. - **Deleting.** After the instances disappeared from `GET /v2/instances`, `DELETE /v2/vpcs/{id}` still failed for 14–20 s with `400 The following servers are attached to this VPC network: ` (spike 2026-09-25). Retry it. On 2026-09-28 the delete succeeded 12 s after the instance was gone, on the second request; the `` in the @@ -1808,6 +1830,10 @@ Facts dated 2026-09-27 were read in the v3.33.0 source. - `type`: the same as `ip_type`. - `direction`: `"in"`. - `loadbalancer_id`: `""`. +- **A creation date without a zone (run `w28exz`, 2026-10-14).** The firewall list gave a group made at 00:46:46 UTC + on 2026-10-14 the `date_created` `2026-10-13 20:46:46`: no zone, and 4 hours behind UTC, which is US Eastern time + (UTC-4 in summer). Other objects gave RFC 3339 with an offset. The E2E suite's client reads a date of that form as US + Eastern time; tent's own Vultr code skips a date it cannot read. - **A second copy of a rule is refused (checked 2026-09-27):** the same rule sent twice gets `400 {"error":"This rule is already defined ","status":400}` the second time, with a trailing space in the text. tent counts it as done. @@ -2670,6 +2696,44 @@ server and one worker, at most 3 machines at once. A job of one allocation and o with no node in the rebuilt cluster it waited until the maintainer's session stopped it. The script now uses `-detach`, and that cluster was deleted. +**E2E runs of M3.9 (2026-10-14)**, `ha` on ubuntu-24.04 (3 servers and 2 workers) and `upgrade` on ubuntu-26.04 (3 +combined nodes, Nomad 2.0.6 to 2.0.7) at once, at most 10 machines +([ADR-0043](adr/0043-e2e-scenarios-ha-and-upgrade.md)); `w28exz` and `ae5dhs` ran `make e2e +E2E_RUN='TestHA|TestUpgrade'`, `qkmdrs` and `hik34s` the full `make e2e`: +- **`w28exz`** (00:46 to 01:09 UTC; `ha` and `upgrade`): `upgrade` passed every step but the leak check. The roll of + three combined nodes from 2.0.6 to 2.0.7 took 14 min 27 s and its probe passed. `ha` passed `create` in 7 min 56 s, + which shows that autopilot's report held a version for every peer at the read before the first client + ([ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)). Its server roll ended with exit 0 in 13 min 12 s, and + the probe failed it: `service unreached for 37.2 s`, 495 polls `led` and `served`, 4 `led` and `unreached`, no + `leaderless` poll, the leader moved once (10.64.0.3 to .8, at 01:06:07). The two faults were in the suite: Vultr's + zoneless creation date (see [3.6](#36-firewall-groups)) and the checks reads that fail after a hard stop of a server + (see [1.6](#16-the-agent-on-a-node)). Both are fixed. +- **`ae5dhs`** (about 01:19 to 01:58 UTC, at the fixed code): both passed. + - `upgrade`: `create` 4 min 2 s, with tent's warning that 2.0.6 is untested; the roll 14 min 46 s; the probe saw 632 + polls, all `led`, 630 `served` and 2 `unchecked` (one stretch of 20.1 s at 01:27:31, a hard stop of a combined + node), no `leaderless`, `unserved` or `unreached` poll and 10.1 s as the longest gap between polls; the leader moved + once (10.64.0.4 to .6); `delete` 21 s; no leftovers. + - `ha`: `create` 7 min 43 s; the server roll 12 min 14 s with 476 polls, all `led`, 475 `served` and one `unreached` + of 7.1 s (the read of the service's registrations through the server that had just stopped), the leader moved once + (10.64.0.4 to .8); the restart of `workers-0`, the worker roll 5 min 45 s with 336 polls, all `led` and `served`; + the backup step 9 min 36 s (backup create 3 s, 13,876 bytes; delete of the servers 3 s; `update` 3 min 12 s; + restore 10 s); `delete` 24 s; no leftovers. The restart and the restore are in [1.6](#16-the-agent-on-a-node) and + [3.5](#35-vpc). +- **`qkmdrs`** (about 02:04 to 02:49 UTC, at `0f16490`, the full `make e2e`): every scenario passed. `smoke` passed + on both images (`create` 3 min 42 s and 4 min 38 s). `upgrade`: `create` 4 min 7 s without a warning, since the + channel now tests 2.0.6; the roll 14 min 53 s with 629 polls, all `led`, 628 `served` and one `unchecked` of 7.1 s; + the leader moved once. `ha`: `create` 7 min 26 s; the server roll 12 min 12 s with 478 polls, all `led` and `served`; + the restart of `workers-0` (a new boot after 2 min 18 s, the registration 16 s later, the same node ID, a join file + that named the three new servers); the worker roll 5 min 44 s with 336 polls, all `led` and `served`; the backup step + 9 min 36 s (13,856 bytes; `update` 3 min 16 s; restore 9 s; the rebuilt servers at 10.64.0.4, .6 and .7, one old + address back; the workers back on their own, no forced roll); no leftovers. +- **`hik34s`** (about 03:25 to 04:11 UTC, the full `make e2e` with the fixes of the final review): every scenario + passed, and every poll of the three probes was `led` and `served` (624 in `upgrade`'s roll of 14 min 57 s, 488 in + `ha`'s server roll of 12 min 30 s, 335 in its worker roll of 5 min 44 s); the probes found a good poll at once + after tent exited. One refresh of the addresses timed out after 5 s on Vultr's instance list, and the probe kept + the last addresses. The restart: a new boot after 2 min 18 s, the registration 11 s later. The backup step + 10 min 12 s, no forced roll; no leftovers. + **Still open:** - A halt whose call gets no answer, and how late Vultr may carry one out. - Object Storage conditional writes ([3.12](#312-object-storage-)). diff --git a/docs/roadmap.md b/docs/roadmap.md index bb3f9a3..89d9421 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,7 +1,7 @@ # Roadmap -> **Current status (2026-10-13):** M0 Foundation, M1 Vultr infrastructure and M2 Nomad bootstrap are complete. M3 Day-2 -> operations is in progress. +> **Current status (2026-10-14):** M0 Foundation, M1 Vultr infrastructure, M2 Nomad bootstrap and M3 Day-2 +> operations are complete. > > M0 built the Go module, `tent version`, the Makefile, the import rules in golangci-lint, CI on Linux, macOS and > Windows, the release pipeline for `tent`, the API types with defaults and validation, the spec reader and writer, @@ -34,7 +34,7 @@ > maintainer's machine. Its runs found four faults, all fixed, and on 2026-10-07 `smoke` passed on ubuntu-24.04 and > ubuntu-26.04: M2 is done. > -> M3 is in progress, in parts M3.1 to M3.9. M3.1 built the rollout decisions +> M3 was built in parts M3.1 to M3.9. M3.1 built the rollout decisions > ([ADR-0035](adr/0035-rollout-decisions.md)): `internal/rollout` returns the next step of a rolling update or of a > removal of nodes from what the cloud and Nomad report, and a simulator proves it from every state a roll passes > through. It also added the `rollingUpdate` settings of a node group. M3.2 built the Nomad calls of a roll @@ -59,14 +59,22 @@ > version at a time, `update` pins a version only after its release files verify and refuses a node older than one > that runs, and it stops before a client that would run a newer Nomad than a server. The upgrade ran on the fakes. > Nomad 2.0.6 and 2.0.7 ran together in a local lab without tent, where the servers were rolled by hand in tent's -> order ([platform notes §1.3](platform-notes.md#13-upgrades-and-version-skew)); M3.9's `upgrade` scenario runs it on +> order ([platform notes §1.3](platform-notes.md#13-upgrades-and-version-skew)); M3.9's `upgrade` scenario ran it on > Vultr. M3.8 built `tent backup create`, `list` and `restore` > ([ADR-0042](adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md)): > a backup is a snapshot of Nomad's state in the state store, under `backups//`, and a restore puts one > into servers that `update` built again from the same state store, only while they run no job and once for each > bootstrap. A restore that is cut is finished by running the command again, `delete cluster` deletes the backups, and > the run on Vultr gave the rebuilt server its old private address, so the old worker came back by itself ([platform -> notes §3.16](platform-notes.md#316-spike-runs)). +> notes §3.16](platform-notes.md#316-spike-runs)). M3.9 added the E2E scenarios `ha` and `upgrade` +> ([ADR-0043](adr/0043-e2e-scenarios-ha-and-upgrade.md)): `ha` rolls the servers, restarts an old worker that must find +> the new servers on its own, rolls the workers and restores a backup into rebuilt servers; `upgrade` rolls a combined +> cluster from Nomad 2.0.6 to 2.0.7. A probe asks the servers for a leader and the service for a healthy allocation at +> every second of each roll. Run `w28exz` (2026-10-14) found two faults of the suite, which are fixed. Run `ae5dhs` +> (2026-10-14) passed both scenarios: no poll was leaderless or unserved, the longest stretch in which the checks could +> not be read was 20.1 s and the longest in which a read of the service failed was 7.1 s. The full runs `qkmdrs` +> and `hik34s` (`smoke`, `ha` and `upgrade`) passed on 2026-10-14. The channel lists 2.0.6 as tested since `ae5dhs`. +> M3 is done. > > **Work items live in GitHub:** each milestone below links to its GitHub milestone, and the > [tent roadmap project][project] shows the open issues. This file keeps the goals and exit criteria; close issues as @@ -81,8 +89,8 @@ These come from [architecture §18](architecture.md#18-open-questions). The firs seventh to the fifteenth on 2026-09-28, the sixteenth to the twenty-first on 2026-09-29, the twenty-second to the twenty-fourth on 2026-09-30, the twenty-fifth to the twenty-seventh on 2026-10-05, and the twenty-eighth to the thirty-second on 2026-10-06, the thirty-third to the thirty-fifth on 2026-10-07, the thirty-seventh to the -forty-fourth on 2026-10-08, the forty-fifth to the forty-eighth on 2026-10-09, and the forty-ninth and the fiftieth on -2026-10-12. The maintainer extended the twenty-third on 2026-10-02. +forty-fourth on 2026-10-08, the forty-fifth to the forty-eighth on 2026-10-09, the forty-ninth and the fiftieth on +2026-10-12, and the fifty-first on 2026-10-13. The maintainer extended the twenty-third on 2026-10-02. | # | Question | Decision | |---|---|---| @@ -135,6 +143,7 @@ forty-fourth on 2026-10-08, the forty-fifth to the forty-eighth on 2026-10-09, a | 48 | tent remembers the highest index of each server and combined group | in the state store at `names/`, so that the names of such a group only grow, also after a shrink or an operator's delete (2026-10-09); built in M3.4 for the roll, where a new server takes the index above it, and in M3.6 for the planner of `update` ([ADR-0038](adr/0038-rolling-update-of-server-groups.md), item 22, [ADR-0040](adr/0040-scale-down-through-the-rollout.md)) | | 49 | A node group keeps its role | `replace` and `edit` refuse to change the `spec.role` of a node group that the store holds (2026-10-12); built in M3.5: tent has no way to change the role of a stored group, since a cluster has exactly one server or combined group and no command deletes a node group ([ADR-0039](adr/0039-rolling-update-of-combined-groups.md)) | | 50 | No channel file from a URL | the channel is the one embedded in tent; an operator who wants another Nomad version than tent recommends sets `spec.nomad.version` (2026-10-12); architecture §13.5 and ADR-0026 had planned a URL that overrides the embedded file ([ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)) | +| 51 | No command takes an upgrade back | once the roll's first new server of an upgrade has joined, no command takes the upgrade back; the way back is by hand, and ADR-0041 names it; no lab for taking it back in `update` (2026-10-13); nothing to build ([ADR-0041](adr/0041-upgrading-a-clusters-nomad-version.md)) | New questions for the maintainer are issues with the `decision` label. @@ -211,7 +220,7 @@ it: [`hack/vultr-spike/README.md`](../hack/vultr-spike/README.md). **Exit criteria:** E2E `ha` and `upgrade` pass on Vultr, with no quorum loss and no job downtime beyond the configured migrations. -**Status:** in progress; see the note at the top of this file. +**Status:** done on 2026-10-14; see the note at the top of this file. ## M4 Hetzner provider @@ -317,6 +326,9 @@ Work after M6 or not scheduled yet: [issues with the `later` label][later]. Maintainer decisions 46 and 47 are built. It ran on Vultr: a cluster of one server and one worker, a new server with the old private address, and a restore in 4 s ([platform notes §3.16](platform-notes.md#316-spike-runs)). Recorded in [ADR-0042](adr/0042-backups-in-the-state-store-and-restore-after-a-loss-of-the-servers.md). Closes #105. +- 2026-10-14: M3.9 is built: the E2E scenarios `ha` and `upgrade` with a probe of the leader and the service, run on + Vultr (see the status block above). Maintainer decision 51 is recorded. Recorded in + [ADR-0043](adr/0043-e2e-scenarios-ha-and-upgrade.md). Closes #106 and #107. M3 is complete. [project]: https://github.com/users/ingvarch/projects/2 [later]: https://github.com/ingvarch/tent/issues?q=is%3Aissue%20label%3Alater diff --git a/hack/e2e-janitor/main.go b/hack/e2e-janitor/main.go index 0e6e81c..d51c123 100644 --- a/hack/e2e-janitor/main.go +++ b/hack/e2e-janitor/main.go @@ -38,7 +38,7 @@ func run( ) int { flags := flag.NewFlagSet("e2e-janitor", flag.ContinueOnError) flags.SetOutput(stderr) - olderThan := flags.Duration("older-than", 3*time.Hour, "act on the clusters whose oldest object is older than this") + olderThan := flags.Duration("older-than", 4*time.Hour, "act on the clusters whose oldest object is older than this") yes := flags.Bool("yes", false, "delete; without it the command only lists what it would delete") if err := flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { diff --git a/hack/e2e-janitor/main_test.go b/hack/e2e-janitor/main_test.go index 277c361..016d0ec 100644 --- a/hack/e2e-janitor/main_test.go +++ b/hack/e2e-janitor/main_test.go @@ -115,7 +115,7 @@ func TestRunWithoutYesListsAndDeletesNothing(t *testing.T) { line("instance", "i1", "e2e-a-servers-0", "e2e-a"), line("vpc", "v1", "tent:cluster=e2e-a;kind=vpc", "e2e-a"), line("vpc", "v2", "tent:cluster=e2e-b;kind=vpc", "e2e-b"), - "would delete 3 objects of 2 clusters older than 3h0m0s; run again with --yes", + "would delete 3 objects of 2 clusters older than 4h0m0s; run again with --yes", }, "\n") + "\n" if res.code != 0 || res.stdout != want || res.stderr != "" { t.Errorf("got exit %d, stdout\n%q\nstderr %q\nwant stdout\n%q", res.code, res.stdout, res.stderr, want) @@ -125,6 +125,21 @@ func TestRunWithoutYesListsAndDeletesNothing(t *testing.T) { } } +// TestRunActsOnClustersOlderThanFourHoursByDefault: the run of make e2e may last 3 hours, so a cluster that is 3 hours +// and 59 minutes old is left alone and one a minute past 4 hours is listed. +func TestRunActsOnClustersOlderThanFourHoursByDefault(t *testing.T) { + api := &janitortest.API{Machines: []vultrapi.Instance{ + {ID: "i-old", Label: "e2e-old-servers-0", Tags: []string{"tent/cluster=e2e-old"}, + Created: time.Now().Add(-4*time.Hour - time.Minute)}, + {ID: "i-new", Label: "e2e-new-servers-0", Tags: []string{"tent/cluster=e2e-new"}, + Created: time.Now().Add(-3*time.Hour - 59*time.Minute)}, + }} + res := runWith(t, api, withKey()) + if !strings.Contains(res.stdout, "e2e-old") || strings.Contains(res.stdout, "e2e-new") { + t.Errorf("stdout = %q, want the cluster of 4h1m and not that of 3h59m", res.stdout) + } +} + func TestRunSaysWhenNothingIsOlder(t *testing.T) { for _, args := range [][]string{{"--older-than", "100h"}, {"--older-than", "100h", "--yes"}} { api := account() diff --git a/internal/buildconfig/makefile_test.go b/internal/buildconfig/makefile_test.go index b97778c..a7b4907 100644 --- a/internal/buildconfig/makefile_test.go +++ b/internal/buildconfig/makefile_test.go @@ -274,14 +274,29 @@ func TestMakeE2ERunsTheSuiteWithTheTentOfTheUpload(t *testing.T) { if root, err = filepath.EvalSymlinks(root); err != nil { t.Fatal(err) } - stdout, stderr, err := runMake(t, map[string]string{"go": fakeGo}, "e2e") - want := "tested test -tags e2e -count=1 -v -timeout 90m ./test/e2e url=https://u sha=abc tent=" + - filepath.Join(root, "bin", "tent") + "\n" - if err != nil || stdout != want { - t.Errorf("make e2e: err %v, stdout %q, want %q\nstderr:\n%s", err, stdout, want, stderr) - } - if !strings.Contains(stderr, "built build") || !strings.Contains(stderr, "shell=/bin/sh") { - t.Errorf("make e2e: stderr lacks the build or the upload for sh:\n%s", stderr) + for _, tc := range []struct { + name, run, flags string + }{ + {"every scenario", "", ""}, + // E2E_RUN reaches go test as one -run pattern, alternatives included. + {"the scenarios of E2E_RUN", "TestHA|TestUpgrade", "-run TestHA|TestUpgrade "}, + } { + t.Run(tc.name, func(t *testing.T) { + args := []string{"e2e"} + if tc.run != "" { + args = append(args, "E2E_RUN="+tc.run) + } + stdout, stderr, err := runMake(t, map[string]string{"go": fakeGo}, args...) + want := "tested test -tags e2e -count=1 -v -timeout 180m " + tc.flags + + "./test/e2e url=https://u sha=abc tent=" + filepath.Join(root, "bin", "tent") + "\n" + if err != nil || stdout != want { + t.Errorf("make %s: err %v, stdout %q, want %q\nstderr:\n%s", strings.Join(args, " "), err, stdout, + want, stderr) + } + if !strings.Contains(stderr, "built build") || !strings.Contains(stderr, "shell=/bin/sh") { + t.Errorf("make e2e: stderr lacks the build or the upload for sh:\n%s", stderr) + } + }) } } diff --git a/internal/channels/channels_test.go b/internal/channels/channels_test.go index 59862cf..5f37950 100644 --- a/internal/channels/channels_test.go +++ b/internal/channels/channels_test.go @@ -68,6 +68,18 @@ func TestStableIsEmbedded(t *testing.T) { } } +func TestStableTestsTheVersionsOfTheUpgradeRun(t *testing.T) { + c, err := Load("stable") + if err != nil { + t.Fatal(err) + } + for _, v := range []string{"2.0.6", "2.0.7"} { + if !c.Tested(v) { + t.Errorf("stable.nomad.tested = %v, want %s among them", c.Nomad.Tested, v) + } + } +} + func TestNamesAreSorted(t *testing.T) { if names := Names(); !slices.IsSorted(names) { t.Errorf("Names() = %v, want them sorted", names) diff --git a/internal/channels/stable.yaml b/internal/channels/stable.yaml index 9cc6173..bb8550d 100644 --- a/internal/channels/stable.yaml +++ b/internal/channels/stable.yaml @@ -2,11 +2,13 @@ # Nomad 2.0.7 (released 2026-09-18): https://api.releases.hashicorp.com/v1/releases/nomad/latest # CNI plugins v1.9.1 (released 2026-03-16): the cni-plugins-linux--v1.9.1.tgz.sha256 files of # https://github.com/containernetworking/plugins/releases/tag/v1.9.1 +# Tested: 2.0.6 and 2.0.7 by the E2E scenario upgrade (run ae5dhs, 2026-10-14), which rolled a combined +# cluster from 2.0.6 to 2.0.7 on Vultr. name: stable nomad: minimum: 2.0.0 recommended: 2.0.7 - tested: [2.0.7] + tested: [2.0.6, 2.0.7] cni: version: 1.9.1 sha256: diff --git a/test/e2e/README.md b/test/e2e/README.md index c5f980a..563fdf2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -7,9 +7,16 @@ service user's key works only from the maintainer's two addresses. It is not par ## What it does -The scenario `smoke` makes one cluster per image, `ubuntu-24.04` and `ubuntu-26.04`, in parallel. Each has 1 server and -1 client, so a run makes 4 machines. A cluster is named `e2e--`, such as `e2e-k3x9qz-2404`. The -steps run in order, and the first that fails stops the cluster's steps: +A run has three scenarios, in this order. `smoke` runs first. `ha` and `upgrade` run after it, in parallel with each +other, so a run has at most 10 machines at once: 4 for `smoke`, then up to 6 for `ha` and 4 for `upgrade`. A cluster of +`ha` or `upgrade` is named `e2e--`, and before it creates anything the suite deletes the leftovers of +earlier runs (see the janitor below). In each scenario the first step that fails stops the cluster's steps, and the +cleanup deletes the cluster. + +### `smoke` + +The scenario makes one cluster per image, `ubuntu-24.04` and `ubuntu-26.04`, in parallel. Each has 1 server and 1 +client. A cluster is named `e2e--`, such as `e2e-k3x9qz-2404`. The steps: 1. **create:** `tent create cluster ... --yes` with the run's SSH key and the runner's address as the only address in `--ssh-access` and `--api-access`. @@ -28,7 +35,66 @@ steps run in order, and the first that fails stops the cluster's steps: 8. **delete:** `tent delete cluster --yes` exits 0, then the Vultr API lists no object of the cluster and `tent get clusters` prints `no clusters in` (`delete` leaves nothing behind). -Before it creates anything, the suite deletes the leftovers of earlier runs (see the janitor below). +### `ha` + +The cluster `e2e--ha` has 3 servers and 2 workers on the first image of `E2E_IMAGES` and runs the job +`e2e-web` with two allocations. The steps: + +1. **create, validate, export, service:** as in `smoke`, with `--servers 3 --workers 2`, and the job `e2e-web` with two + allocations running and its service healthy. +2. **change:** `tent replace` adds an extra Nomad configuration to the specs (a `client` meta value `e2e_roll` with the + run's id), so every machine is outdated. **update** applies it, and `tent rolling-update cluster --exit-code` + exits 2. +3. **roll servers:** `tent rolling-update cluster --yes --nodegroups servers` while a probe polls the servers for a + leader and the service for a healthy allocation (see below). Afterwards Vultr lists 3 new servers and the 2 old + workers. +4. **restart client:** the old worker with the lowest name restarts over SSH. It must boot again, register with the + same node ID (the journal of the new boot holds `client: node registration complete`), be `ready` and `eligible`, + and the service must be served. Its seed servers are dead by now, so only what `refresh-join` wrote leads it to the + new servers. +5. **roll workers:** `tent rolling-update cluster --yes` with the probe, then `--exit-code` exits 0, Vultr lists only + new workers, and each worker's node has `e2e_roll` set to the run's id. +6. **validate again.** +7. **backup:** `tent backup create`, then the suite deletes the cluster's 3 server machines through the Vultr API, + `tent update cluster --yes` builds new ones, and `tent backup restore --yes` restores the backup. The suite waits 6 + minutes, since Nomad lists the backup's nodes as ready for about 5 minutes whether they answer or not, and runs + `validate cluster`. Only when that fails, it rolls the workers with `--force` and validates again. At the end two + allocations run and the service is healthy. +8. **delete** and the leak check. + +### `upgrade` + +The cluster `e2e--upgrade` has 3 combined nodes on the last image of `E2E_IMAGES`, on the Nomad version of +`E2E_UPGRADE_FROM`. The steps: + +1. **create, validate, export, service:** as in `ha`, with `--combined --servers 3 --nomad-version `. A warning + that the old version is untested is logged and does not fail the step. +2. **versions before:** every node runs the old version. +3. **upgrade:** `tent upgrade cluster --yes` moves the spec to the version that tent's channel recommends, which must + differ from the old one; `update` applies it, and `rolling-update --exit-code` exits 2. +4. **roll:** `tent rolling-update cluster --yes` with the probe, then `--exit-code` exits 0. +5. **versions after:** Nomad lists 3 nodes on the new version, all `ready` and `eligible`, with 3 `alive` members that + carry the new version, and Vultr lists only new machines. +6. **validate again:** `tent validate cluster` exits 0 and names the new version. +7. **delete** and the leak check. + +### The probe + +During each roll a probe polls once a second. It asks every server for the leader (`/v1/status/leader?stale=true`) and +reads the job's allocations, their checks and the service's registrations. A poll counts as `led` or `leaderless` or +`unreached`, and as `served`, `unserved`, `unchecked` (the checks could not be read, which happened for up to 37 s +after a server stopped hard, while the allocations ran and were registered) or `unreached`. The step fails when: + +- `leaderless` or `unserved` lasts more than 10 s, `unreached` more than 30 s or `unchecked` more than 60 s; +- one of these outcomes lasts more than twice its limit in all during one roll; +- two polls start more than 30 s apart; +- there are fewer than 10 polls, or the first or the last poll is not both `led` and `served`. + +The worker roll of `ha` stops no server, so its `unchecked` limit is 10 s, like `unserved`. When tent exits 0, the +probe goes on polling until a poll is `led` and `served`, for at most the roll's longest limit: 60 s, or 30 s in the +worker roll of `ha`. + +The reasons for each limit are in [ADR-0043](../../docs/adr/0043-e2e-scenarios-ha-and-upgrade.md). ## Run it @@ -52,14 +118,29 @@ env VULTR_API_KEY=() \ `make e2e` builds `bin/tent` and `bin/tent-node_linux_amd64` with one version, uploads the tent-node (`make dev-upload`), sets `TENT_NODE_URL`, `TENT_NODE_SHA256` and `E2E_TENT` (the absolute path of `bin/tent`) and runs -`go test -tags e2e -count=1 -v -timeout 90m ./test/e2e`. A node refuses a tent-node of another version than its tent's, -so the suite stops at once when `TENT_NODE_SHA256` is not the sha256 of the tent-node next to `E2E_TENT`. Run the suite -with `make e2e` for that reason. +`go test -tags e2e -count=1 -v -timeout 180m ./test/e2e`, with `-run "$(E2E_RUN)"` added when `E2E_RUN` is set. A node +refuses a tent-node of another version than its tent's, so the suite stops at once when `TENT_NODE_SHA256` is not the +sha256 of the tent-node next to `E2E_TENT`. Run the suite with `make e2e` for that reason. + +To run some scenarios, set `E2E_RUN` to a `go test -run` pattern: + +```fish +env VULTR_API_KEY=() \ + make e2e E2E_RUN='TestHA|TestUpgrade' +``` + +`make -n e2e` is not a dry run: the recipe calls `$(MAKE)`, so `dev-upload` runs. -The timeout is 90 minutes, since a `go test` that times out runs no cleanup. Ctrl-C or SIGTERM ends the running step -within seconds; the suite prints `e2e: interrupted; deleting the clusters of this run` and its cleanups delete the -clusters (unless `E2E_KEEP` is set) and the exported files. Ctrl-C reaches the test process through the terminal; send -SIGTERM to the `e2e.test` process itself, since `make` and `go test` do not pass it on. +The timeout is 180 minutes, since a `go test` that times out runs no cleanup. The context of every step ends 20 minutes +before `go test`'s deadline, so a step that runs long fails and the cleanups still have time to delete the clusters. +Run the suite through `make e2e` for that too: a `-timeout` under 20 minutes (go test's default of 10 included) ends +every step at once, and `-timeout 0` sets no end. The cleanup also deletes a cluster when the end of the step's context +cut its delete step short. + +Ctrl-C or SIGTERM ends the running step within seconds; the suite prints `e2e: interrupted; deleting the clusters of +this run` and its cleanups delete the clusters (unless `E2E_KEEP` is set) and the exported files. Ctrl-C reaches the +test process through the terminal; send SIGTERM to the `e2e.test` process itself, since `make` and `go test` do not +pass it on. ### Optional variables @@ -67,7 +148,9 @@ SIGTERM to the `e2e.test` process itself, since `make` and `go test` do not pass |---|---|---| | `E2E_REGION` | `ams` | the Vultr region; there is no automatic fallback, so pick another by hand after an incident | | `E2E_PLAN` | `vc2-1c-1gb` | the machine type | -| `E2E_IMAGES` | `ubuntu-24.04,ubuntu-26.04` | the images, comma-separated, each of the form `ubuntu-NN.NN` | +| `E2E_IMAGES` | `ubuntu-24.04,ubuntu-26.04` | the images, comma-separated, each of the form `ubuntu-NN.NN`; `ha` takes the first and `upgrade` the last | +| `E2E_RUN` | empty | the `go test -run` pattern that `make e2e` passes, such as `TestHA\|TestUpgrade`; empty runs every scenario | +| `E2E_UPGRADE_FROM` | `2.0.6` | the Nomad version that `upgrade` builds its cluster on, of the form `N.N.N` | | `E2E_KEEP` | empty | when set, the suite keeps the clusters, skips the delete step and the leak check, and prints how to delete them | | `RUNNER_ADDR` | what `https://api.ipify.org` answers | the address that gets SSH and API access to the clusters | @@ -77,8 +160,19 @@ Each run has an id of 6 characters and a directory `test/e2e/results//`, wh deletes: - `-.log`: one file for each tent command, with the arguments, stdout, stderr, the exit code and the - time it took (steps `create`, `validate`, `export`, `validate-again`, `delete`, `get-clusters`, `cleanup-delete`); + time it took (`smoke` has the steps `create`, `validate`, `export`, `validate-again`, `delete`, `get-clusters` and + `cleanup-delete`; `ha` and `upgrade` add steps such as `roll-servers`, `roll-workers`, `roll`, `roll-due`, + `roll-done`, `upgrade`, `backup-create`, `backup-restore` and an `export-...` after each roll); - `-intro-token.txt`: the record of the intro-token step; +- `-roll-change.json`: the specs that `ha` replaced, with the extra Nomad configuration; +- `-probe-servers.tsv` and `-probe-workers.tsv` (`ha`), `-probe-nodes.tsv` (`upgrade`): the + probe's polls, one line each, written as they come, with the columns `start` (UTC), `took_ms`, `leader` (`led`, + `leaderless` or `unreached`), `leader_address`, `service` (`served`, `unserved`, `unchecked` or `unreached`) and + `detail`; +- `-restart.txt` (`ha`): the machine, the node ID, the boot IDs before and after, the waits, the registration + lines of the new boot and the content of `/etc/nomad.d/05-join.hcl`; +- `-backup.txt` (`ha`): the backup's name and size, the time of each step, the private addresses of the + servers and workers before and after, and whether the forced roll ran; - `state-/`: the cluster's `file://` state store; after a delete it holds no cluster, only tent's lock files; - `ssh/`: the run's SSH key pair, with the private key at mode 0600; @@ -99,11 +193,11 @@ env VULTR_API_KEY=() \ --state file://(pwd)/test/e2e/results/k3x9qz/state-e2e-k3x9qz-2404 ``` -A kept cluster that nobody deletes goes when the janitor finds it older than 3 hours, in the next run or by hand. +A kept cluster that nobody deletes goes when the janitor finds it older than 4 hours, in the next run or by hand. ## The janitor -The janitor deletes the objects of clusters whose name starts with `e2e-` and whose oldest object is older than 3 hours: +The janitor deletes the objects of clusters whose name starts with `e2e-` and whose oldest object is older than 4 hours: all the cluster's objects, instances first, then firewall groups, VPCs and SSH keys. It finds them by tent's tags and markers. Objects that carry no tent marker, such as the account's own SSH key `main`, stay. @@ -114,12 +208,18 @@ env VULTR_API_KEY=() \ go run ./hack/e2e-janitor ``` +The janitor reads a creation date that Vultr gives without a zone as US Eastern time: on 2026-10-14 the firewall list +gave `2026-10-13 20:46:46` for a group made at 00:46:46 UTC. + Add `--yes` to delete, and `--older-than 1h` to change the age. `--older-than 0` takes every E2E cluster, also one of a run in progress. ## Cost and time -A run makes 4 machines of `vc2-1c-1gb`, and Vultr bills a machine for at least one hour. A green run took 6 to 7 -minutes on 2026-10-07: `create` 4 to 6.5 minutes, `delete` about 20 s, the leak check up to 2 minutes on the cluster -whose check runs while the other is still being built, every other step seconds +Vultr bills a machine for at least one hour, and the machines are `vc2-1c-1gb`. `smoke` makes 4 machines and a green +run took 6 to 7 minutes on 2026-10-07: `create` 4 to 6.5 minutes, `delete` about 20 s, the leak check up to 2 minutes on +the cluster whose check runs while the other is still being built, every other step seconds. In run `ae5dhs` +(2026-10-14) `ha` and `upgrade` took about 40 minutes together, with at most 10 machines: `ha` ran `create` 7 min 43 s, +the server roll 12 min 14 s, the restart of a worker 2 min 35 s, the worker roll 5 min 45 s and the backup step 9 min 36 +s (6 minutes of it the wait); `upgrade` ran `create` 4 min 2 s and the roll 14 min 46 s; each `delete` took under 25 s ([platform notes §3.16](../../docs/platform-notes.md#316-spike-runs)). diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go new file mode 100644 index 0000000..7e5fbdd --- /dev/null +++ b/test/e2e/cluster_test.go @@ -0,0 +1,307 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ingvarch/tent/test/e2e/vultrapi" +) + +const ( + validateTimeout = 7 * time.Minute + deleteTimeout = 15 * time.Minute + purgeTimeout = time.Minute + serviceEvery = 5 * time.Second + serviceTimeout = 5 * time.Minute + leftoversEvery = 10 * time.Second + leftoversTimeout = 2 * time.Minute + noClustersText = "no clusters in" + // previewTimeout bounds a run of a command that reads the cloud and Nomad and changes nothing. + previewTimeout = 5 * time.Minute + // pairAllocations is the number of allocations of the job in testdata/web-pair.nomad.hcl. + pairAllocations = 2 + updateTimeout = 25 * time.Minute + // rolledEvery and rolledTimeout bound the poll for what a roll changes in Vultr's and Nomad's lists. + rolledEvery = 10 * time.Second + rolledTimeout = 2 * time.Minute + // specTimeout bounds a command that reads or writes the specs in the state store and reaches no cloud. + specTimeout = time.Minute +) + +// cluster is a cluster of a scenario and what its steps hand to each other. +type cluster struct { + name string + image string + nomad *nomadAPI + // ctxErr is the error of the context of the latest tent command after tent returned. It tells a command that a + // deadline or a signal cut short from one that ended on its own. + ctxErr error + // deleteRan is true once the delete step has started; deleteDone is true once tent delete exited 0; deleteErr is + // ctxErr after the delete step. The cleanup deletes the cluster only when no delete ran or the delete was cut + // short (deleteInCleanup). + deleteRan bool + deleteDone bool + deleteErr error +} + +// newCluster returns the cluster and registers its cleanups on t: the removal of its export directory and, when its +// steps did not delete it, the delete. +func newCluster(t *testing.T, name, image string) *cluster { + c := &cluster{name: name, image: image} + t.Cleanup(func() { + if err := os.RemoveAll(c.exportDir()); err != nil { + t.Logf("remove the export of %s, which holds its token and client key: %v", c.name, err) + } + }) + t.Cleanup(func() { c.cleanup(t) }) + return c +} + +func (c *cluster) exportDir() string { + return filepath.Join(suite.Dir, "export-"+c.name) +} + +// clusterStep is one subtest of a cluster. +type clusterStep struct { + name string + run func(*testing.T) +} + +// runSteps runs the steps of a cluster in order, each as a subtest that logs the time it took. The first step that +// fails ends the test. +func runSteps(t *testing.T, c *cluster, steps []clusterStep) { + for _, step := range steps { + ok := t.Run(step.name, func(t *testing.T) { + start := time.Now() + defer func() { t.Logf("%s %s took %s", c.name, step.name, time.Since(start).Round(time.Second)) }() + step.run(t) + }) + if !ok { + t.FailNow() + } + } +} + +// tentCode runs a tent command of the cluster and returns its result, whatever its exit code, for the commands +// whose code a scenario reads. It fails the step when tent cannot be run. +func (c *cluster) tentCode(t *testing.T, timeout time.Duration, step string, args ...string) tentResult { + t.Helper() + ctx, cancel := context.WithTimeout(stepContext(suite.Ctx, t), timeout) + defer cancel() + res, err := suite.Tent.run(ctx, c.name, step, args...) + c.ctxErr = ctx.Err() + if err != nil { + t.Fatalf("tent %s: %v", step, err) + } + return res +} + +// tent runs a tent command of the cluster and fails the step when it does not exit with 0. +func (c *cluster) tent(t *testing.T, timeout time.Duration, step string, args ...string) tentResult { + t.Helper() + res := c.tentCode(t, timeout, step, args...) + if res.Code != 0 { + t.Fatal(tentFailure(step, res, suite.Tent.logPath(c.name, step))) + } + return res +} + +func (c *cluster) validate(t *testing.T, step string) { + c.tent(t, validateTimeout, step, validateArgs(c.name)...) +} + +// export has tent write the files that reach Nomad and builds the client from them. The step names its log file. +func (c *cluster) export(t *testing.T, step string) { + res := c.tent(t, time.Minute, step, "export", "nomad", c.name, "--dir", c.exportDir(), "-o", "json") + x, err := parseExport(res.Stdout) + if err != nil { + t.Fatal(err) + } + if c.nomad, err = newNomad(x); err != nil { + t.Fatal(err) + } +} + +// submitJob submits the job of a file in testdata and leaves it in the cluster. +func (c *cluster) submitJob(ctx context.Context, t *testing.T, file string) { + t.Helper() + text, err := os.ReadFile(filepath.Join("testdata", file)) + if err != nil { + t.Fatalf("read the job file: %v", err) + } + if err := submitJob(ctx, c.nomad, string(text)); err != nil { + t.Fatal(err) + } +} + +// runJob submits the job of a file in testdata and purges it when the step ends. +func (c *cluster) runJob(ctx context.Context, t *testing.T, file, job string) { + t.Helper() + c.submitJob(ctx, t, file) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), purgeTimeout) + defer cancel() + if err := purgeJob(ctx, c.nomad, job); err != nil { + t.Errorf("purge the job %s: %v", job, err) + } + }) +} + +// service runs the web job and waits until its allocation, its check and its registration agree. +func (c *cluster) service(t *testing.T) { + ctx := stepContext(suite.Ctx, t) + c.runJob(ctx, t, "web.nomad.hcl", webJob) + if err := waitService(ctx, c.nomad, webJob, webService, serviceEvery, serviceTimeout); err != nil { + t.Fatalf("service %s: %v", webService, err) + } +} + +// pairService submits the job of two allocations, leaves it in the cluster, and waits until both run and the service +// is healthy. +func (c *cluster) pairService(t *testing.T) { + ctx := stepContext(suite.Ctx, t) + c.submitJob(ctx, t, "web-pair.nomad.hcl") + if err := c.waitPair(ctx, serviceTimeout); err != nil { + t.Fatalf("service %s: %v", webService, err) + } +} + +// waitPair waits until both allocations of the job run and the service is healthy. +func (c *cluster) waitPair(ctx context.Context, timeout time.Duration) error { + return pollUntil(ctx, serviceEvery, timeout, func(ctx context.Context) string { + running, err := runningAllocations(ctx, c.nomad, webJob) + if err != nil { + return err.Error() + } + if problem := allocationsProblem(running, webJob, pairAllocations); problem != "" { + return problem + } + return serviceProblem(ctx, c.nomad, webJob, webService) + }) +} + +// machines lists every machine of the Vultr account; machinesOf picks those of a cluster. +func (c *cluster) machines(ctx context.Context) ([]vultrapi.Instance, error) { + return suite.Vultr.Instances(ctx) +} + +// addresses returns the function that lists the API addresses of the cluster's machines of a role. +func (c *cluster) addresses(role string) func(context.Context) ([]string, error) { + return func(ctx context.Context) ([]string, error) { + instances, err := c.machines(ctx) + if err != nil { + return nil, fmt.Errorf("list the machines: %w", err) + } + return serverAddresses(instances, c.name, role), nil + } +} + +// rollWithProbe runs tent with args as the step while a probe polls the machines of a role, and writes the probe's +// records to -probe-.tsv. When tent exits with 0 the probe goes on polling until a poll is led and +// served, for at most the longest stretch limit, since tent can exit while the checks are still unreadable. The step +// fails when tent does not exit with 0 or when the verdict on the limits is an error. +func (c *cluster) rollWithProbe(t *testing.T, role, roll, step string, limits probeLimits, timeout time.Duration, + args ...string) { + t.Helper() + ctx := stepContext(suite.Ctx, t) + addresses := c.addresses(role) + first, err := addresses(ctx) + if err != nil { + t.Fatal(err) + } + if len(first) == 0 { + t.Fatalf("no %s machine of %s has an address to probe", role, c.name) + } + path := filepath.Join(suite.Dir, c.name+"-probe-"+roll+".tsv") + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + t.Fatalf("open the probe's records: %v", err) + } + defer func() { + if err := out.Close(); err != nil { + t.Errorf("close the probe's records %s: %v", path, err) + } + }() + cfg := newProbeConfig(c.nomad, addresses, webJob, webService, out) + cfg.First = first + p := startProbe(ctx, cfg) + defer p.stop() + + res := c.tentCode(t, timeout, step, args...) + if res.Code == 0 { + t.Logf("%s probe polled %s after tent exited", c.name, p.settle(ctx, limits.longest()).Round(time.Second)) + } + summary, verr := probeVerdict(p.stop(), limits) + t.Logf("%s probe during %s: %s\nrecords: %s", c.name, roll, summary, path) + if verr != nil { + t.Errorf("%s: %v", roll, verr) + } + if res.Code != 0 { + t.Fatal(tentFailure(step, res, suite.Tent.logPath(c.name, step))) + } + if verr != nil { + t.FailNow() + } +} + +// rollDue checks that tent rolling-update without --yes exits with want: 2 while a roll is due, 0 when none is. +func (c *cluster) rollDue(t *testing.T, step string, want int) { + t.Helper() + res := c.tentCode(t, previewTimeout, step, "rolling-update", "cluster", c.name, "--exit-code") + if res.Code != want { + t.Fatalf("tent %s: want exit code %d\n%s", step, want, tentFailure(step, res, suite.Tent.logPath(c.name, step))) + } +} + +func (c *cluster) delete(t *testing.T) { + c.deleteRan = true + defer func() { c.deleteErr = c.ctxErr }() + c.tent(t, deleteTimeout, "delete", "delete", "cluster", c.name, "--yes") + c.deleteDone = true +} + +// leftovers checks that Vultr holds nothing of the cluster and that its store is empty. +func (c *cluster) leftovers(t *testing.T) { + ctx := stepContext(suite.Ctx, t) + if err := waitNoLeftovers(ctx, suite.Vultr, c.name, leftoversEvery, leftoversTimeout, t.Logf); err != nil { + t.Fatalf("after the delete of %s: %v", c.name, err) + } + res := c.tent(t, time.Minute, "get-clusters", "get", "clusters") + if !strings.Contains(string(res.Stderr), noClustersText) { + t.Fatalf("tent get clusters: stderr does not hold %q: %s\nlog: %s", noClustersText, + lastLines(string(res.Stderr), stderrTailLines), suite.Tent.logPath(c.name, "get-clusters")) + } +} + +// cleanup deletes the cluster when the delete step did not run or was cut short (deleteInCleanup), with a context of +// its own because the test's has ended. A run that keeps its clusters deletes nothing and logs the command that +// does. +func (c *cluster) cleanup(t *testing.T) { + if !deleteInCleanup(c.deleteRan, c.deleteDone, suite.Ctx.Err() != nil, c.deleteErr) { + return + } + if suite.Settings.Keep { + t.Logf("keeping cluster %s; delete it with: %s", c.name, suite.Tent.deleteCommand(c.name)) + return + } + ctx, cancel := context.WithTimeout(context.Background(), deleteTimeout) + defer cancel() + step := "cleanup-delete" + res, err := suite.Tent.run(ctx, c.name, step, "delete", "cluster", c.name, "--yes") + switch { + case err != nil: + t.Errorf("cleanup: delete cluster %s: %v", c.name, err) + case res.Code != 0: + t.Errorf("cleanup: delete cluster %s failed:\n%s", c.name, + tentFailure(step, res, suite.Tent.logPath(c.name, step))) + default: + t.Logf("cleanup: deleted cluster %s in %s", c.name, res.Took.Round(time.Second)) + } +} diff --git a/test/e2e/ha_test.go b/test/e2e/ha_test.go new file mode 100644 index 0000000..8a733bb --- /dev/null +++ b/test/e2e/ha_test.go @@ -0,0 +1,422 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ingvarch/tent/test/e2e/vultrapi" +) + +const ( + rollServersTimeout = 60 * time.Minute + rollWorkersTimeout = 45 * time.Minute + bootEvery = 5 * time.Second + bootTimeout = 6 * time.Minute + registrationEvery = 5 * time.Second + registrationTimeout = 5 * time.Minute + backupTimeout = 10 * time.Minute + serversGoneEvery = 5 * time.Second + serversGoneTimeout = 3 * time.Minute + // restoreSettle is the wait after a restore: Nomad lists the backup's nodes as ready for about 5 minutes, whether + // they answer or not. + restoreSettle = 6 * time.Minute + pairBackTimeout = 10 * time.Minute +) + +// TestHA builds a cluster of three servers and two workers, changes its specs so that every machine is outdated, and +// rolls it: the servers first, then, after a restart of an old worker that must find the new servers on its own, the +// workers. A probe asks the servers for a leader and the service for a healthy allocation during each roll. At the +// end it backs up Nomad, deletes the servers, has update build new ones and restores the backup into them. +// +// It is parallel, unlike TestSmoke: Go runs the tests that are not parallel first and releases the parallel ones +// after them, so a full run builds the clusters of smoke, and only then those of this test and of TestUpgrade +// together. That keeps at most ten machines up at once. +func TestHA(t *testing.T) { + t.Parallel() + h := &haScenario{cluster: newCluster(t, scenarioName(suite.ID, "ha"), suite.Settings.haImage())} + steps := []clusterStep{ + {"create", h.createHA}, + {"validate", func(t *testing.T) { h.validate(t, "validate") }}, + {"export", func(t *testing.T) { h.export(t, "export") }}, + {"service", h.pairService}, + {"change", h.change}, + {"update", h.update}, + {"roll-servers", h.rollServers}, + {"restart-client", h.restartClient}, + {"roll-workers", h.rollWorkers}, + {"validate-again", func(t *testing.T) { h.validate(t, "validate-again") }}, + {"backup", h.backup}, + } + if !suite.Settings.Keep { + steps = append(steps, clusterStep{"delete", h.delete}, clusterStep{"leftovers", h.leftovers}) + } + runSteps(t, h.cluster, steps) +} + +// haScenario is the cluster of TestHA and the machines that its steps recorded before the rolls. +type haScenario struct { + *cluster + servers []vultrapi.Instance + workers []vultrapi.Instance +} + +func (h *haScenario) createHA(t *testing.T) { + args := haCreateArgs(suite.Settings, h.name, h.image, suite.SSHKey+".pub", suite.Runner) + h.tent(t, createTimeout, "create", args...) +} + +// change adds an extra Nomad configuration to the specs, which changes the spec hash of every server and every worker, +// and records the machines that the rolls must replace. +func (h *haScenario) change(t *testing.T) { + res := h.tent(t, specTimeout, "get-cluster", "get", "clusters", h.name, "-o", "json") + changed, err := withRollChange(res.Stdout, suite.ID) + if err != nil { + t.Fatal(err) + } + file := filepath.Join(suite.Dir, h.name+"-roll-change.json") + if err := os.WriteFile(file, changed, 0o600); err != nil { + t.Fatalf("write the changed specs: %v", err) + } + h.tent(t, specTimeout, "replace", "replace", "-f", file) + + instances, err := h.machines(stepContext(suite.Ctx, t)) + if err != nil { + t.Fatalf("list the machines: %v", err) + } + if err := checkCounts(instances, h.name, haServers, haWorkers); err != nil { + t.Fatal(err) + } + h.servers = machinesOf(instances, h.name, "server") + h.workers = machinesOf(instances, h.name, "client") +} + +// update applies the changed specs to the infrastructure, which leaves every machine outdated: a roll is due. +func (h *haScenario) update(t *testing.T) { + h.tent(t, updateTimeout, "update", "update", "cluster", h.name, "--yes") + h.rollDue(t, "roll-due", 2) +} + +// rollServers rolls the servers alone while the probe runs, and waits until Vultr lists only new servers and still +// lists the old workers. +func (h *haScenario) rollServers(t *testing.T) { + h.rollWithProbe(t, "server", "servers", "roll-servers", defaultProbeLimits, rollServersTimeout, + "rolling-update", "cluster", h.name, "--yes", "--nodegroups", "servers") + h.waitMachines(t, "the servers rolled and the workers kept", func(instances []vultrapi.Instance) error { + if err := checkRolled(h.servers, instances); err != nil { + return err + } + if err := checkCounts(instances, h.name, haServers, haWorkers); err != nil { + return err + } + return checkKept(h.workers, instances) + }) +} + +// waitMachines polls the machines of Vultr until check passes; Vultr drops deleted machines from its list late. +func (h *haScenario) waitMachines(t *testing.T, what string, check func([]vultrapi.Instance) error) { + t.Helper() + h.pollMachines(t, what, rolledEvery, rolledTimeout, check) +} + +// pollMachines polls the machines of Vultr every `every` until check passes or timeout has passed. +func (h *haScenario) pollMachines(t *testing.T, what string, every, timeout time.Duration, + check func([]vultrapi.Instance) error, +) { + t.Helper() + ctx := stepContext(suite.Ctx, t) + err := pollUntil(ctx, every, timeout, func(ctx context.Context) string { + instances, err := h.machines(ctx) + if err != nil { + return err.Error() + } + if err := check(instances); err != nil { + return err.Error() + } + return "" + }) + if err != nil { + t.Fatalf("wait for %s: %v", what, err) + } +} + +// restartClient restarts the old worker with the lowest label after every server was replaced. The worker was built +// with the old servers, so only what refresh-join wrote since leads it to the new servers. The step waits until the +// worker's agent of the new boot has registered with the same node ID and the service still runs. +func (h *haScenario) restartClient(t *testing.T) { + h.export(t, "export-servers-rolled") + ctx := stepContext(suite.Ctx, t) + worker, err := lowestLabel(h.workers) + if err != nil { + t.Fatal(err) + } + nodes, err := listNodes(ctx, h.nomad) + if err != nil { + t.Fatal(err) + } + node, err := nodeNamed(nodes, worker.Label) + if err != nil { + t.Fatal(err) + } + out, err := suite.sshRun(ctx, worker.MainIP, bootIDScript()) + if err != nil { + t.Fatalf("read the boot ID of %s: %v", worker.Label, err) + } + e := restartEvidence{Machine: worker.Label, NodeID: node.ID} + if e.BootBefore, err = parseBootID(out); err != nil { + t.Fatal(err) + } + if _, err := suite.sshRun(ctx, worker.MainIP, rebootScript()); err != nil { + t.Fatalf("restart %s: %v", worker.Label, err) + } + + start := time.Now() + err = pollUntil(ctx, bootEvery, bootTimeout, func(ctx context.Context) string { + out, err := suite.sshRun(ctx, worker.MainIP, bootIDScript()) + if problem := bootProblem(out, err, e.BootBefore); problem != "" { + return problem + } + e.BootAfter = strings.TrimSpace(out) + return "" + }) + e.BootWait = time.Since(start).Round(time.Second) + if err != nil { + t.Fatalf("wait for %s to boot again: %v", worker.Label, err) + } + + start = time.Now() + err = pollUntil(ctx, registrationEvery, registrationTimeout, func(ctx context.Context) string { + out, err := suite.sshRun(ctx, worker.MainIP, registrationScript()) + if err != nil { + return err.Error() + } + e.Registrations = registrationLines(out) + nodes, err := listNodes(ctx, h.nomad) + if err != nil { + return err.Error() + } + return registeredProblem(e.Registrations, nodes, worker.Label, e.NodeID) + }) + e.RegisterWait = time.Since(start).Round(time.Second) + e.JoinFile = h.joinFile(ctx, worker) + record := filepath.Join(suite.Dir, h.name+"-restart.txt") + if werr := os.WriteFile(record, []byte(restartRecord(e)), 0o600); werr != nil { + t.Errorf("write the record of the restart: %v", werr) + } + if err != nil { + t.Fatalf("wait for %s to register again: %v\nrecord: %s", worker.Label, err, record) + } + if err := waitService(ctx, h.nomad, webJob, webService, serviceEvery, serviceTimeout); err != nil { + t.Fatalf("service %s after the restart: %v\nrecord: %s", webService, err, record) + } +} + +// joinFile reads the file that tells the agent which servers to join, or says why it could not. +func (h *haScenario) joinFile(ctx context.Context, worker vultrapi.Instance) string { + out, err := suite.sshRun(ctx, worker.MainIP, joinFileScript()) + if err != nil { + return fmt.Sprintf("not read: %v\n", err) + } + return out +} + +// rollWorkers rolls the workers, which are the only outdated machines now, and checks that each new worker carries +// the run's node meta value. +func (h *haScenario) rollWorkers(t *testing.T) { + h.rollWithProbe(t, "server", "workers", "roll-workers", workerProbeLimits, rollWorkersTimeout, + "rolling-update", "cluster", h.name, "--yes") + h.rollDue(t, "roll-done", 0) + h.waitMachines(t, "the workers rolled", func(instances []vultrapi.Instance) error { + if err := checkRolled(h.workers, instances); err != nil { + return err + } + return checkCounts(instances, h.name, haServers, haWorkers) + }) + h.export(t, "export-rolled") + ctx := stepContext(suite.Ctx, t) + err := pollUntil(ctx, rolledEvery, rolledTimeout, func(ctx context.Context) string { + metas, err := h.workerMetas(ctx) + if err != nil { + return err.Error() + } + if err := checkRollMeta(metas, suite.ID, haWorkers); err != nil { + return err.Error() + } + return "" + }) + if err != nil { + t.Fatalf("wait for the meta of the new workers: %v", err) + } +} + +// workerMetas returns the meta values of each node that Nomad lists, by node name. +func (h *haScenario) workerMetas(ctx context.Context) (map[string]map[string]string, error) { + nodes, err := listNodes(ctx, h.nomad) + if err != nil { + return nil, err + } + metas := make(map[string]map[string]string, len(nodes)) + for _, n := range nodes { + if metas[n.Name], err = nodeMeta(ctx, h.nomad, n.ID); err != nil { + return nil, err + } + } + return metas, nil +} + +// backup takes a backup, deletes the cluster's server machines, has update build new servers, restores the backup into +// them and waits until the workers and the service are back; a forced roll of the workers is the way out when +// validate still finds them dead. The record -backup.txt is written whatever happens. +func (h *haScenario) backup(t *testing.T) { + var ev backupEvidence + last := time.Now() + mark := func(step string) { + ev.Times = append(ev.Times, stepTime{step, time.Since(last).Round(time.Second)}) + last = time.Now() + } + defer func() { + record := filepath.Join(suite.Dir, h.name+"-backup.txt") + if err := os.WriteFile(record, []byte(backupRecord(ev)), 0o600); err != nil { + t.Errorf("write the record of the backup: %v", err) + return + } + t.Logf("record of the backup: %s", record) + }() + ctx := stepContext(suite.Ctx, t) + + res := h.tent(t, backupTimeout, "backup-create", "backup", "create", h.name, "-o", "json") + var err error + if ev.Backup, err = parseStoredBackup(res.Stdout); err != nil { + t.Fatal(err) + } + mark("backup create") + res = h.tent(t, specTimeout, "backup-list", "backup", "list", h.name, "-o", "json") + if err := checkBackupList(res.Stdout, ev.Backup.Name, "current"); err != nil { + t.Fatal(err) + } + mark("backup list") + + if ev.ServersBefore, err = h.serverAddresses(ctx); err != nil { + t.Fatal(err) + } + if ev.WorkersBefore, err = h.workerAddresses(ctx); err != nil { + t.Fatal(err) + } + h.deleteServers(t) + mark("delete the servers") + + h.tent(t, updateTimeout, "update-rebuild", "update", "cluster", h.name, "--yes") + mark("update") + res = h.tent(t, specTimeout, "backup-list-rebuilt", "backup", "list", h.name, "-o", "json") + if err := checkBackupList(res.Stdout, ev.Backup.Name, "earlier"); err != nil { + t.Fatal(err) + } + h.export(t, "export-rebuilt") + if ev.ServersAfter, err = h.serverAddresses(ctx); err != nil { + t.Fatal(err) + } + mark("rebuilt servers") + + res = h.tent(t, backupTimeout, "backup-restore", "backup", "restore", h.name, "--yes") + if err := checkRestored(res.Stdout, ev.Backup.Name, h.name); err != nil { + t.Fatal(err) + } + h.export(t, "export-restored") + var job struct{ ID string } + if err := h.nomad.get(ctx, "/v1/job/"+webJob, &job); err != nil { + t.Fatalf("read the job %s after the restore: %v", webJob, err) + } + if job.ID != webJob { + t.Fatalf("the job read after the restore has ID %q, want %s", job.ID, webJob) + } + mark("restore") + + if !h.validRestored(ctx, t) { + ev.Forced = true + h.forceWorkers(t) + } + mark("workers back") + if err := h.waitPair(ctx, pairBackTimeout); err != nil { + t.Fatalf("service %s after the restore: %v", webService, err) + } + if ev.WorkersAfter, err = h.workerAddresses(ctx); err != nil { + t.Fatal(err) + } + mark("service back") +} + +// deleteServers deletes the server machines of the cluster, and no other machine, and waits until Vultr lists none. +func (h *haScenario) deleteServers(t *testing.T) { + t.Helper() + ctx := stepContext(suite.Ctx, t) + instances, err := h.machines(ctx) + if err != nil { + t.Fatalf("list the machines: %v", err) + } + servers, err := serversToDelete(instances, h.name) + if err != nil { + t.Fatal(err) + } + for _, s := range servers { + if err := suite.Vultr.DeleteInstance(ctx, s.ID); err != nil { + t.Fatalf("delete the server %s: %v", s.Label, err) + } + } + h.pollMachines(t, "Vultr to list no server", serversGoneEvery, serversGoneTimeout, + func(instances []vultrapi.Instance) error { + if left := machinesOf(instances, h.name, "server"); len(left) > 0 { + return fmt.Errorf("%d servers are still listed", len(left)) + } + return nil + }) +} + +// validRestored waits restoreSettle, after which Nomad lists as ready only the nodes that answer, and validates the +// cluster. It tells whether validate passed. +func (h *haScenario) validRestored(ctx context.Context, t *testing.T) bool { + t.Helper() + if err := sleepContext(ctx, restoreSettle); err != nil { + t.Fatalf("wait for Nomad to stop listing the nodes of the backup as ready: %v", err) + } + res := h.tentCode(t, previewTimeout, "validate-restored", "validate", "cluster", h.name) + if res.Code == 0 { + return true + } + t.Logf("validate after the restore exited with %d, rolling the workers with --force:\n%s\n%s", res.Code, + lastLines(strings.TrimSpace(string(res.Stdout)), stderrTailLines), + tentFailure("validate-restored", res, suite.Tent.logPath(h.name, "validate-restored"))) + return false +} + +// forceWorkers rolls the workers with --force, validates the cluster and exports its settings. +func (h *haScenario) forceWorkers(t *testing.T) { + t.Helper() + h.tent(t, rollWorkersTimeout, "roll-workers-forced", "rolling-update", "cluster", h.name, "--yes", "--force", + "--nodegroups", "workers") + h.validate(t, "validate-forced") + h.export(t, "export-forced") +} + +// serverAddresses returns the private addresses of the servers that Nomad lists as members. +func (h *haScenario) serverAddresses(ctx context.Context) ([]string, error) { + members, err := listMembers(ctx, h.nomad) + if err != nil { + return nil, err + } + return memberAddresses(members), nil +} + +// workerAddresses returns the private addresses of the nodes that Nomad lists. +func (h *haScenario) workerAddresses(ctx context.Context) ([]string, error) { + nodes, err := listNodes(ctx, h.nomad) + if err != nil { + return nil, err + } + return nodeAddresses(nodes), nil +} diff --git a/test/e2e/intro_test.go b/test/e2e/intro_test.go index 11e80bb..ae99662 100644 --- a/test/e2e/intro_test.go +++ b/test/e2e/intro_test.go @@ -164,8 +164,8 @@ func nonEmptyLines(out string) []string { // listNodeNames returns the names of the nodes /v1/nodes lists. func listNodeNames(ctx context.Context, n *nomadAPI) ([]string, error) { - var nodes []struct{ Name string } - if err := n.get(ctx, "/v1/nodes", &nodes); err != nil { + nodes, err := listNodes(ctx, n) + if err != nil { return nil, err } names := make([]string, 0, len(nodes)) diff --git a/test/e2e/names_test.go b/test/e2e/names_test.go index fba1bcb..b69f364 100644 --- a/test/e2e/names_test.go +++ b/test/e2e/names_test.go @@ -35,6 +35,9 @@ func clusterName(run, image string) string { return "e2e-" + run + "-" + digits } +// scenarioName names the cluster of a scenario in a run: e2e--. +func scenarioName(run, scenario string) string { return "e2e-" + run + "-" + scenario } + // runnerCIDR turns the address of the machine that runs the suite into a CIDR of one address. func runnerCIDR(addr string) (string, error) { ip, err := netip.ParseAddr(strings.TrimSpace(addr)) diff --git a/test/e2e/names_unit_test.go b/test/e2e/names_unit_test.go index 311c326..a037a82 100644 --- a/test/e2e/names_unit_test.go +++ b/test/e2e/names_unit_test.go @@ -42,8 +42,10 @@ func TestNewRunIDReportsAShortRead(t *testing.T) { } } +// nameRule is tent's rule for a cluster name, copied from api/v1alpha1/validation.go. +var nameRule = regexp.MustCompile(`^[a-z][a-z0-9-]{0,18}[a-z0-9]$`) + func TestClusterNameFollowsTentsRuleAndTheJanitorsPrefix(t *testing.T) { - rule := regexp.MustCompile(`^[a-z][a-z0-9-]{0,18}[a-z0-9]$`) for image, want := range map[string]string{ "ubuntu-24.04": "e2e-ab12cd-2404", "ubuntu-26.04": "e2e-ab12cd-2604", @@ -53,7 +55,25 @@ func TestClusterNameFollowsTentsRuleAndTheJanitorsPrefix(t *testing.T) { if got != want { t.Errorf("clusterName(%q) = %q, want %q", image, got, want) } - if !rule.MatchString(got) { + if !nameRule.MatchString(got) { + t.Errorf("cluster name %q breaks tent's name rule", got) + } + if !strings.HasPrefix(got, janitor.ClusterPrefix) { + t.Errorf("cluster name %q does not start with %q", got, janitor.ClusterPrefix) + } + } +} + +func TestScenarioNameFollowsTentsRuleAndTheJanitorsPrefix(t *testing.T) { + for scenario, want := range map[string]string{ + "ha": "e2e-ab12cd-ha", + "upgrade": "e2e-ab12cd-upgrade", + } { + got := scenarioName("ab12cd", scenario) + if got != want { + t.Errorf("scenarioName(%q) = %q, want %q", scenario, got, want) + } + if !nameRule.MatchString(got) { t.Errorf("cluster name %q breaks tent's name rule", got) } if !strings.HasPrefix(got, janitor.ClusterPrefix) { diff --git a/test/e2e/nomad_test.go b/test/e2e/nomad_test.go index 91fa256..631fba1 100644 --- a/test/e2e/nomad_test.go +++ b/test/e2e/nomad_test.go @@ -64,6 +64,14 @@ func newNomad(x exportInfo) (*nomadAPI, error) { }, nil } +// at returns a client for the server at address, in the form "https://:4646". It shares the token and the HTTP +// client, which pins the TLS server name, so it works for any server of the cluster. +func (n *nomadAPI) at(address string) *nomadAPI { + other := *n + other.base = address + return &other +} + // get reads path (with its query) and decodes the JSON answer into out. func (n *nomadAPI) get(ctx context.Context, path string, out any) error { return n.do(ctx, http.MethodGet, path, nil, out) diff --git a/test/e2e/nomad_unit_test.go b/test/e2e/nomad_unit_test.go index 210fd2c..ac1e42d 100644 --- a/test/e2e/nomad_unit_test.go +++ b/test/e2e/nomad_unit_test.go @@ -22,6 +22,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/hashicorp/hcl" ) const ( @@ -414,3 +416,66 @@ func mustWrite(t *testing.T, path, text string) { t.Fatalf("write %s: %v", path, err) } } + +func TestNomadAtReachesAnotherServerWithTheSameTokenAndClient(t *testing.T) { + ca := newTestCA(t, "ca") + var tokenAtOther string + first := startNomad(t, ca, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"Name":"first"}`) + }) + other := startNomad(t, ca, func(w http.ResponseWriter, r *http.Request) { + tokenAtOther = r.Header.Get("X-Nomad-Token") + _, _ = io.WriteString(w, `{"Name":"other"}`) + }) + n, err := newNomad(first) + if err != nil { + t.Fatalf("newNomad: %v", err) + } + m := n.at(other.Address) + if m.client != n.client || m.token != n.token { + t.Errorf("at made another client or token: client shared %v, token shared %v", + m.client == n.client, m.token == n.token) + } + for _, c := range []struct { + api *nomadAPI + want string + }{{m, "other"}, {n, "first"}} { + var out struct{ Name string } + if err := c.api.get(t.Context(), "/v1/status/leader", &out); err != nil { + t.Fatalf("get through %s: %v", c.want, err) + } + if out.Name != c.want { + t.Errorf("a get answered by %q, want %q", out.Name, c.want) + } + } + if tokenAtOther != testToken { + t.Errorf("the token at the other server = %q, want the token of the original", tokenAtOther) + } +} + +func TestPairJobFileHoldsTwoAllocationsAndTheMigrateSettings(t *testing.T) { + data, err := os.ReadFile("testdata/web-pair.nomad.hcl") + if err != nil { + t.Fatalf("read the job file: %v", err) + } + var doc map[string]any + if err := hcl.Decode(&doc, string(data)); err != nil { + t.Fatalf("decode the job file: %v", err) + } + group := block(t, doc, "job", webJob, "group", "web") + if group["count"] != 2 { + t.Errorf("count = %v, want 2", group["count"]) + } + migrate := block(t, group, "migrate") + want := map[string]any{ + "max_parallel": 1, "health_check": "checks", "min_healthy_time": "10s", "healthy_deadline": "5m", + } + for key, value := range want { + if migrate[key] != value { + t.Errorf("migrate %s = %v, want %v", key, migrate[key], value) + } + } + if svc := block(t, group, "service"); svc["name"] != webService || svc["provider"] != "nomad" { + t.Errorf("service = %v, want name %q and provider nomad", svc, webService) + } +} diff --git a/test/e2e/probe_test.go b/test/e2e/probe_test.go new file mode 100644 index 0000000..2d6e336 --- /dev/null +++ b/test/e2e/probe_test.go @@ -0,0 +1,515 @@ +package e2e + +import ( + "context" + "fmt" + "io" + "maps" + "slices" + "strings" + "sync" + "time" +) + +// The outcomes of a poll. The leader outcome is led, leaderless or unreached; the service outcome is served, +// unserved, unchecked or unreached. +const ( + probeLed = "led" + probeLeaderless = "leaderless" + probeUnreached = "unreached" + probeUnchecked = "unchecked" + probeServed = "served" + probeUnserved = "unserved" +) + +const ( + probeEvery = time.Second + probeRefresh = 10 * time.Second + probeRefreshTimeout = 5 * time.Second + // probeLeaderTimeout bounds a stale leader read, which a server answers from its own view at once. + probeLeaderTimeout = 3 * time.Second + // probeServiceTimeout is above Nomad's 5 s hold of a forwarded read. + probeServiceTimeout = 7 * time.Second + + // probeTotalFactor times a bad outcome's stretch limit is how long its stretches may last together in one roll. + probeTotalFactor = 2 + + // noLeaderAnswer ends the error of the read of a server that has no leader to forward to. + noLeaderAnswer = ": 500: No cluster leader" + + probeHeader = "start\ttook_ms\tleader\tleader_address\tservice\tdetail\n" +) + +// probeLimits are what a run may show before probeVerdict fails it. +type probeLimits struct { + // Leaderless is the longest time without a leader: a transfer takes about 1 s, a lost quorum lasts until a + // person acts. + Leaderless time.Duration + // Unserved is the longest time without a healthy allocation: two check intervals of the job. + Unserved time.Duration + // Unreached is the longest time that no server answers the leader read, or that the allocations or the + // registrations read fails. + Unreached time.Duration + // Unchecked is the longest time that every checks read can fail while the allocations and the registrations + // answer: a client keeps its connection to a stopped server until its keepalive fails and its next heartbeat + // reaches another one, which took 37 s on a real run. + Unchecked time.Duration + // Gap is the longest time between the starts of two polls; a longer one means the probe saw nothing. + Gap time.Duration + // MinPolls is the fewest polls that make a run worth judging. + MinPolls int +} + +// defaultProbeLimits are the limits of the scenarios. +var defaultProbeLimits = probeLimits{ + Leaderless: 10 * time.Second, + Unserved: 10 * time.Second, + Unreached: 30 * time.Second, + Unchecked: 60 * time.Second, + Gap: 30 * time.Second, + MinPolls: 10, +} + +// workerProbeLimits are the limits of a roll that stops no server. Every checks read can fail only when no client +// that runs an allocation answers while Nomad still lists them, which is an outage, so a stretch of unchecked polls +// may last as long as one of unserved polls. +var workerProbeLimits = func() probeLimits { + l := defaultProbeLimits + l.Unchecked = l.Unserved + return l +}() + +// longest returns the longest of the stretch limits. +func (l probeLimits) longest() time.Duration { + return max(l.Leaderless, l.Unserved, l.Unreached, l.Unchecked) +} + +// probeRecord is the result of one poll. +type probeRecord struct { + Start time.Time + Took time.Duration + // Leader is led, leaderless or unreached; LeaderAddr is the leader that a server named, if any. + Leader string + LeaderAddr string + // Service is served, unserved, unchecked or unreached. + Service string + // Detail holds the errors of the reads, and the answers about the service when it is not served. + Detail string +} + +// probeLine returns the record as a line of the records file; tabs and line breaks of the detail become spaces. +func probeLine(r probeRecord) string { + return probeRow(r.Start, r.Took, r.Leader, r.LeaderAddr, r.Service, r.Detail) +} + +func probeRow(start time.Time, took time.Duration, leader, leaderAddr, service, detail string) string { + detail = strings.NewReplacer("\t", " ", "\r\n", " ", "\n", " ", "\r", " ").Replace(detail) + return fmt.Sprintf("%s\t%d\t%s\t%s\t%s\t%s\n", start.UTC().Format("2006-01-02T15:04:05.000Z"), + took.Milliseconds(), leader, leaderAddr, service, detail) +} + +// probeConfig says what a probe asks and how often. +type probeConfig struct { + // Nomad holds the token and the HTTP client; the probe asks each server through Nomad.at. + Nomad *nomadAPI + // Addresses returns the current server addresses, each as "https://:4646". It must end when its context + // ends. + Addresses func(ctx context.Context) ([]string, error) + // First is the list of addresses of the first poll; Addresses replaces it every Refresh. + First []string + Job string + Service string + // Every is the time between the starts of two polls, Refresh the time between two calls of Addresses. + Every, Refresh time.Duration + // RefreshTimeout bounds a call of Addresses, LeaderTimeout a leader read, ServiceTimeout a read of the service. + RefreshTimeout, LeaderTimeout, ServiceTimeout time.Duration + // Out gets the header and then each record as it comes. + Out io.Writer +} + +// newProbeConfig returns the configuration of the scenarios for the job and its service. +func newProbeConfig(n *nomadAPI, addresses func(context.Context) ([]string, error), job, service string, + out io.Writer) probeConfig { + return probeConfig{ + Nomad: n, + Addresses: addresses, + Job: job, + Service: service, + Every: probeEvery, + Refresh: probeRefresh, + RefreshTimeout: probeRefreshTimeout, + LeaderTimeout: probeLeaderTimeout, + ServiceTimeout: probeServiceTimeout, + Out: out, + } +} + +// probe polls the servers of a cluster for a leader and for a healthy allocation of a service until it is stopped. +type probe struct { + cfg probeConfig + cancel context.CancelFunc + wg sync.WaitGroup + outMu sync.Mutex + + // mu guards addresses and records. + mu sync.Mutex + addresses []string + records []probeRecord + // recorded gets a value after each record, for settle. + recorded chan struct{} +} + +// startProbe starts a probe and writes the header. The first poll uses cfg.First; a slow list of addresses never +// holds up a poll. +func startProbe(ctx context.Context, cfg probeConfig) *probe { + ctx, cancel := context.WithCancel(ctx) + p := &probe{cfg: cfg, cancel: cancel, addresses: cfg.First, recorded: make(chan struct{}, 1)} + p.write(probeHeader) + p.wg.Go(func() { p.refreshLoop(ctx) }) + p.wg.Go(func() { p.pollLoop(ctx) }) + return p +} + +// stop ends the probe and returns the records of the polls that ran to the end; a poll that stop interrupted is +// dropped, since its reads failed because of the stop. +func (p *probe) stop() []probeRecord { + p.cancel() + p.wg.Wait() + p.mu.Lock() + defer p.mu.Unlock() + return p.records +} + +// settle waits until the latest poll is led and served, for at most limit and until ctx ends, and returns how long it +// waited. The probe goes on polling meanwhile. +func (p *probe) settle(ctx context.Context, limit time.Duration) time.Duration { + start := time.Now() + timer := time.NewTimer(limit) + defer timer.Stop() + for !p.lastIsGood() { + select { + case <-p.recorded: + case <-timer.C: + return time.Since(start) + case <-ctx.Done(): + return time.Since(start) + } + } + return time.Since(start) +} + +func (p *probe) lastIsGood() bool { + p.mu.Lock() + defer p.mu.Unlock() + if len(p.records) == 0 { + return false + } + last := p.records[len(p.records)-1] + return last.Leader == probeLed && last.Service == probeServed +} + +// addRecord keeps a record, tells settle about it and writes it to the records file. +func (p *probe) addRecord(r probeRecord) { + p.mu.Lock() + p.records = append(p.records, r) + p.mu.Unlock() + select { + case p.recorded <- struct{}{}: + default: + } + p.write(probeLine(r)) +} + +// write sends text to the records file. A lost line costs nothing: stop returns the records as well. +func (p *probe) write(text string) { + p.outMu.Lock() + defer p.outMu.Unlock() + _, _ = io.WriteString(p.cfg.Out, text) +} + +func (p *probe) refreshLoop(ctx context.Context) { + ticker := time.NewTicker(p.cfg.Refresh) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + p.refresh(ctx) + } + } +} + +// refresh replaces the addresses with a new list. A failed list keeps the old addresses and is written as a line. +func (p *probe) refresh(ctx context.Context) { + start := time.Now() + listCtx, cancel := context.WithTimeout(ctx, p.cfg.RefreshTimeout) + defer cancel() + addresses, err := p.cfg.Addresses(listCtx) + if err != nil { + if ctx.Err() == nil { + p.write(probeRow(start, time.Since(start), "-", "-", "-", "the list of servers failed: "+err.Error())) + } + return + } + p.mu.Lock() + p.addresses = addresses + p.mu.Unlock() +} + +func (p *probe) pollLoop(ctx context.Context) { + ticker := time.NewTicker(p.cfg.Every) + defer ticker.Stop() + for { + record := p.poll(ctx) + if ctx.Err() != nil { + return + } + p.addRecord(record) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// leaderAnswer is what one server said to the leader read. +type leaderAnswer struct { + address string + // answered is false when the read failed. + answered bool + leader string + err error +} + +// poll asks every known server for the leader at once and then reads the service through one of them. +func (p *probe) poll(ctx context.Context) probeRecord { + start := time.Now() + p.mu.Lock() + addresses := p.addresses + p.mu.Unlock() + record := probeRecord{Start: start} + if len(addresses) == 0 { + record.Leader, record.Service, record.Detail = probeUnreached, probeUnreached, "no server address is known" + record.Took = time.Since(start) + return record + } + answers := make([]leaderAnswer, len(addresses)) + var wg sync.WaitGroup + for i, address := range addresses { + wg.Go(func() { answers[i] = p.readLeader(ctx, address) }) + } + wg.Wait() + + var details []string + via := "" + record.Leader = probeUnreached + for _, a := range answers { + switch { + case !a.answered: + details = append(details, a.address+": "+a.err.Error()) + case a.leader != "" && record.Leader != probeLed: + record.Leader, record.LeaderAddr, via = probeLed, a.leader, a.address + case record.Leader == probeUnreached: + record.Leader = probeLeaderless + via = a.address + } + } + if via == "" { + record.Service = probeUnreached + } else { + var detail string + record.Service, detail = p.readService(ctx, p.cfg.Nomad.at(via)) + if detail != "" { + details = append(details, detail) + } + } + record.Detail = strings.Join(details, "; ") + record.Took = time.Since(start) + return record +} + +// readLeader asks one server for its leader from its own view of the Raft cluster. A server without a leader +// answers either an empty name or a 500, which both count as an answer. +func (p *probe) readLeader(ctx context.Context, address string) leaderAnswer { + ctx, cancel := context.WithTimeout(ctx, p.cfg.LeaderTimeout) + defer cancel() + var leader string + err := p.cfg.Nomad.at(address).get(ctx, "/v1/status/leader?stale=true", &leader) + switch { + case err == nil: + return leaderAnswer{address: address, answered: true, leader: leader} + case strings.Contains(err.Error(), noLeaderAnswer): + return leaderAnswer{address: address, answered: true} + default: + return leaderAnswer{address: address, err: err} + } +} + +// readService reads the allocations of the job, the checks of each running one and the registrations of the +// service, each read under its own timeout, and returns the outcome with what to write about it. +func (p *probe) readService(ctx context.Context, n *nomadAPI) (outcome, detail string) { + r := readServiceReads(ctx, n, p.cfg.Job, p.cfg.Service, p.cfg.ServiceTimeout) + if r.allocationsErr != nil { + return probeUnreached, r.allocationsErr.Error() + } + var errs []string + if r.registrationsErr != nil { + errs = append(errs, r.registrationsErr.Error()) + } + var running []string + for _, alloc := range r.answers.Allocations { + if alloc.ClientStatus != "running" { + continue + } + running = append(running, alloc.ID) + if err := r.checkErrs[alloc.ID]; err != nil { + errs = append(errs, err.Error()) + } + } + switch { + case r.registrationsErr != nil: + return probeUnreached, strings.Join(errs, "; ") + case r.answers.missing(p.cfg.Service) == nil: + return probeServed, strings.Join(errs, "; ") + case len(running) > 0 && len(r.checkErrs) == len(running) && + slices.ContainsFunc(running, r.answers.hasRegistration): + return probeUnchecked, strings.Join(errs, "; ") + default: + return probeUnserved, strings.Join(append(errs, r.answers.String()), "; ") + } +} + +// probeKind is a kind of stretch: a run of polls with the same outcome. +type probeKind struct { + name string + // limit is the longest time one stretch of the kind may last and total the longest time that all its stretches + // may last together; they count only when checked is set. + limit, total time.Duration + checked bool +} + +// bad returns the kind of a bad outcome with its limits. +func bad(name string, limit time.Duration) probeKind { + return probeKind{name: name, limit: limit, total: limit * probeTotalFactor, checked: true} +} + +func (l probeLimits) leaderKinds() map[string]probeKind { + return map[string]probeKind{ + probeLed: {name: "led"}, + probeLeaderless: bad("leaderless", l.Leaderless), + probeUnreached: bad("leader unreached", l.Unreached), + } +} + +func (l probeLimits) serviceKinds() map[string]probeKind { + return map[string]probeKind{ + probeServed: {name: "served"}, + probeUnserved: bad("unserved", l.Unserved), + probeUnreached: bad("service unreached", l.Unreached), + probeUnchecked: bad("service unchecked", l.Unchecked), + } +} + +// probeSummary counts the polls of each kind and names the longest stretch and the total time of each kind and the +// longest gap. +type probeSummary struct { + counts map[string]int + longest map[string]time.Duration + totals map[string]time.Duration + gap time.Duration +} + +func (s probeSummary) String() string { + var counts, longest, totals []string + counted := []string{"led", "leaderless", "leader unreached", "served", "unserved", "service unreached", + "service unchecked"} + stretched := []string{"leaderless", "leader unreached", "unserved", "service unreached", "service unchecked"} + for _, kind := range counted { + counts = append(counts, fmt.Sprintf("%s %d", kind, s.counts[kind])) + } + for _, kind := range stretched { + longest = append(longest, fmt.Sprintf("%s %s", kind, s.longest[kind])) + totals = append(totals, fmt.Sprintf("%s %s", kind, s.totals[kind])) + } + return fmt.Sprintf("polls: %s; longest stretch: %s; longest gap %s; total: %s", strings.Join(counts, ", "), + strings.Join(longest, ", "), s.gap, strings.Join(totals, ", ")) +} + +// probeVerdict judges the records against the limits. A stretch lasts from the first poll of an outcome to the start +// of the next poll with another outcome, or to the last poll. A bad outcome also fails when its stretches add up to +// more than its total limit. The error names every breach; the summary is returned in both cases. +func probeVerdict(records []probeRecord, limits probeLimits) (probeSummary, error) { + s := probeSummary{counts: map[string]int{}, longest: map[string]time.Duration{}, totals: map[string]time.Duration{}} + var breaches []string + if len(records) < limits.MinPolls { + breaches = append(breaches, fmt.Sprintf("%d polls, at least %d are needed", len(records), limits.MinPolls)) + } + if len(records) > 0 { + first, last := records[0], records[len(records)-1] + if first.Leader != probeLed || first.Service != probeServed { + breaches = append(breaches, fmt.Sprintf("the first poll at %s was %s and %s, want led and served", + clock(first.Start), first.Leader, first.Service)) + } + if last.Leader != probeLed || last.Service != probeServed { + breaches = append(breaches, fmt.Sprintf("the last poll at %s was %s and %s, want led and served", + clock(last.Start), last.Leader, last.Service)) + } + } + for i := 1; i < len(records); i++ { + gap := records[i].Start.Sub(records[i-1].Start) + s.gap = max(s.gap, gap) + if gap > limits.Gap { + breaches = append(breaches, fmt.Sprintf("no poll started for %s after %s, want at most %s", gap, + clock(records[i-1].Start), limits.Gap)) + } + } + breaches = append(breaches, s.addStretches(records, func(r probeRecord) string { return r.Leader }, + limits.leaderKinds())...) + breaches = append(breaches, s.addStretches(records, func(r probeRecord) string { return r.Service }, + limits.serviceKinds())...) + if len(breaches) > 0 { + return s, fmt.Errorf("the probe failed: %s", strings.Join(breaches, "; ")) + } + return s, nil +} + +// addStretches walks the runs of equal outcomes, adds them to the counts, the longest stretches and the totals, and +// returns a breach for each run that is longer than its kind allows and for each kind whose runs add up to more than +// its total limit. +func (s *probeSummary) addStretches(records []probeRecord, outcome func(probeRecord) string, + kinds map[string]probeKind) []string { + var breaches []string + for i := 0; i < len(records); { + j := i + for j < len(records) && outcome(records[j]) == outcome(records[i]) { + j++ + } + kind := kinds[outcome(records[i])] + end := records[len(records)-1].Start + if j < len(records) { + end = records[j].Start + } + length := end.Sub(records[i].Start) + s.counts[kind.name] += j - i + s.longest[kind.name] = max(s.longest[kind.name], length) + s.totals[kind.name] += length + if kind.checked && length > kind.limit { + breaches = append(breaches, fmt.Sprintf("%s for %s from %s, want at most %s", kind.name, length, + clock(records[i].Start), kind.limit)) + } + i = j + } + for _, outcome := range slices.Sorted(maps.Keys(kinds)) { + if kind := kinds[outcome]; kind.checked && s.totals[kind.name] > kind.total { + breaches = append(breaches, fmt.Sprintf("%s for %s in all, want at most %s", kind.name, + s.totals[kind.name], kind.total)) + } + } + return breaches +} + +func clock(t time.Time) string { + return t.UTC().Format("15:04:05.000") +} diff --git a/test/e2e/probe_unit_test.go b/test/e2e/probe_unit_test.go new file mode 100644 index 0000000..e1460a2 --- /dev/null +++ b/test/e2e/probe_unit_test.go @@ -0,0 +1,1034 @@ +package e2e + +import ( + "context" + "errors" + "net/http" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +var probeBase = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + +// steadyRecords returns n polls, one second apart, that are all led and served. +func steadyRecords(n int) []probeRecord { + records := make([]probeRecord, n) + for i := range records { + records[i] = probeRecord{ + Start: probeBase.Add(time.Duration(i) * time.Second), + Leader: probeLed, + Service: probeServed, + } + } + return records +} + +func TestDefaultProbeLimitsAreThoseOfTheScenarios(t *testing.T) { + want := probeLimits{ + Leaderless: 10 * time.Second, + Unserved: 10 * time.Second, + Unreached: 30 * time.Second, + Unchecked: 60 * time.Second, + Gap: 30 * time.Second, + MinPolls: 10, + } + if defaultProbeLimits != want { + t.Errorf("defaultProbeLimits = %+v, want %+v", defaultProbeLimits, want) + } +} + +func TestProbeVerdictPassesASteadyRunAndCountsIt(t *testing.T) { + summary, err := probeVerdict(steadyRecords(20), defaultProbeLimits) + if err != nil { + t.Fatalf("probeVerdict: %v", err) + } + want := "polls: led 20, leaderless 0, leader unreached 0, served 20, unserved 0, service unreached 0, " + + "service unchecked 0; longest stretch: leaderless 0s, leader unreached 0s, unserved 0s, service unreached 0s, " + + "service unchecked 0s; longest gap 1s; total: leaderless 0s, leader unreached 0s, unserved 0s, " + + "service unreached 0s, service unchecked 0s" + if got := summary.String(); got != want { + t.Errorf("summary =\n%s\nwant\n%s", got, want) + } +} + +func TestProbeVerdictSummaryNamesTheLongestStretchOfEachKind(t *testing.T) { + records := steadyRecords(30) + for i := 3; i < 6; i++ { + records[i].Leader = probeLeaderless + } + for i := 10; i < 14; i++ { + records[i].Service = probeUnserved + } + for i := 20; i < 25; i++ { + records[i].Leader = probeUnreached + records[i].Service = probeUnreached + } + for i := 26; i < 28; i++ { + records[i].Service = probeUnchecked + } + summary, err := probeVerdict(records, defaultProbeLimits) + if err != nil { + t.Fatalf("probeVerdict: %v", err) + } + want := "polls: led 22, leaderless 3, leader unreached 5, served 19, unserved 4, service unreached 5, " + + "service unchecked 2; longest stretch: leaderless 3s, leader unreached 5s, unserved 4s, " + + "service unreached 5s, service unchecked 2s; longest gap 1s; total: leaderless 3s, leader unreached 5s, " + + "unserved 4s, service unreached 5s, service unchecked 2s" + if got := summary.String(); got != want { + t.Errorf("summary =\n%s\nwant\n%s", got, want) + } +} + +// TestProbeVerdictStretchLimits runs a bad stretch of each kind that is exactly as long as its limit and one second +// longer: the first passes, the second fails and the error names the kind. +func TestProbeVerdictStretchLimits(t *testing.T) { + cases := []struct { + name string + limit time.Duration + set func(*probeRecord) + }{ + {"leaderless", defaultProbeLimits.Leaderless, func(r *probeRecord) { r.Leader = probeLeaderless }}, + {"leader unreached", defaultProbeLimits.Unreached, func(r *probeRecord) { r.Leader = probeUnreached }}, + {"unserved", defaultProbeLimits.Unserved, func(r *probeRecord) { r.Service = probeUnserved }}, + {"service unreached", defaultProbeLimits.Unreached, func(r *probeRecord) { r.Service = probeUnreached }}, + {"service unchecked", defaultProbeLimits.Unchecked, func(r *probeRecord) { r.Service = probeUnchecked }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // The stretch lasts from its first poll to the first good poll after it. + polls := int(tc.limit / time.Second) + at := func(n int) error { + records := steadyRecords(5 + n + 20) + for i := 5; i < 5+n; i++ { + tc.set(&records[i]) + } + _, err := probeVerdict(records, defaultProbeLimits) + return err + } + if err := at(polls); err != nil { + t.Errorf("a stretch of %s: %v, want a pass", tc.limit, err) + } + err := at(polls + 1) + if err == nil { + t.Fatalf("a stretch of %s passed, want a failure", tc.limit+time.Second) + } + for _, want := range []string{tc.name + " for " + (tc.limit + time.Second).String(), "12:00:05.000"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) + } + } + }) + } +} + +// TestProbeVerdictTotalLimits runs bad stretches of each kind that add up to twice the kind's stretch limit and to one +// second more: the first passes, the second fails and the error names the kind and the total. No single stretch of +// either breaches its own limit. +func TestProbeVerdictTotalLimits(t *testing.T) { + cases := []struct { + name string + limit time.Duration + set func(*probeRecord) + }{ + {"leaderless", defaultProbeLimits.Leaderless, func(r *probeRecord) { r.Leader = probeLeaderless }}, + {"leader unreached", defaultProbeLimits.Unreached, func(r *probeRecord) { r.Leader = probeUnreached }}, + {"unserved", defaultProbeLimits.Unserved, func(r *probeRecord) { r.Service = probeUnserved }}, + {"service unreached", defaultProbeLimits.Unreached, func(r *probeRecord) { r.Service = probeUnreached }}, + {"service unchecked", defaultProbeLimits.Unchecked, func(r *probeRecord) { r.Service = probeUnchecked }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + polls := int(tc.limit / time.Second) + // Each stretch of n polls ends with a good poll, so it lasts n seconds. + at := func(stretches ...int) error { + records := steadyRecords(5 + 3*polls + len(stretches) + 20) + i := 5 + for _, n := range stretches { + for range n { + tc.set(&records[i]) + i++ + } + i++ + } + _, err := probeVerdict(records, defaultProbeLimits) + return err + } + if err := at(polls, polls); err != nil { + t.Errorf("two stretches of %s: %v, want a pass", tc.limit, err) + } + err := at(polls, polls, 1) + if err == nil { + t.Fatalf("stretches that add up to %s passed, want a failure", 2*tc.limit+time.Second) + } + want := tc.name + " for " + (2*tc.limit + time.Second).String() + " in all, want at most " + + (2 * tc.limit).String() + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) + } + }) + } +} + +// TestProbeVerdictFailsALeaderThatIsLostEveryOtherSecond is the case that the longest stretch alone lets pass. +func TestProbeVerdictFailsALeaderThatIsLostEveryOtherSecond(t *testing.T) { + records := steadyRecords(600) + for i := 1; i < 599; i += 2 { + records[i].Leader = probeLeaderless + records[i].Service = probeUnserved + } + _, err := probeVerdict(records, defaultProbeLimits) + if err == nil || !strings.Contains(err.Error(), "leaderless for 4m59s in all") || + !strings.Contains(err.Error(), "unserved for 4m59s in all") { + t.Errorf("verdict = %v, want the totals of leaderless and unserved named", err) + } +} + +func TestProbeLimitsOfAWorkerRollTakeTheUncheckedStretchFromTheUnservedOne(t *testing.T) { + want := defaultProbeLimits + want.Unchecked = defaultProbeLimits.Unserved + if workerProbeLimits != want { + t.Errorf("workerProbeLimits = %+v, want %+v", workerProbeLimits, want) + } + records := steadyRecords(40) + for i := 5; i < 16; i++ { + records[i].Service = probeUnchecked + } + if _, err := probeVerdict(records, workerProbeLimits); err == nil || + !strings.Contains(err.Error(), "service unchecked for 11s") { + t.Errorf("11 s unchecked with the worker limits: %v, want a breach", err) + } + if _, err := probeVerdict(records, defaultProbeLimits); err != nil { + t.Errorf("11 s unchecked with the default limits: %v, want a pass", err) + } +} + +func TestProbeLimitsLongestIsTheLongestStretchLimit(t *testing.T) { + if got := defaultProbeLimits.longest(); got != 60*time.Second { + t.Errorf("default longest = %s, want 60s", got) + } + if got := workerProbeLimits.longest(); got != 30*time.Second { + t.Errorf("worker longest = %s, want 30s", got) + } +} + +// readProbeRecords reads a records file as probeLine writes it. +func readProbeRecords(t *testing.T, path string) []probeRecord { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") + if lines[0]+"\n" != probeHeader { + t.Fatalf("header = %q, want %q", lines[0], probeHeader) + } + var records []probeRecord + for _, line := range lines[1:] { + columns := strings.SplitN(line, "\t", 6) + if len(columns) != 6 { + t.Fatalf("line %q has %d columns, want 6", line, len(columns)) + } + start, err := time.Parse(time.RFC3339Nano, columns[0]) + if err != nil { + t.Fatal(err) + } + took, err := strconv.Atoi(columns[1]) + if err != nil { + t.Fatal(err) + } + records = append(records, probeRecord{Start: start, Took: time.Duration(took) * time.Millisecond, + Leader: columns[2], LeaderAddr: columns[3], Service: columns[4], Detail: columns[5]}) + } + return records +} + +// TestProbeVerdictPassesTheServerStopOfRunW28exz judges the records of a hard stop of a server: an allocations read +// through it timed out, then the clients kept their connection to it and every checks read timed out, 37 s in all. +// The file is the run's records from 01:07:20 to 01:08:13, with the service column of the three polls whose +// allocations and registrations reads answered set to unchecked, as the probe now writes it: one unreached poll of +// 7 s and three unchecked polls of 30 s. +func TestProbeVerdictPassesTheServerStopOfRunW28exz(t *testing.T) { + records := readProbeRecords(t, "testdata/probe-w28exz-server-stop.tsv") + summary, err := probeVerdict(records, defaultProbeLimits) + if err != nil { + t.Fatalf("probeVerdict: %v", err) + } + if got := summary.counts["service unreached"]; got != 1 { + t.Errorf("service unreached polls = %d, want 1", got) + } + if got := summary.counts["service unchecked"]; got != 3 { + t.Errorf("service unchecked polls = %d, want 3", got) + } + if got := summary.longest["service unreached"]; got < 7*time.Second || got > 8*time.Second { + t.Errorf("longest unreached stretch = %s, want about 7 s", got) + } + if got := summary.longest["service unchecked"]; got < 30*time.Second || got > 31*time.Second { + t.Errorf("longest unchecked stretch = %s, want about 30 s", got) + } + // A limit just under the stretch fails the same records. + tight := defaultProbeLimits + tight.Unchecked = 29 * time.Second + if _, err := probeVerdict(records, tight); err == nil || + !strings.Contains(err.Error(), "service unchecked for") { + t.Errorf("verdict with a 29 s limit = %v, want a service unchecked breach", err) + } +} + +func TestProbeVerdictKindsHaveTheirOwnLimits(t *testing.T) { + // 20 s is within the unreached limit and past the leaderless and unserved limits. + records := steadyRecords(50) + for i := 5; i < 25; i++ { + records[i].Leader = probeUnreached + records[i].Service = probeUnreached + } + if _, err := probeVerdict(records, defaultProbeLimits); err != nil { + t.Errorf("20 s unreached: %v, want a pass", err) + } + records = steadyRecords(50) + for i := 5; i < 25; i++ { + records[i].Leader = probeLeaderless + records[i].Service = probeUnserved + } + err := func() error { _, err := probeVerdict(records, defaultProbeLimits); return err }() + if err == nil || !strings.Contains(err.Error(), "leaderless for 20s") || + !strings.Contains(err.Error(), "unserved for 20s") { + t.Errorf("20 s leaderless and unserved: %v, want both named", err) + } +} + +func TestProbeVerdictFailsOnAGapBetweenPolls(t *testing.T) { + withGap := func(gap time.Duration) error { + records := steadyRecords(20) + for i := 10; i < len(records); i++ { + records[i].Start = records[i].Start.Add(gap - time.Second) + } + _, err := probeVerdict(records, defaultProbeLimits) + return err + } + if err := withGap(30 * time.Second); err != nil { + t.Errorf("a gap of 30s: %v, want a pass", err) + } + err := withGap(31 * time.Second) + if err == nil || !strings.Contains(err.Error(), "no poll started for 31s after 12:00:09.000") { + t.Errorf("a gap of 31s: %v, want it named with its start", err) + } +} + +func TestProbeVerdictFailsWhenTheFirstOrTheLastPollIsNotLedAndServed(t *testing.T) { + cases := []struct { + name string + edit func(records []probeRecord) + want string + }{ + {"first leaderless", func(r []probeRecord) { r[0].Leader = probeLeaderless }, "the first poll"}, + {"first unserved", func(r []probeRecord) { r[0].Service = probeUnserved }, "the first poll"}, + {"last leaderless", func(r []probeRecord) { r[len(r)-1].Leader = probeLeaderless }, "the last poll"}, + {"last unserved", func(r []probeRecord) { r[len(r)-1].Service = probeUnserved }, "the last poll"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + records := steadyRecords(20) + tc.edit(records) + _, err := probeVerdict(records, defaultProbeLimits) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %v, want it to name %q", err, tc.want) + } + }) + } +} + +func TestProbeVerdictMeasuresABadStretchAtTheStartAndAtTheEnd(t *testing.T) { + records := steadyRecords(30) + for i := 0; i < 12; i++ { + records[i].Leader = probeLeaderless + } + for i := 18; i < 30; i++ { + records[i].Service = probeUnserved + } + _, err := probeVerdict(records, defaultProbeLimits) + if err == nil { + t.Fatal("probeVerdict passed, want a failure") + } + for _, want := range []string{"leaderless for 12s from 12:00:00.000", "unserved for 11s from 12:00:18.000"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) + } + } +} + +func TestProbeVerdictNeedsTenPolls(t *testing.T) { + if _, err := probeVerdict(steadyRecords(10), defaultProbeLimits); err != nil { + t.Errorf("10 polls: %v, want a pass", err) + } + _, err := probeVerdict(steadyRecords(9), defaultProbeLimits) + if err == nil || !strings.Contains(err.Error(), "9 polls, at least 10 are needed") { + t.Errorf("9 polls: %v, want the count named", err) + } + if _, err := probeVerdict(nil, defaultProbeLimits); err == nil { + t.Error("no polls passed, want a failure") + } +} + +func TestProbeVerdictNamesEveryBreach(t *testing.T) { + records := steadyRecords(30) + records[0].Leader = probeUnreached + for i := 5; i < 20; i++ { + records[i].Service = probeUnserved + } + _, err := probeVerdict(records, defaultProbeLimits) + if err == nil { + t.Fatal("probeVerdict passed, want a failure") + } + for _, want := range []string{"the first poll", "unserved for 15s"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) + } + } +} + +func TestProbeLineHasTheColumnsOfTheHeader(t *testing.T) { + r := probeRecord{ + Start: time.Date(2026, 10, 13, 12, 0, 1, 234_000_000, time.UTC), + Took: 1500 * time.Millisecond, + Leader: probeLed, + LeaderAddr: "10.0.0.1:4647", + Service: probeServed, + Detail: "a", + } + want := "2026-10-13T12:00:01.234Z\t1500\tled\t10.0.0.1:4647\tserved\ta\n" + if got := probeLine(r); got != want { + t.Errorf("probeLine = %q, want %q", got, want) + } + if got, want := probeHeader, "start\ttook_ms\tleader\tleader_address\tservice\tdetail\n"; got != want { + t.Errorf("probeHeader = %q, want %q", got, want) + } +} + +func TestProbeLineKeepsTheDetailOnOneLine(t *testing.T) { + r := probeRecord{Start: probeBase, Detail: "one\ttwo\nthree\r\nfour"} + got := probeLine(r) + if strings.Count(got, "\n") != 1 || strings.Count(got, "\t") != 5 { + t.Errorf("probeLine = %q, want one line of six columns", got) + } + if !strings.HasSuffix(got, "\tone two three four\n") { + t.Errorf("probeLine = %q, want the detail with spaces", got) + } +} + +func TestNewProbeConfigHasTheScenarioValues(t *testing.T) { + var out strings.Builder + cfg := newProbeConfig(&nomadAPI{}, nil, "job", "svc", &out) + if cfg.Every != time.Second || cfg.Refresh != 10*time.Second || cfg.RefreshTimeout != 5*time.Second || + cfg.LeaderTimeout != 3*time.Second || cfg.ServiceTimeout != 7*time.Second { + t.Errorf("config = %+v, want every 1s, refresh 10s in 5s, leader reads 3s, service reads 7s", cfg) + } + if cfg.Job != "job" || cfg.Service != "svc" || cfg.Out != &out { + t.Errorf("config = %+v, want the job, the service and the writer", cfg) + } +} + +// newNode starts a fake server with answers. +func newNode(t *testing.T, answers map[string]func() (int, string)) *fakeNomad { + t.Helper() + f, _ := newFakeNomad(t, answers) + return f +} + +// openOnCleanup opens gate when the test ends. A test calls it after it has started the servers, so a blocked answer +// goes before the servers close. +func openOnCleanup(t *testing.T, gate chan struct{}) { + t.Helper() + t.Cleanup(func() { close(gate) }) +} + +func leaderAnswers(leader string) map[string]func() (int, string) { + answers := readyAnswers() + answers["GET /v1/status/leader?stale=true"] = fixed(200, `"`+leader+`"`) + return answers +} + +func noServiceAnswers(leader func() (int, string)) map[string]func() (int, string) { + return map[string]func() (int, string){"GET /v1/status/leader?stale=true": leader} +} + +// deadAddress is the address of a server that was closed. +func deadAddress(t *testing.T) string { + t.Helper() + f := newNode(t, nil) + f.srv.Close() + return f.srv.URL +} + +func testProbeConfig(n *nomadAPI, addresses ...string) probeConfig { + return probeConfig{ + Nomad: n, + First: addresses, + Addresses: func(context.Context) ([]string, error) { + return addresses, nil + }, + Job: webJob, + Service: webService, + Every: 5 * time.Millisecond, + Refresh: 5 * time.Millisecond, + RefreshTimeout: time.Second, + LeaderTimeout: 40 * time.Millisecond, + ServiceTimeout: 40 * time.Millisecond, + Out: &syncBuffer{}, + } +} + +// pollOnce makes one poll of the servers at addresses. +func pollOnce(t *testing.T, addresses ...string) probeRecord { + t.Helper() + p := &probe{cfg: testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}), addresses: addresses} + return p.poll(t.Context()) +} + +func TestPollLeaderIsLedWhenAnyServerNamesOne(t *testing.T) { + dead := deadAddress(t) + empty := newNode(t, noServiceAnswers(fixed(200, `""`))) + leading := newNode(t, leaderAnswers("10.0.0.1:4647")) + got := pollOnce(t, dead, empty.srv.URL, leading.srv.URL) + if got.Leader != probeLed || got.LeaderAddr != "10.0.0.1:4647" { + t.Errorf("leader = %q at %q, want led at 10.0.0.1:4647", got.Leader, got.LeaderAddr) + } + if !strings.Contains(got.Detail, dead) { + t.Errorf("detail = %q, want the failed server named", got.Detail) + } +} + +func TestPollServiceIsReadThroughTheServerThatNamedTheLeader(t *testing.T) { + // The first server leads nothing and answers 404 to every service read. + empty := newNode(t, noServiceAnswers(fixed(200, `""`))) + leading := newNode(t, leaderAnswers("10.0.0.1:4647")) + got := pollOnce(t, empty.srv.URL, leading.srv.URL) + if got.Service != probeServed { + t.Errorf("service = %q (%s), want served", got.Service, got.Detail) + } + for _, r := range empty.requests { + if strings.Contains(r, "/v1/job/") { + t.Errorf("requests of the server without a leader = %q, want no service read", empty.requests) + } + } +} + +func TestPollLeaderlessWhenNoServerNamesALeader(t *testing.T) { + cases := map[string]func() (int, string){ + "empty answer": fixed(200, `""`), + "no cluster leader": fixed(500, "No cluster leader"), + } + for name, answer := range cases { + t.Run(name, func(t *testing.T) { + a := newNode(t, noServiceAnswers(answer)) + b := newNode(t, noServiceAnswers(answer)) + got := pollOnce(t, a.srv.URL, b.srv.URL) + if got.Leader != probeLeaderless || got.LeaderAddr != "" { + t.Errorf("leader = %q at %q, want leaderless", got.Leader, got.LeaderAddr) + } + }) + } +} + +func TestPollServiceIsReadThroughAServerThatAnsweredWhenNoneNamedALeader(t *testing.T) { + answers := readyAnswers() + answers["GET /v1/status/leader?stale=true"] = fixed(200, `""`) + got := pollOnce(t, deadAddress(t), newNode(t, answers).srv.URL) + if got.Leader != probeLeaderless || got.Service != probeServed { + t.Errorf("leader = %q, service = %q (%s), want leaderless and served", got.Leader, got.Service, got.Detail) + } +} + +func TestPollLeaderIsUnreachedWhenNoServerAnswers(t *testing.T) { + failing := newNode(t, noServiceAnswers(fixed(500, "boom"))) + got := pollOnce(t, deadAddress(t), failing.srv.URL) + if got.Leader != probeUnreached || got.Service != probeUnreached { + t.Errorf("leader = %q, service = %q, want both unreached", got.Leader, got.Service) + } + if !strings.Contains(got.Detail, "boom") || strings.Count(got.Detail, "; ") != 1 { + t.Errorf("detail = %q, want the errors of the two servers and no service read", got.Detail) + } +} + +func TestPollWithoutAddressesIsUnreached(t *testing.T) { + got := pollOnce(t) + if got.Leader != probeUnreached || got.Service != probeUnreached || got.Detail != "no server address is known" { + t.Errorf("poll = %+v, want unreached with the reason", got) + } +} + +func TestPollLeaderSlowServerRunsIntoItsTimeoutBesideALeader(t *testing.T) { + gate := make(chan struct{}) + slow := newNode(t, noServiceAnswers(func() (int, string) { <-gate; return 200, `"x"` })) + openOnCleanup(t, gate) + leading := newNode(t, leaderAnswers("10.0.0.1:4647")) + got := pollOnce(t, slow.srv.URL, leading.srv.URL) + if got.Leader != probeLed || !strings.Contains(got.Detail, slow.srv.URL) { + t.Errorf("leader = %q (%s), want led with the slow server named", got.Leader, got.Detail) + } +} + +func TestPollLeaderOfOnlyASlowServerIsUnreached(t *testing.T) { + gate := make(chan struct{}) + slow := newNode(t, noServiceAnswers(func() (int, string) { <-gate; return 200, `"x"` })) + openOnCleanup(t, gate) + got := pollOnce(t, slow.srv.URL) + if got.Leader != probeUnreached || !strings.Contains(got.Detail, "deadline exceeded") { + t.Errorf("leader = %q (%s), want unreached by the timeout", got.Leader, got.Detail) + } +} + +func allocationsOf(ids ...string) func() (int, string) { + var items []string + for _, id := range ids { + items = append(items, `{"ID":"`+id+`","ClientStatus":"running"}`) + } + return fixed(200, "["+strings.Join(items, ",")+"]") +} + +func checksPath(id string) string { return "GET /v1/allocation/" + id + "/checks" } + +func serviceNode(t *testing.T, edit func(answers map[string]func() (int, string))) string { + t.Helper() + answers := leaderAnswers("10.0.0.1:4647") + edit(answers) + return newNode(t, answers).srv.URL +} + +func TestPollServiceOutcomes(t *testing.T) { + const registered = `[{"AllocID":"` + allocRunning + `"},{"AllocID":"` + allocOther + `"}]` + cases := []struct { + name string + edit func(answers map[string]func() (int, string)) + want string + note string + }{ + {"one healthy allocation", func(map[string]func() (int, string)) {}, probeServed, ""}, + {"no running allocation", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = fixed(200, `[{"ID":"`+allocRunning+`","ClientStatus":"pending"}]`) + }, probeUnserved, "allocations: 11111111 pending; checks: none"}, + {"check fails", func(a map[string]func() (int, string)) { + a[checksPath(allocRunning)] = fixed(200, `{"c1":{"Service":"e2e-web","Status":"failure"}}`) + }, probeUnserved, "checks: 11111111 e2e-web failure"}, + {"no registration", func(a map[string]func() (int, string)) { + a["GET /v1/service/e2e-web"] = fixed(200, `[]`) + }, probeUnserved, "registrations: none"}, + {"checks read of one of two fails", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning, allocOther) + a["GET /v1/service/e2e-web"] = fixed(200, registered) + a[checksPath(allocOther)] = fixed(500, "no route to client") + }, probeServed, ""}, + {"checks read of one of two fails and the other has no check", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning, allocOther) + a["GET /v1/service/e2e-web"] = fixed(200, registered) + a[checksPath(allocRunning)] = fixed(200, `{}`) + a[checksPath(allocOther)] = fixed(500, "no route to client") + }, probeUnserved, "allocations:"}, + {"checks reads of every allocation fail", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning, allocOther) + a["GET /v1/service/e2e-web"] = fixed(200, registered) + a[checksPath(allocRunning)] = fixed(500, "no route to client") + a[checksPath(allocOther)] = fixed(500, "no route to client") + }, probeUnchecked, "no route to client"}, + {"checks reads of every allocation fail, one of them registered", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning, allocOther) + a["GET /v1/service/e2e-web"] = fixed(200, `[{"AllocID":"`+allocOther+`"}]`) + a[checksPath(allocRunning)] = fixed(500, "no route to client") + a[checksPath(allocOther)] = fixed(500, "no route to client") + }, probeUnchecked, "no route to client"}, + {"checks reads of every allocation fail, no registration", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning, allocOther) + a["GET /v1/service/e2e-web"] = fixed(200, `[]`) + a[checksPath(allocRunning)] = fixed(500, "no route to client") + a[checksPath(allocOther)] = fixed(500, "no route to client") + }, probeUnserved, "no route to client"}, + {"checks reads fail and only a stopped allocation is registered", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning) + a["GET /v1/service/e2e-web"] = fixed(200, `[{"AllocID":"`+allocOther+`"}]`) + a[checksPath(allocRunning)] = fixed(500, "no route to client") + }, probeUnserved, "no route to client"}, + {"allocations read fails", func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = fixed(500, "allocations down") + }, probeUnreached, "allocations down"}, + {"registrations read fails", func(a map[string]func() (int, string)) { + a["GET /v1/service/e2e-web"] = fixed(500, "registrations down") + }, probeUnreached, "registrations down"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := pollOnce(t, serviceNode(t, tc.edit)) + if got.Service != tc.want { + t.Errorf("service = %q (%s), want %q", got.Service, got.Detail, tc.want) + } + if !strings.Contains(got.Detail, tc.note) { + t.Errorf("detail = %q, want it to contain %q", got.Detail, tc.note) + } + if tc.want == probeServed && strings.Contains(got.Detail, "allocations:") { + t.Errorf("detail = %q, want the answers quoted only when the service is not served", got.Detail) + } + }) + } +} + +func TestPollServiceReadThatIsSlowIsUnreached(t *testing.T) { + gate := make(chan struct{}) + address := serviceNode(t, func(a map[string]func() (int, string)) { + a["GET /v1/job/e2e-web/allocations"] = func() (int, string) { <-gate; return 200, "[]" } + }) + openOnCleanup(t, gate) + got := pollOnce(t, address) + if got.Service != probeUnreached || !strings.Contains(got.Detail, "deadline exceeded") { + t.Errorf("service = %q (%s), want unreached by the timeout", got.Service, got.Detail) + } +} + +// syncBuffer is a writer that a test can read while the probe writes. +type syncBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// eventually waits until cond holds; it fails the test after 5 s. +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } +} + +func leadingProbeConfig(t *testing.T) probeConfig { + t.Helper() + address := newNode(t, leaderAnswers("10.0.0.1:4647")).srv.URL + return testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, address) +} + +func TestProbeWritesTheHeaderFirstAndEachRecordAsItComes(t *testing.T) { + cfg := leadingProbeConfig(t) + out := cfg.Out.(*syncBuffer) + p := startProbe(t.Context(), cfg) + eventually(t, "two lines", func() bool { return strings.Count(out.String(), "\n") >= 3 }) + records := p.stop() + lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n") + if lines[0]+"\n" != probeHeader { + t.Errorf("first line = %q, want the header", lines[0]) + } + polls := 0 + for _, l := range lines[1:] { + if strings.Contains(l, "\tled\t10.0.0.1:4647\tserved\t") { + polls++ + } + } + if polls != len(records) || len(records) < 2 { + t.Errorf("%d poll lines for %d records, want the same number, at least 2:\n%s", polls, len(records), out) + } +} + +func TestProbeStopReturnsEveryRecordAndEndsTheGoroutines(t *testing.T) { + var polls, lists atomic.Int32 + address := newNode(t, map[string]func() (int, string){ + "GET /v1/status/leader?stale=true": func() (int, string) { polls.Add(1); return 200, `"x"` }, + }).srv.URL + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}) + cfg.Addresses = func(context.Context) ([]string, error) { lists.Add(1); return []string{address}, nil } + p := startProbe(t.Context(), cfg) + out := cfg.Out.(*syncBuffer) + eventually(t, "three records", func() bool { return strings.Count(out.String(), "\n") >= 4 }) + records := p.stop() + if len(records) < 3 { + t.Errorf("stop returned %d records, want at least 3", len(records)) + } + polled, listed := polls.Load(), lists.Load() + time.Sleep(30 * time.Millisecond) + if polls.Load() != polled || lists.Load() != listed { + t.Errorf("polls %d to %d, lists %d to %d after stop, want no more", polled, polls.Load(), listed, lists.Load()) + } +} + +func TestProbeStopDropsThePollItInterrupted(t *testing.T) { + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + address := newNode(t, noServiceAnswers(func() (int, string) { + select { + case entered <- struct{}{}: + default: + } + <-gate + return 200, `"x"` + })).srv.URL + openOnCleanup(t, gate) + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, address) + cfg.LeaderTimeout = time.Minute + cfg.Out = &syncBuffer{} + p := startProbe(t.Context(), cfg) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the leader read") + } + if records := p.stop(); len(records) != 0 { + t.Errorf("stop returned %+v, want no record of an interrupted poll", records) + } + if got := cfg.Out.(*syncBuffer).String(); got != probeHeader { + t.Errorf("output = %q, want the header alone", got) + } +} + +// TestProbeFirstPollUsesTheFirstAddressesWhateverTheListDoes has the first list that Addresses returns fail: the first +// poll still reaches the server that First names. +func TestProbeFirstPollUsesTheFirstAddressesWhateverTheListDoes(t *testing.T) { + address := newNode(t, leaderAnswers("10.0.0.1:4647")).srv.URL + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, address) + cfg.Refresh = time.Hour + var calls atomic.Int32 + cfg.Addresses = func(context.Context) ([]string, error) { + calls.Add(1) + return nil, errors.New("vultr: 503") + } + p := startProbe(t.Context(), cfg) + eventually(t, "a record", func() bool { return strings.Count(cfg.Out.(*syncBuffer).String(), "\n") >= 2 }) + records := p.stop() + if records[0].Leader != probeLed || records[0].Service != probeServed { + t.Errorf("first record = %+v, want led and served from the first addresses", records[0]) + } + if got := calls.Load(); got != 0 { + t.Errorf("Addresses was called %d times, want none before the first refresh", got) + } +} + +func TestProbeRefreshThatBlocksDoesNotDelayThePolls(t *testing.T) { + address := newNode(t, leaderAnswers("10.0.0.1:4647")).srv.URL + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, address) + cfg.RefreshTimeout = time.Hour + blocked := make(chan struct{}) + var once sync.Once + cfg.Addresses = func(ctx context.Context) ([]string, error) { + once.Do(func() { close(blocked) }) + <-ctx.Done() + return nil, ctx.Err() + } + p := startProbe(t.Context(), cfg) + select { + case <-blocked: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the list") + } + out := cfg.Out.(*syncBuffer) + before := strings.Count(out.String(), "\tserved\t") + eventually(t, "polls while the list blocks", func() bool { + return strings.Count(out.String(), "\tserved\t") >= before+3 + }) + p.stop() + if strings.Contains(out.String(), "the list of servers failed") { + t.Errorf("output = %q, want no line for a list that the stop ended", out.String()) + } +} + +func TestProbeRefreshEndsAtItsTimeoutAndIsWrittenAsALine(t *testing.T) { + address := newNode(t, leaderAnswers("10.0.0.1:4647")).srv.URL + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}) + cfg.RefreshTimeout = 10 * time.Millisecond + out := cfg.Out.(*syncBuffer) + var calls atomic.Int32 + cfg.Addresses = func(ctx context.Context) ([]string, error) { + if calls.Add(1) == 1 { + <-ctx.Done() + return nil, ctx.Err() + } + // The lists name no server until a poll has run without one. + if !strings.Contains(out.String(), "no server address is known") { + return nil, nil + } + return []string{address}, nil + } + p := startProbe(t.Context(), cfg) + eventually(t, "a served poll", func() bool { return strings.Contains(out.String(), "\tserved\t") }) + records := p.stop() + if !strings.Contains(out.String(), "\t-\t-\t-\tthe list of servers failed: context deadline exceeded\n") { + t.Errorf("output = %q, want a line for the failed list", out.String()) + } + if records[0].Leader != probeUnreached || records[0].Detail != "no server address is known" { + t.Errorf("first record = %+v, want unreached: no list came before the first poll", records[0]) + } + if got := len(records); strings.Count(out.String(), "\n") != got+2 { + t.Errorf("output holds %d lines for %d records, want the header and the failed list besides them", + strings.Count(out.String(), "\n"), got) + } +} + +func TestProbeKeepsTheLastAddressesWhenAListFails(t *testing.T) { + address := newNode(t, leaderAnswers("10.0.0.1:4647")).srv.URL + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, address) + cfg.Addresses = func(context.Context) ([]string, error) { return nil, errors.New("vultr is down") } + p := startProbe(t.Context(), cfg) + out := cfg.Out.(*syncBuffer) + eventually(t, "a failed list", func() bool { return strings.Contains(out.String(), "vultr is down") }) + before := strings.Count(out.String(), "\tserved\t") + eventually(t, "polls after it", func() bool { return strings.Count(out.String(), "\tserved\t") >= before+3 }) + for _, r := range p.stop() { + if r.Leader != probeLed { + t.Errorf("record = %+v, want led: the last addresses are kept", r) + } + } +} + +func TestProbeListThatSucceedsReplacesTheAddresses(t *testing.T) { + gone := deadAddress(t) + address := newNode(t, leaderAnswers("10.0.0.1:4647")).srv.URL + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, gone) + out := cfg.Out.(*syncBuffer) + cfg.Addresses = func(context.Context) ([]string, error) { + // The lists name the dead server until a poll has asked it, so a list before the first poll changes nothing. + if !strings.Contains(out.String(), "\tunreached\t") { + return []string{gone}, nil + } + return []string{address}, nil + } + p := startProbe(t.Context(), cfg) + eventually(t, "a served poll", func() bool { return strings.Contains(out.String(), "\tserved\t") }) + records := p.stop() + if records[0].Leader != probeUnreached { + t.Errorf("first record = %+v, want unreached: the first addresses name a dead server", records[0]) + } +} + +// TestPollAsksTheServersForTheLeaderAtOnce gives each server a leader answer that waits until the other server has +// been asked too: asked one after the other, the first read would run into its timeout. +func TestPollAsksTheServersForTheLeaderAtOnce(t *testing.T) { + gate := make(chan struct{}) + asked := []chan struct{}{make(chan struct{}), make(chan struct{})} + var once [2]sync.Once + address := func(self, other int) string { + answers := leaderAnswers("10.0.0.1:4647") + answers["GET /v1/status/leader?stale=true"] = func() (int, string) { + once[self].Do(func() { close(asked[self]) }) + select { + case <-asked[other]: + return 200, `"10.0.0.1:4647"` + case <-gate: + return 500, "the other server was never asked" + } + } + return newNode(t, answers).srv.URL + } + addresses := []string{address(0, 1), address(1, 0)} + openOnCleanup(t, gate) + p := &probe{cfg: testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}), addresses: addresses} + p.cfg.LeaderTimeout = 2 * time.Second + got := p.poll(t.Context()) + if got.Leader != probeLed || got.Detail != "" { + t.Errorf("leader = %q (%s), want led with no failed read", got.Leader, got.Detail) + } +} + +// settleConfig returns the configuration of a probe whose servers name a leader only while led is set. +func settleConfig(t *testing.T, led *atomic.Bool) probeConfig { + t.Helper() + answers := leaderAnswers("10.0.0.1:4647") + answers["GET /v1/status/leader?stale=true"] = func() (int, string) { + if led.Load() { + return 200, `"10.0.0.1:4647"` + } + return 200, `""` + } + return testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, newNode(t, answers).srv.URL) +} + +// TestProbeSettleKeepsPollingUntilAPollIsLedAndServed starts with a leaderless probe that gets its leader back 30 ms +// later: settle returns after that, and the last record is good. +func TestProbeSettleKeepsPollingUntilAPollIsLedAndServed(t *testing.T) { + var led atomic.Bool + cfg := settleConfig(t, &led) + p := startProbe(t.Context(), cfg) + eventually(t, "a leaderless poll", func() bool { + return strings.Contains(cfg.Out.(*syncBuffer).String(), "\tleaderless\t") + }) + time.AfterFunc(30*time.Millisecond, func() { led.Store(true) }) + waited := p.settle(t.Context(), time.Minute) + records := p.stop() + if len(records) == 0 { + t.Fatal("no records") + } + last := records[len(records)-1] + if last.Leader != probeLed || last.Service != probeServed { + t.Errorf("last record = %+v, want led and served", last) + } + if waited >= time.Minute { + t.Errorf("settle waited %s, want it to end when the leader came back", waited) + } +} + +func TestProbeSettleGivesUpAfterItsLimit(t *testing.T) { + var led atomic.Bool + p := startProbe(t.Context(), settleConfig(t, &led)) + waited := p.settle(t.Context(), 50*time.Millisecond) + records := p.stop() + if waited < 50*time.Millisecond { + t.Errorf("settle waited %s, want the whole 50ms", waited) + } + if len(records) == 0 { + t.Fatal("no records") + } + if last := records[len(records)-1]; last.Leader != probeLeaderless { + t.Errorf("last record = %+v, want the leaderless poll that settle could not improve", last) + } +} + +func TestProbeSettleEndsWithItsContext(t *testing.T) { + var led atomic.Bool + p := startProbe(t.Context(), settleConfig(t, &led)) + defer p.stop() + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond) + defer cancel() + if waited := p.settle(ctx, time.Hour); waited >= time.Hour { + t.Errorf("settle waited %s, want it to end with its context", waited) + } + if ctx.Err() == nil { + t.Error("settle returned before its context ended and before a good poll") + } +} + +func TestProbeSettleReturnsAtOnceWhenTheLastPollIsGood(t *testing.T) { + cfg := leadingProbeConfig(t) + p := startProbe(t.Context(), cfg) + eventually(t, "a record", func() bool { return strings.Count(cfg.Out.(*syncBuffer).String(), "\n") >= 2 }) + if waited := p.settle(t.Context(), time.Hour); waited > time.Second { + t.Errorf("settle waited %s on a probe whose last poll was good", waited) + } + p.stop() +} + +// TestProbeSettleWantsTheServiceServedAsWellAsALeader has a leader and no running allocation. +func TestProbeSettleWantsTheServiceServedAsWellAsALeader(t *testing.T) { + answers := leaderAnswers("10.0.0.1:4647") + answers["GET /v1/job/e2e-web/allocations"] = fixed(200, `[]`) + cfg := testProbeConfig(&nomadAPI{token: "t", client: &http.Client{}}, newNode(t, answers).srv.URL) + p := startProbe(t.Context(), cfg) + waited := p.settle(t.Context(), 50*time.Millisecond) + records := p.stop() + if waited < 50*time.Millisecond { + t.Errorf("settle waited %s, want the whole 50ms for a service that is not served", waited) + } + if len(records) == 0 || records[len(records)-1].Service != probeUnserved { + t.Errorf("records = %+v, want unserved polls", records) + } +} diff --git a/test/e2e/scenario_text_test.go b/test/e2e/scenario_text_test.go new file mode 100644 index 0000000..45b7436 --- /dev/null +++ b/test/e2e/scenario_text_test.go @@ -0,0 +1,659 @@ +package e2e + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/ingvarch/tent/test/e2e/janitor" + "github.com/ingvarch/tent/test/e2e/vultrapi" +) + +// scenarioCreateArgs are the arguments of tent that create a cluster of the given shape on an image, open to the +// runner alone. +func scenarioCreateArgs(cfg settings, cluster, image, sshPub, runner string, shape ...string) []string { + args := []string{ + "create", "cluster", cluster, + "--provider", "vultr", "--region", cfg.Region, "--machine-type", cfg.Plan, + } + args = append(args, shape...) + return append(args, + "--image", image, "--ssh-key", sshPub, + "--ssh-access", runner, "--api-access", runner, "--yes") +} + +// haServers and haWorkers are the sizes of the node groups of the HA scenario. +const ( + haServers = 3 + haWorkers = 2 +) + +// haCreateArgs are the arguments of tent that create the cluster of the HA scenario: haServers servers and haWorkers +// workers on the image. +func haCreateArgs(cfg settings, cluster, image, sshPub, runner string) []string { + return scenarioCreateArgs(cfg, cluster, image, sshPub, runner, + "--servers", strconv.Itoa(haServers), "--workers", strconv.Itoa(haWorkers)) +} + +// upgradeNodes is the size of the combined group of the upgrade scenario. +const upgradeNodes = 3 + +// upgradeCreateArgs are the arguments of tent that create the cluster of the upgrade scenario: a combined group of +// three nodes on the old Nomad version and the image. +func upgradeCreateArgs(cfg settings, cluster, image, sshPub, runner string) []string { + return scenarioCreateArgs(cfg, cluster, image, sshPub, runner, + "--combined", "--servers", strconv.Itoa(upgradeNodes), "--nomad-version", cfg.UpgradeFrom) +} + +// upgradeTarget takes the output of `tent get clusters -o json`, an array of one cluster, and returns its +// spec.nomad.version, which must differ from the version that the cluster was built on. +func upgradeTarget(clustersJSON []byte, from string) (string, error) { + var clusters []struct { + Spec struct { + Nomad struct { + Version string + } + } + } + if err := json.Unmarshal(clustersJSON, &clusters); err != nil { + return "", fmt.Errorf("read the clusters: %w", err) + } + if len(clusters) != 1 { + return "", fmt.Errorf("got %d clusters, want 1", len(clusters)) + } + version := clusters[0].Spec.Nomad.Version + if version == "" { + return "", errors.New("the cluster has no spec.nomad.version") + } + if version == from { + return "", fmt.Errorf("spec.nomad.version is still %s", from) + } + return version, nil +} + +// upgradedVersion returns the version that the line of `tent upgrade cluster --yes` says the spec has now: "cluster +// : spec.nomad.version is now ; the cluster was on Nomad ." +func upgradedVersion(stdout []byte, cluster string) (string, error) { + prefix := "cluster " + cluster + ": spec.nomad.version is now " + for line := range strings.Lines(string(stdout)) { + rest, ok := strings.CutPrefix(strings.TrimSpace(line), prefix) + if !ok { + continue + } + if version, _, found := strings.Cut(rest, ";"); found && version != "" { + return version, nil + } + } + return "", fmt.Errorf("upgrade does not say %q :\n%s", prefix, lastLines(string(stdout), stderrTailLines)) +} + +// untestedVersionPhrase is what tent's warning says about a Nomad version that its channel has not tested. +const untestedVersionPhrase = "is not tested by this tent" + +// untestedVersionWarning returns the line of stderr that warns about an untested Nomad version, or "" when there is +// none. +func untestedVersionWarning(stderr []byte) string { + for line := range strings.Lines(string(stderr)) { + if strings.Contains(line, untestedVersionPhrase) { + return strings.TrimSpace(line) + } + } + return "" +} + +// checkValidVersion checks that the output of tent validate cluster has the line "cluster is valid: ... run +// Nomad ". +func checkValidVersion(stdout []byte, cluster, version string) error { + prefix := "cluster " + cluster + " is valid: " + suffix := " run Nomad " + version + for line := range strings.Lines(string(stdout)) { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, prefix) && strings.HasSuffix(line, suffix) { + return nil + } + } + return fmt.Errorf("validate does not say %q ... %q:\n%s", prefix, suffix, lastLines(string(stdout), stderrTailLines)) +} + +// withRollChange takes the output of `tent get clusters -o json`, an array of one cluster, and returns that +// cluster with a Nomad extra configuration that changes the spec hash of every server and every worker: the +// servers get a comment and the workers a node meta value e2e_roll. It refuses a cluster that already has an extra +// configuration, which the suite did not write. +func withRollChange(clustersJSON []byte, run string) ([]byte, error) { + dec := json.NewDecoder(bytes.NewReader(clustersJSON)) + dec.UseNumber() + var clusters []map[string]any + if err := dec.Decode(&clusters); err != nil { + return nil, fmt.Errorf("read the clusters: %w", err) + } + if len(clusters) != 1 { + return nil, fmt.Errorf("got %d clusters, want 1", len(clusters)) + } + spec, ok := clusters[0]["spec"].(map[string]any) + if !ok { + return nil, errors.New("the cluster has no spec") + } + nomad := map[string]any{} + if v, found := spec["nomad"]; found { + if nomad, ok = v.(map[string]any); !ok { + return nil, errors.New("spec.nomad is not an object") + } + } + if _, found := nomad["extraConfig"]; found { + return nil, errors.New("spec.nomad.extraConfig is already set") + } + nomad["extraConfig"] = map[string]any{ + "server": "# e2e roll " + run + "\n", + "client": "client {\n meta {\n e2e_roll = \"" + run + "\"\n }\n}\n", + } + spec["nomad"] = nomad + out, err := json.MarshalIndent(clusters[0], "", " ") + if err != nil { + return nil, fmt.Errorf("write the cluster: %w", err) + } + return out, nil +} + +// registrationPhrase is what a Nomad client logs after each registration with a server. +const registrationPhrase = "node registration complete" + +// bootIDScript prints the machine's boot ID. +func bootIDScript() string { return "cat /proc/sys/kernel/random/boot_id" } + +// rebootScript reboots the machine two seconds after it returns, so that the SSH session ends cleanly. +func rebootScript() string { return "systemd-run --on-active=2 --unit=e2e-reboot systemctl reboot" } + +// registrationScript prints the registration lines of the current boot's Nomad journal, and exits 0 when there is +// none. +func registrationScript() string { + return "journalctl -b -u nomad --no-pager -o cat | grep '" + registrationPhrase + "' || true" +} + +// joinFilePath is the file that tells the agent which servers to join. +const joinFilePath = "/etc/nomad.d/05-join.hcl" + +// joinFileScript prints joinFilePath. +func joinFileScript() string { return "cat " + joinFilePath } + +var bootIDForm = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// parseBootID reads the answer of bootIDScript. +func parseBootID(out string) (string, error) { + id := strings.TrimSpace(out) + if !bootIDForm.MatchString(id) { + return "", fmt.Errorf("read the boot ID from %q", id) + } + return id, nil +} + +// registrationLines returns the lines of out that report a registration. +func registrationLines(out string) []string { + return slices.DeleteFunc(nonEmptyLines(out), func(line string) bool { + return !strings.Contains(line, registrationPhrase) + }) +} + +// nomadNode is a node as /v1/nodes lists it. +type nomadNode struct { + ID string + Name string + Status string + SchedulingEligibility string + Version string + Address string +} + +// listNodes returns the nodes /v1/nodes lists. +func listNodes(ctx context.Context, n *nomadAPI) ([]nomadNode, error) { + var nodes []nomadNode + if err := n.get(ctx, "/v1/nodes", &nodes); err != nil { + return nil, err + } + return nodes, nil +} + +// nodeMeta returns the meta values of the node with this ID. +func nodeMeta(ctx context.Context, n *nomadAPI, id string) (map[string]string, error) { + var node struct{ Meta map[string]string } + if err := n.get(ctx, "/v1/node/"+url.PathEscape(id), &node); err != nil { + return nil, err + } + return node.Meta, nil +} + +// agentMember is a member of the servers' gossip pool, as /v1/agent/members lists it. +type agentMember struct { + Name string + Addr string + Status string + Tags map[string]string +} + +// listMembers returns the members that /v1/agent/members lists. +func listMembers(ctx context.Context, n *nomadAPI) ([]agentMember, error) { + var answer struct{ Members []agentMember } + if err := n.get(ctx, "/v1/agent/members", &answer); err != nil { + return nil, err + } + return answer.Members, nil +} + +// runningAllocations counts the running allocations of job. +func runningAllocations(ctx context.Context, n *nomadAPI, job string) (int, error) { + var allocs []jobAllocation + if err := n.get(ctx, "/v1/job/"+url.PathEscape(job)+"/allocations", &allocs); err != nil { + return 0, err + } + return len(slices.DeleteFunc(allocs, func(a jobAllocation) bool { return a.ClientStatus != "running" })), nil +} + +// checkVersions checks a cluster of combined nodes after a roll: each machine has a node and an alive member, and +// they are all on version; every node is ready and eligible. A server member is named .. +// Members of other states are left out of the checks and listed apart in the error. The error names every mismatch. +func checkVersions(nodes []nomadNode, members []agentMember, machines []vultrapi.Instance, version string) error { + if len(machines) == 0 { + return errors.New("no machine to check") + } + labels := make([]string, 0, len(machines)) + for _, m := range machines { + labels = append(labels, m.Label) + } + var problems []string + var seen []named + for _, node := range nodes { + seen = append(seen, named{node.Name, node.Name}) + if node.Version != version { + problems = append(problems, fmt.Sprintf("node %s: version %s, want %s", node.Name, node.Version, version)) + } + if node.Status != "ready" { + problems = append(problems, fmt.Sprintf("node %s: status %s, want ready", node.Name, node.Status)) + } + if node.SchedulingEligibility != "eligible" { + problems = append(problems, + fmt.Sprintf("node %s: scheduling %s, want eligible", node.Name, node.SchedulingEligibility)) + } + } + problems = append(problems, matchProblems("node", seen, labels)...) + var other []string + seen = nil + for _, m := range members { + if m.Status != "alive" { + other = append(other, fmt.Sprintf("%s (%s)", m.Name, m.Status)) + continue + } + label, _, hasRegion := strings.Cut(m.Name, ".") + if !hasRegion { + problems = append(problems, fmt.Sprintf("member %s has no region in its name", m.Name)) + } + seen = append(seen, named{m.Name, label}) + if build := cmp.Or(m.Tags["build"], "none"); build != version { + problems = append(problems, fmt.Sprintf("member %s: build %s, want %s", m.Name, build, version)) + } + } + problems = append(problems, matchProblems("alive member", seen, labels)...) + if len(problems) == 0 { + return nil + } + msg := strings.Join(problems, "; ") + if len(other) > 0 { + msg += "; other members: " + strings.Join(other, ", ") + } + return errors.New(msg) +} + +// named is something of a machine that Nomad lists: the name shown, and the label of the machine it belongs to. +type named struct{ shown, label string } + +// matchProblems reports what keeps the things of kind from being one for each machine label: a count that differs, +// a thing of no machine, a machine without a thing. +func matchProblems(kind string, things []named, labels []string) []string { + var problems []string + if len(things) != len(labels) { + problems = append(problems, fmt.Sprintf("%d %ss for %d machines", len(things), kind, len(labels))) + } + have := map[string]bool{} + for _, th := range things { + have[th.label] = true + if !slices.Contains(labels, th.label) { + problems = append(problems, fmt.Sprintf("%s %s is of no current machine", kind, th.shown)) + } + } + for _, label := range labels { + if !have[label] { + problems = append(problems, fmt.Sprintf("machine %s has no %s", label, kind)) + } + } + return problems +} + +// checkRolled fails when a machine of before is still listed in after, and names those machines. An empty before +// fails: the machines to roll were never recorded. +func checkRolled(before, after []vultrapi.Instance) error { + if len(before) == 0 { + return errors.New("no machine was recorded before the roll") + } + listed := listedIDs(after) + var left []string + for _, in := range before { + if listed[in.ID] { + left = append(left, in.Label) + } + } + if len(left) > 0 { + return fmt.Errorf("machines of before the roll are still listed: %s", strings.Join(left, ", ")) + } + return nil +} + +// checkKept fails when a machine of kept is no longer listed in after, and names those machines. An empty kept +// fails: the machines to keep were never recorded. +func checkKept(kept, after []vultrapi.Instance) error { + if len(kept) == 0 { + return errors.New("no machine was recorded to be kept") + } + listed := listedIDs(after) + var gone []string + for _, in := range kept { + if !listed[in.ID] { + gone = append(gone, in.Label) + } + } + if len(gone) > 0 { + return fmt.Errorf("machines that should stay are no longer listed: %s", strings.Join(gone, ", ")) + } + return nil +} + +func listedIDs(instances []vultrapi.Instance) map[string]bool { + ids := make(map[string]bool, len(instances)) + for _, in := range instances { + ids[in.ID] = true + } + return ids +} + +// checkCounts checks that cluster has the given numbers of server and client machines among instances. +func checkCounts(instances []vultrapi.Instance, cluster string, servers, workers int) error { + var problems []string + if got := len(machinesOf(instances, cluster, "server")); got != servers { + problems = append(problems, fmt.Sprintf("%d servers, want %d", got, servers)) + } + if got := len(machinesOf(instances, cluster, "client")); got != workers { + problems = append(problems, fmt.Sprintf("%d workers, want %d", got, workers)) + } + if len(problems) > 0 { + return fmt.Errorf("cluster %s has %s", cluster, strings.Join(problems, "; ")) + } + return nil +} + +// serversToDelete returns the server machines of an e2e cluster: the ones that carry both its cluster tag and the +// server role tag. It refuses a name that is not e2e- followed by more, and a cluster that has no server machine. +func serversToDelete(instances []vultrapi.Instance, cluster string) ([]vultrapi.Instance, error) { + if !strings.HasPrefix(cluster, janitor.ClusterPrefix) || len(cluster) == len(janitor.ClusterPrefix) { + return nil, fmt.Errorf("%q is not the name of a cluster of this suite", cluster) + } + servers := machinesOf(instances, cluster, "server") + if len(servers) == 0 { + return nil, fmt.Errorf("cluster %s has no server machine", cluster) + } + return servers, nil +} + +// nomadPort is the port of a server's HTTP API. +const nomadPort = "4646" + +// serverAddresses returns the API address, "https://:4646", of each machine of a role in a cluster that has a +// public address. +func serverAddresses(instances []vultrapi.Instance, cluster, role string) []string { + var out []string + for _, m := range machinesOf(instances, cluster, role) { + if m.MainIP != "" && m.MainIP != "0.0.0.0" { + out = append(out, "https://"+net.JoinHostPort(m.MainIP, nomadPort)) + } + } + return out +} + +// lowestLabel returns the machine whose label sorts first. +func lowestLabel(machines []vultrapi.Instance) (vultrapi.Instance, error) { + if len(machines) == 0 { + return vultrapi.Instance{}, errors.New("no machine to choose from") + } + return slices.MinFunc(machines, func(a, b vultrapi.Instance) int { return strings.Compare(a.Label, b.Label) }), nil +} + +// nodeNamed returns the one node that has name; tent names a node after its machine. +func nodeNamed(nodes []nomadNode, name string) (nomadNode, error) { + var found []nomadNode + for _, n := range nodes { + if n.Name == name { + found = append(found, n) + } + } + if len(found) != 1 { + return nomadNode{}, fmt.Errorf("%d nodes are named %s, want 1", len(found), name) + } + return found[0], nil +} + +// rollMetaKey is the node meta key that the roll change gives every worker. +const rollMetaKey = "e2e_roll" + +// checkRollMeta checks that there are want nodes and that each of them, by name, has the run's value in its meta. +func checkRollMeta(metas map[string]map[string]string, run string, want int) error { + var problems []string + if len(metas) != want { + problems = append(problems, fmt.Sprintf("%d nodes, want %d", len(metas), want)) + } + for _, name := range slices.Sorted(maps.Keys(metas)) { + if got := metas[name][rollMetaKey]; got != run { + problems = append(problems, fmt.Sprintf("node %s has %s %q, want %q", name, rollMetaKey, got, run)) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// bootProblem is "" when the answer of bootIDScript shows a boot ID other than before; otherwise it says why not. +func bootProblem(out string, err error, before string) string { + if err != nil { + return err.Error() + } + id, err := parseBootID(out) + switch { + case err != nil: + return err.Error() + case id == before: + return "the machine still has the boot ID " + before + } + return "" +} + +// registeredProblem is "" when the journal of this boot holds a registration line, Nomad lists the node with id as +// ready and eligible, and the node named after the machine is that node; otherwise it says what is missing. +func registeredProblem(lines []string, nodes []nomadNode, machine, id string) string { + if len(lines) == 0 { + return "the journal of this boot holds no registration" + } + i := slices.IndexFunc(nodes, func(n nomadNode) bool { return n.ID == id }) + switch { + case i < 0: + return "Nomad does not list the node " + id + case nodes[i].Status != "ready": + return "the node " + id + " is " + nodes[i].Status + case nodes[i].SchedulingEligibility != "eligible": + return "the node " + id + " is " + nodes[i].SchedulingEligibility + } + named, err := nodeNamed(nodes, machine) + switch { + case err != nil: + return err.Error() + case named.ID != id: + return fmt.Sprintf("the node named %s has the ID %s, want %s", machine, named.ID, id) + } + return "" +} + +// restartEvidence is what the restart of a client leaves to read. +type restartEvidence struct { + Machine, NodeID string + BootBefore, BootAfter string + BootWait, RegisterWait time.Duration + Registrations []string + JoinFile string +} + +// restartRecord writes the evidence as the text of the step's record file. +func restartRecord(e restartEvidence) string { + var b strings.Builder + fmt.Fprintf(&b, "machine: %s\nnode ID: %s\nboot ID before: %s\nboot ID after: %s\n", e.Machine, e.NodeID, + e.BootBefore, e.BootAfter) + fmt.Fprintf(&b, "wait for the new boot: %s\nwait for the registration: %s\n", e.BootWait, e.RegisterWait) + b.WriteString("registration lines of this boot:\n") + for _, line := range e.Registrations { + b.WriteString(line + "\n") + } + b.WriteString(joinFilePath + ":\n" + e.JoinFile) + return b.String() +} + +// allocationsProblem is "" when running is the wanted number of running allocations of job. +func allocationsProblem(running int, job string, want int) string { + if running == want { + return "" + } + return fmt.Sprintf("%d running allocations of %s, want %d", running, job, want) +} + +// storedBackup is the answer of `tent backup create -o json`, as far as the scenario reads it. +type storedBackup struct { + Name string `json:"name"` + Size int `json:"size"` +} + +// parseStoredBackup reads the answer of `tent backup create -o json`; a backup without a name or a size is no backup. +func parseStoredBackup(out []byte) (storedBackup, error) { + var b storedBackup + if err := json.Unmarshal(out, &b); err != nil { + return storedBackup{}, fmt.Errorf("read the stored backup: %w", err) + } + if b.Name == "" || b.Size <= 0 { + return storedBackup{}, fmt.Errorf("the stored backup has name %q and size %d", b.Name, b.Size) + } + return b, nil +} + +// checkBackupList checks that `tent backup list -o json` lists the backup name alone and that it is of servers, +// "current" or "earlier". +func checkBackupList(out []byte, name, servers string) error { + var backups []struct { + Name string `json:"name"` + Servers string `json:"servers"` + } + if err := json.Unmarshal(out, &backups); err != nil { + return fmt.Errorf("read the backups: %w", err) + } + if len(backups) != 1 { + return fmt.Errorf("%d backups listed, want 1", len(backups)) + } + if got := backups[0]; got.Name != name || got.Servers != servers { + return fmt.Errorf("listed backup %s of %s servers, want %s of %s servers", got.Name, got.Servers, name, servers) + } + return nil +} + +// checkRestored checks that the output of `tent backup restore --yes` reports the restore of backup into cluster. +func checkRestored(out []byte, backup, cluster string) error { + want := "Restored: backup " + backup + " into cluster " + cluster + "." + if !strings.Contains(string(out), want) { + return fmt.Errorf("the output of the restore does not hold %q: %s", want, strings.TrimSpace(string(out))) + } + return nil +} + +// memberAddresses returns the addresses of the members, sorted. +func memberAddresses(members []agentMember) []string { + out := make([]string, 0, len(members)) + for _, m := range members { + out = append(out, m.Addr) + } + slices.Sort(out) + return out +} + +// nodeAddresses returns the addresses of the nodes, sorted. +func nodeAddresses(nodes []nomadNode) []string { + out := make([]string, 0, len(nodes)) + for _, n := range nodes { + out = append(out, n.Address) + } + slices.Sort(out) + return out +} + +// stepTime is how long a part of a step took. +type stepTime struct { + Step string + Took time.Duration +} + +// backupEvidence is what the backup step leaves to read: the stored backup, the time of each part, the private +// addresses of the servers and workers that Nomad listed before the servers were deleted and at the end, and whether +// the workers had to be rolled with --force. +type backupEvidence struct { + Backup storedBackup + Times []stepTime + ServersBefore, WorkersBefore []string + ServersAfter, WorkersAfter []string + Forced bool +} + +// backupRecord writes the evidence as the text of the step's record file. +func backupRecord(e backupEvidence) string { + var b strings.Builder + fmt.Fprintf(&b, "backup: %s\nsize: %d\n", e.Backup.Name, e.Backup.Size) + for _, st := range e.Times { + fmt.Fprintf(&b, "%s: %s\n", st.Step, st.Took) + } + for _, l := range []struct { + label string + addrs []string + }{ + {"servers before", e.ServersBefore}, {"workers before", e.WorkersBefore}, + {"servers after", e.ServersAfter}, {"workers after", e.WorkersAfter}, + } { + fmt.Fprintf(&b, "%s: %s\n", l.label, cmp.Or(strings.Join(l.addrs, " "), "none")) + } + forced := "no" + if e.Forced { + forced = "yes" + } + fmt.Fprintf(&b, "forced roll of the workers: %s\n", forced) + return b.String() +} + +// sleepContext waits for d and returns nil, or returns the context's error as soon as ctx ends. +func sleepContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/test/e2e/scenario_text_unit_test.go b/test/e2e/scenario_text_unit_test.go new file mode 100644 index 0000000..5ac1e01 --- /dev/null +++ b/test/e2e/scenario_text_unit_test.go @@ -0,0 +1,889 @@ +package e2e + +import ( + "context" + "encoding/json" + "errors" + "os" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/ingvarch/tent/test/e2e/vultrapi" +) + +func scenarioSettings() settings { + return settings{ + Region: "ams", Plan: "vc2-1c-1gb", Images: []string{"ubuntu-24.04", "ubuntu-26.04"}, UpgradeFrom: "2.0.6", + } +} + +func TestHACreateArgsBuildThreeServersAndTwoWorkersOnTheImage(t *testing.T) { + args := haCreateArgs(scenarioSettings(), "e2e-abc123-ha", "ubuntu-24.04", "/k/id.pub", "203.0.113.7/32") + want := []string{ + "create", "cluster", "e2e-abc123-ha", + "--provider", "vultr", "--region", "ams", "--machine-type", "vc2-1c-1gb", + "--servers", "3", "--workers", "2", + "--image", "ubuntu-24.04", "--ssh-key", "/k/id.pub", + "--ssh-access", "203.0.113.7/32", "--api-access", "203.0.113.7/32", "--yes", + } + if !reflect.DeepEqual(args, want) { + t.Errorf("args =\n%v\nwant\n%v", args, want) + } + if slices.Contains(args, "--allow-single-server") || slices.Contains(args, "--combined") { + t.Errorf("args %v allow a single server or a combined group", args) + } +} + +func TestUpgradeCreateArgsBuildACombinedGroupOfThreeOnTheOldVersionAndTheImage(t *testing.T) { + args := upgradeCreateArgs(scenarioSettings(), "e2e-abc123-upgrade", "ubuntu-26.04", "/k/id.pub", "203.0.113.7/32") + want := []string{ + "create", "cluster", "e2e-abc123-upgrade", + "--provider", "vultr", "--region", "ams", "--machine-type", "vc2-1c-1gb", + "--combined", "--servers", "3", "--nomad-version", "2.0.6", + "--image", "ubuntu-26.04", "--ssh-key", "/k/id.pub", + "--ssh-access", "203.0.113.7/32", "--api-access", "203.0.113.7/32", "--yes", + } + if !reflect.DeepEqual(args, want) { + t.Errorf("args =\n%v\nwant\n%v", args, want) + } + if slices.Contains(args, "--workers") { + t.Errorf("args %v name workers, which a combined group refuses", args) + } +} + +func decodeCluster(t *testing.T, data []byte) map[string]any { + t.Helper() + var out map[string]any + if err := json.Unmarshal(data, &out); err != nil { + t.Fatalf("the result is not one JSON object: %v\n%s", err, data) + } + return out +} + +func readClusterJSON(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile("testdata/cluster.json") + if err != nil { + t.Fatalf("read the fixture: %v", err) + } + return data +} + +func TestWithRollChangeSetsBothPartsOfTheExtraConfigAndKeepsTheRest(t *testing.T) { + out, err := withRollChange(readClusterJSON(t), "abc123") + if err != nil { + t.Fatalf("withRollChange: %v", err) + } + got := decodeCluster(t, out) + spec := got["spec"].(map[string]any) + extra := spec["nomad"].(map[string]any)["extraConfig"].(map[string]any) + if extra["server"] != "# e2e roll abc123\n" { + t.Errorf("server = %q", extra["server"]) + } + wantClient := "client {\n meta {\n e2e_roll = \"abc123\"\n }\n}\n" + if extra["client"] != wantClient { + t.Errorf("client = %q, want %q", extra["client"], wantClient) + } + if len(extra) != 2 || len(spec["nomad"].(map[string]any)) != 1 { + t.Errorf("spec.nomad = %v, want extraConfig with server and client alone", spec["nomad"]) + } + if got["apiVersion"] != "tent/v1alpha1" || got["kind"] != "Cluster" || + got["metadata"].(map[string]any)["name"] != "e2e-abc123-ha" { + t.Errorf("the header of the cluster changed: %v", got) + } + cloud := spec["cloud"].(map[string]any) + if cloud["provider"] != "vultr" || cloud["region"] != "ams" || len(spec) != 2 { + t.Errorf("the rest of the spec changed: %v", spec) + } +} + +func TestWithRollChangeKeepsOtherFieldsOfNomadAndNumbers(t *testing.T) { + in := `[{"spec":{"nomad":{"version":"2.0.7"},"big":12345678901234567890}}]` + out, err := withRollChange([]byte(in), "r") + if err != nil { + t.Fatalf("withRollChange: %v", err) + } + spec := decodeCluster(t, out)["spec"].(map[string]any) + if spec["nomad"].(map[string]any)["version"] != "2.0.7" { + t.Errorf("spec.nomad.version was lost: %v", spec["nomad"]) + } + if _, ok := spec["nomad"].(map[string]any)["extraConfig"]; !ok { + t.Errorf("spec.nomad has no extraConfig: %v", spec["nomad"]) + } + if !strings.Contains(string(out), "12345678901234567890") { + t.Errorf("a number lost its digits:\n%s", out) + } +} + +func TestWithRollChangeRefusesWhatTheSuiteDidNotMake(t *testing.T) { + for name, in := range map[string]string{ + "an empty array": `[]`, + "two clusters": `[{"spec":{}},{"spec":{}}]`, + "not an array": `{"spec":{}}`, + "not JSON": `spec: {}`, + "an element that is not an object": `[1]`, + "no spec": `[{"metadata":{}}]`, + "a spec that is a list": `[{"spec":[]}]`, + "a nomad that is a string": `[{"spec":{"nomad":"x"}}]`, + "an extraConfig": `[{"spec":{"nomad":{"extraConfig":{"server":"x"}}}}]`, + "an empty extraConfig": `[{"spec":{"nomad":{"extraConfig":{}}}}]`, + } { + t.Run(name, func(t *testing.T) { + if out, err := withRollChange([]byte(in), "r"); err == nil { + t.Errorf("withRollChange accepted %s and gave\n%s", in, out) + } + }) + } +} + +func TestScenarioScriptsAreTheOnesTheRestartStepRuns(t *testing.T) { + if got := bootIDScript(); got != "cat /proc/sys/kernel/random/boot_id" { + t.Errorf("bootIDScript = %q", got) + } + if got := rebootScript(); got != "systemd-run --on-active=2 --unit=e2e-reboot systemctl reboot" { + t.Errorf("rebootScript = %q", got) + } + if got := joinFileScript(); got != "cat /etc/nomad.d/05-join.hcl" { + t.Errorf("joinFileScript = %q", got) + } + want := "journalctl -b -u nomad --no-pager -o cat | grep 'node registration complete' || true" + if got := registrationScript(); got != want { + t.Errorf("registrationScript = %q, want %q", got, want) + } +} + +func TestParseBootID(t *testing.T) { + const id = "0f8fad5b-d9cb-469f-a165-70867728950e" + got, err := parseBootID(" " + id + "\n") + if err != nil || got != id { + t.Errorf("parseBootID = %q, %v; want %q", got, err, id) + } + for name, in := range map[string]string{ + "empty": "", + "blank": " \n", + "two lines": id + "\n" + id + "\n", + "not a UUID": "boot-1", + "too long": id + "0", + "upper case": strings.ToUpper(id), + "a banner": "Warning: Permanently added\n" + id, + "no dashes": strings.ReplaceAll(id, "-", ""), + "bad letters": "0f8fad5b-d9cb-469f-a165-70867728950g", + } { + t.Run(name, func(t *testing.T) { + if got, err := parseBootID(in); err == nil { + t.Errorf("parseBootID(%q) = %q, want an error", in, got) + } + }) + } +} + +func TestRegistrationLinesKeepsOnlyTheRegistrationLines(t *testing.T) { + out := "\nclient: node registration complete\n \nsomething else\nclient: node registration complete\n" + got := registrationLines(out) + want := []string{"client: node registration complete", "client: node registration complete"} + if !reflect.DeepEqual(got, want) { + t.Errorf("registrationLines = %q, want %q", got, want) + } + if got := registrationLines(""); len(got) != 0 { + t.Errorf("registrationLines of nothing = %q", got) + } +} + +const ( + nodesAnswer = `[{"ID":"n1","Name":"c-1","Status":"ready","SchedulingEligibility":"eligible",` + + `"Version":"2.0.7","Address":"10.0.0.5","Drain":false},` + + `{"ID":"n2","Name":"c-2","Status":"down","SchedulingEligibility":"ineligible","Version":"2.0.6",` + + `"Address":"10.0.0.6"}]` + membersAnswer = `{"ServerName":"s.global","Members":[` + + `{"Name":"s-1.global","Addr":"10.0.0.1","Port":4648,"Status":"alive",` + + `"Tags":{"build":"2.0.7","role":"nomad"}},` + + `{"Name":"s-2.global","Addr":"10.0.0.2","Port":4648,"Status":"left","Tags":null}]}` +) + +func TestListNodesReadsTheFieldsTheScenariosUse(t *testing.T) { + _, n := newFakeNomad(t, map[string]func() (int, string){"GET /v1/nodes": fixed(200, nodesAnswer)}) + got, err := listNodes(context.Background(), n) + if err != nil { + t.Fatalf("listNodes: %v", err) + } + want := []nomadNode{ + {ID: "n1", Name: "c-1", Status: "ready", SchedulingEligibility: "eligible", Version: "2.0.7", + Address: "10.0.0.5"}, + {ID: "n2", Name: "c-2", Status: "down", SchedulingEligibility: "ineligible", Version: "2.0.6", + Address: "10.0.0.6"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("listNodes = %+v, want %+v", got, want) + } +} + +func TestListNodesNamesTheRequestOnAnError(t *testing.T) { + _, n := newFakeNomad(t, map[string]func() (int, string){"GET /v1/nodes": fixed(500, "boom")}) + if _, err := listNodes(context.Background(), n); err == nil || !strings.Contains(err.Error(), "GET /v1/nodes") { + t.Errorf("listNodes error = %v, want one that names the request", err) + } +} + +func TestNodeMetaReadsTheMetaOfOneNode(t *testing.T) { + _, n := newFakeNomad(t, map[string]func() (int, string){ + "GET /v1/node/n1": fixed(200, `{"ID":"n1","Meta":{"e2e_roll":"abc123","connect.sidecar_image":"x"}}`), + "GET /v1/node/n2": fixed(200, `{"ID":"n2","Meta":null}`), + "GET /v1/node/a%2Fb": fixed(200, `{"Meta":{"k":"v"}}`), + }) + got, err := nodeMeta(context.Background(), n, "n1") + if err != nil || got["e2e_roll"] != "abc123" || len(got) != 2 { + t.Errorf("nodeMeta(n1) = %v, %v", got, err) + } + if got, err := nodeMeta(context.Background(), n, "n2"); err != nil || len(got) != 0 { + t.Errorf("nodeMeta(n2) = %v, %v; want no meta and no error", got, err) + } + if got, err := nodeMeta(context.Background(), n, "a/b"); err != nil || got["k"] != "v" { + t.Errorf("nodeMeta(a/b) = %v, %v; want the ID escaped in the path", got, err) + } + if _, err := nodeMeta(context.Background(), n, "missing"); err == nil { + t.Errorf("nodeMeta(missing) succeeded on a 404") + } +} + +func TestListMembersReadsTheMembersOfTheAnswer(t *testing.T) { + _, n := newFakeNomad(t, map[string]func() (int, string){"GET /v1/agent/members": fixed(200, membersAnswer)}) + got, err := listMembers(context.Background(), n) + if err != nil { + t.Fatalf("listMembers: %v", err) + } + want := []agentMember{ + {Name: "s-1.global", Addr: "10.0.0.1", Status: "alive", + Tags: map[string]string{"build": "2.0.7", "role": "nomad"}}, + {Name: "s-2.global", Addr: "10.0.0.2", Status: "left"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("listMembers = %+v, want %+v", got, want) + } +} + +func TestListMembersNamesTheRequestOnAnError(t *testing.T) { + _, n := newFakeNomad(t, map[string]func() (int, string){"GET /v1/agent/members": fixed(403, "denied")}) + if _, err := listMembers(context.Background(), n); err == nil || + !strings.Contains(err.Error(), "GET /v1/agent/members") { + t.Errorf("listMembers error = %v, want one that names the request", err) + } +} + +func TestRunningAllocationsCountsTheRunningOnesOfTheJob(t *testing.T) { + _, n := newFakeNomad(t, map[string]func() (int, string){ + "GET /v1/job/e2e-web/allocations": fixed(200, `[{"ID":"a","ClientStatus":"running"},`+ + `{"ID":"b","ClientStatus":"pending"},{"ID":"c","ClientStatus":"running"},`+ + `{"ID":"d","ClientStatus":"complete"}]`), + "GET /v1/job/none/allocations": fixed(200, `[]`), + "GET /v1/job/a%2Fb/allocations": fixed(200, `[{"ID":"a","ClientStatus":"running"}]`), + }) + if got, err := runningAllocations(context.Background(), n, "e2e-web"); err != nil || got != 2 { + t.Errorf("runningAllocations(e2e-web) = %d, %v; want 2", got, err) + } + if got, err := runningAllocations(context.Background(), n, "none"); err != nil || got != 0 { + t.Errorf("runningAllocations(none) = %d, %v; want 0", got, err) + } + if got, err := runningAllocations(context.Background(), n, "a/b"); err != nil || got != 1 { + t.Errorf("runningAllocations(a/b) = %d, %v; want 1 with the job escaped in the path", got, err) + } + if _, err := runningAllocations(context.Background(), n, "missing"); err == nil { + t.Errorf("runningAllocations(missing) succeeded on a 404") + } +} + +func machine(id, label, role, cluster string) vultrapi.Instance { + return vultrapi.Instance{ + ID: id, Label: label, MainIP: "192.0.2.1", + Tags: []string{"tent/cluster=" + cluster, "tent/role=" + role}, + } +} + +func upgradeMachines() []vultrapi.Instance { + return []vultrapi.Instance{ + machine("i1", "up-nodes-4", "combined", "up"), + machine("i2", "up-nodes-5", "combined", "up"), + machine("i3", "up-nodes-6", "combined", "up"), + } +} + +func readyNodes(version string) []nomadNode { + return []nomadNode{ + {ID: "n4", Name: "up-nodes-4", Status: "ready", SchedulingEligibility: "eligible", Version: version}, + {ID: "n5", Name: "up-nodes-5", Status: "ready", SchedulingEligibility: "eligible", Version: version}, + {ID: "n6", Name: "up-nodes-6", Status: "ready", SchedulingEligibility: "eligible", Version: version}, + } +} + +func aliveMembers(version string) []agentMember { + var out []agentMember + for _, name := range []string{"up-nodes-4", "up-nodes-5", "up-nodes-6"} { + out = append(out, agentMember{ + Name: name + ".global", Status: "alive", Tags: map[string]string{"build": version}, + }) + } + return out +} + +func TestCheckVersionsPassesWhenEveryNodeAndMemberIsOnTheVersion(t *testing.T) { + if err := checkVersions(readyNodes("2.0.7"), aliveMembers("2.0.7"), upgradeMachines(), "2.0.7"); err != nil { + t.Errorf("checkVersions: %v", err) + } +} + +func TestCheckVersionsIgnoresMembersThatAreNotAlive(t *testing.T) { + members := append(aliveMembers("2.0.7"), + agentMember{Name: "up-nodes-1.global", Status: "left", Tags: map[string]string{"build": "2.0.6"}}, + agentMember{Name: "up-nodes-2.global", Status: "failed"}) + if err := checkVersions(readyNodes("2.0.7"), members, upgradeMachines(), "2.0.7"); err != nil { + t.Errorf("checkVersions: %v", err) + } +} + +func TestCheckVersionsNamesEveryMismatch(t *testing.T) { + for _, tc := range []struct { + name string + nodes func([]nomadNode) []nomadNode + members func([]agentMember) []agentMember + want []string + }{ + {"a node on the old version", func(n []nomadNode) []nomadNode { n[1].Version = "2.0.6"; return n }, nil, + []string{"up-nodes-5", "version 2.0.6"}}, + {"a node that is down", func(n []nomadNode) []nomadNode { n[0].Status = "down"; return n }, nil, + []string{"up-nodes-4", "status down"}}, + {"a node that is ineligible", func(n []nomadNode) []nomadNode { + n[2].SchedulingEligibility = "ineligible" + return n + }, nil, []string{"up-nodes-6", "ineligible"}}, + {"a node of a machine that is gone", func(n []nomadNode) []nomadNode { + n[0].Name = "up-nodes-1" + return n + }, nil, []string{"up-nodes-1", "up-nodes-4"}}, + {"a missing node", func(n []nomadNode) []nomadNode { return n[:2] }, nil, + []string{"2 nodes", "3 machines"}}, + {"a member on the old version", nil, func(m []agentMember) []agentMember { + m[1].Tags = map[string]string{"build": "2.0.6"} + return m + }, []string{"up-nodes-5", "build 2.0.6"}}, + {"a member without a build tag", nil, func(m []agentMember) []agentMember { m[0].Tags = nil; return m }, + []string{"up-nodes-4", "build none"}}, + {"a missing member", nil, func(m []agentMember) []agentMember { return m[:2] }, + []string{"2 alive members", "3 machines", "up-nodes-6"}}, + {"a member of no machine", nil, func(m []agentMember) []agentMember { + m[2].Name = "old-9.global" + return m + }, []string{"old-9.global", "up-nodes-6"}}, + {"a member named without a region", nil, func(m []agentMember) []agentMember { + m[0].Name = "up-nodes-4" + return m + }, []string{"up-nodes-4", "no region"}}, + } { + t.Run(tc.name, func(t *testing.T) { + nodes, members, machines := readyNodes("2.0.7"), aliveMembers("2.0.7"), upgradeMachines() + if tc.nodes != nil { + nodes = tc.nodes(nodes) + } + if tc.members != nil { + members = tc.members(members) + } + err := checkVersions(nodes, members, machines, "2.0.7") + if err == nil { + t.Fatalf("checkVersions passed") + } + for _, w := range tc.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("error %q does not mention %q", err, w) + } + } + }) + } +} + +func TestCheckVersionsReportsAllProblemsAndTheOtherMembersApart(t *testing.T) { + nodes := readyNodes("2.0.6") + members := append(aliveMembers("2.0.6"), agentMember{Name: "up-nodes-1.global", Status: "left"}) + err := checkVersions(nodes, members, upgradeMachines(), "2.0.7") + if err == nil { + t.Fatal("checkVersions passed") + } + msg := err.Error() + if strings.Count(msg, "version 2.0.6") != 3 || strings.Count(msg, "build 2.0.6") != 3 { + t.Errorf("error %q does not name all six mismatches", msg) + } + if !strings.Contains(msg, "other members: up-nodes-1.global (left)") { + t.Errorf("error %q does not list the other members apart", msg) + } +} + +func TestCheckVersionsWithoutMachinesFails(t *testing.T) { + if err := checkVersions(nil, nil, nil, "2.0.7"); err == nil { + t.Error("checkVersions passed with no machine") + } +} + +func TestCheckRolledNamesTheMachinesThatAreStillListed(t *testing.T) { + before := []vultrapi.Instance{{ID: "a", Label: "x-1"}, {ID: "b", Label: "x-2"}} + if err := checkRolled(before, []vultrapi.Instance{{ID: "c", Label: "x-3"}, {ID: "d", Label: "x-4"}}); err != nil { + t.Errorf("checkRolled: %v", err) + } + err := checkRolled(before, []vultrapi.Instance{{ID: "b", Label: "x-2"}, {ID: "c", Label: "x-3"}}) + if err == nil || !strings.Contains(err.Error(), "x-2") || strings.Contains(err.Error(), "x-1") { + t.Errorf("checkRolled = %v, want an error that names x-2 alone", err) + } + if err := checkRolled(nil, nil); err == nil { + t.Error("checkRolled passed with nothing before; the machines to roll were never recorded") + } +} + +func TestCheckKeptNamesTheMachinesThatAreGone(t *testing.T) { + kept := []vultrapi.Instance{{ID: "a", Label: "w-1"}, {ID: "b", Label: "w-2"}} + if err := checkKept(kept, []vultrapi.Instance{{ID: "b"}, {ID: "a"}, {ID: "z"}}); err != nil { + t.Errorf("checkKept: %v", err) + } + err := checkKept(kept, []vultrapi.Instance{{ID: "a"}}) + if err == nil || !strings.Contains(err.Error(), "w-2") || strings.Contains(err.Error(), "w-1") { + t.Errorf("checkKept = %v, want an error that names w-2 alone", err) + } + if err := checkKept(nil, nil); err == nil { + t.Error("checkKept passed with nothing kept; the machines were never recorded") + } +} + +func haMachines() []vultrapi.Instance { + return []vultrapi.Instance{ + machine("s1", "ha-servers-1", "server", "ha"), + machine("s2", "ha-servers-2", "server", "ha"), + machine("s3", "ha-servers-3", "server", "ha"), + machine("w1", "ha-workers-1", "client", "ha"), + machine("w2", "ha-workers-2", "client", "ha"), + machine("o1", "other-servers-1", "server", "other"), + } +} + +func TestCheckCountsCountsTheServersAndWorkersOfTheCluster(t *testing.T) { + if err := checkCounts(haMachines(), "ha", 3, 2); err != nil { + t.Errorf("checkCounts: %v", err) + } + for name, tc := range map[string]struct{ servers, workers int }{ + "too few servers": {4, 2}, + "too many servers": {2, 2}, + "too few workers": {3, 3}, + "too many workers": {3, 1}, + } { + t.Run(name, func(t *testing.T) { + err := checkCounts(haMachines(), "ha", tc.servers, tc.workers) + if err == nil || !strings.Contains(err.Error(), "ha") { + t.Errorf("checkCounts = %v, want an error that names the cluster", err) + } + }) + } +} + +func TestCheckCountsNamesBothCountsWhenBothAreWrong(t *testing.T) { + err := checkCounts(nil, "ha", 3, 2) + if err == nil { + t.Fatal("checkCounts passed with no machine") + } + for _, want := range []string{"0 servers, want 3", "0 workers, want 2"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("checkCounts = %v, want %q named", err, want) + } + } +} + +func TestServersToDeleteReturnsOnlyTheServersOfTheCluster(t *testing.T) { + machines := []vultrapi.Instance{ + machine("s1", "a-1", "server", "e2e-abc123-ha"), + machine("s2", "a-2", "server", "e2e-abc123-ha"), + machine("w1", "a-w", "client", "e2e-abc123-ha"), + machine("c1", "a-c", "combined", "e2e-abc123-ha"), + machine("x1", "b-1", "server", "e2e-abc123-ha2"), + machine("x2", "b-2", "server", "e2e-abc123"), + machine("x3", "b-3", "server", "prod"), + {ID: "u1", Label: "untagged"}, + {ID: "u2", Label: "role-only", Tags: []string{"tent/role=server"}}, + {ID: "u3", Label: "cluster-only", Tags: []string{"tent/cluster=e2e-abc123-ha"}}, + } + got, err := serversToDelete(machines, "e2e-abc123-ha") + if err != nil { + t.Fatalf("serversToDelete: %v", err) + } + var ids []string + for _, in := range got { + ids = append(ids, in.ID) + } + if !reflect.DeepEqual(ids, []string{"s1", "s2"}) { + t.Errorf("serversToDelete returned %v, want s1 and s2", ids) + } +} + +func TestServersToDeleteRefusesWhatIsNotAnE2ECluster(t *testing.T) { + machines := []vultrapi.Instance{machine("p1", "p-1", "server", "prod"), machine("e1", "e-1", "server", ""), + machine("e2", "e-2", "server", "e2e-"), machine("e3", "e-3", "server", "e2e")} + for _, name := range []string{"", "prod", "e2e-", "e2e"} { + if got, err := serversToDelete(machines, name); err == nil { + t.Errorf("serversToDelete(%q) = %v, want an error", name, got) + } + } +} + +func TestServersToDeleteRefusesAnEmptyResult(t *testing.T) { + machines := []vultrapi.Instance{machine("w1", "w", "client", "e2e-abc123-ha")} + got, err := serversToDelete(machines, "e2e-abc123-ha") + if err == nil || !strings.Contains(err.Error(), "e2e-abc123-ha") { + t.Errorf("serversToDelete = %v, %v, want an error that names the cluster", got, err) + } +} + +func TestServerAddressesListsTheUsableAddressesOfTheRoleInTheCluster(t *testing.T) { + all := []vultrapi.Instance{ + instance("s1", "server", "192.0.2.1", "e2e-a"), + instance("s2", "server", "0.0.0.0", "e2e-a"), + instance("s3", "server", "", "e2e-a"), + instance("s4", "server", "192.0.2.4", "e2e-a"), + instance("c1", "client", "192.0.2.5", "e2e-a"), + instance("s5", "server", "192.0.2.6", "e2e-b"), + } + want := []string{"https://192.0.2.1:4646", "https://192.0.2.4:4646"} + if got := serverAddresses(all, "e2e-a", "server"); !slices.Equal(got, want) { + t.Errorf("serverAddresses = %v, want %v", got, want) + } + if got := serverAddresses(all, "e2e-c", "server"); len(got) != 0 { + t.Errorf("serverAddresses of a cluster without machines = %v, want none", got) + } +} + +func TestLowestLabelPicksTheMachineThatSortsFirst(t *testing.T) { + got, err := lowestLabel([]vultrapi.Instance{{Label: "ha-workers-2"}, {Label: "ha-workers-1"}, {Label: "ha-workers-3"}}) + if err != nil || got.Label != "ha-workers-1" { + t.Errorf("lowestLabel = %q, %v; want ha-workers-1", got.Label, err) + } + if _, err := lowestLabel(nil); err == nil { + t.Error("lowestLabel passed with no machine") + } +} + +func TestNodeNamedFindsTheNodeOfAMachine(t *testing.T) { + nodes := []nomadNode{{ID: "n1", Name: "ha-workers-1"}, {ID: "n2", Name: "ha-workers-2"}} + got, err := nodeNamed(nodes, "ha-workers-2") + if err != nil || got.ID != "n2" { + t.Errorf("nodeNamed = %+v, %v; want n2", got, err) + } + if _, err := nodeNamed(nodes, "ha-workers-3"); err == nil || !strings.Contains(err.Error(), "ha-workers-3") { + t.Errorf("nodeNamed of a missing node = %v, want an error that names it", err) + } + twin := append(slices.Clone(nodes), nomadNode{ID: "n3", Name: "ha-workers-1"}) + if _, err := nodeNamed(twin, "ha-workers-1"); err == nil { + t.Error("nodeNamed passed with two nodes of one name") + } +} + +func TestCheckRollMetaNamesTheNodesWithoutTheRunsValue(t *testing.T) { + good := map[string]map[string]string{"w-1": {"e2e_roll": "abc123"}, "w-2": {"e2e_roll": "abc123", "x": "y"}} + if err := checkRollMeta(good, "abc123", 2); err != nil { + t.Errorf("checkRollMeta: %v", err) + } + bad := map[string]map[string]string{"w-1": {"e2e_roll": "old"}, "w-2": {}, "w-3": {"e2e_roll": "abc123"}} + err := checkRollMeta(bad, "abc123", 3) + if err == nil || !strings.Contains(err.Error(), "w-1") || !strings.Contains(err.Error(), "w-2") || + strings.Contains(err.Error(), "w-3") { + t.Errorf("checkRollMeta = %v, want an error that names w-1 and w-2 alone", err) + } + if err := checkRollMeta(good, "abc123", 3); err == nil { + t.Error("checkRollMeta passed with fewer nodes than wanted") + } +} + +func TestBootProblemWaitsForAnotherBootID(t *testing.T) { + const before = "0f8fad5b-d9cb-469f-a165-70867728950e" + const after = "1f8fad5b-d9cb-469f-a165-70867728950e" + for _, tc := range []struct { + name string + out string + err error + wantErr bool + }{ + {"another boot", after + "\n", nil, false}, + {"the same boot", before + "\n", nil, true}, + {"ssh fails", "", errors.New("connection refused"), true}, + {"garbage", "hello", nil, true}, + } { + t.Run(tc.name, func(t *testing.T) { + got := bootProblem(tc.out, tc.err, before) + if (got != "") != tc.wantErr { + t.Errorf("bootProblem = %q, want a problem: %t", got, tc.wantErr) + } + }) + } +} + +func TestRegisteredProblemNeedsALineAndTheNodeOfTheMachineReadyAndEligibleWithTheSameID(t *testing.T) { + lines := []string{"2026-10-13T10:00:00Z client: node registration complete"} + node := func(id, name, status, eligibility string) nomadNode { + return nomadNode{ID: id, Name: name, Status: status, SchedulingEligibility: eligibility} + } + ready := []nomadNode{node("n1", "w-0", "ready", "eligible")} + if got := registeredProblem(lines, ready, "w-0", "n1"); got != "" { + t.Errorf("registeredProblem = %q, want none", got) + } + for _, tc := range []struct { + name string + lines []string + nodes []nomadNode + want string + }{ + {"no line", nil, ready, "registration"}, + {"another node ID", lines, []nomadNode{node("n2", "w-0", "ready", "eligible")}, "n1"}, + {"down", lines, []nomadNode{node("n1", "w-0", "down", "eligible")}, "down"}, + {"ineligible", lines, []nomadNode{node("n1", "w-0", "ready", "ineligible")}, "ineligible"}, + {"the old ID is listed under another name", lines, []nomadNode{node("n1", "w-9", "ready", "eligible")}, "w-0"}, + {"the machine's node has another ID", lines, []nomadNode{ + node("n1", "w-1", "ready", "eligible"), node("n2", "w-0", "ready", "eligible")}, "n2"}, + {"two nodes are named after the machine", lines, []nomadNode{ + node("n1", "w-0", "ready", "eligible"), node("n2", "w-0", "down", "eligible")}, "2 nodes"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := registeredProblem(tc.lines, tc.nodes, "w-0", "n1"); !strings.Contains(got, tc.want) { + t.Errorf("registeredProblem = %q, want it to hold %q", got, tc.want) + } + }) + } +} + +func TestRestartRecordHoldsTheEvidenceOfTheRestart(t *testing.T) { + got := restartRecord(restartEvidence{ + Machine: "ha-workers-1", NodeID: "n1", BootBefore: "b1", BootAfter: "b2", + BootWait: 90 * time.Second, RegisterWait: 30 * time.Second, + Registrations: []string{"line one", "line two"}, JoinFile: "retry_join = [\"10.0.0.9\"]\n", + }) + for _, want := range []string{ + "machine: ha-workers-1", "node ID: n1", "boot ID before: b1", "boot ID after: b2", + "wait for the new boot: 1m30s", "wait for the registration: 30s", + "line one\nline two\n", "retry_join = [\"10.0.0.9\"]\n", + } { + if !strings.Contains(got, want) { + t.Errorf("record does not hold %q:\n%s", want, got) + } + } +} + +func TestAllocationsProblemNeedsExactlyTheWantedRunningAllocations(t *testing.T) { + if got := allocationsProblem(2, "e2e-web", 2); got != "" { + t.Errorf("allocationsProblem(2, 2) = %q, want none", got) + } + for _, running := range []int{0, 1, 3} { + got := allocationsProblem(running, "e2e-web", 2) + if !strings.Contains(got, "e2e-web") || !strings.Contains(got, "want 2") { + t.Errorf("allocationsProblem(%d, 2) = %q, want a problem that names the job and the count", running, got) + } + } +} + +func TestParseStoredBackupReadsNameAndSize(t *testing.T) { + got, err := parseStoredBackup([]byte(`{"cluster":"c","name":"b-1","time":"t","path":"p","size":2048}`)) + if err != nil { + t.Fatal(err) + } + if want := (storedBackup{Name: "b-1", Size: 2048}); got != want { + t.Errorf("parseStoredBackup = %+v, want %+v", got, want) + } +} + +func TestParseStoredBackupRefusesWhatIsNoBackup(t *testing.T) { + for name, in := range map[string]string{ + "not JSON": `Backup stored`, + "no name": `{"size":10}`, + "empty file": `{"name":"b-1","size":0}`, + "no size": `{"name":"b-1"}`, + } { + if _, err := parseStoredBackup([]byte(in)); err == nil { + t.Errorf("%s: parseStoredBackup(%s) passed", name, in) + } + } +} + +func TestCheckBackupListNeedsOneBackupOfTheNameWithTheServers(t *testing.T) { + one := `[{"name":"b-1","servers":"current"}]` + if err := checkBackupList([]byte(one), "b-1", "current"); err != nil { + t.Errorf("checkBackupList(one current backup) = %v", err) + } + for name, tc := range map[string]struct{ in, servers, want string }{ + "another servers": {one, "earlier", "current"}, + "another name": {`[{"name":"b-2","servers":"current"}]`, "current", "b-2"}, + "two backups": { + `[{"name":"b-1","servers":"current"},{"name":"b-2","servers":"current"}]`, "current", "2 backups", + }, + "no backup": {`[]`, "current", "0 backups"}, + "not JSON": {`no backups`, "current", "read the backups"}, + } { + err := checkBackupList([]byte(tc.in), "b-1", tc.servers) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: checkBackupList = %v, want an error that holds %q", name, err, tc.want) + } + } +} + +func TestCheckRestoredNeedsTheLineOfTheBackupAndTheCluster(t *testing.T) { + line := "Restored: backup b-1 into cluster e2e-x-ha.\n" + if err := checkRestored([]byte(line), "b-1", "e2e-x-ha"); err != nil { + t.Errorf("checkRestored(%q) = %v", line, err) + } + for _, in := range []string{ + "", "Restored: backup b-2 into cluster e2e-x-ha.\n", "Restored: backup b-1 into cluster e2e-y-ha.\n", + "Would restore: backup b-1 into cluster e2e-x-ha.\n", + } { + if err := checkRestored([]byte(in), "b-1", "e2e-x-ha"); err == nil { + t.Errorf("checkRestored(%q) passed", in) + } + } +} + +func TestAddressesOfMembersAndNodesAreSortedAndComplete(t *testing.T) { + members := []agentMember{{Name: "s2.global", Addr: "10.0.0.9"}, {Name: "s1.global", Addr: "10.0.0.3"}} + if got, want := memberAddresses(members), []string{"10.0.0.3", "10.0.0.9"}; !slices.Equal(got, want) { + t.Errorf("memberAddresses = %v, want %v", got, want) + } + nodes := []nomadNode{{Name: "w2", Address: "10.0.0.8"}, {Name: "w1", Address: "10.0.0.4"}} + if got, want := nodeAddresses(nodes), []string{"10.0.0.4", "10.0.0.8"}; !slices.Equal(got, want) { + t.Errorf("nodeAddresses = %v, want %v", got, want) + } + if got := memberAddresses(nil); len(got) != 0 { + t.Errorf("memberAddresses(nil) = %v, want none", got) + } +} + +func TestBackupRecordHoldsTheBackupTheTimesTheAddressesAndTheForcedRoll(t *testing.T) { + e := backupEvidence{ + Backup: storedBackup{Name: "b-1", Size: 4096}, + Times: []stepTime{{"create", 20 * time.Second}, {"rebuild", 9*time.Minute + 30*time.Second}}, + ServersBefore: []string{"10.0.0.3", "10.0.0.9"}, WorkersBefore: []string{"10.0.0.4"}, + ServersAfter: []string{"10.0.0.5"}, WorkersAfter: []string{"10.0.0.4"}, + } + got := backupRecord(e) + for _, want := range []string{ + "backup: b-1\nsize: 4096\n", "create: 20s\n", "rebuild: 9m30s\n", + "servers before: 10.0.0.3 10.0.0.9\n", "workers before: 10.0.0.4\n", + "servers after: 10.0.0.5\n", "workers after: 10.0.0.4\n", "forced roll of the workers: no\n", + } { + if !strings.Contains(got, want) { + t.Errorf("record does not hold %q:\n%s", want, got) + } + } + e.Forced = true + if got := backupRecord(e); !strings.Contains(got, "forced roll of the workers: yes\n") { + t.Errorf("record of a forced roll:\n%s", got) + } +} + +func TestBackupRecordOfAnEmptyEvidenceSaysNoneForEveryList(t *testing.T) { + got := backupRecord(backupEvidence{}) + for _, want := range []string{ + "servers before: none\n", "workers before: none\n", "servers after: none\n", "workers after: none\n", + } { + if !strings.Contains(got, want) { + t.Errorf("record does not hold %q:\n%s", want, got) + } + } +} + +func TestSleepContextWaitsTheTimeAndEndsWithTheContext(t *testing.T) { + const d = 20 * time.Millisecond + begin := time.Now() + if err := sleepContext(context.Background(), d); err != nil { + t.Errorf("sleepContext = %v, want nil after the time", err) + } + if took := time.Since(begin); took < d { + t.Errorf("sleepContext returned after %s, want at least %s", took, d) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + start := time.Now() + err := sleepContext(ctx, time.Hour) + if !errors.Is(err, context.Canceled) { + t.Errorf("sleepContext of an ended context = %v, want context.Canceled", err) + } + if time.Since(start) > 5*time.Second { + t.Errorf("sleepContext kept waiting for the time after the context ended") + } +} + +func TestUpgradeTargetReadsTheNomadVersionOfTheSpec(t *testing.T) { + got, err := upgradeTarget([]byte(`[{"spec":{"nomad":{"version":"2.0.7"}}}]`), "2.0.6") + if err != nil || got != "2.0.7" { + t.Errorf("upgradeTarget = %q, %v; want 2.0.7", got, err) + } +} + +func TestUpgradeTargetRefusesWhatIsNotANewVersion(t *testing.T) { + for name, in := range map[string]string{ + "the old version": `[{"spec":{"nomad":{"version":"2.0.6"}}}]`, + "no version": `[{"spec":{"nomad":{}}}]`, + "no nomad": `[{"spec":{}}]`, + "a version that is a number": `[{"spec":{"nomad":{"version":2}}}]`, + "an empty array": `[]`, + "two clusters": `[{"spec":{}},{"spec":{}}]`, + "not JSON": `spec: {}`, + } { + t.Run(name, func(t *testing.T) { + if got, err := upgradeTarget([]byte(in), "2.0.6"); err == nil { + t.Errorf("upgradeTarget = %q, want an error", got) + } + }) + } +} + +func TestUntestedVersionWarningFindsTheLineOfTheWarning(t *testing.T) { + stderr := "WARNING: another warning\nWARNING: Nomad 2.0.6 is not tested by this tent; channel stable tests 2.0.7\n" + want := "WARNING: Nomad 2.0.6 is not tested by this tent; channel stable tests 2.0.7" + if got := untestedVersionWarning([]byte(stderr)); got != want { + t.Errorf("warning = %q, want %q", got, want) + } + if got := untestedVersionWarning([]byte("WARNING: another warning\n")); got != "" { + t.Errorf("warning = %q for stderr without one, want none", got) + } +} + +func TestCheckValidVersionWantsTheVersionOnTheValidLine(t *testing.T) { + ok := "cluster e2e-a-upgrade is valid: 3 servers and 0 clients run Nomad 2.0.7\n" + if err := checkValidVersion([]byte(ok), "e2e-a-upgrade", "2.0.7"); err != nil { + t.Errorf("checkValidVersion: %v", err) + } + for name, out := range map[string]string{ + "another version": "cluster e2e-a-upgrade is valid: 3 servers and 0 clients run Nomad 2.0.6\n", + "a longer version": "cluster e2e-a-upgrade is valid: 3 servers and 0 clients run Nomad 2.0.70\n", + "another cluster": "cluster other is valid: 3 servers and 0 clients run Nomad 2.0.7\n", + "no valid line": "cluster e2e-a-upgrade has problems\n", + "the version elsewhere": "Nomad 2.0.7 cluster e2e-a-upgrade is valid: 3 servers and 0 clients\n", + "empty": "", + } { + t.Run(name, func(t *testing.T) { + if err := checkValidVersion([]byte(out), "e2e-a-upgrade", "2.0.7"); err == nil { + t.Errorf("checkValidVersion accepted %q", out) + } + }) + } +} + +func TestUpgradedVersionReadsTheVersionOfTheLineThatUpgradeClusterPrints(t *testing.T) { + out := "cluster e2e-a-upgrade: spec.nomad.version is now 2.0.7; the cluster was on Nomad 2.0.6.\n" + + "Next: tent update cluster, then tent rolling-update cluster.\n" + if got, err := upgradedVersion([]byte(out), "e2e-a-upgrade"); err != nil || got != "2.0.7" { + t.Errorf("upgradedVersion = %q, %v; want 2.0.7", got, err) + } + for name, in := range map[string]string{ + "another cluster": "cluster other: spec.nomad.version is now 2.0.7; the cluster was on Nomad 2.0.6.\n", + "the plan, not the result": "cluster e2e-a-upgrade: upgrade Nomad from 2.0.6 to 2.0.7, " + + "which channel stable recommends.\n", + "nothing to upgrade": "cluster e2e-a-upgrade: nothing to upgrade: its Nomad version is 2.0.7.\n", + "no version": "cluster e2e-a-upgrade: spec.nomad.version is now ; the cluster was on Nomad 2.0.6.\n", + "empty": "", + } { + t.Run(name, func(t *testing.T) { + if got, err := upgradedVersion([]byte(in), "e2e-a-upgrade"); err == nil { + t.Errorf("upgradedVersion = %q, want an error", got) + } + }) + } +} diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 3679721..d673a5c 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -8,6 +8,7 @@ import ( "slices" "sort" "strings" + "sync" "time" ) @@ -118,26 +119,88 @@ func listOrNone(items []string) string { return strings.Join(items, ", ") } -// readServiceAnswers asks Nomad for the allocations of job, the checks of each running one and the registrations of -// service. On an error it returns what it had read until then. -func readServiceAnswers(ctx context.Context, n *nomadAPI, job, service string) (serviceAnswers, error) { - var a serviceAnswers - if err := n.get(ctx, "/v1/job/"+url.PathEscape(job)+"/allocations", &a.Allocations); err != nil { - return a, err +// serviceReads is what the reads of a service's answers returned: the answers that were read and the error of each +// read that failed. +type serviceReads struct { + answers serviceAnswers + // allocationsErr is the error of the read of the job's allocations; no check is read then. + allocationsErr error + registrationsErr error + // checkErrs holds the error of the checks read of each running allocation that failed, by allocation ID. + checkErrs map[string]error +} + +// readServiceReads reads the allocations of job, then the checks of each running allocation and the registrations of +// service at once. Each read ends at timeout; a timeout of 0 sets none. answers holds only what was read. +func readServiceReads(ctx context.Context, n *nomadAPI, job, service string, timeout time.Duration) serviceReads { + var r serviceReads + within := func(read func(context.Context) error) error { + readCtx := ctx + if timeout > 0 { + var cancel context.CancelFunc + readCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + return read(readCtx) } - a.Checks = map[string]map[string]allocationCheck{} - for _, alloc := range a.Allocations { - if alloc.ClientStatus != "running" { + if r.allocationsErr = within(func(ctx context.Context) error { + return n.get(ctx, "/v1/job/"+url.PathEscape(job)+"/allocations", &r.answers.Allocations) + }); r.allocationsErr != nil { + return r + } + var running []string + for _, alloc := range r.answers.Allocations { + if alloc.ClientStatus == "running" { + running = append(running, alloc.ID) + } + } + checks := make([]map[string]allocationCheck, len(running)) + errs := make([]error, len(running)) + var wg sync.WaitGroup + wg.Go(func() { + r.registrationsErr = within(func(ctx context.Context) error { + return n.get(ctx, "/v1/service/"+url.PathEscape(service), &r.answers.Registrations) + }) + }) + for i, id := range running { + wg.Go(func() { + errs[i] = within(func(ctx context.Context) error { + return n.get(ctx, "/v1/allocation/"+url.PathEscape(id)+"/checks", &checks[i]) + }) + }) + } + wg.Wait() + r.answers.Checks = map[string]map[string]allocationCheck{} + r.checkErrs = map[string]error{} + for i, id := range running { + if errs[i] != nil { + r.checkErrs[id] = errs[i] continue } - var checks map[string]allocationCheck - if err := n.get(ctx, "/v1/allocation/"+url.PathEscape(alloc.ID)+"/checks", &checks); err != nil { - return a, err + r.answers.Checks[id] = checks[i] + } + return r +} + +// err returns the first error in this order: the allocations, the checks in the order of the allocations, then the +// registrations. +func (r serviceReads) err() error { + if r.allocationsErr != nil { + return r.allocationsErr + } + for _, alloc := range r.answers.Allocations { + if err := r.checkErrs[alloc.ID]; err != nil { + return err } - a.Checks[alloc.ID] = checks } - err := n.get(ctx, "/v1/service/"+url.PathEscape(service), &a.Registrations) - return a, err + return r.registrationsErr +} + +// readServiceAnswers asks Nomad for the allocations of job, the checks of each running one and the registrations of +// service. On an error it returns what it had read. +func readServiceAnswers(ctx context.Context, n *nomadAPI, job, service string) (serviceAnswers, error) { + r := readServiceReads(ctx, n, job, service, 0) + return r.answers, r.err() } // waitService reads the answers about service every `every` until it is ready or timeout has passed. diff --git a/test/e2e/service_unit_test.go b/test/e2e/service_unit_test.go index efdbe98..5179444 100644 --- a/test/e2e/service_unit_test.go +++ b/test/e2e/service_unit_test.go @@ -356,3 +356,26 @@ func TestWebJobFileDefinesTheJobAndTheServiceTheStepWaitsFor(t *testing.T) { t.Errorf("resources = %v, want cpu 50 and memory 32", res) } } + +func TestReadServiceReadsKeepsTheErrorOfEachReadAndBoundsEachByTheTimeout(t *testing.T) { + gate := make(chan struct{}) + answers := readyAnswers() + answers["GET /v1/job/e2e-web/allocations"] = allocationsOf(allocRunning, allocOther) + answers["GET /v1/service/e2e-web"] = fixed(200, `[{"AllocID":"`+allocRunning+`"}]`) + answers[checksPath(allocOther)] = func() (int, string) { <-gate; return 200, `{}` } + _, n := newFakeNomad(t, answers) + openOnCleanup(t, gate) + got := readServiceReads(t.Context(), n, "e2e-web", "e2e-web", 40*time.Millisecond) + if got.allocationsErr != nil || got.registrationsErr != nil { + t.Errorf("errors = %v, %v; want none for the reads that answered", got.allocationsErr, got.registrationsErr) + } + if err := got.checkErrs[allocOther]; err == nil || !strings.Contains(err.Error(), "deadline exceeded") { + t.Errorf("check error of the slow allocation = %v, want the timeout", err) + } + if err := got.checkErrs[allocRunning]; err != nil { + t.Errorf("check error of the fast allocation = %v, want none", err) + } + if got.answers.missing("e2e-web") != nil { + t.Errorf("answers = %v, want the fast allocation ready", got.answers) + } +} diff --git a/test/e2e/settings_test.go b/test/e2e/settings_test.go index 3e1684d..620f53d 100644 --- a/test/e2e/settings_test.go +++ b/test/e2e/settings_test.go @@ -19,16 +19,27 @@ type settings struct { Plan string Runner string Images []string - Keep bool + // UpgradeFrom is the Nomad version that the upgrade scenario builds its cluster on. + UpgradeFrom string + Keep bool } // String describes the run without the key, the node URL or its digest. func (s settings) String() string { - return fmt.Sprintf("region %s, plan %s, images %s, keep %t", s.Region, s.Plan, strings.Join(s.Images, ","), s.Keep) + return fmt.Sprintf("region %s, plan %s, images %s, upgrade from %s, keep %t", + s.Region, s.Plan, strings.Join(s.Images, ","), s.UpgradeFrom, s.Keep) } +// haImage is the image of the HA scenario: the first of the list. +func (s settings) haImage() string { return s.Images[0] } + +// upgradeImage is the image of the upgrade scenario: the last of the list. +func (s settings) upgradeImage() string { return s.Images[len(s.Images)-1] } + var imageForm = regexp.MustCompile(`^ubuntu-\d\d\.\d\d$`) +var versionForm = regexp.MustCompile(`^\d+\.\d+\.\d+$`) + // makeE2EHint ends the error about missing variables. const makeE2EHint = "; make e2e sets all but VULTR_API_KEY" @@ -60,6 +71,10 @@ func readSettings(getenv func(string) string) (settings, error) { return settings{}, err } s.Images = images + s.UpgradeFrom = valueOr(getenv("E2E_UPGRADE_FROM"), "2.0.6") + if !versionForm.MatchString(s.UpgradeFrom) { + return settings{}, fmt.Errorf("E2E_UPGRADE_FROM: %q is not of the form N.N.N", s.UpgradeFrom) + } return s, nil } diff --git a/test/e2e/settings_unit_test.go b/test/e2e/settings_unit_test.go index 175bd48..aa18a99 100644 --- a/test/e2e/settings_unit_test.go +++ b/test/e2e/settings_unit_test.go @@ -28,13 +28,14 @@ func TestReadSettingsDefaults(t *testing.T) { t.Fatalf("readSettings: %v", err) } want := settings{ - Key: "key-value-for-tests", - NodeURL: "https://node.example/tent-node", - NodeSHA: "0123abcd", - Tent: "/repo/bin/tent", - Region: "ams", - Plan: "vc2-1c-1gb", - Images: []string{"ubuntu-24.04", "ubuntu-26.04"}, + Key: "key-value-for-tests", + NodeURL: "https://node.example/tent-node", + NodeSHA: "0123abcd", + Tent: "/repo/bin/tent", + Region: "ams", + Plan: "vc2-1c-1gb", + Images: []string{"ubuntu-24.04", "ubuntu-26.04"}, + UpgradeFrom: "2.0.6", } if !reflect.DeepEqual(got, want) { t.Errorf("settings = %+v, want %+v", got, want) @@ -47,21 +48,23 @@ func TestReadSettingsOverrides(t *testing.T) { vars["E2E_PLAN"] = "vc2-2c-4gb" vars["E2E_IMAGES"] = "ubuntu-22.04" vars["E2E_KEEP"] = "1" + vars["E2E_UPGRADE_FROM"] = "2.0.5" vars["RUNNER_ADDR"] = "203.0.113.7" got, err := readSettings(envOf(vars)) if err != nil { t.Fatalf("readSettings: %v", err) } want := settings{ - Key: "key-value-for-tests", - NodeURL: "https://node.example/tent-node", - NodeSHA: "0123abcd", - Tent: "/repo/bin/tent", - Region: "fra", - Plan: "vc2-2c-4gb", - Images: []string{"ubuntu-22.04"}, - Keep: true, - Runner: "203.0.113.7", + Key: "key-value-for-tests", + NodeURL: "https://node.example/tent-node", + NodeSHA: "0123abcd", + Tent: "/repo/bin/tent", + Region: "fra", + Plan: "vc2-2c-4gb", + Images: []string{"ubuntu-22.04"}, + Keep: true, + Runner: "203.0.113.7", + UpgradeFrom: "2.0.5", } if !reflect.DeepEqual(got, want) { t.Errorf("settings = %+v, want %+v", got, want) @@ -170,9 +173,10 @@ func TestSettingsNeverPrintTheKey(t *testing.T) { } } -func TestSettingsStringNamesRegionPlanImagesAndKeep(t *testing.T) { +func TestSettingsStringNamesRegionPlanImagesUpgradeVersionAndKeep(t *testing.T) { s := settings{Key: "k", Region: "ams", Plan: "vc2-1c-1gb", Images: []string{"ubuntu-24.04", "ubuntu-26.04"}} - want := "region ams, plan vc2-1c-1gb, images ubuntu-24.04,ubuntu-26.04, keep false" + s.UpgradeFrom = "2.0.6" + want := "region ams, plan vc2-1c-1gb, images ubuntu-24.04,ubuntu-26.04, upgrade from 2.0.6, keep false" if got := s.String(); got != want { t.Errorf("String() = %q, want %q", got, want) } @@ -184,3 +188,57 @@ func TestParseImagesRefusesAnImageNamedTwice(t *testing.T) { t.Errorf("parseImages error = %v, want the image named twice", err) } } + +func TestReadSettingsUpgradeFrom(t *testing.T) { + for _, tt := range []struct { + value string + want string + wantErr bool + }{ + {"", "2.0.6", false}, + {"2.0.5", "2.0.5", false}, + {"10.20.30", "10.20.30", false}, + {"2.0", "", true}, + {"2.0.6.1", "", true}, + {"v2.0.6", "", true}, + {"2.0.6-rc1", "", true}, + {"2.0.x", "", true}, + {" 2.0.6", "", true}, + {"2.0.6\n", "", true}, + } { + vars := requiredEnv() + vars["E2E_UPGRADE_FROM"] = tt.value + got, err := readSettings(envOf(vars)) + if tt.wantErr { + if err == nil || !strings.Contains(err.Error(), "E2E_UPGRADE_FROM") || + !strings.Contains(err.Error(), fmt.Sprintf("%q", tt.value)) { + t.Errorf("E2E_UPGRADE_FROM=%q: error = %v, want one naming the variable and the value", tt.value, err) + } + continue + } + if err != nil { + t.Errorf("E2E_UPGRADE_FROM=%q: %v", tt.value, err) + } else if got.UpgradeFrom != tt.want { + t.Errorf("E2E_UPGRADE_FROM=%q: UpgradeFrom = %q, want %q", tt.value, got.UpgradeFrom, tt.want) + } + } +} + +func TestHAImageIsTheFirstAndUpgradeImageTheLast(t *testing.T) { + for _, tt := range []struct { + images []string + wantHA, wantUp string + }{ + {[]string{"ubuntu-24.04", "ubuntu-26.04"}, "ubuntu-24.04", "ubuntu-26.04"}, + {[]string{"ubuntu-22.04"}, "ubuntu-22.04", "ubuntu-22.04"}, + {[]string{"ubuntu-22.04", "ubuntu-24.04", "ubuntu-26.04"}, "ubuntu-22.04", "ubuntu-26.04"}, + } { + s := settings{Images: tt.images} + if got := s.haImage(); got != tt.wantHA { + t.Errorf("haImage() of %q = %q, want %q", tt.images, got, tt.wantHA) + } + if got := s.upgradeImage(); got != tt.wantUp { + t.Errorf("upgradeImage() of %q = %q, want %q", tt.images, got, tt.wantUp) + } + } +} diff --git a/test/e2e/smoke_test.go b/test/e2e/smoke_test.go index 7ccb61b..544b876 100644 --- a/test/e2e/smoke_test.go +++ b/test/e2e/smoke_test.go @@ -12,19 +12,11 @@ import ( ) const ( - createTimeout = 25 * time.Minute - validateTimeout = 7 * time.Minute - deleteTimeout = 15 * time.Minute - purgeTimeout = time.Minute - serviceEvery = 5 * time.Second - serviceTimeout = 5 * time.Minute - metadataEvery = 5 * time.Second - metadataTimeout = 4 * time.Minute - introEvery = 5 * time.Second - introTimeout = 90 * time.Second - leftoversEvery = 10 * time.Second - leftoversTimeout = 2 * time.Minute - noClustersText = "no clusters in" + createTimeout = 25 * time.Minute + metadataEvery = 5 * time.Second + metadataTimeout = 4 * time.Minute + introEvery = 5 * time.Second + introTimeout = 90 * time.Second ) // TestSmoke builds a Nomad cluster for each image, checks it from outside and deletes it. The images run in @@ -38,127 +30,31 @@ func TestSmoke(t *testing.T) { } } -// smokeCluster is the cluster of one image and what its steps hand to each other. -type smokeCluster struct { - name string - image string - nomad *nomadAPI - // deleteRan is true once the delete step has started; deleteDone is true once tent delete exited 0. The cleanup - // deletes the cluster only when no delete ran or a signal ended it (deleteInCleanup). - deleteRan bool - deleteDone bool -} - -func (c *smokeCluster) exportDir() string { - return filepath.Join(suite.Dir, "export-"+c.name) -} - -// smokeStep is one subtest of an image. -type smokeStep struct { - name string - run func(*testing.T) -} - -// smokeImage runs the steps of one image. Each step logs the time it took. +// smokeImage runs the steps of one image. func smokeImage(t *testing.T, image string) { - c := &smokeCluster{name: clusterName(suite.ID, image), image: image} - t.Cleanup(func() { - if err := os.RemoveAll(c.exportDir()); err != nil { - t.Logf("remove the export of %s, which holds its token and client key: %v", c.name, err) - } - }) - t.Cleanup(func() { c.cleanup(t) }) - - steps := []smokeStep{ + c := newCluster(t, clusterName(suite.ID, image), image) + steps := []clusterStep{ {"create", c.create}, {"validate", func(t *testing.T) { c.validate(t, "validate") }}, - {"export", c.export}, + {"export", func(t *testing.T) { c.export(t, "export") }}, {"service", c.service}, {"metadata", c.metadata}, {"intro-token", c.introToken}, {"validate-again", func(t *testing.T) { c.validate(t, "validate-again") }}, } if !suite.Settings.Keep { - steps = append(steps, smokeStep{"delete", c.delete}, smokeStep{"leftovers", c.leftovers}) - } - for _, step := range steps { - ok := t.Run(step.name, func(t *testing.T) { - start := time.Now() - defer func() { t.Logf("%s %s took %s", c.name, step.name, time.Since(start).Round(time.Second)) }() - step.run(t) - }) - if !ok { - t.FailNow() - } + steps = append(steps, clusterStep{"delete", c.delete}, clusterStep{"leftovers", c.leftovers}) } + runSteps(t, c, steps) } -// tent runs a tent command of the cluster and fails the step when it does not exit with 0. -func (c *smokeCluster) tent(t *testing.T, timeout time.Duration, step string, args ...string) tentResult { - t.Helper() - ctx, cancel := context.WithTimeout(stepContext(suite.Ctx, t), timeout) - defer cancel() - res, err := suite.Tent.run(ctx, c.name, step, args...) - if err != nil { - t.Fatalf("tent %s: %v", step, err) - } - if res.Code != 0 { - t.Fatal(tentFailure(step, res, suite.Tent.logPath(c.name, step))) - } - return res -} - -func (c *smokeCluster) create(t *testing.T) { +func (c *cluster) create(t *testing.T) { args := createArgs(suite.Settings, c.name, c.image, suite.SSHKey+".pub", suite.Runner) c.tent(t, createTimeout, "create", args...) } -func (c *smokeCluster) validate(t *testing.T, step string) { - c.tent(t, validateTimeout, step, validateArgs(c.name)...) -} - -// export has tent write the files that reach Nomad and builds the client from them. -func (c *smokeCluster) export(t *testing.T) { - res := c.tent(t, time.Minute, "export", "export", "nomad", c.name, "--dir", c.exportDir(), "-o", "json") - x, err := parseExport(res.Stdout) - if err != nil { - t.Fatal(err) - } - if c.nomad, err = newNomad(x); err != nil { - t.Fatal(err) - } -} - -// runJob submits the job of a file in testdata and purges it when the step ends. -func (c *smokeCluster) runJob(ctx context.Context, t *testing.T, file, job string) { - t.Helper() - text, err := os.ReadFile(filepath.Join("testdata", file)) - if err != nil { - t.Fatalf("read the job file: %v", err) - } - if err := submitJob(ctx, c.nomad, string(text)); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), purgeTimeout) - defer cancel() - if err := purgeJob(ctx, c.nomad, job); err != nil { - t.Errorf("purge the job %s: %v", job, err) - } - }) -} - -// service runs the web job and waits until its allocation, its check and its registration agree. -func (c *smokeCluster) service(t *testing.T) { - ctx := stepContext(suite.Ctx, t) - c.runJob(ctx, t, "web.nomad.hcl", webJob) - if err := waitService(ctx, c.nomad, webJob, webService, serviceEvery, serviceTimeout); err != nil { - t.Fatalf("service %s: %v", webService, err) - } -} - // metadata runs a probe on each network a container can use and checks that none reaches the metadata service. -func (c *smokeCluster) metadata(t *testing.T) { +func (c *cluster) metadata(t *testing.T) { ctx := stepContext(suite.Ctx, t) c.runJob(ctx, t, "metadata.nomad.hcl", metadataJob) results, err := collectMetadata(ctx, c.nomad, metadataEvery, metadataTimeout) @@ -174,7 +70,7 @@ func (c *smokeCluster) metadata(t *testing.T) { // introToken starts a second agent without an intro token on the client machine and checks that the server // refuses it. The agent is stopped and its files are removed when the step ends. -func (c *smokeCluster) introToken(t *testing.T) { +func (c *cluster) introToken(t *testing.T) { ctx := stepContext(suite.Ctx, t) instances, err := suite.Vultr.Instances(ctx) if err != nil { @@ -219,7 +115,7 @@ func (c *smokeCluster) introToken(t *testing.T) { // readIntro reads the three places that show whether the rogue agent was refused: the node list, the server's // journal since epoch and the rogue's log on the client machine. It returns what it read when a read fails. -func (c *smokeCluster) readIntro(ctx context.Context, serverIP, clientIP string, epoch int64) (introEvidence, error) { +func (c *cluster) readIntro(ctx context.Context, serverIP, clientIP string, epoch int64) (introEvidence, error) { var ev introEvidence var err error if ev.Nodes, err = listNodeNames(ctx, c.nomad); err != nil { @@ -233,48 +129,3 @@ func (c *smokeCluster) readIntro(ctx context.Context, serverIP, clientIP string, ev.RogueLog, err = suite.sshRun(ctx, clientIP, rogueLogScript()) return ev, err } - -func (c *smokeCluster) delete(t *testing.T) { - c.deleteRan = true - c.tent(t, deleteTimeout, "delete", "delete", "cluster", c.name, "--yes") - c.deleteDone = true -} - -// leftovers checks that Vultr holds nothing of the cluster and that its store is empty. -func (c *smokeCluster) leftovers(t *testing.T) { - ctx := stepContext(suite.Ctx, t) - if err := waitNoLeftovers(ctx, suite.Vultr, c.name, leftoversEvery, leftoversTimeout, t.Logf); err != nil { - t.Fatalf("after the delete of %s: %v", c.name, err) - } - res := c.tent(t, time.Minute, "get-clusters", "get", "clusters") - if !strings.Contains(string(res.Stderr), noClustersText) { - t.Fatalf("tent get clusters: stderr does not hold %q: %s\nlog: %s", noClustersText, - lastLines(string(res.Stderr), stderrTailLines), suite.Tent.logPath(c.name, "get-clusters")) - } -} - -// cleanup deletes the cluster when the delete step did not run or a signal ended it (deleteInCleanup), with a context -// of its own because the test's has ended. A run that keeps its clusters deletes nothing and logs the command that -// does. -func (c *smokeCluster) cleanup(t *testing.T) { - if !deleteInCleanup(c.deleteRan, c.deleteDone, suite.Ctx.Err() != nil) { - return - } - if suite.Settings.Keep { - t.Logf("keeping cluster %s; delete it with: %s", c.name, suite.Tent.deleteCommand(c.name)) - return - } - ctx, cancel := context.WithTimeout(context.Background(), deleteTimeout) - defer cancel() - step := "cleanup-delete" - res, err := suite.Tent.run(ctx, c.name, step, "delete", "cluster", c.name, "--yes") - switch { - case err != nil: - t.Errorf("cleanup: delete cluster %s: %v", c.name, err) - case res.Code != 0: - t.Errorf("cleanup: delete cluster %s failed:\n%s", c.name, - tentFailure(step, res, suite.Tent.logPath(c.name, step))) - default: - t.Logf("cleanup: deleted cluster %s in %s", c.name, res.Took.Round(time.Second)) - } -} diff --git a/test/e2e/smoke_text_test.go b/test/e2e/smoke_text_test.go index dbd146b..e53eb21 100644 --- a/test/e2e/smoke_text_test.go +++ b/test/e2e/smoke_text_test.go @@ -1,6 +1,8 @@ package e2e import ( + "context" + "errors" "fmt" "path/filepath" "strings" @@ -49,25 +51,22 @@ func (r tentRunner) deleteCommand(cluster string) string { // createArgs are the arguments of tent that create a cluster of one server and one worker, open to the runner alone. func createArgs(cfg settings, cluster, image, sshPub, runner string) []string { - return []string{ - "create", "cluster", cluster, - "--provider", "vultr", "--region", cfg.Region, "--machine-type", cfg.Plan, - "--servers", "1", "--allow-single-server", "--workers", "1", - "--image", image, "--ssh-key", sshPub, - "--ssh-access", runner, "--api-access", runner, "--yes", - } + return scenarioCreateArgs(cfg, cluster, image, sshPub, runner, + "--servers", "1", "--allow-single-server", "--workers", "1") } // validateWait is how long the validate steps let tent wait for a valid cluster. const validateWait = "5m" -// validateArgs are the arguments of tent that check a cluster of this suite, which has one server, until it is valid. +// validateArgs are the arguments of tent that check a cluster of this suite until it is valid. The flag +// --allow-single-server matters only to the clusters of one server and changes nothing for the others. func validateArgs(cluster string) []string { return []string{"validate", "cluster", cluster, "--wait", validateWait, "--allow-single-server"} } -// deleteInCleanup tells whether the cleanup deletes the cluster: when no delete ran, or when a signal ended the -// delete before it finished. A delete that failed on its own is not repeated. -func deleteInCleanup(ran, done, interrupted bool) bool { - return !done && (!ran || interrupted) +// deleteInCleanup tells whether the cleanup deletes the cluster: when no delete ran, or when the delete did not +// finish because a signal or the deadline of its context ended it (deleteErr is that context's error). A delete that +// failed on its own is not repeated. +func deleteInCleanup(ran, done, interrupted bool, deleteErr error) bool { + return !done && (!ran || interrupted || errors.Is(deleteErr, context.DeadlineExceeded)) } diff --git a/test/e2e/smoke_text_unit_test.go b/test/e2e/smoke_text_unit_test.go index 4eb21d2..1deeabe 100644 --- a/test/e2e/smoke_text_unit_test.go +++ b/test/e2e/smoke_text_unit_test.go @@ -1,6 +1,8 @@ package e2e import ( + "context" + "fmt" "path/filepath" "slices" "strings" @@ -104,21 +106,27 @@ func TestValidateArgsAcceptTheSingleServer(t *testing.T) { } func TestDeleteInCleanup(t *testing.T) { + deadline := fmt.Errorf("tent delete: %w", context.DeadlineExceeded) cases := []struct { name string ran, done, interrupted bool + err error want bool }{ - {"no delete ran", false, false, false, true}, - {"delete exited 0", true, true, false, false}, - {"delete failed on its own", true, false, false, false}, - {"a signal ended the delete", true, false, true, true}, - {"a signal after the delete exited 0", true, true, true, false}, + {"no delete ran", false, false, false, nil, true}, + {"delete exited 0", true, true, false, nil, false}, + {"delete failed on its own", true, false, false, nil, false}, + {"delete failed and its context was canceled", true, false, false, context.Canceled, false}, + {"a signal ended the delete", true, false, true, nil, true}, + {"the step's deadline ended the delete", true, false, false, deadline, true}, + {"a signal after the delete exited 0", true, true, true, nil, false}, + {"the deadline after the delete exited 0", true, true, false, deadline, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if got := deleteInCleanup(tc.ran, tc.done, tc.interrupted); got != tc.want { - t.Fatalf("deleteInCleanup(%v, %v, %v) = %v, want %v", tc.ran, tc.done, tc.interrupted, got, tc.want) + if got := deleteInCleanup(tc.ran, tc.done, tc.interrupted, tc.err); got != tc.want { + t.Fatalf("deleteInCleanup(%v, %v, %v, %v) = %v, want %v", + tc.ran, tc.done, tc.interrupted, tc.err, got, tc.want) } }) } diff --git a/test/e2e/stepctx_test.go b/test/e2e/stepctx_test.go index ef1b62e..79bd54b 100644 --- a/test/e2e/stepctx_test.go +++ b/test/e2e/stepctx_test.go @@ -4,8 +4,13 @@ import ( "context" "io" "testing" + "time" ) +// cleanupReserve is the time before a test's deadline that a step keeps free for the cleanups: the delete of a cluster +// and a margin. +const cleanupReserve = 20 * time.Minute + // interruptedText is what the suite prints once when a signal ends the run. const interruptedText = "e2e: interrupted; deleting the clusters of this run\n" @@ -21,13 +26,26 @@ func joinContext(parent, other context.Context) (ctx context.Context, release co } // stepContext returns the context for the calls of a test. It ends when the test's own context or suiteCtx ends, and -// suiteCtx ends when a signal interrupts the run. It is released when the test ends. +// suiteCtx ends when a signal interrupts the run. When the test has a deadline, it also ends cleanupReserve before it, +// so the cleanups can still run. It is released when the test ends. func stepContext(suiteCtx context.Context, t testing.TB) context.Context { ctx, release := joinContext(t.Context(), suiteCtx) t.Cleanup(release) + if d, ok := t.(interface{ Deadline() (time.Time, bool) }); ok { + if end, ok := stepEnd(d.Deadline()); ok { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(ctx, end) + t.Cleanup(cancel) + } + } return ctx } +// stepEnd is the end of a step for a test that ends at deadline, if it has one. +func stepEnd(deadline time.Time, has bool) (time.Time, bool) { + return deadline.Add(-cleanupReserve), has +} + // announceInterrupt writes interruptedText to out when ctx ends, unless the returned stop is called first. func announceInterrupt(ctx context.Context, out io.Writer) (stop func() bool) { return context.AfterFunc(ctx, func() { _, _ = io.WriteString(out, interruptedText) }) diff --git a/test/e2e/stepctx_unit_test.go b/test/e2e/stepctx_unit_test.go index 9f31984..1164075 100644 --- a/test/e2e/stepctx_unit_test.go +++ b/test/e2e/stepctx_unit_test.go @@ -2,6 +2,7 @@ package e2e import ( "context" + "errors" "strings" "sync" "sync/atomic" @@ -115,6 +116,61 @@ func TestStepContextEndsWhenTheSuiteIsInterruptedAndTheCleanupOfTheTestStillRuns } } +func TestStepEndIsTheDeadlineMinusTheCleanupReserve(t *testing.T) { + deadline := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + got, ok := stepEnd(deadline, true) + if want := deadline.Add(-20 * time.Minute); !ok || !got.Equal(want) { + t.Errorf("stepEnd = %v, %t, want %v, true", got, ok, want) + } + if _, ok := stepEnd(deadline, false); ok { + t.Error("a test without a deadline got an end") + } +} + +// deadlineTB is a test whose deadline the unit test chooses. +type deadlineTB struct { + testing.TB + deadline time.Time +} + +func (d deadlineTB) Deadline() (time.Time, bool) { return d.deadline, true } + +func TestStepContextEndsBeforeTheDeadlineOfTheTest(t *testing.T) { + deadline := time.Now().Add(time.Hour) + var got time.Time + var ok bool + t.Run("step", func(t *testing.T) { + ctx := stepContext(newWatchedContext(), deadlineTB{TB: t, deadline: deadline}) + got, ok = ctx.Deadline() + }) + if want := deadline.Add(-cleanupReserve); !ok || !got.Equal(want) { + t.Errorf("deadline of the step = %v, %t, want %v, true", got, ok, want) + } +} + +func TestStepContextEndsWhenItsEndHasPassed(t *testing.T) { + t.Run("step", func(t *testing.T) { + ctx := stepContext(newWatchedContext(), deadlineTB{TB: t, deadline: time.Now().Add(cleanupReserve / 2)}) + waitDone(ctx, t) + if err := ctx.Err(); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("the step's context ended with %v, want %v", err, context.DeadlineExceeded) + } + }) +} + +// plainTB hides the Deadline method of a *testing.T, which a TB does not have. +type plainTB struct{ testing.TB } + +func TestStepContextOfATestWithoutADeadlineHasNone(t *testing.T) { + var ok bool + t.Run("step", func(t *testing.T) { + _, ok = stepContext(newWatchedContext(), plainTB{t}).Deadline() + }) + if ok { + t.Error("the step's context has a deadline although the test has none") + } +} + // lockedBuffer is a writer that the AfterFunc goroutine and the test can use together. type lockedBuffer struct { mu sync.Mutex diff --git a/test/e2e/sweep_test.go b/test/e2e/sweep_test.go index 574f733..ff9e96e 100644 --- a/test/e2e/sweep_test.go +++ b/test/e2e/sweep_test.go @@ -10,7 +10,7 @@ import ( ) // leftoverAge is how old a cluster must be for the sweep before a run to delete it. -const leftoverAge = 3 * time.Hour +const leftoverAge = 4 * time.Hour // sweepLeftovers deletes the clusters of earlier runs that are older than leftoverAge at now. func sweepLeftovers(ctx context.Context, api janitor.API, out io.Writer, now time.Time) error { diff --git a/test/e2e/sweep_unit_test.go b/test/e2e/sweep_unit_test.go index 0b62197..b81338a 100644 --- a/test/e2e/sweep_unit_test.go +++ b/test/e2e/sweep_unit_test.go @@ -14,10 +14,10 @@ import ( var sweepNow = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) -func TestSweepDeletesOnlyClustersOlderThanThreeHours(t *testing.T) { +func TestSweepDeletesOnlyClustersOlderThanFourHours(t *testing.T) { api := &janitortest.API{Machines: []vultrapi.Instance{ - {ID: "i-old", Tags: []string{"tent/cluster=e2e-old000-2404"}, Created: sweepNow.Add(-3*time.Hour - time.Minute)}, - {ID: "i-new", Tags: []string{"tent/cluster=e2e-new000-2404"}, Created: sweepNow.Add(-2*time.Hour - 59*time.Minute)}, + {ID: "i-old", Tags: []string{"tent/cluster=e2e-old000-2404"}, Created: sweepNow.Add(-4*time.Hour - time.Minute)}, + {ID: "i-new", Tags: []string{"tent/cluster=e2e-new000-2404"}, Created: sweepNow.Add(-3*time.Hour - 59*time.Minute)}, }} if err := sweepLeftovers(t.Context(), api, &bytes.Buffer{}, sweepNow); err != nil { t.Fatalf("sweepLeftovers: %v", err) @@ -50,7 +50,7 @@ func TestSweepReportsAFailedDelete(t *testing.T) { boom := errors.New("boom") api := &janitortest.API{ Keys: []vultrapi.SSHKey{{ - ID: "k1", Name: "tent:cluster=e2e-old000-2404;kind=ssh", Created: sweepNow.Add(-4 * time.Hour), + ID: "k1", Name: "tent:cluster=e2e-old000-2404;kind=ssh", Created: sweepNow.Add(-5 * time.Hour), }}, Err: map[string]error{"delete-ssh-key k1": boom}, } diff --git a/test/e2e/testdata/cluster.json b/test/e2e/testdata/cluster.json new file mode 100644 index 0000000..0c7bb6f --- /dev/null +++ b/test/e2e/testdata/cluster.json @@ -0,0 +1,16 @@ +[ + { + "apiVersion": "tent/v1alpha1", + "kind": "Cluster", + "metadata": { + "name": "e2e-abc123-ha" + }, + "spec": { + "cloud": { + "provider": "vultr", + "region": "ams", + "vultr": {} + } + } + } +] diff --git a/test/e2e/testdata/probe-w28exz-server-stop.tsv b/test/e2e/testdata/probe-w28exz-server-stop.tsv new file mode 100644 index 0000000..3d78213 --- /dev/null +++ b/test/e2e/testdata/probe-w28exz-server-stop.tsv @@ -0,0 +1,17 @@ +start took_ms leader leader_address service detail +2026-10-14T01:07:20.471Z 185 led 10.64.0.8:4647 served +2026-10-14T01:07:21.469Z 192 led 10.64.0.8:4647 served +2026-10-14T01:07:22.471Z 271 led 10.64.0.8:4647 served +2026-10-14T01:07:23.471Z 186 led 10.64.0.8:4647 served +2026-10-14T01:07:24.471Z 224 led 10.64.0.8:4647 served +2026-10-14T01:07:25.469Z 196 led 10.64.0.8:4647 served +2026-10-14T01:07:26.471Z 211 led 10.64.0.8:4647 served +2026-10-14T01:07:27.471Z 199 led 10.64.0.8:4647 served +2026-10-14T01:07:28.471Z 203 led 10.64.0.8:4647 served +2026-10-14T01:07:29.471Z 7063 led 10.64.0.8:4647 unreached nomad: GET /v1/job/e2e-web/allocations: Get "https://199.247.24.188:4646/v1/job/e2e-web/allocations": context deadline exceeded +2026-10-14T01:07:36.534Z 10053 led 10.64.0.8:4647 unchecked https://199.247.24.188:4646: nomad: GET /v1/status/leader?stale=true: Get "https://199.247.24.188:4646/v1/status/leader?stale=true": context deadline exceeded; nomad: GET /v1/allocation/0456946b-2847-41bb-c71f-1e89ed879f67/checks: Get "https://136.244.110.51:4646/v1/allocation/0456946b-2847-41bb-c71f-1e89ed879f67/checks": context deadline exceeded; nomad: GET /v1/allocation/19d8eebe-0749-7cfa-681e-55d16102ee2a/checks: Get "https://136.244.110.51:4646/v1/allocation/19d8eebe-0749-7cfa-681e-55d16102ee2a/checks": context deadline exceeded +2026-10-14T01:07:46.588Z 10059 led 10.64.0.8:4647 unchecked https://199.247.24.188:4646: nomad: GET /v1/status/leader?stale=true: Get "https://199.247.24.188:4646/v1/status/leader?stale=true": context deadline exceeded; nomad: GET /v1/allocation/0456946b-2847-41bb-c71f-1e89ed879f67/checks: Get "https://136.244.110.51:4646/v1/allocation/0456946b-2847-41bb-c71f-1e89ed879f67/checks": context deadline exceeded; nomad: GET /v1/allocation/19d8eebe-0749-7cfa-681e-55d16102ee2a/checks: Get "https://136.244.110.51:4646/v1/allocation/19d8eebe-0749-7cfa-681e-55d16102ee2a/checks": context deadline exceeded +2026-10-14T01:07:56.648Z 10055 led 10.64.0.8:4647 unchecked https://199.247.24.188:4646: nomad: GET /v1/status/leader?stale=true: Get "https://199.247.24.188:4646/v1/status/leader?stale=true": context deadline exceeded; nomad: GET /v1/allocation/0456946b-2847-41bb-c71f-1e89ed879f67/checks: Get "https://136.244.110.51:4646/v1/allocation/0456946b-2847-41bb-c71f-1e89ed879f67/checks": context deadline exceeded; nomad: GET /v1/allocation/19d8eebe-0749-7cfa-681e-55d16102ee2a/checks: Get "https://136.244.110.51:4646/v1/allocation/19d8eebe-0749-7cfa-681e-55d16102ee2a/checks": context deadline exceeded +2026-10-14T01:08:06.704Z 3201 led 10.64.0.8:4647 served https://199.247.24.188:4646: nomad: GET /v1/status/leader?stale=true: Get "https://199.247.24.188:4646/v1/status/leader?stale=true": context deadline exceeded +2026-10-14T01:08:09.906Z 3142 led 10.64.0.8:4647 served https://199.247.24.188:4646: nomad: GET /v1/status/leader?stale=true: Get "https://199.247.24.188:4646/v1/status/leader?stale=true": context deadline exceeded +2026-10-14T01:08:13.048Z 187 led 10.64.0.8:4647 served diff --git a/test/e2e/testdata/web-pair.nomad.hcl b/test/e2e/testdata/web-pair.nomad.hcl new file mode 100644 index 0000000..53d261c --- /dev/null +++ b/test/e2e/testdata/web-pair.nomad.hcl @@ -0,0 +1,41 @@ +job "e2e-web" { + group "web" { + count = 2 + migrate { + max_parallel = 1 + health_check = "checks" + min_healthy_time = "10s" + healthy_deadline = "5m" + } + network { + mode = "bridge" + port "http" { + to = 8080 + } + } + service { + name = "e2e-web" + provider = "nomad" + port = "http" + check { + type = "http" + path = "/" + interval = "5s" + timeout = "2s" + } + } + task "web" { + driver = "docker" + config { + image = "busybox:1.38" + command = "sh" + args = ["-c", "mkdir -p /www && echo hello >/www/index.html && exec httpd -f -p 8080 -h /www"] + ports = ["http"] + } + resources { + cpu = 50 + memory = 32 + } + } + } +} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go new file mode 100644 index 0000000..5d7a80b --- /dev/null +++ b/test/e2e/upgrade_test.go @@ -0,0 +1,164 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ingvarch/tent/test/e2e/vultrapi" +) + +const ( + // upgradeRollTimeout bounds the roll of the combined group, one new node first for each of its nodes. + upgradeRollTimeout = 60 * time.Minute + // upgradeTimeout bounds tent upgrade cluster, which downloads and verifies the signed checksums of a Nomad + // release. + upgradeTimeout = 5 * time.Minute +) + +// TestUpgrade builds a combined cluster of three nodes on the old Nomad version, has tent upgrade cluster move the +// spec to the version that the channel recommends, and rolls the nodes while a probe asks for a leader and a healthy +// allocation of the service. At the end every node and member runs the new version. +// +// It is parallel for the reason that TestHA is. +func TestUpgrade(t *testing.T) { + t.Parallel() + u := &upgradeScenario{cluster: newCluster(t, scenarioName(suite.ID, "upgrade"), suite.Settings.upgradeImage())} + steps := []clusterStep{ + {"create", u.create}, + {"validate", func(t *testing.T) { u.validate(t, "validate") }}, + {"export", func(t *testing.T) { u.export(t, "export") }}, + {"service", u.pairService}, + {"versions-before", u.versionsBefore}, + {"upgrade", u.upgrade}, + {"roll", u.roll}, + {"versions-after", u.versionsAfter}, + {"validate-again", u.validateAgain}, + } + if !suite.Settings.Keep { + steps = append(steps, clusterStep{"delete", u.delete}, clusterStep{"leftovers", u.leftovers}) + } + runSteps(t, u.cluster, steps) +} + +// upgradeScenario is the cluster of TestUpgrade, the machines that its steps recorded before the roll and the Nomad +// version that tent upgrade cluster chose. +type upgradeScenario struct { + *cluster + before []vultrapi.Instance + target string +} + +// create builds the cluster on the old version. When tent warns that the version is untested, the step logs the +// warning and goes on. +func (u *upgradeScenario) create(t *testing.T) { + args := upgradeCreateArgs(suite.Settings, u.name, u.image, suite.SSHKey+".pub", suite.Runner) + res := u.tent(t, createTimeout, "create", args...) + if warning := untestedVersionWarning(res.Stderr); warning != "" { + t.Logf("tent warns about the old version: %s", warning) + } +} + +// versionsBefore checks that every node runs the old version and records the machines that the roll must replace. +func (u *upgradeScenario) versionsBefore(t *testing.T) { + ctx := stepContext(suite.Ctx, t) + nodes, err := listNodes(ctx, u.nomad) + if err != nil { + t.Fatal(err) + } + if len(nodes) != upgradeNodes { + t.Fatalf("Nomad lists %d nodes, want %d", len(nodes), upgradeNodes) + } + for _, n := range nodes { + if n.Version != suite.Settings.UpgradeFrom { + t.Fatalf("node %s runs Nomad %s, want %s", n.Name, n.Version, suite.Settings.UpgradeFrom) + } + } + instances, err := u.machines(ctx) + if err != nil { + t.Fatalf("list the machines: %v", err) + } + u.before = machinesOf(instances, u.name, "combined") + if len(u.before) != upgradeNodes { + t.Fatalf("Vultr lists %d combined machines of %s, want %d", len(u.before), u.name, upgradeNodes) + } +} + +// upgrade moves the spec to the version that the channel recommends, applies it to the infrastructure, which leaves +// every node outdated, and checks that a roll is due. +func (u *upgradeScenario) upgrade(t *testing.T) { + res := u.tent(t, upgradeTimeout, "upgrade", "upgrade", "cluster", u.name, "--yes") + said, err := upgradedVersion(res.Stdout, u.name) + if err != nil { + t.Fatal(err) + } + res = u.tent(t, specTimeout, "get-cluster", "get", "clusters", u.name, "-o", "json") + if u.target, err = upgradeTarget(res.Stdout, suite.Settings.UpgradeFrom); err != nil { + t.Fatal(err) + } + if u.target != said { + t.Fatalf("tent upgrade says the spec has Nomad %s now, get clusters gives %s", said, u.target) + } + t.Logf("%s upgrades from Nomad %s to %s", u.name, suite.Settings.UpgradeFrom, u.target) + u.tent(t, updateTimeout, "update", "update", "cluster", u.name, "--yes") + u.rollDue(t, "roll-due", 2) +} + +// roll rolls the combined group while the probe runs and checks that no roll is due afterwards. +func (u *upgradeScenario) roll(t *testing.T) { + u.rollWithProbe(t, "combined", "nodes", "roll", defaultProbeLimits, upgradeRollTimeout, + "rolling-update", "cluster", u.name, "--yes") + u.rollDue(t, "roll-done", 0) +} + +// versionsAfter exports again, because the roll deleted the server that the first export named, and waits until the +// old machines are gone and every node and member runs the target version. +func (u *upgradeScenario) versionsAfter(t *testing.T) { + u.export(t, "export-rolled") + ctx := stepContext(suite.Ctx, t) + err := pollUntil(ctx, rolledEvery, rolledTimeout, func(ctx context.Context) string { + if err := u.versionsProblem(ctx); err != nil { + return err.Error() + } + return "" + }) + if err != nil { + t.Fatalf("wait for every node and member to run Nomad %s: %v", u.target, err) + } +} + +// versionsProblem reads the machines, the nodes and the members and returns what keeps them from being the new +// nodes on the target version. +func (u *upgradeScenario) versionsProblem(ctx context.Context) error { + instances, err := u.machines(ctx) + if err != nil { + return err + } + if err := checkRolled(u.before, instances); err != nil { + return err + } + current := machinesOf(instances, u.name, "combined") + if len(current) != upgradeNodes { + return fmt.Errorf("Vultr lists %d combined machines of %s, want %d", len(current), u.name, upgradeNodes) + } + nodes, err := listNodes(ctx, u.nomad) + if err != nil { + return err + } + members, err := listMembers(ctx, u.nomad) + if err != nil { + return err + } + return checkVersions(nodes, members, current, u.target) +} + +// validateAgain checks that validate exits with 0 and says that the cluster runs the target version. +func (u *upgradeScenario) validateAgain(t *testing.T) { + res := u.tent(t, validateTimeout, "validate-again", validateArgs(u.name)...) + if err := checkValidVersion(res.Stdout, u.name, u.target); err != nil { + t.Fatal(err) + } +} diff --git a/test/e2e/vultrapi/client.go b/test/e2e/vultrapi/client.go index 9440b5c..b34c980 100644 --- a/test/e2e/vultrapi/client.go +++ b/test/e2e/vultrapi/client.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "time" + _ "time/tzdata" // the zone of a zoneless date must load on Windows too ) const ( @@ -171,13 +172,24 @@ func (c *Client) DeleteSSHKey(ctx context.Context, id string) error { return c.delete(ctx, "/ssh-keys/"+url.PathEscape(id)) } -// parseCreated reads the creation date of an object; Vultr writes RFC 3339 with an offset. +// zonelessCreated is the form of a creation date without a zone. +const zonelessCreated = "2006-01-02 15:04:05" + +// parseCreated reads the creation date of an object. Vultr writes RFC 3339 with an offset. On 2026-10-14 its +// firewall list gave a group made at 00:46:46 UTC the date "2026-10-13 20:46:46": no zone, in US Eastern time. func parseCreated(kind, id, value string) (time.Time, error) { t, err := time.Parse(time.RFC3339, value) - if err != nil { - return time.Time{}, fmt.Errorf("vultr: %s %s: creation date %q: %w", kind, id, value, err) + if err == nil { + return t, nil + } + eastern, zoneErr := time.LoadLocation("America/New_York") + if zoneErr != nil { + return time.Time{}, fmt.Errorf("vultr: %s %s: creation date %q: %w", kind, id, value, zoneErr) + } + if t, zonelessErr := time.ParseInLocation(zonelessCreated, value, eastern); zonelessErr == nil { + return t, nil } - return t, nil + return time.Time{}, fmt.Errorf("vultr: %s %s: creation date %q: %w", kind, id, value, err) } // listAll reads every page of a list and turns each element of the array under key into a T. It fails when a diff --git a/test/e2e/vultrapi/client_test.go b/test/e2e/vultrapi/client_test.go index e7d5230..c4c22b8 100644 --- a/test/e2e/vultrapi/client_test.go +++ b/test/e2e/vultrapi/client_test.go @@ -217,6 +217,54 @@ func TestUnparsableDateNamesTheObject(t *testing.T) { } } +// TestParseCreatedReadsAZonelessDateAsUSEastern checks the date that Vultr's firewall list gave without a zone: it +// is US Eastern time, UTC-4 in summer and UTC-5 in winter. A date with an offset keeps it. +func TestParseCreatedReadsAZonelessDateAsUSEastern(t *testing.T) { + cases := []struct { + name, value string + want time.Time + }{ + {"summer", "2026-10-13 20:46:46", time.Date(2026, 10, 14, 0, 46, 46, 0, time.UTC)}, + {"winter", "2026-01-15 20:46:46", time.Date(2026, 1, 16, 1, 46, 46, 0, time.UTC)}, + {"rfc 3339", "2022-10-19T12:15:38+00:00", time.Date(2022, 10, 19, 12, 15, 38, 0, time.UTC)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseCreated("firewall group", "f-1", tc.value) + if err != nil { + t.Fatalf("parseCreated(%q): %v", tc.value, err) + } + if !got.Equal(tc.want) { + t.Errorf("parseCreated(%q) = %s, want %s", tc.value, got.UTC().Format(time.RFC3339), + tc.want.Format(time.RFC3339)) + } + }) + } +} + +func TestParseCreatedRefusesAnotherFormWithTheKindTheIDAndTheValue(t *testing.T) { + _, err := parseCreated("firewall group", "f-1", "13/10/2026 20:46") + if err == nil { + t.Fatal("parseCreated accepted another form, want an error") + } + for _, want := range []string{"firewall group", "f-1", `"13/10/2026 20:46"`} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %q, want it to contain %q", err, want) + } + } +} + +func TestFirewallGroupsListTakesAZonelessDate(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 200, `{"firewall_groups": [{"id": "f-1", "date_created": "2026-10-13 20:46:46"}], + "meta": {"links": {"next": ""}}}`) + }) + got, err := c.FirewallGroups(context.Background()) + if err != nil || len(got) != 1 || !got[0].Created.Equal(time.Date(2026, 10, 14, 0, 46, 46, 0, time.UTC)) { + t.Errorf("FirewallGroups = %+v, %v, want one group made at 2026-10-14T00:46:46Z", got, err) + } +} + func TestMalformedJSONIsAnError(t *testing.T) { c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200, `{"instances": [`) }) _, err := c.Instances(context.Background())