fix(check): stop rebooting the router on transient probe failures

CLAUDE.md

@@ -91,9 +91,33 @@ recorded:
   `8.8.8.8:53`. Bypasses DNS; proves raw link.
 - **`domain`** — HTTP GET to `generate_204`-style endpoints. Exercises the DNS path too.
 
-The verdict is deliberately conservative: **down ⇔ (all `ip` targets fail) OR (all
-`domain` targets fail)**. A single failed target is never a trigger — that guards
-against rebooting the router because one remote server hiccuped.
+`domain.Verdict` grades one round into three states (issue #6): **`down` ⇔ all `ip`
+targets fail** (the only verdict a reboot may follow); **`degraded` ⇔ all `domain`
+targets fail while an `ip` target still answers** — the DNS/HTTPS path is broken, the
+raw link is not; everything else is `up`. A single failed target within a group is never
+a trigger, and an empty group is "not applicable" rather than vacuously failed — except
+that with no `ip` targets configured at all, the `domain` group becomes the only evidence
+there is and a full failure of it counts as `down`.
+
+The split is measured, not theoretical: of 29 "down" verdicts recorded in 21 days, 28 had
+both `ip` targets connecting in 19-1118 ms and ICMP answering 4/4 while only the two
+HTTPS probes timed out — they share one DNS path, in production through the very router
+being rebooted. Four reboots in seven hours on 2026-07-30 did not stop them.
+
+Two further guards sit between a `down` verdict and a reboot, both in
+`services/orchestrator.go`:
+
+- **In-run confirmation** — `confirmDown` re-probes `CHECK_CONFIRMATIONS` times,
+  `CHECK_CONFIRM_DELAY` apart (persisted under `PhaseConfirm`); one healthy sample ends
+  the run as `unconfirmed`, correcting the `internet_ok` it wrote from the first sample.
+- **Cross-run corroboration** — `outageSustained` requires the previous
+  `REBOOT_MIN_STREAK - 1` runs to have seen the outage too, via
+  `RunRepository.RecentRuns`. A run that falls short ends as `unsustained`. Fail-closed:
+  an unreadable history withholds the reboot rather than granting it.
+
+Neither guard sends a Telegram message — silence is the point; the `runs` row and a log
+line carry the record. `degraded` and cooldown-skip notices are edge-triggered
+(`shouldAnnounce`), so a state that lasts hours is announced once, not once per run.
 
 ### Data model
 
@@ -186,7 +210,8 @@ prepended.
 
 `README.md` holds the authoritative table. Required: `LABEL`, `TELEGRAMBOT_DSN`,
 `ROUTER_USER`, `ROUTER_PASS`. Optional (with defaults): `ROUTER_ADDR`, `ROUTER_KIND`,
-`DB_PATH`, `CHECK_IPS`, `CHECK_DOMAINS`, `CHECK_TIMEOUT`, `RECOVERY_INTERVAL`,
+`DB_PATH`, `CHECK_IPS`, `CHECK_DOMAINS`, `CHECK_TIMEOUT`, `CHECK_CONFIRMATIONS`,
+`CHECK_CONFIRM_DELAY`, `REBOOT_MIN_STREAK`, `RECOVERY_INTERVAL`,
 `RECOVERY_TIMEOUT`, `REBOOT_COOLDOWN`, `RETENTION_DAYS`, `REBOOT_ENABLED` (monitor-only
 when `false`: check/record/notify but never reboot), `METRICS_ENABLED`,
 `METRICS_TEMPERATURE`, `METRICS_FANS`, `METRICS_CPU`, `METRICS_MEMORY`, `METRICS_DISK`,
@@ -220,6 +245,7 @@ response (`writeError500`). The contract has these surfaces:
   | 3 | reboot request failed (login / `reboot.cgi`) |
   | 4 | reboot done, internet not restored within the timeout |
   | 5 | reboot skipped due to cooldown |
+  | 6 | a failure was observed but deliberately not acted on (`degraded`, `unsustained`) |
 
 - **Daemon exit codes** — its own disjoint contract: `0` clean shutdown (SIGINT/SIGTERM,
   drained), `1` configuration error (missing `DAEMON_TOKEN`, unparseable value), `2` the

README.md

@@ -13,6 +13,9 @@ API and status dashboard over the LAN.
 - checks connectivity against independent IP and domain targets
 - soft mode (default) reboots only when the uplink is actually down; hard mode
   (`--hard-reboot`) reboots unconditionally
+- a down verdict must survive re-probing within the run and be corroborated by the previous
+  run before anything reboots; a failure of the domain targets alone, while raw IP
+  connectivity holds, is reported as degraded and never reboots
 - monitor-only mode (`YARDDOG_REBOOT_ENABLED=false`): watch and record, never reboot
 - records every run, check and recovery phase in SQLite, with optional per-run
   host-telemetry and ping metrics

cmd/yarddog/main.go

@@ -128,13 +128,16 @@ func run() int {
 	}
 
 	settings := services.Settings{
-		Label:            cfg.Label,
-		RebootCooldown:   cfg.RebootCooldown,
-		RecoveryInterval: cfg.RecoveryInterval,
-		RecoveryTimeout:  cfg.RecoveryTimeout,
-		RebootEnabled:    cfg.RebootEnabled,
-		MetricsEnabled:   cfg.MetricsEnabled,
-		PingEnabled:      len(cfg.PingHosts) > 0,
+		Label:              cfg.Label,
+		RebootCooldown:     cfg.RebootCooldown,
+		RecoveryInterval:   cfg.RecoveryInterval,
+		RecoveryTimeout:    cfg.RecoveryTimeout,
+		CheckConfirmations: cfg.CheckConfirmations,
+		CheckConfirmDelay:  cfg.CheckConfirmDelay,
+		RebootMinStreak:    cfg.RebootMinStreak,
+		RebootEnabled:      cfg.RebootEnabled,
+		MetricsEnabled:     cfg.MetricsEnabled,
+		PingEnabled:        len(cfg.PingHosts) > 0,
 	}
 
 	code := services.Execute(ctx, settings, st, st, st, chk, rb, nt, mc, pc, clk, mode)

domain/check.go

@@ -10,9 +10,31 @@ const (
 	CheckKindGateway = "gateway"
 )
 
-// Verdict applies the conservative down rule (design §4.2): the internet is down when
-// every ip target fails OR every domain target fails. A single failed target within a
-// group never trips it; an empty group is "not applicable", never vacuously down.
+// StateUp, StateDegraded, and StateDown are the tri-state verdict Verdict
+// produces (issue #6). The two failure states are deliberately not the same
+// event: StateDown means the raw link is gone and a router reboot is on the
+// table, while StateDegraded means only the name-resolution/HTTPS path broke
+// while raw connectivity provably held — recorded and announced, never
+// rebooted.
+const (
+	StateUp       = "up"
+	StateDegraded = "degraded"
+	StateDown     = "down"
+)
+
+// Verdict grades one round of probes into StateUp, StateDegraded, or StateDown
+// (issue #6, superseding design §4.2's single "all ip OR all domain failed"
+// rule). Only a fully failed ip group — raw TCP dials to IP literals, which
+// need no DNS — is StateDown. A fully failed domain group while any ip target
+// still connects is StateDegraded: both domain probes are HTTPS GETs sharing
+// one DNS path (in production, through the very router this watchdog reboots),
+// so they fail together as a matter of course, and 28 of 29 historical "down"
+// verdicts were exactly that with the ip group answering in 19-1118ms.
+//
+// When no ip targets are configured at all, the domain group is the only
+// evidence there is and a fully failed one is StateDown rather than a verdict
+// nothing could ever act on. A single failed target within a group never trips
+// anything; an empty group is "not applicable", never vacuously failed.
 func Verdict(targets []TargetResult) Result {
 	var ip, dom []TargetResult
 	for _, t := range targets {
@@ -23,16 +45,35 @@ func Verdict(targets []TargetResult) Result {
 			dom = append(dom, t)
 		}
 	}
-	return Result{Down: allFailed(ip) || allFailed(dom), Targets: targets}
+
+	state := StateUp
+	switch {
+	case allFailed(ip):
+		state = StateDown
+	case allFailed(dom) && len(ip) == 0:
+		state = StateDown
+	case allFailed(dom):
+		state = StateDegraded
+	}
+
+	return Result{State: state, Targets: targets}
 }
 
 // Result is the outcome of one connectivity check: every probed target's
-// individual result plus the aggregate quorum verdict (design §4.2).
+// individual result plus the aggregate verdict (design §4.2, issue #6).
 type Result struct {
-	Down    bool
+	State   string
 	Targets []TargetResult
 }
 
+// IsDegraded reports whether the application-layer path is broken while raw
+// connectivity holds — worth recording and announcing, never worth a reboot.
+func (r Result) IsDegraded() bool { return r.State == StateDegraded }
+
+// IsDown reports whether the raw link is gone, the only verdict a reboot may
+// ever follow from.
+func (r Result) IsDown() bool { return r.State == StateDown }
+
 // TargetResult is one probed target's outcome, shaped to map directly onto a
 // checks row (design §9) once the caller supplies RunID and Phase.
 type TargetResult struct {

domain/check_test.go

@@ -5,80 +5,106 @@ import "testing"
 func TestVerdict(t *testing.T) {
 	t.Parallel()
 
-	t.Run("one down target among several is not a down verdict", func(t *testing.T) {
-		t.Parallel()
-
-		targets := []TargetResult{
-			{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: true},
-			{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
-			{Target: "https://example.com/generate_204", Kind: CheckKindDomain, OK: true},
-		}
-
-		got := Verdict(targets)
-
-		if got.Down {
-			t.Fatalf("Down = true, want false (only one ip target failed): %+v", got.Targets)
-		}
-	})
-
-	t.Run("all ip targets down is a down verdict regardless of domains", func(t *testing.T) {
-		t.Parallel()
-
-		targets := []TargetResult{
-			{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: false},
-			{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
-			{Target: "https://example.com/generate_204", Kind: CheckKindDomain, OK: true},
-		}
-
-		got := Verdict(targets)
-
-		if !got.Down {
-			t.Fatalf("Down = false, want true (all ip targets failed): %+v", got.Targets)
-		}
-	})
-
-	t.Run("ip up but all domains down is a down verdict", func(t *testing.T) {
-		t.Parallel()
-
-		targets := []TargetResult{
-			{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: true},
-			{Target: "https://a.example/", Kind: CheckKindDomain, OK: false},
-			{Target: "https://b.example/", Kind: CheckKindDomain, OK: false},
-		}
-
-		got := Verdict(targets)
-
-		if !got.Down {
-			t.Fatalf("Down = false, want true (all domain targets failed): %+v", got.Targets)
-		}
-	})
-
-	t.Run("at least one ip and one domain up is an up verdict", func(t *testing.T) {
-		t.Parallel()
-
-		targets := []TargetResult{
-			{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: true},
-			{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
-			{Target: "https://a.example/", Kind: CheckKindDomain, OK: true},
-			{Target: "https://b.example/", Kind: CheckKindDomain, OK: false},
-		}
-
-		got := Verdict(targets)
-
-		if got.Down {
-			t.Fatalf("Down = true, want false: %+v", got.Targets)
-		}
-	})
-
-	t.Run("empty group is not applicable, never vacuously down", func(t *testing.T) {
-		t.Parallel()
-
-		got := Verdict(nil)
-
-		if got.Down {
-			t.Fatal("Down = true, want false: an empty target list must never be a vacuous down verdict")
-		}
-	})
+	cases := []struct {
+		name    string
+		targets []TargetResult
+		want    string
+	}{
+		{
+			name: "one down target among several is not a failure verdict",
+			targets: []TargetResult{
+				{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: true},
+				{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
+				{Target: "https://example.com/generate_204", Kind: CheckKindDomain, OK: true},
+			},
+			want: StateUp,
+		},
+		{
+			name: "all ip targets down is a down verdict regardless of domains",
+			targets: []TargetResult{
+				{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: false},
+				{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
+				{Target: "https://example.com/generate_204", Kind: CheckKindDomain, OK: true},
+			},
+			want: StateDown,
+		},
+		{
+			name: "every target down is a down verdict",
+			targets: []TargetResult{
+				{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: false},
+				{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
+				{Target: "https://a.example/", Kind: CheckKindDomain, OK: false},
+			},
+			want: StateDown,
+		},
+		{
+			name: "ip up but all domains down is degraded, not down (issue #6)",
+			targets: []TargetResult{
+				{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: true},
+				{Target: "https://a.example/", Kind: CheckKindDomain, OK: false},
+				{Target: "https://b.example/", Kind: CheckKindDomain, OK: false},
+			},
+			want: StateDegraded,
+		},
+		{
+			name: "one ip up among failures still keeps a domain wipeout at degraded",
+			targets: []TargetResult{
+				{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: false},
+				{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: true},
+				{Target: "https://a.example/", Kind: CheckKindDomain, OK: false},
+				{Target: "https://b.example/", Kind: CheckKindDomain, OK: false},
+			},
+			want: StateDegraded,
+		},
+		{
+			name: "with no ip targets configured, all domains down is the only evidence there is",
+			targets: []TargetResult{
+				{Target: "https://a.example/", Kind: CheckKindDomain, OK: false},
+				{Target: "https://b.example/", Kind: CheckKindDomain, OK: false},
+			},
+			want: StateDown,
+		},
+		{
+			name: "at least one ip and one domain up is an up verdict",
+			targets: []TargetResult{
+				{Target: "1.1.1.1:443", Kind: CheckKindIP, OK: true},
+				{Target: "8.8.8.8:53", Kind: CheckKindIP, OK: false},
+				{Target: "https://a.example/", Kind: CheckKindDomain, OK: true},
+				{Target: "https://b.example/", Kind: CheckKindDomain, OK: false},
+			},
+			want: StateUp,
+		},
+		{
+			name:    "empty group is not applicable, never vacuously down",
+			targets: nil,
+			want:    StateUp,
+		},
+		{
+			name: "a gateway probe alone decides nothing",
+			targets: []TargetResult{
+				{Target: "192.168.1.1:80", Kind: CheckKindGateway, OK: false},
+			},
+			want: StateUp,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := Verdict(c.targets)
+
+			if got.State != c.want {
+				t.Fatalf("State = %q, want %q: %+v", got.State, c.want, got.Targets)
+			}
+			if got.IsDown() != (c.want == StateDown) {
+				t.Fatalf("IsDown() = %v, want %v", got.IsDown(), c.want == StateDown)
+			}
+			if got.IsDegraded() != (c.want == StateDegraded) {
+				t.Fatalf("IsDegraded() = %v, want %v", got.IsDegraded(), c.want == StateDegraded)
+			}
+		})
+	}
 
 	t.Run("targets slice is carried through unmodified", func(t *testing.T) {
 		t.Parallel()