Fix upload data-usage counting, Quick Setup retries, and large downloads

- A speed test whose upload fails now records the data it actually used
  again (a library update had quietly stopped counting it).
- Retrying the first-run Quick Setup after it turns on a login now
  succeeds instead of showing an error.
- Downloading oversized logs or CSV exports shows a clear "too large"
  message (and points to the database file) instead of failing silently.
- The log-download link always goes through the app's own login, never
  the browser's basic-auth popup.
- Accessibility touch-ups on the first-run setup dialog.
- Internal test and CI hardening; no other change in behavior.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: cansofgrease

.github/workflows/deep-test.yml

@@ -245,7 +245,7 @@ jobs:
           # No host networking here: under QEMU we only care that the arm64
           # binary starts, opens its port and serves - the measurements it takes
           # inside an emulated container are meaningless either way.
-          docker run -d --name ping-a64 -p 9111:9000 ping-arm64
+          docker run -d --name ping-a64 -e PINGULARITY_ACCESS=network -p 9111:9000 ping-arm64
           ok=""
           for i in $(seq 1 30); do
             curl -fsS --max-time 3 http://127.0.0.1:9111/metrics >/dev/null 2>&1 && { ok=1; break; }
@@ -308,6 +308,8 @@ jobs:
               image: ghcr.io/pingular/pingularity:0.7.0
               container_name: pingularity
               network_mode: host
+              environment:
+                - PINGULARITY_ACCESS=network
               cap_add:
                 - NET_RAW
               volumes:
@@ -357,7 +359,8 @@ jobs:
             # (-metrics-token) and must never surface in any log; the allow-host
             # part is what the inertness 403 is asserted against. Do NOT sentinel
             # on the domain - the Host guard legitimately logs rejected Hosts.
-            sed -i 's/^    command: \["-allow-host=ping.example.com"\]$/    environment:\n      - PINGULARITY_OPTS=-allow-host=ping.example.com -metrics-token=DEEP_SECRET_SENTINEL/' compose.yaml
+            sed -i '/^    command: \["-allow-host=ping.example.com"\]$/d' compose.yaml
+            sed -i 's/^      - PINGULARITY_ACCESS=network$/      - PINGULARITY_ACCESS=network\n      - PINGULARITY_OPTS=-allow-host=ping.example.com -metrics-token=DEEP_SECRET_SENTINEL/' compose.yaml
             docker compose up -d
             for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
             eh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9000/ || echo FAIL)
@@ -408,7 +411,7 @@ jobs:
             [ "$perm" = "700" ] || note "iperf image data dir is $perm, want 700"
             docker run --rm --entrypoint iperf3 ping-iperf --version \
               || note "iperf3 does not run in the image that exists to carry it"
-            docker run -d --name ping-iperf -p 9112:9000 ping-iperf
+            docker run -d --name ping-iperf -e PINGULARITY_ACCESS=network -p 9112:9000 ping-iperf
             for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9112/metrics >/dev/null 2>&1 && break; sleep 2; done
             im=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:9112/metrics || echo FAIL)
             echo "iperf image metrics -> $im"
@@ -419,7 +422,7 @@ jobs:
             echo "--- iperf container logs ---"; tail -15 <<<"$ilogs"
             docker rm -fv ping-iperf
             echo "--- issue #16 parity: same Host-guard pair on the variant the report used ---"
-            docker run -d --name ping-iperf-cmd -p 9113:9000 ping-iperf -allow-host=ping.example.com
+            docker run -d --name ping-iperf-cmd -e PINGULARITY_ACCESS=network -p 9113:9000 ping-iperf -allow-host=ping.example.com
             for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9113/metrics >/dev/null 2>&1 && break; sleep 2; done
             iah=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9113/ || echo FAIL)
             ioh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: other.example.com' http://127.0.0.1:9113/ || echo FAIL)
@@ -429,7 +432,7 @@ jobs:
             [ "$ioh" = "403" ] || note "iperf image: unlisted public Host not rejected (got $ioh, want 403)"
             case "$iargs" in *"-allow-host=ping.example.com"*) ;; *) note "iperf image: run-arg flag missing from container argv";; esac
             docker rm -fv ping-iperf-cmd
-            docker run -d --name ping-iperf-env -p 9113:9000 -e "PINGULARITY_OPTS=-allow-host=ping.example.com -metrics-token=DEEP_SECRET_SENTINEL" ping-iperf
+            docker run -d --name ping-iperf-env -p 9113:9000 -e PINGULARITY_ACCESS=network -e "PINGULARITY_OPTS=-allow-host=ping.example.com -metrics-token=DEEP_SECRET_SENTINEL" ping-iperf
             for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9113/metrics >/dev/null 2>&1 && break; sleep 2; done
             ieh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9113/ || echo FAIL)
             echo "iperf env-var: Host -> $ieh (must stay 403)"

firstrun_wiring_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"io"
 	"log/slog"
+	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -110,3 +112,60 @@ func TestFirstRunReadyFnComposition(t *testing.T) {
 		t.Errorf("iperf3 engine readiness = %v, want %v (IperfAvailable on this host)", got, want)
 	}
 }
+
+// The pending->PUBLISHED half of the readiness truth table. TestFirstRunReadyFnComposition
+// only pins the pending (false) side, so a mutation that leaves readiness
+// permanently false AFTER netinfo publishes slips past it. This drives a real
+// publish (even a failed lookup stamps UpdatedAt) and asserts readiness flips.
+func TestFirstRunReadyFnReleasesAfterPublish(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	st, err := store.Open(":memory:")
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer st.Close()
+	set, err := settings.New(ctx, st, settings.Values{Monitoring: true, NetinfoEnabled: true})
+	if err != nil {
+		t.Fatalf("settings: %v", err)
+	}
+	ni := netinfo.NewManager(slog.New(slog.NewTextHandler(io.Discard, nil)))
+	ni.EnabledFn = func() bool { return set.Monitoring() && set.NetinfoEnabled() }
+	ready := newFirstRunReadyFn(set, ni)
+
+	if ready() {
+		t.Fatal("must NOT be ready before netinfo publishes (UpdatedAt still 0)")
+	}
+	loopCtx, loopCancel := context.WithCancel(ctx)
+	done := make(chan struct{})
+	defer func() { loopCancel(); <-done }()
+	go func() { defer close(done); ni.Loop(loopCtx, time.Hour) }()
+
+	deadline := time.Now().Add(15 * time.Second)
+	for !ready() {
+		if time.Now().After(deadline) {
+			t.Fatalf("readiness never released after netinfo published (UpdatedAt=%d)", ni.Get().UpdatedAt)
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+}
+
+// The wiring lives in main.go's run() assembly, which isn't unit-testable, and
+// the tests above re-create it locally - so deleting it from main.go leaves them
+// green (the exact gap that shipped). This source-presence guard fails if the
+// production wiring is removed or renamed. It is deliberately literal.
+func TestMainWiresFirstRunAndOptsHooks(t *testing.T) {
+	src, err := os.ReadFile("main.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"ni.WakeFn = set.Changed",                     // #3: first speedtest must not race an empty netinfo
+		"sched.ReadyFn = newFirstRunReadyFn(set, ni)", // #3: selection readiness predicate
+		"replayIgnoredOpts(",                          // #7: ignored-PINGULARITY_OPTS warning is replayed at boot
+	} {
+		if !strings.Contains(string(src), want) {
+			t.Errorf("main.go no longer wires %q - package tests won't catch it; the behavior is silently dead", want)
+		}
+	}
+}

internal/netinfo/looptimer_test.go

@@ -0,0 +1,59 @@
+package netinfo
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+)
+
+// Loop sizes its backstop timer from the staleness cap, and the cap is recomputed
+// AFTER the refresh (netinfo.go:435-443): a boot/refresh can flip the snapshot
+// healthy->error, and the error path wants the fast errRetryStale retry, not the
+// full maxStale the pre-refresh healthy read implied. This drives that exact flip
+// with no network - the boot snapshot is empty (Error==""), and the first refresh
+// fails every lookup deterministically ("ip lookup failed"), which CarriedIdentity
+// short-circuits before any traceroute. The armed wait must be the short
+// errRetryStale-based cap, not maxStale. Deleting the post-refresh recompute makes
+// wait ~maxStale (this test fails).
+func TestLoopArmsFastRetryWhenRefreshFlipsToError(t *testing.T) {
+	oldV4, oldV6, oldRE, oldAfter := ipv4Client, ipv6Client, resolverEgress, afterFn
+	defer func() { ipv4Client, ipv6Client, resolverEgress, afterFn = oldV4, oldV6, oldRE, oldAfter }()
+	ipv4Client = canned(500, "")
+	ipv6Client = canned(500, "")
+	resolverEgress = func(context.Context) string { return "" } // no DNS egress -> no real lookups
+
+	waits := make(chan time.Duration, 4)
+	block := make(chan time.Time) // never fires: Loop parks in the select until ctx cancel
+	afterFn = func(d time.Duration) <-chan time.Time { waits <- d; return block }
+
+	m := NewManager(slog.New(slog.NewTextHandler(io.Discard, nil)))
+	m.http = canned(500, "") // cfColo / geo answer nothing
+
+	const maxStale = time.Hour // >> errRetryStale (5m), so the two caps are unmistakable
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { defer close(done); m.Loop(ctx, maxStale) }()
+
+	var wait time.Duration
+	select {
+	case wait = <-waits:
+	case <-time.After(3 * time.Second):
+		cancel()
+		<-done
+		t.Fatal("Loop never armed its backstop timer")
+	}
+	cancel()
+	<-done
+
+	// The refresh flipped the snapshot to an error; the timer must be armed off
+	// that post-refresh error state (errRetryStale ~5m), not the pre-refresh
+	// healthy maxStale (1h).
+	if got := m.Get().Error; got == "" {
+		t.Fatalf("boot refresh did not flip the snapshot to an error; got Error=%q", got)
+	}
+	if wait > 10*time.Minute {
+		t.Fatalf("post-error retry armed at %v; want the short errRetryStale-based cap (~%v), not the pre-refresh maxStale (%v) - the post-refresh cap recompute (netinfo.go:440-443) is gone", wait, errRetryStale, maxStale)
+	}
+}

internal/netinfo/netinfo.go

@@ -360,6 +360,11 @@ func (m *Manager) RefreshNow(ctx context.Context) Info {
 // shrink it.
 var refreshRetryDelay = 2 * time.Second
 
+// afterFn is Loop's backstop sleep timer, injectable so a test can read the wait
+// Loop computes from the POST-refresh staleness cap without sleeping it out.
+// Production is the real timer.
+var afterFn = func(d time.Duration) <-chan time.Time { return time.After(d) }
+
 // enabled reports whether connection-info lookups may run. Nil means yes, so a
 // Manager built without the hook (tests, callers that always want it) behaves
 // as it always did.
@@ -463,7 +468,7 @@ func (m *Manager) Loop(ctx context.Context, maxStale time.Duration) {
 			// refresh NOW.
 		case <-m.nudge:
 			// A broadcast-less enable edge (see Nudge) - same re-evaluation.
-		case <-time.After(wait):
+		case <-afterFn(wait):
 		}
 	}
 }

internal/speedtest/iperf.go

@@ -130,14 +130,17 @@ func congestionForOS(requested, goos string) (effective string, dropped bool) {
 	return requested, false
 }
 
-func AvailableCongestionControl() []string {
-	// Only Linux iperf3 has the -C socket option; on macOS and Windows passing
-	// it aborts the whole run at startup ("an option you are trying to set is
-	// not implemented yet"). Offering a curated list there advertised choices
-	// that BREAK every test, so offer nothing and let the field mean "system
-	// default". (setCongestionSkipped mirrors this at run time for a value that
-	// arrived via an imported backup from a Linux box.)
-	switch runtime.GOOS {
+func AvailableCongestionControl() []string { return availableCongestionControlFor(runtime.GOOS) }
+
+// availableCongestionControlFor is AvailableCongestionControl with the OS as a
+// parameter, so the FreeBSD sysctl-reading + table-parsing branch is testable on
+// ANY host (through the injectable ccSysctl seam) rather than only when
+// GOOS==freebsd. Only Linux iperf3 has the -C socket option; on macOS and Windows
+// passing it aborts the whole run at startup, so offer nothing there and let the
+// field mean "system default". (setCongestionSkipped mirrors this at run time for
+// a value that arrived via an imported backup from a Linux box.)
+func availableCongestionControlFor(goos string) []string {
+	switch goos {
 	case "linux":
 		b, err := os.ReadFile("/proc/sys/net/ipv4/tcp_allowed_congestion_control")
 		if err != nil {

internal/speedtest/iperf_test.go

@@ -831,10 +831,8 @@ func TestParseFreeBSDCC(t *testing.T) {
 	}
 }
 
-// TestAvailableCongestionControlFreeBSDInjected drives the FreeBSD branch's
-// sysctl reader through the injectable ccSysctl seam so the table-parsing path
-// is covered on any host (the switch only reaches it when GOOS==freebsd, so we
-// test the reader+parser composition directly here).
+// Parser-only: pins that table noise never leaks (see TestAvailableCongestionControlFreeBSD
+// for the reader+branch composition through the ccSysctl seam).
 func TestParseFreeBSDCC_DropsAllTableNoise(t *testing.T) {
 	got := parseFreeBSDCC("CCmod   D PCB\nnewreno  *  1\ncubic       0\n")
 	for _, bad := range []string{"CCmod", "D", "PCB", "*", "1", "0"} {
@@ -848,3 +846,38 @@ func TestParseFreeBSDCC_DropsAllTableNoise(t *testing.T) {
 		t.Fatalf("want 2 algorithms, got %v", got)
 	}
 }
+
+// TestAvailableCongestionControlFreeBSD actually drives the FreeBSD branch
+// (reader + table parser) through the injectable ccSysctl seam, on any host - the
+// coverage the "injected" comment previously claimed but never delivered.
+// Disconnecting the freebsd case now fails here instead of leaving the suite green.
+func TestAvailableCongestionControlFreeBSD(t *testing.T) {
+	orig := ccSysctl
+	defer func() { ccSysctl = orig }()
+
+	// 14.x table shape, via the seam -> parsed algorithm names, noise dropped.
+	ccSysctl = func() ([]byte, error) {
+		return []byte("CCmod   D PCB\nnewreno  *  1\ncubic       0\nhtcp        0\n"), nil
+	}
+	got := availableCongestionControlFor("freebsd")
+	want := []string{"newreno", "cubic", "htcp"}
+	if len(got) != len(want) {
+		t.Fatalf("freebsd branch: got %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("freebsd branch: got %v, want %v", got, want)
+		}
+	}
+
+	// A read failure yields no dropdown, not a crash.
+	ccSysctl = func() ([]byte, error) { return nil, errors.New("sysctl unavailable") }
+	if got := availableCongestionControlFor("freebsd"); got != nil {
+		t.Errorf("read error: got %v, want nil", got)
+	}
+
+	// macOS/Windows offer nothing (the -C option aborts there).
+	if got := availableCongestionControlFor("darwin"); got != nil {
+		t.Errorf("darwin: got %v, want nil", got)
+	}
+}

internal/speedtest/ookla.go

@@ -939,6 +939,24 @@ var (
 	}
 )
 
+// ooklaPing is the ranking latency probe, a swap-a-var seam (like ooklaDownload/
+// ooklaUpload) so rankedServers' selection logic - which reachable/failed server
+// wins - is testable without a live server.
+var ooklaPing = func(ctx context.Context, srv *ookla.Server, cb func(time.Duration)) error {
+	return srv.PingTestContext(ctx, cb)
+}
+
+// uploadSpent is the data an upload attempt actually pushed across the (possibly
+// metered) link, for data-usage accounting. It is confirmed bytes (GetTotalUpload)
+// PLUS the backlog (GetUploadBacklog): bytes read into the socket but not
+// server-confirmed. speedtest-go v1.7.11 counts only confirmed bytes in
+// GetTotalUpload - correct for the RATE (ULSpeed) but it drops the bytes a FAILED
+// or aborted attempt already sent, which "data used" must still include. Download
+// has no equivalent gap (GetTotalDownload counts bytes actually received).
+func uploadSpent(srv *ookla.Server) int64 {
+	return srv.Context.GetTotalUpload() + srv.Context.GetUploadBacklog()
+}
+
 // errTransferAbandoned marks a transfer we walked away from because the context
 // died first (see runTransfer). It travels alongside the context error - both are
 // wrapped, so callers testing for context.Canceled still match - and it is what
@@ -1125,7 +1143,7 @@ func (o *Ookla) measure(ctx context.Context, srv *ookla.Server, dir string, retr
 	// Same ten samples the library already sends - the callback only keeps the
 	// fastest alongside the mean it returns, so this costs no extra probe.
 	var bestPing time.Duration
-	if err := srv.PingTestContext(ctx, keepFastestPing(&bestPing)); err != nil {
+	if err := ooklaPing(ctx, srv, keepFastestPing(&bestPing)); err != nil {
 		return Result{}, fmt.Errorf("ping: %w", err)
 	}
 	// Idle baseline for latency-under-load: same method/target as the loaded
@@ -1187,7 +1205,7 @@ func (o *Ookla) measure(ctx context.Context, srv *ookla.Server, dir string, retr
 			if !finished { // abandoned: srv belongs to the orphan now (see above)
 				return e
 			}
-			upBytes += srv.Context.GetTotalUpload() // count this attempt before the next Reset zeroes it
+			upBytes += uploadSpent(srv) // confirmed + backlog: a FAILED attempt's pushed bytes aren't in GetTotalUpload alone
 			if e == nil && ctx.Err() != nil {
 				e = ctx.Err()
 			}
@@ -1524,7 +1542,7 @@ func rankedServers(ctx context.Context, servers ookla.Servers, isp string) (ookl
 			// value - so "err == nil && Latency > 0" would record a one-shot
 			// echo as an answered ten-sample ranking ping.
 			sampled := false
-			err := s.PingTestContext(ctx, func(time.Duration) { sampled = true }) // sets s.Latency on success
+			err := ooklaPing(ctx, s, func(time.Duration) { sampled = true }) // sets s.Latency on success
 			pings[i] = applyRankPing(s, err, sampled)
 		}(i, s)
 	}

internal/speedtest/rankping_test.go

@@ -1,7 +1,9 @@
 package speedtest
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
@@ -68,3 +70,34 @@ func TestApplyRankPingAndRanking(t *testing.T) {
 		t.Error("an unreachable server must never rank ahead of a reachable one")
 	}
 }
+
+// rankedServers (the call site) must not let a candidate whose ranking ping
+// FAILED win on its stale discovery latency. Drives the site through the ooklaPing
+// seam: a reachable server measured at 11ms vs an unreachable one holding a stale
+// 1ms. The fix ranks the reachable one first; reverting applyRankPing's zeroing
+// lets the stale 1ms win (test fails) - the call-site coverage the helper-only
+// tests were missing.
+func TestRankedServersDropsUnreachableStaleLatency(t *testing.T) {
+	orig := ooklaPing
+	defer func() { ooklaPing = orig }()
+	ooklaPing = func(ctx context.Context, srv *ookla.Server, cb func(time.Duration)) error {
+		if srv.ID == "reachable" {
+			srv.Latency = 11 * time.Millisecond // the library sets the measured mean on success
+			cb(srv.Latency)
+			return nil
+		}
+		return errors.New("connection refused") // no sample -> failed ranking ping
+	}
+	servers := ookla.Servers{
+		{ID: "unreachable", Sponsor: "A", Name: "near", Distance: 0, Latency: 1 * time.Millisecond}, // stale, low
+		{ID: "reachable", Sponsor: "B", Name: "far", Distance: 1, Latency: 0},
+	}
+	out, _ := rankedServers(context.Background(), servers, "")
+	if len(out) == 0 || out[0].ID != "reachable" {
+		var ids []string
+		for _, s := range out {
+			ids = append(ids, fmt.Sprintf("%s(%v)", s.ID, s.Latency))
+		}
+		t.Fatalf("reachable must rank first; a failed ping's stale 1ms must not win. order=%v", ids)
+	}
+}