Skip to content
CI

Fix what the dashboard reports and saves #312

Sign in to view logs

Fix what the dashboard reports and saves

Fix what the dashboard reports and saves #312

Workflow file for this run

.github/workflows/ci.yml at 4031fbc
name: CI
on:
push:
branches: [main]
pull_request:
# Weekly re-run so govulncheck re-scans the shipped code against newly disclosed
# vulnerabilities even when nothing changed - the vuln DB moves, the code doesn't.
schedule:
- cron: '17 4 * * 1' # Mondays 04:17 UTC
# Callable so the release workflow can gate a tag on this exact-SHA CI before it
# publishes anything (see .github/workflows/release.yml).
workflow_call:
permissions:
contents: read
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22'
- name: gofmt
run: |
unformatted=$(gofmt -l .)
if [ -n "$unformatted" ]; then
echo "gofmt needs to run on:"; echo "$unformatted"; exit 1
fi
- name: vet
run: go vet ./...
- name: dead code
# With tests as entry points (-test) the legitimate test seams
# (raceCities, bestResult, resultScore, stats.ResetForTest, ...) are
# reachable and only truly-orphaned functions fail the gate.
run: |
out=$(go run golang.org/x/tools/cmd/deadcode@v0.48.0 -test ./...)
if [ -n "$out" ]; then
echo "dead code found:"; echo "$out"; exit 1
fi
- name: build
run: go build ./...
- name: test
# -race is the gate: three test files only observe the unsynchronized
# access they were written to catch when the race detector is on.
run: go test ./... -count=1 -race
- name: test (file-backed store)
# The ordinary run's ":memory:" stores pin the SQLite pool to ONE
# connection, which makes every multi-connection WAL failure mode
# structurally invisible - the snapshot-upgrade class busy_timeout
# cannot absorb (SQLITE_BUSY_SNAPSHOT) shipped a measurement-loss bug
# past this entire matrix exactly that way. This leg reruns the
# database-heavy packages against real files and the production
# four-connection pool (see PINGULARITY_TEST_DB_DIR in store.go).
# speedtest is excluded deliberately: its wall clock is capture
# windows, not database work, and its file-backed coverage rides its
# own concurrency hammer test.
env:
PINGULARITY_TEST_DB_DIR: ${{ runner.temp }}
run: go test ./internal/store/ ./internal/settings/ . -count=1
- name: ui test
run: node --test internal/web/ui/*.test.mjs
- name: promtool /metrics lint
# Validate the live /metrics exposition against the real Prometheus parser,
# so a malformed label/escape or type mismatch fails CI instead of a
# scraper. Runs a real daemon briefly (probing on) to populate the dynamic
# families (targets, latency histograms, stat counters) before scraping.
run: |
set -euo pipefail
PROM_VER=2.53.3
curl -fsSL -o prom.tgz "https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}/prometheus-${PROM_VER}.linux-amd64.tar.gz"
tar xzf prom.tgz
PROMTOOL="$PWD/prometheus-${PROM_VER}.linux-amd64/promtool"
go build -o pingularity-ci .
DBDIR=$(mktemp -d)
./pingularity-ci -listen 127.0.0.1:19099 -db "$DBDIR/ci.db" -speedtest=false -speedtest-on-reconnect=false &
PID=$!
trap 'kill $PID 2>/dev/null || true' EXIT
sleep 6 # let a couple of probe rounds populate targets + histograms
curl -fsS http://127.0.0.1:19099/metrics | "$PROMTOOL" check metrics
curl -fsS -o /dev/null -w 'healthz=%{http_code}\n' http://127.0.0.1:19099/healthz
- name: govulncheck
# Pin the version for reproducibility: a floating @latest can change the
# analyzer (and thus pass/fail) between two runs of the same commit.
run: |
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
"$(go env GOPATH)/bin/govulncheck" ./...
# Cheap cross-compile floor: catches platform-specific build regressions.
xbuild:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
goos: [windows, darwin]
goarch: [amd64, arm64]
# Two BSD builds keep the stub files honest: freebsd exercises the
# !linux/!darwin/!windows stubs (resolver_stub, trace_stub, bytes_other)
# plus diskfree_unix's syscall use; openbsd exercises diskfree_other
# (its Statfs_t has different field names, which once broke the build).
include:
- goos: freebsd
goarch: amd64
- goos: openbsd
goarch: amd64
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 0
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- name: build
run: go build ./...
- name: vet
run: go vet ./...
# Native run: macOS, Windows and arm64-Linux runners EXECUTE the OS-tagged tests (which xbuild
# only cross-compiles - trace_darwin/windows, resolver, netstat, disk_free, the
# Windows DACL), run the frontend tests, and smoke-boot the binary. The two
# privileged paths - raw-socket traceroute and service install - are exercised
# by the manually-dispatched deep-test workflow (.github/workflows/deep-test.yml),
# not here.
native:
strategy:
fail-fast: false
matrix:
# ubuntu-24.04-arm is the closest CI gets to the Raspberry Pi class
# this daemon most often runs on: before it, linux/arm64 was BUILT here
# but never EXECUTED anywhere in the pipeline.
os: [macos-latest, windows-latest, ubuntu-24.04-arm]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22'
- name: test
run: go test ./... -count=1
- name: ui test
run: node --test internal/web/ui/*.test.mjs
- name: smoke (boot + serve /metrics)
shell: bash
run: |
exe=$(go env GOEXE)
go build -o "pingularity_smoke$exe" .
"./pingularity_smoke$exe" -listen 127.0.0.1:9000 -db smoke.db &
pid=$!
ok=
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9000/metrics >/dev/null 2>&1; then ok=1; break; fi
sleep 1
done
kill "$pid" 2>/dev/null || true
[ -n "$ok" ] || { echo "binary did not serve /metrics within 30s"; exit 1; }
echo "smoke OK: booted and served /metrics"
# Validate the release config so a broken .goreleaser.yaml is caught on a PR
# instead of at tag-push time (when it would abort a release mid-flight). Pinned
# Browser smoke: ui.test.mjs proves the page's FUNCTIONS; nothing proved the
# page PAINTS. A real Chromium loads the dashboard from a live daemon and
# asserts the floor - panels and chart render, zero console errors, zero
# failed requests. Coarse anchors only; fine-grained selectors rot into
# flakes. release.yml gates on this via its `ci` job like everything here.
browser-smoke:
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22'
- name: install playwright chromium
run: |
npm install --no-save playwright@1.55.0
npx playwright install --with-deps chromium
- name: boot the daemon and drive the browser
run: |
go build -o smoke_bin .
./smoke_bin run -listen 127.0.0.1:9109 -db "$RUNNER_TEMP/browser-smoke.db" &
pid=$!
ok=
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9109/healthz >/dev/null 2>&1; then ok=1; break; fi
sleep 1
done
[ -n "$ok" ] || { echo "daemon did not come up"; exit 1; }
node internal/web/ui/browser_smoke.mjs 9109
rc=$?
kill "$pid" 2>/dev/null || true
exit $rc
# to the exact goreleaser version release.yml publishes with, so what CI checks is
# what the release runs.
goreleaser-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- name: goreleaser check
uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0
with:
version: 'v2.17.0'
args: check
# Artifact smoke: the one distribution class nothing else executes. The native
# jobs test SOURCE builds and the docker job compiles its own binary - the
# goreleaser artifacts users actually download (release ldflags, -s -w, the
# stamped version) were never run before publish. This builds the snapshot
# artifacts with the same pinned goreleaser the release uses, asserts the
# version stamp landed, and boots the Linux binary until /healthz answers.
# release.yml gates on this via its `ci` job, like every job in this file.
artifact-smoke:
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- name: build snapshot artifacts (binaries, archives, deb/rpm)
uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0
with:
version: 'v2.17.0'
# The docker pipeline has its own dedicated gate (the `docker` job);
# everything publish-shaped is skipped, everything package-shaped is
# built - the deb/rpm below are installed for real further down.
args: release --snapshot --clean --skip=docker,publish,announce,homebrew,winget,sign,sbom
- name: smoke the Linux artifact (version stamp + boot)
run: |
bin=$(echo dist/pingularity_linux_amd64*/pingularity)
v=$("$bin" version)
echo "artifact reports: $v"
# An unstamped artifact reports the source default; the ldflags path
# is exactly what source-built test binaries never exercise.
case "$v" in
*SNAPSHOT*) ;;
*) echo "version stamp missing - ldflags did not apply"; exit 1;;
esac
"$bin" run -listen 127.0.0.1:9100 -db "$RUNNER_TEMP/artifact-smoke.db" &
pid=$!
ok=
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9100/healthz >/dev/null 2>&1; then ok=1; break; fi
sleep 1
done
kill "$pid" 2>/dev/null || true
[ -n "$ok" ] || { echo "the shipped-shape binary did not serve /healthz within 30s"; exit 1; }
echo "artifact smoke OK: stamped, booted, healthy"
- name: install the deb and run the service lifecycle
# The packages were built on every push and installed by no one until a
# user did it. This does what a Debian user does: install (postinstall
# creates the service account and auto-starts the unit), verify the
# daemon answers AND runs de-rooted as the dedicated account with only
# CAP_NET_RAW, stop it, remove the package (preremove stops/disables).
run: |
sudo dpkg -i dist/pingularity_*_linux_amd64.deb
ok=
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9000/healthz >/dev/null 2>&1; then ok=1; break; fi
sleep 1
done
[ -n "$ok" ] || { echo "packaged service did not serve /healthz"; sudo systemctl status pingularity || true; sudo journalctl -u pingularity --no-pager | tail -30 || true; exit 1; }
svcuser=$(ps -o user= -p "$(systemctl show -p MainPID --value pingularity)")
[ "$svcuser" = "pingularity" ] || { echo "service runs as '$svcuser', want the dedicated 'pingularity' account"; exit 1; }
sudo systemctl stop pingularity
sudo dpkg -r pingularity
systemctl is-active pingularity >/dev/null 2>&1 && { echo "service still active after package removal"; exit 1; }
echo "deb lifecycle OK: installed, de-rooted, healthy, removed cleanly"
- name: install the rpm in a Rocky Linux container
# No systemd in the container - the scriptlets are written to degrade
# gracefully there (verified in packaging/postinstall.sh) - so this leg
# asserts the rpm installs, the account is created, and the binary runs.
run: |
docker run --rm -v "$PWD/dist:/dist:ro" rockylinux:9 bash -ec '
rpm -i /dist/pingularity_*_linux_amd64.rpm
getent passwd pingularity >/dev/null
pingularity version
'
echo "rpm install OK"
# Downgrade gate: the previous RELEASED binary must open a database created by
# this commit's schema. Migration tests synthesize old schemas in-process; this
# is the other direction with the real prior artifact - the safety net for a
# user who upgrades, hits trouble, and steps back. Rehearsed against v0.70.1:
# healthz answers and the log stays clean.
downgrade:
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- name: create a current-schema database
run: |
go build -o new_pingularity .
./new_pingularity run -listen 127.0.0.1:9107 -db "$RUNNER_TEMP/downgrade.db" &
pid=$!
ok=
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9107/healthz >/dev/null 2>&1; then ok=1; break; fi
sleep 1
done
kill "$pid" 2>/dev/null || true
[ -n "$ok" ] || { echo "current build failed to create its own database"; exit 1; }
- name: previous release opens it
env:
GH_TOKEN: ${{ github.token }}
run: |
if ! gh release download --pattern '*linux_amd64.tar.gz' -D prev; then
echo "no prior release to test against - skipping"
exit 0
fi
tar xzf prev/*.tar.gz -C prev
./prev/pingularity version
./prev/pingularity run -listen 127.0.0.1:9108 -db "$RUNNER_TEMP/downgrade.db" > old.log 2>&1 &
pid=$!
ok=
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9108/healthz >/dev/null 2>&1; then ok=1; break; fi
sleep 1
done
kill "$pid" 2>/dev/null || true
[ -n "$ok" ] || { echo "the PREVIOUS release cannot open a database this commit creates - a user who downgrades is stranded"; tail -30 old.log; exit 1; }
if grep -ciE 'panic' old.log >/dev/null 2>&1 && [ "$(grep -ciE 'panic' old.log)" -gt 0 ]; then
echo "previous release panicked on the new schema:"; tail -30 old.log; exit 1
fi
echo "downgrade OK: previous release serves healthz on the new schema"
# Image gate: builds BOTH Dockerfiles from the exact context layout goreleaser
# stages (linux/<arch>/pingularity, COPY'd via $TARGETPLATFORM) and proves the
# three properties a broken image would otherwise first show in a user's
# release: the CAP_NET_RAW xattr survives the COPY chain, the data dir ships
# 0700 65532:65532 with the volume-lineage marker BEFORE any daemon run, and a
# hardened `docker run` actually becomes healthy. Runs under buildx (the
# builder the release uses) because plain `docker build` was observed dropping
# COPY --chmod on directories - this job must measure what ships, not what a
# different builder happens to produce. release.yml gates on this via its `ci`
# job (workflow_call runs every job in this file).
docker:
runs-on: ubuntu-latest
timeout-minutes: 20
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: go.mod
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- name: build both images from the goreleaser context layout
run: |
mkdir -p linux/amd64
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o linux/amd64/pingularity .
docker buildx build --load -t ping-ci:default .
docker buildx build --load -f Dockerfile.iperf -t ping-ci:iperf .
- name: xattr chain (security.capability must survive COPY into the final layer)
run: |
# setcap stamps the security.capability xattr in a throwaway stage and
# the final stage COPYs the binary across. If any link in that chain
# drops the xattr, the shipped binary silently loses CAP_NET_RAW and
# raw-socket traceroute dies for every non-root container. docker save
# exposes the layer tars, where the xattr rides as a PAX
# SCHILY.xattr.security.capability record on the /pingularity entry -
# assert it is present in every such entry, in BOTH images.
docker save ping-ci:default -o default.tar
docker save ping-ci:iperf -o iperf.tar
python3 - default.tar iperf.tar <<'EOF'
import sys, tarfile
failed = False
for path in sys.argv[1:]:
checked = missing = 0
with tarfile.open(path) as outer:
for m in outer.getmembers():
if not m.isfile():
continue
f = outer.extractfile(m)
try:
inner = tarfile.open(fileobj=f, mode="r:*")
except tarfile.ReadError:
continue # manifest/config JSON, not a layer tar
with inner:
for e in inner:
if e.isfile() and e.name.lstrip("./") == "pingularity":
checked += 1
if "SCHILY.xattr.security.capability" not in e.pax_headers:
missing += 1
print(f"{path}: layer {m.name}: /pingularity has NO security.capability PAX record")
if checked == 0:
print(f"{path}: no /pingularity layer entry found - the check matched nothing")
failed = True
elif missing:
failed = True
else:
print(f"{path}: {checked} /pingularity layer entry(ies), all carry security.capability")
sys.exit(1 if failed else 0)
EOF
- name: shipped permissions (docker export, BEFORE any daemon run)
run: |
# Measured on created-but-never-started containers on purpose: the
# daemon tightens its own container data dir at boot (store.go), so
# any post-run measurement passes even when the image ships 0755.
# docker export reads what the layers actually say.
docker create --name ci-default ping-ci:default
docker export ci-default -o default-rootfs.tar
docker rm ci-default
docker create --name ci-iperf ping-ci:iperf
docker export ci-iperf -o iperf-rootfs.tar
docker rm ci-iperf
python3 - default-rootfs.tar iperf-rootfs.tar <<'EOF'
import sys, tarfile
failed = False
for path in sys.argv[1:]:
entries = {}
with tarfile.open(path) as tf:
for e in tf:
entries[e.name.lstrip("./").rstrip("/")] = e
ok = True
d = entries.get("var/lib/pingularity")
if d is None or not d.isdir():
print(f"{path}: /var/lib/pingularity missing from the exported image"); ok = False
else:
mode = d.mode & 0o7777
if mode != 0o700:
print(f"{path}: /var/lib/pingularity mode {oct(mode)}, want 0700"); ok = False
if (d.uid, d.gid) != (65532, 65532):
print(f"{path}: /var/lib/pingularity owner {d.uid}:{d.gid}, want 65532:65532"); ok = False
if "var/lib/pingularity/.pingularity-image-dir" not in entries:
print(f"{path}: volume-lineage marker .pingularity-image-dir missing"); ok = False
if ok:
print(f"{path}: /var/lib/pingularity 0700 65532:65532 with marker - as shipped")
failed = failed or not ok
sys.exit(1 if failed else 0)
EOF
- name: smoke (hardened run must reach healthy via the baked-in HEALTHCHECK)
run: |
# distroless has no shell, so `docker exec ... curl` cannot exist and a
# CMD-string healthcheck could never run. The liveness mechanism is the
# image's own exec-form HEALTHCHECK (/pingularity healthz), which hits
# /healthz from inside the container's netns. Polling docker inspect's
# .State.Health proves the HEALTHCHECK wiring, the healthz subcommand,
# and the endpoint in one assertion - and fails if the HEALTHCHECK
# instruction is ever dropped (no .State.Health at all then). The
# interval is tightened at run time only so the poll converges fast.
smoke() {
name=$1 image=$2
docker run -d --name "$name" --cap-drop=ALL --cap-add=NET_RAW \
--health-interval=2s --health-timeout=5s --health-start-period=2s \
-v "$name-data:/var/lib/pingularity" "$image"
s=none
for i in $(seq 1 45); do
s=$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null || echo none)
[ "$s" = "healthy" ] && break
sleep 2
done
echo "--- $name health state ---"
docker inspect -f '{{json .State.Health}}' "$name" 2>/dev/null || true
echo "--- $name daemon logs ---"
docker logs "$name" 2>&1 | tail -10
docker rm -f "$name" >/dev/null
if [ "$s" != "healthy" ]; then
echo "::error::$image never reported healthy (last state: $s) - the HEALTHCHECK or the healthz subcommand is broken"
return 1
fi
echo "$image: healthy under --cap-drop=ALL --cap-add=NET_RAW"
}
smoke ping-ci-default ping-ci:default
smoke ping-ci-iperf ping-ci:iperf