diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0348e77..67fd54d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,8 @@ jobs: with: components: rustfmt - run: cargo fmt --all -- --check + - run: cargo fmt --all -- --check + working-directory: bench/tools clippy: runs-on: ubuntu-22.04 @@ -30,6 +32,8 @@ jobs: components: clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --all-targets --locked -- -D warnings + - run: cargo clippy --all-targets --locked -- -D warnings + working-directory: bench/tools test: runs-on: ubuntu-22.04 @@ -38,3 +42,9 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --locked + + shellcheck: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v7 + - run: git ls-files '*.sh' | xargs shellcheck diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..d663b4d --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,4 @@ +# Resolve `. "$SCRIPT_DIR/lib.sh"` style sources relative to each script's own +# directory so shellcheck follows them (clears SC1091) instead of guessing CWD. +external-sources=true +source-path=SCRIPTDIR diff --git a/Cargo.lock b/Cargo.lock index 8c49fe1..45c4f13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -526,15 +526,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -1231,12 +1222,6 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1514,7 +1499,6 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", - "encoding_rs", "futures-core", "futures-util", "h2", @@ -1526,7 +1510,6 @@ dependencies = [ "hyper-util", "js-sys", "log", - "mime", "percent-encoding", "pin-project-lite", "quinn", diff --git a/Cargo.toml b/Cargo.toml index a9a19a2..249e67c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ hex = "0.4" url = "2" percent-encoding = "2" base64 = "0.22" -reqwest = { version = "0.13", default-features = false, features = ["rustls", "stream", "json", "form", "query", "charset", "http2"] } +reqwest = { version = "0.13", default-features = false, features = ["rustls", "stream", "json", "form", "query", "http2"] } crc32c = "0.6" roaring = "0.11" aws-lc-rs = "1" diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 0000000..c22fb29 --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,15 @@ +# local config (secrets) + run artifacts + python caches +/config.env +/results/ +/.venv/ +__pycache__/ + +# rust build artifacts +/tools/target/ + +# terraform state + generated SSH key + run log (never commit) +/terraform/.terraform/ +/terraform/*.tfstate +/terraform/*.tfstate.* +/terraform/walrus_bench_key.pem +/terraform/apply.log diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..403d2c7 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,181 @@ +# bench — WAL-archiving benchmark harness + +Reproducible single-host benchmark comparing three PostgreSQL 18 WAL archivers on +**throughput** and **memory** under heavy write load: + +- **walrus** (this repo, Rust) — serial wal-push daemon +- **wal-g** (Go) — fan-out daemon (`WALG_UPLOAD_CONCURRENCY`) +- **pgbackrest** (C) — daemonless; PG forks `archive-push`, async `process-max` workers + +All three are driven identically: PG `archive_command` → the tool's own client → S3. +Memory/throughput are sampled on the **archiver** process (not Postgres) at 1 Hz. + +`run_op.sh` extends the same harness to the rest of walrus's data paths — full +backup send/fetch, delta backups (archived-WAL and PG17 WAL-summary sourced), and +streaming WAL receive — benchmarked cross-tool where an equivalent exists (see +[Operation benchmarks](#operation-benchmarks)). + +## Layout + +``` +config.env.example copy to config.env; bucket, creds, PG role, sizing +setup.sh bootstrap THIS host (install PG18, build tools, units) +run.sh run ONE archive cell (daemon × run-id) +matrix.sh loop pgbackrest/walg/walrus × repeats +run_op.sh run ONE operation cell (op × tool × run-id) +op_matrix.sh loop backup-send/backup-fetch/wal-receive × tools × repeats +scripts/lib.sh shared driver scaffolding sourced by run.sh + run_op.sh +scripts/sut/ per-host bootstrap steps (00..40, systemd units) +scripts/driver/ pgbench workload: schema, seed, burst (FPI storm) +tools/ Rust crate (standalone): bench-sampler + bench-analyze bins +``` + +`bench-sampler` (1 Hz mem/CPU/WAL/backlog sampler; `--daemon walg|walrus|pgbackrest` +picks unit-MainPID vs proc-match) and `bench-analyze` (aggregated plots + +self-describing CSV/JSON exports) are built from `tools/` by `setup.sh` and installed +to `/usr/local/bin`. + +## Prerequisites + +- **Debian or Ubuntu + systemd**, x86_64, and `sudo`. The scripts install PostgreSQL + 18 (PGDG apt, codename via `lsb_release -cs`), Go, Rust, `pgbackrest`, and build + `wal-g` + `walrus` + `walg_archive`. Built/tested on Ubuntu 24.04 (the EC2 AMI) and + Debian 13; any PGDG-supported release should work. +- An **S3 bucket** and credentials. `walrus` reads credentials from the **environment + only** (no IMDS, no shared-config profiles), so `config.env` must carry explicit keys. +- Conventional paths the scripts assume: `PGDATA=/dat/18/data`, daemon env file + `/etc/postgresql/wal-g.env`, daemon socket `/tmp/wal-g`, PG binaries under + `/usr/lib/postgresql/18/bin`. A spare NVMe is mounted at `/dat` by + `scripts/sut/00_mount_nvme.sh` (AWS instance-store oriented) — on other hosts set + `SKIP_MOUNT=1` and provide `/dat` on a fast disk yourself. + +## Run it + +```sh +cd bench +cp config.env.example config.env # fill BUCKET, AWS keys, PGUSER/PGPASSWORD, sizing + +# 1. bootstrap this host (PG18 + build all three tools + systemd units) +sudo ./setup.sh # SKIP_MOUNT=1 sudo ./setup.sh if /dat already exists + +# 2. confirm the active daemon archives to S3 +bash scripts/sut/40_smoke_test.sh + +# 3. seed the bench DB once (shared across cells; large at full scale) +set -a; . ./config.env; set +a +PGHOST=127.0.0.1 ./scripts/driver/pgbench_init.sh + +# 4. run one cell, or the whole matrix +./run.sh pgbackrest r1 +./matrix.sh # pgbackrest, walg, walrus (once each) + +# 5. plots + raw CSV/JSON exports (installed by setup.sh) +bench-analyze --run results/walrus-r1 --label walrus --out results/plots +``` + +`run.sh` and `matrix.sh` run as a normal user (they `sudo` for the root steps); do not +run pgbench as root. Results land under `results/-/` (gitignored). + +## What `run.sh` does (the run contract) + +1. **Select the daemon** — write `wal-g.env` with this cell's `UPLOAD_CONCURRENCY` + (`11_write_walg_env.sh`), start its systemd unit, point `archive_command` at the + tool's own client (`30_select_daemon.sh`), pre-drain leftover `.ready` backlog. For + pgbackrest: set `process-max`, (re)create the stanza, set `archive-push`, drain. +2. **Normalize PG state** — force a checkpoint before the measured burst so + full-page-image WAL is comparable across cells. +3. **Reset** `pg_stat_archiver`, start the 1 Hz sampler into the results dir. +4. **Drive the workload** — `run_workload.sh`: the high-WAL burst (FPI-heavy random + UPDATEs on a wide indexed table + bulk COPY) — the heavy-load measurement. +5. **Capture** the final S3 inventory and `provenance.txt` (tool versions + binary + SHA-256, harness git SHA, run parameters). + +## Metrics (sampled at 1 Hz, written as CSV per run) + +| File | Metric | +|---|---| +| `mem.csv` | archiver `VmRSS` (resident) and `VmPeak` (virtual — the no-overcommit metric) | +| `cpu.csv` | archiver CPU % | +| `wal.csv` | cumulative WAL generated vs archived | +| `archive.csv` | `pg_stat_archiver` counters + `.ready` backlog (does the archiver keep up?) | +| `net.csv` | tx bytes (upload rate) | + +`bench-analyze` aggregates replicas of the same variant (median line + min..max band) +and also exports `samples_.csv` / `summary_.csv` / `summary.json` — every +row carries its run metadata, so the raw data is self-describing for external analysis. + +## Operation benchmarks + +`run_op.sh OP TOOL RUN_ID` extends the harness past the `archive_command` path to the +rest of walrus's data movement, single-host, reusing the same 1 Hz sampler — here +attached by `--proc-match` on the tool's process name, since these are one-shot CLI +runs, not daemons. `backup-fetch` and `wal-receive` run with the archive daemons +stopped, so the sample is the op process alone. The backup-push ops +(`backup-send`/`backup-delta`/`backup-delta-summaries`) keep the tool's own archive +daemon live — a base backup's `pg_backup_stop` blocks on `BackupWaitWalArchive` until +its WAL is archived — so for those the sample is the op process plus the mostly-idle +daemon (~27 MB for walrus; wal-g's fan-out daemon adds more baseline). + +| OP | walrus / wal-g | pgbackrest | measures | +|---|---|---|---| +| `backup-send` | `backup-push --full` | `backup --type=full` | full base backup → S3 | +| `backup-fetch` | `backup-fetch LATEST` | `restore` | restore ← S3 | +| `backup-delta` | `backup-push` (delta, `wi1`) | `backup --type=incr` | delta backup → S3 | +| `backup-delta-summaries` | `backup-push --delta-from-wal-summaries` | — (walrus-only) | delta from PG17 WAL summaries → S3 | +| `wal-receive` | `wal-receive ` | — (no equivalent) | stream WAL from PG | + +walrus's walsender (serving WAL over the replication protocol) has no CLI entry point +yet, so `wal-send` is intentionally absent. + +The two delta cells exercise walrus's incremental backup. `backup-delta` builds the +delta map by walking **archived WAL** (the default source; wal-g-comparable `wi1` +wire format, so it stays cross-tool). `backup-delta-summaries` instead sources the map +from `$PGDATA/pg_wal/summaries` (PG17 `summarize_wal=on`, enabled by `10_init_pg.sh`); +wal-g and pgbackrest have no WAL-summary delta, so it is walrus-only. Both first force +a checkpoint, then drive a `DELTA_CHURN_SECONDS` burst — with the archiver live so the +churn WAL is in the repo — drain, keep the archiver live, then time the delta push. +They need a parent full, so +`backup-send` must precede them. `DELTA_ORIGIN=LATEST_FULL` keeps each delta cell +anchored to the chain root by default; `DELTA_MAX_STEPS` still caps chain depth. + +```sh +# one cell (assumes setup.sh ran; non-fetch ops need the seeded DB) +./run_op.sh backup-send walrus r1 +./run_op.sh backup-fetch walrus r1 # fetches LATEST; run a backup-send first +./run_op.sh backup-delta walrus r1 # churn → delta push; needs a parent full +./run_op.sh backup-delta-summaries walrus r1 # walrus-only; needs summarize_wal=on +./run_op.sh wal-receive walrus r1 # streams for WAL_RECEIVE_SECONDS + +# whole sweep: send → fetch → delta → delta-summaries → wal-receive +./op_matrix.sh +``` + +Each cell writes the sampler CSVs plus `op_metrics.txt`: + +| Field | Meaning | +|---|---| +| `elapsed_s` | wall-clock of the operation | +| `bytes_processed` | backup-send: on-disk cluster size (excl. `pg_wal`); backup-fetch: restored bytes; backup-delta(-summaries): S3-inventory byte growth across the push (the delta's stored size); wal-receive: S3-inventory byte growth while receiver drains | +| `throughput_mb_s` | `bytes_processed / elapsed_s / 1e6` | +| `checkpoint_before_workload` | `1` when cell forced a checkpoint before FPI-sensitive work (backup-send, delta churn, wal-receive); else `0` | +| `delta_origin` | delta parent policy passed as `WALG_DELTA_ORIGIN` for walrus / wal-g delta cells | + +Notes: +- `backup-fetch` fetches `LATEST` from the tool's repo. `run_op.sh` scopes walrus / + wal-g and pgbackrest prefixes by tool and run ID, so `LATEST` and implicit delta + parents cannot come from another tool or a prior sweep. Op order runs + `backup-fetch` before the delta cells, so it restores a clean full. +- `backup-send`/`backup-fetch` use `RESTORE_DIR`/`WAL_RECV_DIR` (wiped per run); keep + them on the fast disk. `wal-receive` and the delta cells drive the burst workload as + their WAL source (`WAL_RECEIVE_SECONDS` / `DELTA_CHURN_SECONDS`). +- pgbackrest `backup` (full or `incr`) needs live archiving, so those cells point + `archive_command` at pgbackrest and drain first, as the archive bench does. + +## Config knobs + +See `config.env.example`. Common ones: `UPLOAD_CONCURRENCY` (wal-g concurrency / +pgbackrest `process-max`), `SCALE` (pgbench DB size), `CHURN_ROWS`, `BURST_SECONDS`, +`BURST_WORKERS`. `matrix.sh` honors `DAEMONS` (and `RUN_ID`). Operation benchmarks add +`RESTORE_DIR`, `WAL_RECV_DIR`, `WAL_RECEIVE_SECONDS`, `DELTA_CHURN_SECONDS`, +`DELTA_MAX_STEPS`, `DELTA_ORIGIN`; `op_matrix.sh` honors `OPS`, `TOOLS` (and +`RUN_ID`). diff --git a/bench/config.env.example b/bench/config.env.example new file mode 100644 index 0000000..8f1c057 --- /dev/null +++ b/bench/config.env.example @@ -0,0 +1,49 @@ +# bench config — copy to config.env (gitignored) and fill in. +# Sourced by setup.sh, run.sh, matrix.sh. Plain bash; no quoting tricks needed. + +# --- S3 target --------------------------------------------------------------- +BUCKET=my-wal-bench-bucket +AWS_REGION=us-east-1 + +# AWS credentials. walrus (wal-rs) reads credentials from the ENVIRONMENT ONLY +# — no IMDS, no shared-config profiles — so explicit keys are REQUIRED here; +# 11_write_walg_env.sh writes them into the daemons' env file. The same keys +# are used by the `aws` CLI for the run's S3 inventory + smoke test. +# (AWS_PROFILE would cover the CLI but NOT walrus, so set the keys.) +AWS_ACCESS_KEY_ID=AKIA... +AWS_SECRET_ACCESS_KEY=... +# Only for temporary/STS credentials; leave unset for long-lived keys. +#AWS_SESSION_TOKEN= + +# --- Postgres bench role ----------------------------------------------------- +PGUSER=walbench +PGPASSWORD=change-me + +# --- daemon / archiver tuning ------------------------------------------------ +UPLOAD_CONCURRENCY=4 # wal-g WALG_UPLOAD_CONCURRENCY / pgbackrest process-max +WALG_COMPRESSION_METHOD=lz4 + +# --- workload sizing --------------------------------------------------------- +SCALE=5000 # pgbench scale (~15 MB/unit; 5000 ≈ 75 GB) +CHURN_ROWS=2000000 # burst-phase churn table rows (must match pgbench_init) +BURST_SECONDS=300 # high-WAL burst phase (the heavy-load measurement) +#BURST_WORKERS= # defaults to driver nproc + +# --- operation benchmarks (run_op.sh / op_matrix.sh) ------------------------- +# backup-send / backup-fetch / backup-delta / backup-delta-summaries / +# wal-receive, cross-tool. Both dirs are wiped per run; point them at the fast +# disk (under /dat by convention). +#RESTORE_DIR=/dat/restore # backup-fetch restores here +#WAL_RECV_DIR=/dat/walrecv # wal-receive assembles segments here +WAL_RECEIVE_SECONDS=300 # wal-receive streaming window (burst drives WAL) +# Delta cells: churn window that dirties pages between the parent full and the +# delta push, and the delta-chain depth (WALG_DELTA_MAX_STEPS) for walrus/wal-g. +DELTA_CHURN_SECONDS=300 +DELTA_MAX_STEPS=7 +DELTA_ORIGIN=LATEST_FULL + +# --- host conventions (override only if your box differs) -------------------- +# BUILD_USER owns the rust/cargo toolchain (defaults to sudo invoker, then ubuntu). +#BUILD_USER=ubuntu +# CIDR allowed in pg_hba; single-host driver == this box. +#DRIVER_CIDR=127.0.0.1/32 diff --git a/bench/matrix.sh b/bench/matrix.sh new file mode 100755 index 0000000..f57ad90 --- /dev/null +++ b/bench/matrix.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# matrix.sh [RUN_ID] +# +# RUN_ID - label for this sweep's result dirs (default r1). +# +# Sweeps comparison on this host: pgbackrest, wal-g, walrus, once each, +# calling run.sh per cell. Single-host counterpart of the external fleet's +# orchestrate/run_matrix.sh (no SSH, and no GOMEMLIMIT-cap cell — that was a +# GC-policy experiment, not intrinsic footprint). +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +RUN_ID="${1:-${RUN_ID:-r1}}" +read -r -a DAEMONS <<< "${DAEMONS:-pgbackrest walg walrus}" + +log() { printf '[matrix %s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } + +log "start: run_id=${RUN_ID} daemons='${DAEMONS[*]}'" + +for daemon in "${DAEMONS[@]}"; do + log "=== run ${daemon} ${RUN_ID} ===" + "${SCRIPT_DIR}/run.sh" "${daemon}" "${RUN_ID}" +done + +log "DONE" diff --git a/bench/op_matrix.sh b/bench/op_matrix.sh new file mode 100755 index 0000000..77abfaa --- /dev/null +++ b/bench/op_matrix.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# op_matrix.sh [RUN_ID] +# +# RUN_ID - label for this sweep's result dirs (default r1). +# +# Sweeps data-movement operation benchmarks (run_op.sh) on this host: +# backup-send, backup-fetch, backup-delta, backup-delta-summaries, then +# wal-receive — each across the tools that implement it, once. Op order matters: +# backup-send runs first so a parent full exists for backup-fetch to restore and +# for the delta cells to extend. +# +# Skipped cells: pgbackrest has no wal-receive equivalent; backup-delta-summaries +# is walrus-only (no wal-g / pgbackrest WAL-summary delta). Override OPS / TOOLS +# via env. Counterpart of matrix.sh (archive path). +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +RUN_ID="${1:-${RUN_ID:-r1}}" +read -r -a OPS <<< "${OPS:-backup-send backup-fetch backup-delta backup-delta-summaries wal-receive}" +read -r -a TOOLS <<< "${TOOLS:-pgbackrest walg walrus}" + +log() { printf '[op-matrix %s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } + +log "start: run_id=${RUN_ID} ops='${OPS[*]}' tools='${TOOLS[*]}'" + +for op in "${OPS[@]}"; do + for tool in "${TOOLS[@]}"; do + if [[ "${op}" == "wal-receive" && "${tool}" == "pgbackrest" ]]; then + log "skip ${op}/${tool} (no equivalent)" + continue + fi + if [[ "${op}" == "backup-delta-summaries" && "${tool}" != "walrus" ]]; then + log "skip ${op}/${tool} (walrus-only)" + continue + fi + log "=== run ${op} ${tool} ${RUN_ID} ===" + "${SCRIPT_DIR}/run_op.sh" "${op}" "${tool}" "${RUN_ID}" + done +done + +log "DONE" diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 0000000..eb3c8a4 --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# +# run.sh DAEMON RUN_ID +# +# DAEMON - walg | walrus | pgbackrest (which archiver to exercise) +# RUN_ID - free-form label, e.g. r1 / 2026-06-22 +# +# Drives ONE benchmark cell end to end on THIS host (PG + daemon + pgbench all +# local). Single-host counterpart of the external fleet's orchestrate/run_one.sh, +# with SSH and IMDS removed: credentials come from config.env, the workload runs +# against the local cluster. +# +# 1. Select the daemon: write wal-g.env (this cell's concurrency), start its +# unit, point archive_command at the tool's own client, pre-drain backlog. +# (pgbackrest is daemonless: set process-max, stanza, archive-push, drain.) +# 2. Reset pg_stat_archiver, start the 1 Hz sampler into the results dir. +# 3. Run the workload (high-WAL burst) against local PG. +# 4. Stop the sampler, capture the S3 inventory + provenance. +# +# Results land under bench/results/-/ (override RESULTS_ROOT). +# Assumes ./setup.sh has run and the bench DB is seeded (pgbench_init.sh). +# Run as a normal user (it uses sudo for the root steps); do not run pgbench as +# root. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +LOG_TAG=run +# shellcheck source=scripts/lib.sh +. "${SCRIPT_DIR}/scripts/lib.sh" +load_config + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi +DAEMON="$1" +RUN_ID="$2" + +case "${DAEMON}" in + walg|walrus|pgbackrest) ;; + *) echo "error: DAEMON must be walg|walrus|pgbackrest, got '${DAEMON}'" >&2; exit 2 ;; +esac + +: "${BUCKET:?set BUCKET in config.env}" +: "${PGUSER:?set PGUSER in config.env}" +: "${PGPASSWORD:?set PGPASSWORD in config.env}" +: "${UPLOAD_CONCURRENCY:?set UPLOAD_CONCURRENCY in config.env}" + +# --- fixed contract constants ------------------------------------------------ +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd)" +PG_ENV_FILE="/etc/postgresql/wal-g.env" +AWS_REGION="${AWS_REGION:-us-east-1}" +COMPRESSION="${WALG_COMPRESSION_METHOD:-lz4}" +# Scope the archive prefix per daemon+run (same bucket = same destination storage, +# fair comparison) so cells of a sweep do not pile WAL into one shared prefix. +WALG_PREFIX="s3://${BUCKET}/walg-bench/${DAEMON}/${RUN_ID}" +PGBACKREST_REPO_PATH="/pgbackrest-bench/${DAEMON}/${RUN_ID}" +PGBACKREST_STANZA="walbench" +PGDATA_DIR="/dat/18/data" +PGBIN="/usr/lib/postgresql/18/bin" +PGHOST_DRIVER="${PGHOST_DRIVER:-127.0.0.1}" +RESULTS_ROOT="${RESULTS_ROOT:-${SCRIPT_DIR}/results}" +RESULT_DIR="${RESULTS_ROOT}/${DAEMON}-${RUN_ID}" +SAMPLER="/usr/local/bin/bench-sampler" +WORKLOAD="${SCRIPT_DIR}/scripts/driver/run_workload.sh" +if [[ "${DAEMON}" == "pgbackrest" ]]; then + INV_PREFIX="s3://${BUCKET}${PGBACKREST_REPO_PATH}" +else + INV_PREFIX="${WALG_PREFIX}" +fi + +# --- pre-flight: DB seeded? -------------------------------------------------- +require_seeded + +log "daemon=${DAEMON} run_id=${RUN_ID} concurrency=${UPLOAD_CONCURRENCY}" + +# --- step 1: select + configure the daemon ----------------------------------- +if [[ "${DAEMON}" == "pgbackrest" ]]; then + log "configuring pgbackrest (stanza=${PGBACKREST_STANZA}, process-max=${UPLOAD_CONCURRENCY})" + run_root "${UPLOAD_CONCURRENCY}" "${PGBACKREST_STANZA}" "${PGDATA_DIR}" "${PGBIN}" \ + "${PGBACKREST_REPO_PATH}" <<'REMOTE' +set -euo pipefail +CONCURRENCY="$1"; STANZA="$2"; PGDATA_DIR="$3"; PGBIN="$4"; REPO_PATH="$5" +CONF="/etc/pgbackrest/pgbackrest.conf" + +# Daemonless: stop wal-g/walrus so only pgbackrest archives this cell. +systemctl stop wal-g.service walrus.service 2>/dev/null || true +rm -f /tmp/wal-g + +[[ -f "${CONF}" ]] || { echo "error: ${CONF} missing (run 05_install_pgbackrest.sh)" >&2; exit 1; } +sed -i -E "s/^process-max=.*/process-max=${CONCURRENCY}/" "${CONF}" +if grep -qE '^repo1-path=' "${CONF}"; then + sed -i -E "s#^repo1-path=.*#repo1-path=${REPO_PATH}#" "${CONF}" +else + printf 'repo1-path=%s\n' "${REPO_PATH}" >>"${CONF}" +fi +echo "process-max -> $(grep -E '^process-max=' "${CONF}")" +echo "repo1-path -> $(grep -E '^repo1-path=' "${CONF}")" + +sudo -u postgres pgbackrest --stanza="${STANZA}" stanza-create +ARCHIVE_CMD="pgbackrest --stanza=${STANZA} archive-push %p" +sudo -u postgres "${PGBIN}/psql" -p 5432 -tA \ + -c "ALTER SYSTEM SET archive_library = '';" \ + -c "ALTER SYSTEM SET archive_command = '${ARCHIVE_CMD}';" \ + -c "SELECT pg_reload_conf();" >/dev/null +sleep 2 +echo "archive_command set for pgbackrest: ${ARCHIVE_CMD}" + +if sudo -u postgres pgbackrest --stanza="${STANZA}" check; then + echo "pgbackrest check OK" +else + echo "warning: pgbackrest check non-zero; relying on workload metrics" >&2 +fi +REMOTE + log "pre-drain leftover backlog" + drain_backlog 10 300 +else + log "writing ${PG_ENV_FILE} + selecting ${DAEMON} (own daemon-client)" + # 11 writes the shared env file from the env credentials; 30 stops the other + # unit, starts this one, points archive_command at its own client, waits the + # socket. Both inherit BUCKET/creds/region from this shell. ENV_FILE is pinned + # to the daemon env path so a caller-set ENV_FILE (config selector) preserved + # by sudo -E cannot redirect 11's OUTPUT onto the config file. + ENV_FILE="${PG_ENV_FILE}" \ + BUCKET="${BUCKET}" UPLOAD_CONCURRENCY="${UPLOAD_CONCURRENCY}" \ + WALG_S3_PREFIX="${WALG_PREFIX}" \ + AWS_REGION="${AWS_REGION}" WALG_COMPRESSION_METHOD="${COMPRESSION}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}" AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}" \ + AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}" \ + sudo -E bash "${SCRIPT_DIR}/scripts/sut/11_write_walg_env.sh" + sudo bash "${SCRIPT_DIR}/scripts/sut/30_select_daemon.sh" "${DAEMON}" + + log "pre-drain leftover backlog" + drain_backlog 10 300 +fi + +log "checkpoint before measured burst" +checkpoint_pg + +# --- step 2: reset archiver stats + start sampler ---------------------------- +start_sampler --daemon "${DAEMON}" +trap stop_sampler EXIT + +# --- step 3: drive the workload (local) -------------------------------------- +log "starting workload against PG ${PGHOST_DRIVER}" +WL_ENV=(PGHOST="${PGHOST_DRIVER}" PGUSER="${PGUSER}" PGPASSWORD="${PGPASSWORD}" RUN_ID="${DAEMON}-${RUN_ID}") +for v in BURST_SECONDS BURST_WORKERS COPY_WORKERS COPY_BLOB_REPEAT CHURN_ROWS; do + if [[ -n "${!v:-}" ]]; then WL_ENV+=("${v}=${!v}"); fi +done +if env "${WL_ENV[@]}" bash "${WORKLOAD}"; then + log "workload complete" +else + mark_invalid "burst degraded (failed workers or failed transactions)" +fi + +# --- step 4a: stop sampler --------------------------------------------------- +stop_sampler +trap - EXIT + +# --- step 4b: capture S3 inventory + provenance ------------------------------ +log "capturing S3 inventory and provenance into ${RESULT_DIR}" +HARNESS_GIT="$(git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || echo 'no-git')" + +write_provenance "${RESULT_DIR}" "${INV_PREFIX}" "${AWS_REGION}" \ + "daemon=${DAEMON}" \ + "run_id=${RUN_ID}" \ + "upload_concurrency=${UPLOAD_CONCURRENCY}" \ + "scale=${SCALE:-unset}" \ + "churn_rows=${CHURN_ROWS:-unset}" \ + "burst_seconds=${BURST_SECONDS:-unset}" \ + "checkpoint_before_burst=1" \ + "harness_git=${HARNESS_GIT}" + +log "DONE: ${DAEMON}-${RUN_ID}" diff --git a/bench/run_op.sh b/bench/run_op.sh new file mode 100755 index 0000000..73aa769 --- /dev/null +++ b/bench/run_op.sh @@ -0,0 +1,442 @@ +#!/usr/bin/env bash +# +# run_op.sh OP TOOL RUN_ID +# +# OP - backup-send | backup-fetch | backup-delta | +# backup-delta-summaries | wal-receive (data-movement operation) +# TOOL - walrus | walg | pgbackrest (implementation) +# RUN_ID - free-form label, e.g. r1 / 2026-06-22 +# +# Benchmarks ONE data-movement operation with ONE tool, single-host (PG + tool +# local), cross-tool where an equivalent exists. Counterpart of run.sh, which +# benches the archive_command (wal-push) path; this covers the rest of walrus: +# +# backup-send base backup -> S3 walrus/wal-g backup-push --full | pgbackrest backup --type=full +# backup-fetch restore <- S3 walrus/wal-g backup-fetch | pgbackrest restore +# backup-delta delta backup -> S3 walrus/wal-g backup-push (wi1) | pgbackrest backup --type=incr +# backup-delta-summaries delta from WAL walrus backup-push | (walrus-only) +# summaries -> S3 --delta-from-wal-summaries +# wal-receive stream WAL from PG walrus/wal-g wal-receive | (no pgbackrest peer) +# +# Delta cells need a parent full backup (backup-send must precede them) and a +# churn phase: they configure the tool, checkpoint, drive a DELTA_CHURN_SECONDS +# burst with the archiver live (the default delta map walks archived WAL), +# drain, then time the delta push while archiver stays live. backup-delta-summaries +# instead sources the delta map from $PGDATA/pg_wal/summaries (needs +# summarize_wal=on, set by 10_init_pg.sh) and is walrus-only (no wal-g / +# pgbackrest peer). DELTA_ORIGIN defaults to LATEST_FULL so both delta paths +# anchor to chain root. Delta size is S3-inventory byte growth across the push, +# not on-disk cluster size. +# +# walrus's walsender (serving WAL via the replication protocol) has no CLI entry +# point yet, so wal-send is intentionally absent. +# +# The 1 Hz sampler is reused, here attached with --proc-match : these +# ops are one-shot CLI processes, not systemd units. Both archive daemons are +# stopped first; backup-push ops (NEEDS_ARCHIVE) then start ONLY the tool's own +# daemon and leave it up across the push (pg_backup_stop blocks on WAL archival), +# so for those the sample is the op process plus the mostly-idle daemon (~27 MB +# for walrus). backup-fetch / wal-receive run with no daemon — op process only. +# +# Results: bench/results/--/ — sampler CSVs, op_metrics.txt +# (elapsed, bytes processed, MB/s), provenance.txt, s3_inventory.txt. Override +# RESULTS_ROOT to relocate. +# +# Assumes ./setup.sh has run. backup-send and wal-receive also assume the bench +# DB is seeded (pgbench_init.sh); backup-fetch assumes a compatible backup-send +# already produced a backup to fetch. Run as a normal user (uses sudo for root +# steps); do not run pgbench as root. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +LOG_TAG=op +# shellcheck source=scripts/lib.sh +. "${SCRIPT_DIR}/scripts/lib.sh" +load_config + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi +OP="$1" +TOOL="$2" +RUN_ID="$3" + +case "${OP}" in + backup-send|backup-fetch|backup-delta|backup-delta-summaries|wal-receive) ;; + *) echo "error: OP must be backup-send|backup-fetch|backup-delta|backup-delta-summaries|wal-receive, got '${OP}'" >&2; exit 2 ;; +esac +case "${TOOL}" in + walrus|walg|pgbackrest) ;; + *) echo "error: TOOL must be walrus|walg|pgbackrest, got '${TOOL}'" >&2; exit 2 ;; +esac +if [[ "${OP}" == "wal-receive" && "${TOOL}" == "pgbackrest" ]]; then + echo "error: pgbackrest has no wal-receive equivalent (skip this cell)" >&2 + exit 2 +fi +# WAL-summary-sourced delta is a walrus-only path (no wal-g / pgbackrest peer). +if [[ "${OP}" == "backup-delta-summaries" && "${TOOL}" != "walrus" ]]; then + echo "error: backup-delta-summaries is walrus-only (skip this cell)" >&2 + exit 2 +fi + +# Delta ops drive a churn phase, then a delta push; group them for branch tests. +IS_DELTA=0 +[[ "${OP}" == "backup-delta" || "${OP}" == "backup-delta-summaries" ]] && IS_DELTA=1 + +# Backup-push ops (full + delta) take a base backup, whose pg_backup_stop blocks +# on BackupWaitWalArchive until the backup's WAL is archived. So the tool's +# archiver MUST stay live across these cells (the sampler then sees the op +# process plus the mostly-idle daemon; for walrus that baseline is ~27 MB). +# backup-fetch (restore) and wal-receive need no archiver. +NEEDS_ARCHIVE=0 +case "${OP}" in backup-send|backup-delta|backup-delta-summaries) NEEDS_ARCHIVE=1 ;; esac + +: "${BUCKET:?set BUCKET in config.env}" +: "${PGUSER:?set PGUSER in config.env}" +: "${PGPASSWORD:?set PGPASSWORD in config.env}" +: "${UPLOAD_CONCURRENCY:?set UPLOAD_CONCURRENCY in config.env}" + +# --- fixed contract constants ------------------------------------------------ +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd)" +AWS_REGION="${AWS_REGION:-us-east-1}" +COMPRESSION="${WALG_COMPRESSION_METHOD:-lz4}" +# Scope the prefix per tool+run (same bucket = same destination storage, fair +# comparison) so fetch LATEST / implicit delta-parent resolve only within current +# tool/run backups, never another tool's or a prior sweep's. +WALG_PREFIX="s3://${BUCKET}/walg-bench/${TOOL}/${RUN_ID}" +PGBACKREST_REPO_PATH="/pgbackrest-bench/${TOOL}/${RUN_ID}" +PGBACKREST_STANZA="walbench" +PGDATA_DIR="/dat/18/data" +PGBIN="/usr/lib/postgresql/18/bin" +PGHOST_DRIVER="${PGHOST_DRIVER:-127.0.0.1}" +WALRUS_BIN="/usr/local/bin/walrus" +WALG_BIN="/usr/bin/wal-g" +RESULTS_ROOT="${RESULTS_ROOT:-${SCRIPT_DIR}/results}" +RESULT_DIR="${RESULTS_ROOT}/${OP}-${TOOL}-${RUN_ID}" +SAMPLER="/usr/local/bin/bench-sampler" +# Where backup-fetch restores into and wal-receive assembles segments. +RESTORE_DIR="${RESTORE_DIR:-/dat/restore}" +WAL_RECV_DIR="${WAL_RECV_DIR:-/dat/walrecv}" +WAL_RECEIVE_SECONDS="${WAL_RECEIVE_SECONDS:-300}" +# Delta cells: churn window that dirties pages between the parent full and the +# delta push, and the delta-chain depth handed to walrus/wal-g (WALG_DELTA_MAX_STEPS). +DELTA_CHURN_SECONDS="${DELTA_CHURN_SECONDS:-300}" +DELTA_MAX_STEPS="${DELTA_MAX_STEPS:-7}" +DELTA_ORIGIN="${DELTA_ORIGIN:-LATEST_FULL}" + +case "${TOOL}" in + walrus) COMM="walrus" ;; + walg) COMM="wal-g" ;; + pgbackrest) COMM="pgbackrest" ;; +esac +if [[ "${TOOL}" == "pgbackrest" ]]; then + INV_PREFIX="s3://${BUCKET}${PGBACKREST_REPO_PATH}" +else + INV_PREFIX="${WALG_PREFIX}" +fi + +# Run a walrus/wal-g command as postgres with the daemon env file sourced +# (WALG_S3_PREFIX, AWS creds, region, compression, PGHOST). Absolute paths, so +# no reliance on the postgres login PATH. +run_tool() { + sudo -u postgres bash -c ' + set -a + . /etc/postgresql/wal-g.env + set +a + exec "$@" + ' _ "$@" +} + +# Current WAL position as an absolute byte offset (for wal-receive throughput). +lsn_bytes() { + PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST_DRIVER}" -U "${PGUSER}" -d walbench \ + -tAc "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(),'0/0')" +} + +# Total bytes stored under the tool's S3 prefix (delta cells diff before/after +# the push to size the increment). Empty/zero when the prefix has no objects. +inv_size() { + sudo aws s3 ls --recursive --summarize "${INV_PREFIX}/" --region "${AWS_REGION}" 2>/dev/null \ + | awk '/Total Size:/ {print $3}' | tail -1 +} + +# --- pre-flight: DB seeded? (backup-send + wal-receive need a populated DB) --- +[[ "${OP}" == "backup-fetch" ]] || require_seeded + +log "op=${OP} tool=${TOOL} run_id=${RUN_ID} concurrency=${UPLOAD_CONCURRENCY}" +CHECKPOINT_BEFORE_WORKLOAD=0 + +# --- step 1: tool config ----------------------------------------------------- +# Stop both archive daemons so neither pollutes proc-match (they share the +# 'walrus'/'wal-g' comm with the op process) and so they do not race archiving. +run_root <<'REMOTE' +set -euo pipefail +systemctl stop wal-g.service walrus.service 2>/dev/null || true +rm -f /tmp/wal-g +REMOTE + +if [[ "${TOOL}" == "pgbackrest" ]]; then + log "configuring pgbackrest (stanza=${PGBACKREST_STANZA}, process-max=${UPLOAD_CONCURRENCY})" + run_root "${UPLOAD_CONCURRENCY}" "${PGBACKREST_STANZA}" "${PGDATA_DIR}" "${PGBIN}" \ + "${OP}" "${PGBACKREST_REPO_PATH}" <<'REMOTE' +set -euo pipefail +CONCURRENCY="$1"; STANZA="$2"; PGDATA_DIR="$3"; PGBIN="$4"; OP="$5"; REPO_PATH="$6" +CONF="/etc/pgbackrest/pgbackrest.conf" +[[ -f "${CONF}" ]] || { echo "error: ${CONF} missing (run 05_install_pgbackrest.sh)" >&2; exit 1; } +sed -i -E "s/^process-max=.*/process-max=${CONCURRENCY}/" "${CONF}" +if grep -qE '^repo1-path=' "${CONF}"; then + sed -i -E "s#^repo1-path=.*#repo1-path=${REPO_PATH}#" "${CONF}" +else + printf 'repo1-path=%s\n' "${REPO_PATH}" >>"${CONF}" +fi +echo "process-max -> $(grep -E '^process-max=' "${CONF}")" +echo "repo1-path -> $(grep -E '^repo1-path=' "${CONF}")" +sudo -u postgres pgbackrest --stanza="${STANZA}" stanza-create || true + +# backup (full or incr) needs WAL archiving live (pgbackrest blocks on the +# start-WAL archive), so point archive_command at pgbackrest and drain. restore +# reads only the repo. backup-delta (incr) churns + drains in the delta-prep step. +if [[ "${OP}" == "backup-send" || "${OP}" == "backup-delta" ]]; then + ARCHIVE_CMD="pgbackrest --stanza=${STANZA} archive-push %p" + sudo -u postgres "${PGBIN}/psql" -p 5432 -tA \ + -c "ALTER SYSTEM SET archive_library = '';" \ + -c "ALTER SYSTEM SET archive_command = '${ARCHIVE_CMD}';" \ + -c "SELECT pg_reload_conf();" >/dev/null + sleep 2 + sudo -u postgres pgbackrest --stanza="${STANZA}" check || \ + echo "warning: pgbackrest check non-zero" >&2 +fi +REMOTE + [[ "${NEEDS_ARCHIVE}" -eq 1 ]] && { log "pre-drain leftover backlog"; drain_backlog 10 300; } +else + log "writing /etc/postgresql/wal-g.env for ${TOOL}" + # Pin ENV_FILE to the daemon env path: 11_write_walg_env.sh reads ENV_FILE as + # its OUTPUT target, and sudo -E would otherwise leak a caller-set ENV_FILE + # (our config-file selector) and clobber it. + ENV_FILE="/etc/postgresql/wal-g.env" \ + BUCKET="${BUCKET}" UPLOAD_CONCURRENCY="${UPLOAD_CONCURRENCY}" \ + WALG_S3_PREFIX="${WALG_PREFIX}" \ + AWS_REGION="${AWS_REGION}" WALG_COMPRESSION_METHOD="${COMPRESSION}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}" AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}" \ + AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}" \ + sudo -E bash "${SCRIPT_DIR}/scripts/sut/11_write_walg_env.sh" + + # backup-push ops need a live archiver (see NEEDS_ARCHIVE): start the tool's + # daemon and point archive_command at its own client, then pre-drain leftover + # backlog. backup-fetch / wal-receive skip this (no archiving needed). + if [[ "${NEEDS_ARCHIVE}" -eq 1 ]]; then + log "starting ${TOOL} archive daemon (backup-push waits on WAL archival at stop)" + sudo bash "${SCRIPT_DIR}/scripts/sut/30_select_daemon.sh" "${TOOL}" + drain_backlog 10 300 + fi +fi + +if [[ "${OP}" == "backup-send" || "${OP}" == "wal-receive" ]]; then + log "checkpoint before measured ${OP}" + checkpoint_pg + CHECKPOINT_BEFORE_WORKLOAD=1 +fi + +# --- step 1b: delta prep — churn between the parent full and the delta push --- +# The default delta map walks ARCHIVED WAL, so the churn WAL must reach the repo +# before the push. The tool's archiver is already live (step 1, NEEDS_ARCHIVE) +# and STAYS up through the push: pg_backup_stop blocks on WAL archival, so a push +# without a live archiver hangs. Just churn, then drain so the map is complete. +# (backup-delta-summaries sources the map from local pg_wal/summaries instead, +# but archiving the churn still lets pg_wal recycle and keeps the parent valid.) +if [[ "${IS_DELTA}" -eq 1 ]]; then + log "delta-prep: checkpoint before churn" + checkpoint_pg + CHECKPOINT_BEFORE_WORKLOAD=1 + log "delta-prep: churn ${DELTA_CHURN_SECONDS}s (dirties pages for the delta)" + CH_ENV=(PGHOST="${PGHOST_DRIVER}" PGUSER="${PGUSER}" PGPASSWORD="${PGPASSWORD}" + DURATION="${DELTA_CHURN_SECONDS}" CHURN_ROWS="${CHURN_ROWS:-2000000}") + [[ -n "${BURST_WORKERS:-}" ]] && CH_ENV+=("WORKERS=${BURST_WORKERS}") + if ! env "${CH_ENV[@]}" bash "${SCRIPT_DIR}/scripts/driver/workload_burst.sh"; then + mark_invalid "delta-prep churn degraded (weaker dirtying -> non-comparable delta)" + fi + + log "delta-prep: draining archive backlog so the churn WAL is in the repo" + drain_backlog 5 600 +fi + +# --- step 2: start the sampler (proc-match on the tool's comm) ---------------- +start_sampler --proc-match "${COMM}" +trap stop_sampler EXIT + +# --- step 3: run the operation, timed ---------------------------------------- +BYTES=0 +START="$(date +%s.%N)" +case "${OP}" in + backup-send) + log "base backup -> ${INV_PREFIX} (full)" + case "${TOOL}" in + walrus) run_tool "${WALRUS_BIN}" backup-push --full ;; + walg) run_tool "${WALG_BIN}" backup-push "${PGDATA_DIR}" --full ;; + pgbackrest) sudo -u postgres pgbackrest --stanza="${PGBACKREST_STANZA}" backup --type=full ;; + esac + # bytes processed = on-disk cluster size, excluding WAL (the backup payload) + BYTES="$(sudo du -sb --exclude=pg_wal "${PGDATA_DIR}" | awk '{print $1}')" + ;; + backup-delta) + inv_before="$(inv_size)"; inv_before="${inv_before:-0}" + log "delta backup -> ${INV_PREFIX} (wi1; origin=${DELTA_ORIGIN}; parent inventory ${inv_before} B)" + case "${TOOL}" in + walrus) run_tool env WALG_DELTA_MAX_STEPS="${DELTA_MAX_STEPS}" \ + WALG_DELTA_ORIGIN="${DELTA_ORIGIN}" \ + "${WALRUS_BIN}" backup-push --pgdata "${PGDATA_DIR}" ;; + walg) run_tool env WALG_DELTA_MAX_STEPS="${DELTA_MAX_STEPS}" \ + WALG_DELTA_ORIGIN="${DELTA_ORIGIN}" \ + "${WALG_BIN}" backup-push "${PGDATA_DIR}" ;; + pgbackrest) sudo -u postgres pgbackrest --stanza="${PGBACKREST_STANZA}" backup --type=incr ;; + esac + # bytes processed = inventory growth = the delta's stored (compressed) size + inv_after="$(inv_size)"; inv_after="${inv_after:-0}" + BYTES=$(( inv_after - inv_before )); (( BYTES < 0 )) && BYTES=0 + ;; + backup-delta-summaries) + inv_before="$(inv_size)"; inv_before="${inv_before:-0}" + log "delta-from-wal-summaries backup -> ${INV_PREFIX} (origin=${DELTA_ORIGIN}; parent inventory ${inv_before} B)" + run_tool env WALG_DELTA_MAX_STEPS="${DELTA_MAX_STEPS}" \ + WALG_DELTA_ORIGIN="${DELTA_ORIGIN}" \ + "${WALRUS_BIN}" backup-push --pgdata "${PGDATA_DIR}" --delta-from-wal-summaries + inv_after="$(inv_size)"; inv_after="${inv_after:-0}" + BYTES=$(( inv_after - inv_before )); (( BYTES < 0 )) && BYTES=0 + ;; + backup-fetch) + log "restore LATEST -> ${RESTORE_DIR}" + run_root "${RESTORE_DIR}" <<'REMOTE' +set -euo pipefail +RESTORE_DIR="$1" +rm -rf "${RESTORE_DIR}" +install -d -o postgres -g postgres "${RESTORE_DIR}" +REMOTE + case "${TOOL}" in + walrus) run_tool "${WALRUS_BIN}" backup-fetch "${RESTORE_DIR}" LATEST ;; + walg) run_tool "${WALG_BIN}" backup-fetch "${RESTORE_DIR}" LATEST ;; + pgbackrest) + sudo -u postgres pgbackrest --stanza="${PGBACKREST_STANZA}" \ + --pg1-path="${RESTORE_DIR}" --type=none restore ;; + esac + BYTES="$(sudo du -sb "${RESTORE_DIR}" | awk '{print $1}')" + log "cleaning ${RESTORE_DIR}" + sudo rm -rf "${RESTORE_DIR}" + ;; + wal-receive) + log "wal-receive for ${WAL_RECEIVE_SECONDS}s while burst generates WAL" + RECV_LOG="${RESULT_DIR}/wal-receive.log" + run_root "${WAL_RECV_DIR}" <<'REMOTE' +set -euo pipefail +WAL_RECV_DIR="$1" +rm -rf "${WAL_RECV_DIR}" +install -d -o postgres -g postgres "${WAL_RECV_DIR}" +REMOTE + if [[ "${TOOL}" == "walrus" ]]; then + # archive_dir is a rotation buffer: walrus uploads each rotated segment to + # WALG_S3_PREFIX, the SAME S3 destination wal-g streams to. Both are scored + # by what lands in storage (below), not where they stage locally. + recv_cmd=("${WALRUS_BIN}" wal-receive "${WAL_RECV_DIR}") + else + recv_cmd=("${WALG_BIN}" wal-receive) + fi + # Launch as postgres with the env file sourced; redirect INSIDE sudo so the + # log lands in the postgres-owned results dir. Background the sudo wrapper. + sudo -u postgres bash -c ' + set -a; . /etc/postgresql/wal-g.env; set +a + log="$1"; shift + exec "$@" >"${log}" 2>&1 + ' _ "${RECV_LOG}" "${recv_cmd[@]}" & + RECV_PID=$! + sleep 2 + if ! kill -0 "${RECV_PID}" 2>/dev/null; then + echo "error: wal-receive exited early; see ${RECV_LOG}" >&2 + sudo cat "${RECV_LOG}" >&2 || true + exit 1 + fi + recv_before="$(inv_size)"; recv_before="${recv_before:-0}" + lsn_start="$(lsn_bytes)" + log "generating WAL (burst) for ${WAL_RECEIVE_SECONDS}s" + WL_ENV=(PGHOST="${PGHOST_DRIVER}" PGUSER="${PGUSER}" PGPASSWORD="${PGPASSWORD}" + DURATION="${WAL_RECEIVE_SECONDS}" CHURN_ROWS="${CHURN_ROWS:-2000000}") + [[ -n "${BURST_WORKERS:-}" ]] && WL_ENV+=("WORKERS=${BURST_WORKERS}") + if ! env "${WL_ENV[@]}" bash "${SCRIPT_DIR}/scripts/driver/workload_burst.sh"; then + mark_invalid "wal-receive burst degraded" + fi + lsn_end="$(lsn_bytes)" + + # Throughput = WAL that actually LANDED in the S3 destination, not WAL + # generated by PG (pg_current_wal_lsn advances regardless of receiver lag). + # Uploads are async, so keep the receiver alive and poll the inventory until + # it stops growing before sizing receipt. + log "draining receiver uploads into ${INV_PREFIX}" + recv_after="${recv_before}"; prev="" + for _ in $(seq 1 30); do + kill -0 "${RECV_PID}" 2>/dev/null || break + recv_after="$(inv_size)"; recv_after="${recv_after:-0}" + [[ "${recv_after}" == "${prev}" ]] && break + prev="${recv_after}" + sleep 5 + done + BYTES=$(( recv_after - recv_before )); (( BYTES < 0 )) && BYTES=0 + + gen=$(( lsn_end - lsn_start )) + log "wal-receive: generated=${gen} B (uncompressed) received=${BYTES} B (stored)" + # WAL generated but nothing stored => measured generation, not receipt. + if (( gen > 0 && BYTES == 0 )); then + mark_invalid "wal-receive stored 0 B to ${INV_PREFIX} while ${gen} B WAL generated" + fi + log "stopping wal-receive" + kill "${RECV_PID}" 2>/dev/null || true + sudo pkill -TERM -x "${COMM}" 2>/dev/null || true + for _ in $(seq 1 10); do sudo pkill -0 -x "${COMM}" 2>/dev/null || break; sleep 1; done + sudo pkill -KILL -x "${COMM}" 2>/dev/null || true + ;; +esac +END="$(date +%s.%N)" + +# --- step 4a: stop sampler --------------------------------------------------- +stop_sampler +trap - EXIT + +# --- step 4b: metrics + inventory + provenance ------------------------------- +ELAPSED="$(awk -v a="${START}" -v b="${END}" 'BEGIN{printf "%.3f", b-a}')" +MBPS="$(awk -v by="${BYTES}" -v s="${ELAPSED}" 'BEGIN{printf "%.2f", (s>0)? by/1e6/s : 0}')" +log "elapsed=${ELAPSED}s bytes=${BYTES} throughput=${MBPS} MB/s" + +HARNESS_GIT="$(git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || echo 'no-git')" + +log "writing op metrics into ${RESULT_DIR}" +run_root "${RESULT_DIR}" "${OP}" "${TOOL}" "${RUN_ID}" "${ELAPSED}" "${BYTES}" \ + "${MBPS}" "${UPLOAD_CONCURRENCY}" "${WAL_RECEIVE_SECONDS}" \ + "${CHECKPOINT_BEFORE_WORKLOAD}" "${DELTA_ORIGIN}" <<'REMOTE' +set -euo pipefail +RESULT_DIR="$1"; OP="$2"; TOOL="$3"; RUN_ID="$4"; ELAPSED="$5"; BYTES="$6" +MBPS="$7"; CONCURRENCY="$8"; WAL_RECEIVE_SECONDS="$9" +CHECKPOINT_BEFORE_WORKLOAD="${10}" +DELTA_ORIGIN="${11}" +{ + echo "op=${OP}" + echo "tool=${TOOL}" + echo "run_id=${RUN_ID}" + echo "elapsed_s=${ELAPSED}" + echo "bytes_processed=${BYTES}" + echo "throughput_mb_s=${MBPS}" + echo "upload_concurrency=${CONCURRENCY}" + echo "wal_receive_seconds=${WAL_RECEIVE_SECONDS}" + echo "checkpoint_before_workload=${CHECKPOINT_BEFORE_WORKLOAD}" + echo "delta_origin=${DELTA_ORIGIN:-}" +} >"${RESULT_DIR}/op_metrics.txt" +cat "${RESULT_DIR}/op_metrics.txt" +REMOTE + +log "capturing S3 inventory and provenance into ${RESULT_DIR}" +write_provenance "${RESULT_DIR}" "${INV_PREFIX}" "${AWS_REGION}" \ + "op=${OP}" \ + "tool=${TOOL}" \ + "run_id=${RUN_ID}" \ + "checkpoint_before_workload=${CHECKPOINT_BEFORE_WORKLOAD}" \ + "delta_origin=${DELTA_ORIGIN}" \ + "harness_git=${HARNESS_GIT}" + +log "DONE: ${OP}-${TOOL}-${RUN_ID}" diff --git a/bench/scripts/driver/calibrate.sh b/bench/scripts/driver/calibrate.sh new file mode 100755 index 0000000..0e81ea2 --- /dev/null +++ b/bench/scripts/driver/calibrate.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# calibrate.sh +# +# Calibration helper: generate ~5 minutes of burst load and report how much WAL +# the SUT produced, so the operator can size burst WORKERS against the measured +# single-daemon drain rate (which the SUT-side sampler records in wal.csv / +# archive.csv at the same time). +# +# It records pg_current_wal_lsn() before and after, then prints WAL bytes, +# WAL MB/s generated, and sizing guidance: to satisfy the "generation >= ~2x +# drain" target, compare generated MB/s here to the drain MB/s the sampler saw. +# +# Env vars (with defaults): +# PGHOST (required) SUT private IP / host +# PGPORT 5432 +# PGUSER (required) login role +# PGPASSWORD (required) password (or ~/.pgpass) +# PGDATABASE walbench +# CAL_DURATION 300 calibration burst duration in seconds (~5 min) +# WORKERS burst workers to calibrate with (passed through) +# (any other workload_burst.sh env vars are passed through unchanged) + +set -euo pipefail + +PGPORT="${PGPORT:-5432}" +PGDATABASE="${PGDATABASE:-walbench}" +CAL_DURATION="${CAL_DURATION:-300}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BURST="${SCRIPT_DIR}/workload_burst.sh" + +: "${PGHOST:?Set PGHOST to the SUT private IP}" +: "${PGUSER:?Set PGUSER to the login role}" + +export PGPORT PGDATABASE + +if [[ ! -x "${BURST}" ]]; then + echo "FATAL: ${BURST} not found or not executable" >&2 + exit 1 +fi + +# Helper: absolute WAL byte position (LSN distance from origin). +wal_bytes() { + psql -d "${PGDATABASE}" -At -c \ + "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0')" +} + +echo "==> Calibration burst for ${CAL_DURATION}s against ${PGHOST}:${PGPORT}/${PGDATABASE}" +echo "==> Make sure the SUT sampler is running so drain (wal.csv/archive.csv) is captured concurrently." + +start_bytes="$(wal_bytes)" +start_epoch="$(date +%s)" + +# Run the burst at the calibration duration. WORKERS and other tunables are +# inherited from the environment by workload_burst.sh. +DURATION="${CAL_DURATION}" "${BURST}" + +end_bytes="$(wal_bytes)" +end_epoch="$(date +%s)" + +elapsed=$(( end_epoch - start_epoch )) +if (( elapsed <= 0 )); then + elapsed=1 +fi +gen_bytes=$(( end_bytes - start_bytes )) + +# Report in MB and MB/s using awk for floating point. +awk -v b="${gen_bytes}" -v s="${elapsed}" 'BEGIN { + mb = b / 1048576.0; + mbs = mb / s; + printf "\n===== CALIBRATION RESULT =====\n"; + printf "Elapsed: %d s\n", s; + printf "WAL generated: %.1f MB (%d bytes)\n", mb, b; + printf "WAL generation rate: %.1f MB/s\n", mbs; + printf "==============================\n\n"; + printf "Sizing guidance:\n"; + printf " * Read the single-daemon drain rate (DRAIN MB/s) from the SUT\n"; + printf " sampler (wal.csv slope, or archive.csv archived_count rate).\n"; + printf " * Target generation >= 2x drain so *.ready backlog climbs.\n"; + printf " * If generated %.1f MB/s < 2 * DRAIN, scale WORKERS up by about\n", mbs; + printf " ceil( (2 * DRAIN) / (%.1f / current_WORKERS) ).\n", mbs; + printf " * If it is already well above 2x drain, you can lower WORKERS to\n"; + printf " reduce driver-side cost while still backing up the archiver.\n"; +}' + +echo "==> Calibration complete." diff --git a/bench/scripts/driver/gen_schema.sql b/bench/scripts/driver/gen_schema.sql new file mode 100644 index 0000000..6150d14 --- /dev/null +++ b/bench/scripts/driver/gen_schema.sql @@ -0,0 +1,70 @@ +-- gen_schema.sql +-- Workload schema for the wal-g vs walrus WAL-archiving benchmark. +-- +-- Design goal: maximize WAL volume, dominated by full-page images (FPI). +-- * PostgreSQL writes a full-page image the first time a page is modified +-- after a checkpoint (full_page_writes is on by default). Every B-tree +-- index whose page is touched by an UPDATE also emits its own FPI. +-- * A WIDE row (so few rows per heap page, many distinct pages get dirtied) +-- plus MANY indexes (so a single UPDATE fans out into several index-page +-- FPIs) turns random UPDATE storms into a WAL firehose. +-- * fillfactor is lowered so HOT-update pruning still rewrites pages and the +-- heap stays spread across many pages rather than packing tightly. +-- +-- Run against the 'walbench' database (created by pgbench_init.sh). + +-- --------------------------------------------------------------------------- +-- wal_churn: wide, heavily-indexed table targeted by random UPDATE storms. +-- --------------------------------------------------------------------------- +DROP TABLE IF EXISTS wal_churn; + +CREATE TABLE wal_churn ( + id bigint PRIMARY KEY, + k1 bigint NOT NULL, -- indexed, frequently updated + k2 bigint NOT NULL, -- indexed, frequently updated + k3 integer NOT NULL, -- indexed, frequently updated + tag text NOT NULL, -- indexed, frequently updated + updated_at timestamptz NOT NULL, -- indexed, frequently updated + counter bigint NOT NULL DEFAULT 0, + payload text NOT NULL -- wide filler -> few rows per page +) WITH (fillfactor = 70); + +-- Four secondary B-tree indexes (+ the primary key = five B-trees total). +-- Each UPDATE that changes an indexed column dirties the matching index pages, +-- multiplying FPI WAL output per modified row. +CREATE INDEX wal_churn_k1_idx ON wal_churn (k1); +CREATE INDEX wal_churn_k2_idx ON wal_churn (k2); +CREATE INDEX wal_churn_k3_idx ON wal_churn (k3); +CREATE INDEX wal_churn_tag_idx ON wal_churn (tag); +CREATE INDEX wal_churn_updated_at_idx ON wal_churn (updated_at); + +-- Seed rows. ROWS is substituted by the loader; default 2,000,000 (~ a few GB +-- of heap once the payload filler is included). Spread across many pages thanks +-- to the wide payload and fillfactor 70. +INSERT INTO wal_churn (id, k1, k2, k3, tag, updated_at, counter, payload) +SELECT + g, + (random() * 1e9)::bigint, + (random() * 1e9)::bigint, + (random() * 1e6)::integer, + md5(g::text), + now(), + 0, + repeat(md5(random()::text), 8) -- ~256 bytes of filler per row +FROM generate_series(1, :rows) AS g; + +-- --------------------------------------------------------------------------- +-- wal_bulk: append/truncate target for large COPY bursts. Logged so every +-- COPY is fully WAL-logged. No indexes: COPY here is pure heap-insert WAL. +-- --------------------------------------------------------------------------- +DROP TABLE IF EXISTS wal_bulk; + +CREATE TABLE wal_bulk ( + id bigint NOT NULL, + batch integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + blob text NOT NULL +); + +ANALYZE wal_churn; +ANALYZE wal_bulk; diff --git a/bench/scripts/driver/pgbench_init.sh b/bench/scripts/driver/pgbench_init.sh new file mode 100755 index 0000000..7982efe --- /dev/null +++ b/bench/scripts/driver/pgbench_init.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# pgbench_init.sh +# +# Initialize the benchmark database on the SUT (driven over the network): +# 1. create the 'walbench' database if it is absent +# 2. pgbench -i -s "$SCALE" to lay down the standard TPC-B tables +# 3. apply gen_schema.sql to add the WAL-churn / bulk-COPY workload tables +# +# All connection parameters come from libpq env vars so no IPs/passwords are +# hardcoded. Set PGHOST to the SUT private IP before running. +# +# Env vars (with defaults): +# PGHOST (required) SUT private IP / host +# PGPORT 5432 +# PGUSER (required) login role +# PGPASSWORD (required) password for PGUSER (or use ~/.pgpass) +# PGDATABASE walbench target database +# SCALE 5000 pgbench scaling factor (-s) +# CHURN_ROWS 2000000 rows seeded into wal_churn by gen_schema.sql +# PGBENCH_INIT_JOBS 8 parallel client jobs for pgbench load phase (-j) + +set -euo pipefail + +PGPORT="${PGPORT:-5432}" +PGDATABASE="${PGDATABASE:-walbench}" +SCALE="${SCALE:-5000}" +CHURN_ROWS="${CHURN_ROWS:-2000000}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCHEMA_SQL="${SCRIPT_DIR}/gen_schema.sql" + +: "${PGHOST:?Set PGHOST to the SUT private IP}" +: "${PGUSER:?Set PGUSER to the login role}" + +export PGPORT + +if [[ ! -f "${SCHEMA_SQL}" ]]; then + echo "FATAL: schema file not found: ${SCHEMA_SQL}" >&2 + exit 1 +fi + +echo "==> Target: ${PGUSER}@${PGHOST}:${PGPORT}, database '${PGDATABASE}', scale ${SCALE}" + +# 1. Create the database if it does not already exist. Connect to 'postgres' +# for the existence check / CREATE DATABASE. +db_exists="$(psql -d postgres -At -c \ + "SELECT 1 FROM pg_database WHERE datname = '${PGDATABASE}'")" + +if [[ "${db_exists}" == "1" ]]; then + echo "==> Database '${PGDATABASE}' already exists; skipping createdb." +else + echo "==> Creating database '${PGDATABASE}'." + createdb "${PGDATABASE}" +fi + +# 2. Standard pgbench TPC-B tables at the requested scale. Init mode is +# single-threaded — pgbench rejects -j here ("cannot be used in init mode"). +echo "==> pgbench -i -s ${SCALE} (this can take a while at large scale)." +pgbench -i -s "${SCALE}" "${PGDATABASE}" + +# 3. WAL-churn workload schema. CHURN_ROWS is passed as the :rows variable. +echo "==> Applying gen_schema.sql with ${CHURN_ROWS} churn rows." +psql -d "${PGDATABASE}" -v ON_ERROR_STOP=1 \ + -v rows="${CHURN_ROWS}" \ + -f "${SCHEMA_SQL}" + +echo "==> Initialization complete." diff --git a/bench/scripts/driver/run_workload.sh b/bench/scripts/driver/run_workload.sh new file mode 100755 index 0000000..dffd64b --- /dev/null +++ b/bench/scripts/driver/run_workload.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# run_workload.sh +# +# Driver-side orchestrator for ONE benchmark cell. run.sh invokes it locally as: +# PGHOST=.. PGUSER=.. PGPASSWORD=.. RUN_ID=.. bash scripts/driver/run_workload.sh +# +# Runs the measured workload: the high-WAL burst phase, and BLOCKS until it +# finishes. The burst is the heavy-load phase we care about. Assumes the +# 'walbench' DB is already initialized (pgbench_init.sh ran once during setup) — +# it does NOT re-init. +# +# Env (with defaults): +# PGHOST/PGUSER/PGPASSWORD (required; passed by run.sh) +# PGDATABASE walbench +# BURST_SECONDS 300 burst-phase duration +# BURST_WORKERS concurrent burst workers (passed through) +# CHURN_ROWS 2000000 must match pgbench_init.sh CHURN_ROWS +# RUN_ID label, for logs only +# +# Runs from scripts/driver/, next to the workload_burst.sh phase script. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +export PGDATABASE="${PGDATABASE:-walbench}" +BURST_SECONDS="${BURST_SECONDS:-300}" +CHURN_ROWS="${CHURN_ROWS:-2000000}" + +: "${PGHOST:?run.sh must pass PGHOST}" +: "${PGUSER:?run.sh must pass PGUSER}" + +echo "==> run_workload RUN_ID=${RUN_ID:-?}: burst=${BURST_SECONDS}s db=${PGDATABASE}" + +echo "==> high-WAL burst" +if [[ -n "${BURST_WORKERS:-}" ]]; then + DURATION="${BURST_SECONDS}" CHURN_ROWS="${CHURN_ROWS}" WORKERS="${BURST_WORKERS}" \ + bash "${SCRIPT_DIR}/workload_burst.sh" +else + DURATION="${BURST_SECONDS}" CHURN_ROWS="${CHURN_ROWS}" \ + bash "${SCRIPT_DIR}/workload_burst.sh" +fi + +echo "==> run_workload complete RUN_ID=${RUN_ID:-?}" diff --git a/bench/scripts/driver/workload_burst.sh b/bench/scripts/driver/workload_burst.sh new file mode 100755 index 0000000..437a05f --- /dev/null +++ b/bench/scripts/driver/workload_burst.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +# workload_burst.sh +# +# High-WAL burst load, driven over the network at the SUT's PostgreSQL. +# Goal: generate WAL >= ~2x the single-daemon archive drain rate so the +# pg_wal *.ready backlog climbs into the hundreds/thousands and we can measure +# how each archiver daemon keeps up (or falls behind). +# +# Strategy: +# * UPDATE storm -- random-row UPDATEs on wal_churn that mutate *indexed* +# columns (k1,k2,k3,tag,updated_at). After each checkpoint, the first touch +# of every heap + index page emits a full-page image (FPI), so random +# scatter across a wide, 5-B-tree table maximizes WAL bytes per row. +# * COPY storm -- a fraction of workers run large COPY batches into the +# unindexed wal_bulk table for bursty heap-insert WAL on top of the UPDATEs. +# +# Both are expressed as pgbench custom scripts run by N concurrent workers +# (one pgbench process per worker so COPY \. blocks behave) for DURATION. +# +# Env vars (with defaults): +# PGHOST (required) SUT private IP / host +# PGPORT 5432 +# PGUSER (required) login role +# PGPASSWORD (required) password (or ~/.pgpass) +# PGDATABASE walbench +# WORKERS number of concurrent burst workers (default vCPUs) +# COPY_WORKERS how many of WORKERS run COPY bursts +# DURATION 600 total burst duration in seconds +# CHURN_ROWS 2000000 id range to scatter random UPDATEs over +# (must match pgbench_init.sh CHURN_ROWS) +# UPDATE_BATCH 25 UPDATEs per pgbench transaction (update workers) +# COPY_ROWS 50000 rows per COPY batch (copy workers) +# PROTOCOL prepared pgbench query protocol for the UPDATE workers + +set -euo pipefail + +PGPORT="${PGPORT:-5432}" +PGDATABASE="${PGDATABASE:-walbench}" +DURATION="${DURATION:-600}" +CHURN_ROWS="${CHURN_ROWS:-2000000}" +UPDATE_BATCH="${UPDATE_BATCH:-25}" +COPY_ROWS="${COPY_ROWS:-50000}" +COPY_BLOB_REPEAT="${COPY_BLOB_REPEAT:-8}" # md5 repeats per blob; raises WAL bytes/row at ~same CPU +PROTOCOL="${PROTOCOL:-prepared}" + +host_cpus="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" +WORKERS="${WORKERS:-${host_cpus}}" +COPY_WORKERS="${COPY_WORKERS:-$(( WORKERS / 4 > 0 ? WORKERS / 4 : 1 ))}" + +: "${PGHOST:?Set PGHOST to the SUT private IP}" +: "${PGUSER:?Set PGUSER to the login role}" + +if (( COPY_WORKERS > WORKERS )); then + echo "FATAL: COPY_WORKERS (${COPY_WORKERS}) > WORKERS (${WORKERS})" >&2 + exit 1 +fi +UPDATE_WORKERS=$(( WORKERS - COPY_WORKERS )) + +export PGPORT + +echo "==> Burst load: ${PGUSER}@${PGHOST}:${PGPORT}/${PGDATABASE}" +echo "==> workers=${WORKERS} (update=${UPDATE_WORKERS}, copy=${COPY_WORKERS}) duration=${DURATION}s" +echo "==> update_batch=${UPDATE_BATCH} copy_rows=${COPY_ROWS} churn_rows=${CHURN_ROWS}" + +# --- temp pgbench script files --------------------------------------------- +WORKDIR="$(mktemp -d)" +cleanup() { + # Kill any still-running worker pgbench processes, then remove temp scripts. + if [[ -n "${WORKER_PIDS:-}" ]]; then + # shellcheck disable=SC2086 + kill ${WORKER_PIDS} 2>/dev/null || true + fi + rm -rf "${WORKDIR}" +} +trap cleanup EXIT INT TERM + +UPDATE_SQL="${WORKDIR}/update.sql" +COPY_SQL="${WORKDIR}/copy.sql" + +# UPDATE worker script: one random id per :rid, repeated UPDATE_BATCH times in a +# single transaction. Every mutated column is indexed so each row dirties the +# heap page plus several index pages -> heavy FPI WAL. +{ + echo "\\set rmax ${CHURN_ROWS}" + echo "BEGIN;" + for _ in $(seq 1 "${UPDATE_BATCH}"); do + cat <<'SQL' +\set rid random(1, :rmax) +UPDATE wal_churn + SET k1 = (random() * 1e9)::bigint, + k2 = (random() * 1e9)::bigint, + k3 = (random() * 1e6)::integer, + tag = md5(random()::text), + updated_at = clock_timestamp(), + counter = counter + 1 + WHERE id = :rid; +SQL + done + echo "END;" +} > "${UPDATE_SQL}" + +# COPY worker script: build a COPY_ROWS-row batch on the server with +# generate_series feeding an INSERT...SELECT. This is the COPY-equivalent bulk +# heap-insert burst into the unindexed wal_bulk table, fully WAL-logged. +{ + echo "\\set batch random(1, 1000000000)" + cat < "${COPY_SQL}" + +# Reset wal_bulk so the COPY storm does not accumulate across cells/runs and +# fill the data volume — at COPY_ROWS=50000 a 10-min burst adds tens of GB, and +# nothing reclaimed it, so a multi-cell matrix on a finite NVMe eventually hit +# ENOSPC (PG crash + aborted workload). wal_bulk is pure churn (only there to +# emit heap-insert WAL), so truncating loses nothing and each cell starts clean. +echo "==> TRUNCATE wal_bulk (bound data-volume growth across cells/runs)" +psql -X -v ON_ERROR_STOP=1 -c "TRUNCATE TABLE wal_bulk;" + +# --- launch workers --------------------------------------------------------- +# One pgbench process per worker (-c 1 -j 1), each looping its script for the +# whole DURATION. Running them as separate processes keeps a clean 1:1 mapping +# between workers and backends and isolates COPY workers from UPDATE workers. +WORKER_PIDS="" + +launch_worker() { + local label="$1" script="$2" proto="$3" + pgbench \ + -c 1 -j 1 \ + -T "${DURATION}" \ + -M "${proto}" \ + --no-vacuum \ + -f "${script}" \ + "${PGDATABASE}" \ + > "${WORKDIR}/${label}.log" 2>&1 & + WORKER_PIDS="${WORKER_PIDS} $!" +} + +for i in $(seq 1 "${UPDATE_WORKERS}"); do + launch_worker "update-${i}" "${UPDATE_SQL}" "${PROTOCOL}" +done +for i in $(seq 1 "${COPY_WORKERS}"); do + # COPY/INSERT-SELECT workers use the simple protocol; no benefit from prepared. + launch_worker "copy-${i}" "${COPY_SQL}" "simple" +done + +echo "==> Launched ${WORKERS} workers; running for ${DURATION}s ..." + +# Wait for all workers; tally failures without aborting mid-burst so every log is +# still summarized below. +failed_workers=0 +for pid in ${WORKER_PIDS}; do + if ! wait "${pid}"; then + failed_workers=$(( failed_workers + 1 )) + fi +done +WORKER_PIDS="" # all reaped; nothing for cleanup() to kill + +echo "==> Per-worker pgbench summaries:" +for log in "${WORKDIR}"/*.log; do + [[ -e "${log}" ]] || continue + echo "---- $(basename "${log}" .log) ----" + grep -E "tps|number of transactions actually processed|failed" "${log}" || cat "${log}" +done + +# pgbench prints "number of failed transactions" only when >0 (deadlock / +# serialization / mid-run disconnect): a worker can exit 0 yet still drop work, so +# a clean exit code alone does not prove a full-strength workload. +failed_txns="$(grep -hoE 'number of failed transactions: [0-9]+' "${WORKDIR}"/*.log 2>/dev/null \ + | awk '{s+=$NF} END{print s+0}' || true)" + +echo "==> Burst finished: failed_workers=${failed_workers} failed_txns=${failed_txns}" + +# A degraded burst means this cell saw a weaker workload than its peers and is not +# comparable. Exit non-zero; callers record an explicit invalid-run marker. +if (( failed_workers != 0 || failed_txns != 0 )); then + echo "FATAL: burst degraded (${failed_workers} worker(s) failed, ${failed_txns} failed txn(s))" >&2 + exit 1 +fi diff --git a/bench/scripts/lib.sh b/bench/scripts/lib.sh new file mode 100644 index 0000000..9c47a6b --- /dev/null +++ b/bench/scripts/lib.sh @@ -0,0 +1,169 @@ +# shellcheck shell=bash +# +# lib.sh — shared scaffolding for the single-host benchmark drivers. +# +# Sourced (not executed) by run.sh (archive_command path) and run_op.sh +# (data-movement ops). Holds only the plumbing both share verbatim: config load, +# logging, the local root-exec wrapper, the seeded-DB preflight, archive-backlog +# drain, sampler start/stop, and inventory+provenance capture. The two drivers +# keep their own measurement models (daemon-as-signal vs daemon-as-noise, burst +# vs one-shot op); this file is scaffolding, not policy. +# +# Relies on globals set by sourcing driver before each call: SCRIPT_DIR, +# LOG_TAG, PGUSER, PGPASSWORD, PGHOST_DRIVER, PGDATA_DIR, PGBIN, RESULT_DIR, +# SAMPLER, AWS_REGION. + +# Source config.env (or ENV_FILE) with auto-export so child sudo blocks inherit. +load_config() { + set -a + # shellcheck source=../config.env.example + . "${ENV_FILE:-${SCRIPT_DIR}/config.env}" + set +a +} + +log() { printf '[%s %s] %s\n' "${LOG_TAG}" "$(date -u +%H:%M:%S)" "$*" >&2; } + +# Run a bash snippet as root locally (fed on stdin; positional args after --). +run_root() { sudo bash -s -- "$@"; } + +# Abort unless the bench DB is seeded (wal_churn present). Callers that do not +# need a populated DB (e.g. restore) skip this. +require_seeded() { + local seeded + seeded="$(PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST_DRIVER}" -U "${PGUSER}" \ + -d walbench -tAc "SELECT to_regclass('wal_churn') IS NOT NULL" 2>/dev/null || true)" + if [[ "${seeded}" != "t" ]]; then + echo "error: bench DB not seeded (wal_churn missing). Seed once, e.g.:" >&2 + echo " PGHOST=${PGHOST_DRIVER} PGUSER=${PGUSER} PGPASSWORD=*** SCALE=${SCALE:-5000} \\" >&2 + echo " CHURN_ROWS=${CHURN_ROWS:-2000000} ${SCRIPT_DIR}/scripts/driver/pgbench_init.sh" >&2 + exit 1 + fi +} + +# drain_backlog THRESHOLD ITERS — wait until the .ready archive backlog falls to +# THRESHOLD, polling every 2s up to ITERS times. Settles leftover WAL before a +# measured window so the sample is not contaminated by prior load. Aborts the +# cell (exit nonzero) if backlog still exceeds THRESHOLD after ITERS: a timed-out +# drain leaks prior load into the measured start, so fail rather than sample it. +drain_backlog() { + local threshold="$1" iters="$2" + run_root "${PGDATA_DIR}" "${threshold}" "${iters}" <<'REMOTE' +set -euo pipefail +PGDATA_DIR="$1"; THRESHOLD="$2"; ITERS="$3" +rb=0 +for _ in $(seq 1 "${ITERS}"); do + rb="$(ls "${PGDATA_DIR}/pg_wal/archive_status/" 2>/dev/null | grep -c '\.ready$' || true)" + [[ "${rb}" -le "${THRESHOLD}" ]] && break + sleep 2 +done +if [[ "${rb}" -gt "${THRESHOLD}" ]]; then + echo "error: drain timeout: ready backlog = ${rb} > ${THRESHOLD} (contaminated start; aborting cell)" >&2 + exit 1 +fi +echo "drain complete: ready backlog = ${rb}" +REMOTE +} + +# Normalize FPI state before burst workloads. CHECKPOINT is superuser-only. +checkpoint_pg() { + run_root "${PGBIN:-/usr/lib/postgresql/18/bin}" <<'REMOTE' +set -euo pipefail +PGBIN="$1" +sudo -u postgres "${PGBIN}/psql" -p 5432 -d walbench -X -q \ + -c "CHECKPOINT;" >/dev/null +echo "checkpoint complete" +REMOTE +} + +# Reset archiver stats and launch the 1 Hz sampler as postgres into RESULT_DIR. +# MODE_FLAG/MODE_VALUE select the attach mode: --daemon (run.sh, the +# daemon IS the measurement) or --proc-match (run_op.sh, match the op +# process). Aborts if the sampler does not come up. +start_sampler() { + local mode_flag="$1" mode_value="$2" + log "starting sampler (${mode_flag} ${mode_value}) -> ${RESULT_DIR}" + run_root "${RESULT_DIR}" "${SAMPLER}" "${mode_flag}" "${mode_value}" "${PGDATA_DIR}" <<'REMOTE' +set -euo pipefail +RESULT_DIR="$1"; SAMPLER="$2"; MODE_FLAG="$3"; MODE_VALUE="$4"; PGDATA="$5" +install -d -o postgres -g postgres "${RESULT_DIR}" +sudo -u postgres psql -X -q -c "SELECT pg_stat_reset_shared('archiver');" >/dev/null 2>&1 || true +sudo -u postgres bash -c " + nohup '${SAMPLER}' ${MODE_FLAG} '${MODE_VALUE}' --pgdata '${PGDATA}' \ + --outdir '${RESULT_DIR}' >'${RESULT_DIR}/sampler.log' 2>&1 & + echo \$! >'${RESULT_DIR}/sampler.pid' +" +sleep 1 +SPID="$(cat "${RESULT_DIR}/sampler.pid")" +if ! kill -0 "${SPID}" 2>/dev/null; then + echo "error: sampler failed to start; see ${RESULT_DIR}/sampler.log" >&2 + cat "${RESULT_DIR}/sampler.log" >&2 || true + exit 1 +fi +echo "sampler running pid=${SPID}" +REMOTE +} + +# Stop the sampler (TERM, then KILL after 10s grace). Safe to call twice and from +# an EXIT trap; never fails the caller. +stop_sampler() { + log "stopping sampler" + run_root "${RESULT_DIR}" <<'REMOTE' || true +set -euo pipefail +RESULT_DIR="$1" +if [[ -f "${RESULT_DIR}/sampler.pid" ]]; then + SPID="$(cat "${RESULT_DIR}/sampler.pid")" + kill "${SPID}" 2>/dev/null || true + for _ in $(seq 1 10); do kill -0 "${SPID}" 2>/dev/null || break; sleep 1; done + kill -9 "${SPID}" 2>/dev/null || true +fi +REMOTE +} + +# Record an explicit invalid-run marker in RESULT_DIR. bench-analyze skips any run +# dir containing INVALID, so a degraded cell is excluded from comparison instead +# of being silently averaged in. Reason is free text. Relies on RESULT_DIR. +mark_invalid() { + log "INVALID run: $*" + run_root "${RESULT_DIR}" "$*" <<'REMOTE' || true +set -euo pipefail +RESULT_DIR="$1"; REASON="$2" +install -d -o postgres -g postgres "${RESULT_DIR}" +printf 'reason=%s\ncaptured_at=%s\n' "${REASON}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + >"${RESULT_DIR}/INVALID" +chown postgres:postgres "${RESULT_DIR}/INVALID" +REMOTE +} + +# Capture the S3 inventory and write provenance.txt. Args: RESULT_DIR INV_PREFIX +# REGION then any number of leading "key=value" lines (driver-specific identity: +# daemon/op/tool, run_id, sizing, harness_git). The shared tool version/sha block +# and captured_at are appended. +write_provenance() { + run_root "$@" <<'REMOTE' +set -euo pipefail +RESULT_DIR="$1"; INV_PREFIX="$2"; AWS_REGION="$3"; shift 3 + +aws s3 ls --recursive --summarize "${INV_PREFIX}/" --region "${AWS_REGION}" \ + >"${RESULT_DIR}/s3_inventory.txt" 2>&1 || \ + echo "warning: aws s3 ls failed (see file)" >&2 + +WALG_VER="$(/usr/bin/wal-g --version 2>&1 | head -1 || echo 'unknown')" +WALRUS_VER="$(/usr/local/bin/walrus --version 2>&1 | head -1 || echo 'unknown')" +WALRUS_SHA="$(sha256sum /usr/local/bin/walrus 2>/dev/null | awk '{print $1}' || echo 'unknown')" +WALG_SHA="$(sha256sum /usr/bin/wal-g 2>/dev/null | awk '{print $1}' || echo 'unknown')" +PGBR_VER="$(pgbackrest version 2>&1 | head -1 || echo 'unknown')" +PGBR_SHA="$(sha256sum "$(command -v pgbackrest)" 2>/dev/null | awk '{print $1}' || echo 'unknown')" +{ + for line in "$@"; do printf '%s\n' "${line}"; done + echo "wal_g_version=${WALG_VER}" + echo "wal_g_binary_sha256=${WALG_SHA}" + echo "walrus_version=${WALRUS_VER}" + echo "walrus_binary_sha256=${WALRUS_SHA}" + echo "pgbackrest_version=${PGBR_VER}" + echo "pgbackrest_binary_sha256=${PGBR_SHA}" + echo "captured_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +} >"${RESULT_DIR}/provenance.txt" +echo "results persisted to ${RESULT_DIR}" +cat "${RESULT_DIR}/provenance.txt" +REMOTE +} diff --git a/bench/scripts/sut/00_mount_nvme.sh b/bench/scripts/sut/00_mount_nvme.sh new file mode 100755 index 0000000..0c8f971 --- /dev/null +++ b/bench/scripts/sut/00_mount_nvme.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Detect the non-root NVMe instance-store device, format it ext4 (only if not +# already formatted), mount it at /dat, and prepare the PG18 data parent dir. +set -euo pipefail + +MOUNT_POINT=/dat +PG_PARENT="${MOUNT_POINT}/18" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo)." >&2 + exit 1 +fi + +# The root filesystem's backing device. lsblk -no PKNAME gives the parent disk +# of whatever device hosts "/", e.g. nvme0n1. +root_src="$(findmnt -no SOURCE / )" +root_disk="$(lsblk -no PKNAME "${root_src}" | head -n1)" +if [[ -z "${root_disk}" ]]; then + # Some setups mount / directly on the whole disk; fall back to basename. + root_disk="$(basename "${root_src}")" +fi +echo "Root device: ${root_src} (disk: ${root_disk})" + +# Pick the first NVMe whole-disk that is not the root disk and has no children. +target="" +while read -r name type; do + [[ "${type}" == "disk" ]] || continue + [[ "${name}" == "${root_disk}" ]] && continue + case "${name}" in + nvme*) + # Skip disks that already have partitions/children mounted as root. + target="${name}" + break + ;; + esac +done < <(lsblk -dno NAME,TYPE) + +if [[ -z "${target}" ]]; then + echo "ERROR: no non-root NVMe instance-store device found." >&2 + lsblk -o NAME,TYPE,SIZE,MOUNTPOINT >&2 + exit 1 +fi + +dev="/dev/${target}" +echo "Selected instance-store device: ${dev}" + +# Guard: only mkfs if there is no existing filesystem on the device. +fstype="$(blkid -o value -s TYPE "${dev}" 2>/dev/null || true)" +if [[ -z "${fstype}" ]]; then + echo "No filesystem detected on ${dev}; creating ext4..." + mkfs.ext4 -F "${dev}" +else + echo "Existing filesystem (${fstype}) on ${dev}; skipping mkfs." +fi + +mkdir -p "${MOUNT_POINT}" + +if mountpoint -q "${MOUNT_POINT}"; then + echo "${MOUNT_POINT} already mounted." +else + echo "Mounting ${dev} at ${MOUNT_POINT}..." + mount "${dev}" "${MOUNT_POINT}" +fi + +mkdir -p "${PG_PARENT}" +# The postgres user is created later by 01_install_pg18.sh, and 10_init_pg.sh +# sets ${PG_PARENT} ownership. Keep /dat root-owned but world-traversable so +# postgres can reach PGDATA underneath it; chown to postgres only once it exists +# (e.g. on a re-run after PG is installed). +chmod 755 "${MOUNT_POINT}" +if id -u postgres >/dev/null 2>&1; then + chown postgres:postgres "${MOUNT_POINT}" "${PG_PARENT}" +fi + +echo "Done." +findmnt "${MOUNT_POINT}" +df -h "${MOUNT_POINT}" diff --git a/bench/scripts/sut/01_install_pg18.sh b/bench/scripts/sut/01_install_pg18.sh new file mode 100755 index 0000000..a51f20c --- /dev/null +++ b/bench/scripts/sut/01_install_pg18.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Install PostgreSQL 18 (PGDG), build toolchain, Go 1.26.2, and Rust 1.89 for +# building wal-g and walrus on Ubuntu 24.04 (noble). +set -euo pipefail + +GO_VERSION="${GO_VERSION:-1.26.2}" +RUST_VERSION="${RUST_VERSION:-1.89.0}" +GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz" +GO_URL="https://go.dev/dl/${GO_TARBALL}" +# User who will run cargo to build walrus. Defaults to the sudo invoker (dev box, +# any distro), then ubuntu (provisioned EC2 SUT) — matches the other sut scripts. +BUILD_USER="${BUILD_USER:-${SUDO_USER:-ubuntu}}" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo)." >&2 + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive + +echo "=== Adding PGDG apt repository ===" +apt-get update -qq +apt-get install -y wget gnupg lsb-release ca-certificates +install -d -m 0755 /usr/share/keyrings +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor --batch --yes -o /usr/share/keyrings/pgdg.gpg +echo "deb [signed-by=/usr/share/keyrings/pgdg.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list + +echo "=== Installing PostgreSQL 18 and build dependencies ===" +apt-get update -qq +apt-get install -y \ + postgresql-18 \ + postgresql-server-dev-18 \ + build-essential \ + git \ + curl \ + pkg-config \ + libssl-dev \ + liblz4-dev \ + cmake \ + sysstat \ + unzip + +echo "=== Enabling sysstat data collection (pidstat) ===" +if [[ -f /etc/default/sysstat ]]; then + sed -i 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat || true +fi +systemctl enable --now sysstat 2>/dev/null || true + +echo "=== Installing AWS CLI v2 (the apt 'awscli' package is gone on noble) ===" +if ! command -v aws >/dev/null 2>&1; then + tmp="$(mktemp -d)" + curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "${tmp}/awscliv2.zip" + unzip -q "${tmp}/awscliv2.zip" -d "${tmp}" + "${tmp}/aws/install" --update + rm -rf "${tmp}" +fi +aws --version + +echo "=== Installing Go ${GO_VERSION} to /usr/local/go ===" +if [[ ! -x /usr/local/go/bin/go ]] || ! /usr/local/go/bin/go version | grep -q "go${GO_VERSION}"; then + tmp="$(mktemp -d)" + curl -fsSL "${GO_URL}" -o "${tmp}/${GO_TARBALL}" + rm -rf /usr/local/go + tar -C /usr/local -xzf "${tmp}/${GO_TARBALL}" + rm -rf "${tmp}" +fi +# Make go available on PATH for all login shells. +cat > /etc/profile.d/go.sh <<'EOF' +export PATH=$PATH:/usr/local/go/bin +EOF +chmod 0644 /etc/profile.d/go.sh +/usr/local/go/bin/go version + +echo "=== Installing rustup + toolchain ${RUST_VERSION} for user ${BUILD_USER} ===" +if ! id "${BUILD_USER}" >/dev/null 2>&1; then + echo "ERROR: build user '${BUILD_USER}' does not exist." >&2 + exit 1 +fi +build_home="$(getent passwd "${BUILD_USER}" | cut -d: -f6)" +if [[ -z "${build_home}" ]]; then + echo "ERROR: cannot resolve home directory for ${BUILD_USER}." >&2 + exit 1 +fi + +if [[ ! -x "${build_home}/.cargo/bin/cargo" ]]; then + sudo -u "${BUILD_USER}" -H bash -c \ + "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain ${RUST_VERSION} --profile minimal" +else + sudo -u "${BUILD_USER}" -H bash -c \ + "${build_home}/.cargo/bin/rustup toolchain install ${RUST_VERSION} --profile minimal && \ + ${build_home}/.cargo/bin/rustup default ${RUST_VERSION}" +fi +sudo -u "${BUILD_USER}" -H bash -c "${build_home}/.cargo/bin/rustc --version" + +echo "Done. PostgreSQL 18, Go ${GO_VERSION}, Rust ${RUST_VERSION} installed." +/usr/lib/postgresql/18/bin/pg_config --version diff --git a/bench/scripts/sut/02_build_walg.sh b/bench/scripts/sut/02_build_walg.sh new file mode 100755 index 0000000..b7f3f38 --- /dev/null +++ b/bench/scripts/sut/02_build_walg.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Build wal-g at the pinned commit and install the PG binary + daemon client. +set -euo pipefail + +WALG_COMMIT="${WALG_COMMIT:-f81943e64bdf97aa66f6c52fec55114703f97af7}" +WALG_REPO="${WALG_REPO:-https://github.com/wal-g/wal-g.git}" +SRC_DIR="${SRC_DIR:-/opt/walbench/src/wal-g}" +GOBIN_DIR="/usr/bin" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo) so installs to /usr/bin succeed." >&2 + exit 1 +fi + +# Idempotent: skip the rebuild if both binaries are already installed. +if [[ -z "${FORCE_REBUILD:-}" && -x /usr/bin/wal-g && -x /usr/bin/walg-daemon-client ]]; then + echo "wal-g already installed ($(/usr/bin/wal-g --version 2>/dev/null | head -1)); skipping (FORCE_REBUILD=1 to rebuild)." + exit 0 +fi + +export PATH="/usr/local/go/bin:${PATH}" +export GOEXPERIMENT=jsonv2 + +if ! command -v go >/dev/null 2>&1; then + echo "ERROR: go not found on PATH (expected /usr/local/go/bin/go)." >&2 + exit 1 +fi +echo "Using $(go version)" + +echo "=== Fetching wal-g @ ${WALG_COMMIT} ===" +mkdir -p "${SRC_DIR}" +cd "${SRC_DIR}" +if [[ ! -d .git ]]; then + git init -q + git remote add origin "${WALG_REPO}" +fi +git remote set-url origin "${WALG_REPO}" +git fetch origin --depth 1 "${WALG_COMMIT}" +git reset --hard FETCH_HEAD + +echo "=== make deps ===" +make deps + +echo "=== make pg_build ===" +make pg_build + +echo "=== make pg_install (GOBIN=${GOBIN_DIR}) ===" +GOBIN="${GOBIN_DIR}" make pg_install + +echo "=== make build_client ===" +make build_client +cp bin/walg-daemon-client "${GOBIN_DIR}/walg-daemon-client" +chmod 0755 "${GOBIN_DIR}/walg-daemon-client" + +echo "=== Installed ===" +ls -l "${GOBIN_DIR}/wal-g" "${GOBIN_DIR}/walg-daemon-client" +"${GOBIN_DIR}/wal-g" --version diff --git a/bench/scripts/sut/03_build_walrus.sh b/bench/scripts/sut/03_build_walrus.sh new file mode 100755 index 0000000..dfe26ba --- /dev/null +++ b/bench/scripts/sut/03_build_walrus.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Build walrus (wal-rs) from the in-repo source and install to +# /usr/local/bin/walrus. +# +# bench/ lives inside the wal-rs repo, so the source is right here: build the +# repo working tree directly (no clone, no uploaded tarball). The git SHA is +# read from the repo when present. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# bench/scripts/sut -> repo root is three levels up. +REPO_ROOT="${WALRUS_REPO:-$(cd -- "${SCRIPT_DIR}/../../.." >/dev/null 2>&1 && pwd)}" +INSTALL_BIN="/usr/local/bin/walrus" +SHA_FILE="/opt/walbench/walrus.sha" +# User that owns the rustup/cargo toolchain installed by 01_install_pg18.sh. +# Defaults to the sudo invoker (dev box) then ubuntu (provisioned SUT). +BUILD_USER="${BUILD_USER:-${SUDO_USER:-ubuntu}}" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo) so installs to /usr/local/bin succeed." >&2 + exit 1 +fi +# Idempotent: skip the (slow) cargo build if walrus is already installed. +if [[ -z "${FORCE_REBUILD:-}" && -x "${INSTALL_BIN}" ]]; then + echo "walrus already installed; skipping build (FORCE_REBUILD=1 to rebuild)." + exit 0 +fi +if [[ ! -f "${REPO_ROOT}/Cargo.toml" ]]; then + echo "ERROR: wal-rs source not found at ${REPO_ROOT} (no Cargo.toml)." >&2 + exit 1 +fi +if ! id "${BUILD_USER}" >/dev/null 2>&1; then + echo "ERROR: build user '${BUILD_USER}' does not exist." >&2 + exit 1 +fi +build_home="$(getent passwd "${BUILD_USER}" | cut -d: -f6)" +cargo_bin="${build_home}/.cargo/bin/cargo" +if [[ ! -x "${cargo_bin}" ]]; then + echo "ERROR: cargo not found at ${cargo_bin} (run 01_install_pg18.sh first)." >&2 + exit 1 +fi + +WALRUS_SHA="$(git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || echo unknown)" +echo "=== Building walrus from ${REPO_ROOT} (SHA ${WALRUS_SHA}) ===" + +# cargo runs as BUILD_USER, which must own the tree (target/ is gitignored, so +# building in place is fine). The binary is installed by root afterwards. +echo "=== cargo build --release ===" +sudo -u "${BUILD_USER}" -H bash -c "cd '${REPO_ROOT}' && '${cargo_bin}' build --release" + +echo "=== Installing to ${INSTALL_BIN} ===" +install -m 0755 "${REPO_ROOT}/target/release/walrus" "${INSTALL_BIN}" + +mkdir -p "$(dirname "${SHA_FILE}")" +printf '%s\n' "${WALRUS_SHA}" > "${SHA_FILE}" +echo "Recorded SHA to ${SHA_FILE}" + +echo "=== Installed ===" +ls -l "${INSTALL_BIN}" +"${INSTALL_BIN}" --version diff --git a/bench/scripts/sut/04_build_walg_archive.sh b/bench/scripts/sut/04_build_walg_archive.sh new file mode 100755 index 0000000..dee2abb --- /dev/null +++ b/bench/scripts/sut/04_build_walg_archive.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Build and install the walg_archive PostgreSQL archive module for PG18 via PGXS. +set -euo pipefail + +ARCHIVE_COMMIT="${ARCHIVE_COMMIT:-ce0d160b8503f98c179646e38cd24b9351ec8c0a}" +ARCHIVE_REPO="${ARCHIVE_REPO:-https://github.com/wal-g/walg_archive}" +SRC_DIR="${SRC_DIR:-/opt/walbench/src/walg_archive}" +PG_CONFIG="${PG_CONFIG:-/usr/lib/postgresql/18/bin/pg_config}" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo) so 'make install' succeeds." >&2 + exit 1 +fi + +if [[ ! -x "${PG_CONFIG}" ]]; then + echo "ERROR: pg_config not found at ${PG_CONFIG} (install postgresql-server-dev-18)." >&2 + exit 1 +fi + +# Idempotent: skip if the module is already installed for this PG. +_pkglibdir="$("${PG_CONFIG}" --pkglibdir)" +if [[ -z "${FORCE_REBUILD:-}" && -f "${_pkglibdir}/walg_archive.so" ]]; then + echo "walg_archive.so already installed at ${_pkglibdir}; skipping build (FORCE_REBUILD=1 to rebuild)." + exit 0 +fi + +echo "=== Fetching walg_archive @ ${ARCHIVE_COMMIT} ===" +mkdir -p "${SRC_DIR}" +cd "${SRC_DIR}" +if [[ ! -d .git ]]; then + git init -q + git remote add origin "${ARCHIVE_REPO}" +fi +git remote set-url origin "${ARCHIVE_REPO}" +git fetch origin --depth 1 "${ARCHIVE_COMMIT}" +git reset --hard FETCH_HEAD + +echo "=== Building (PGXS, PG_CONFIG=${PG_CONFIG}) ===" +make clean USE_PGXS=1 PG_CONFIG="${PG_CONFIG}" 2>/dev/null || true +make USE_PGXS=1 PG_CONFIG="${PG_CONFIG}" + +echo "=== Installing ===" +make USE_PGXS=1 PG_CONFIG="${PG_CONFIG}" install + +pkglibdir="$("${PG_CONFIG}" --pkglibdir)" +echo "=== Installed module ===" +ls -l "${pkglibdir}/walg_archive.so" diff --git a/bench/scripts/sut/05_install_pgbackrest.sh b/bench/scripts/sut/05_install_pgbackrest.sh new file mode 100755 index 0000000..cb130bb --- /dev/null +++ b/bench/scripts/sut/05_install_pgbackrest.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Install pgBackRest (PGDG apt) and configure it as the third WAL archiver under +# test. Unlike wal-g/walrus there is NO long-running daemon: PG's archiver runs +# `pgbackrest archive-push` per segment; in async mode the first invocation forks +# a transient async process that pushes ready segments with up to `process-max` +# worker processes, then exits. Footprint is therefore a process TREE, sampled by +# bench-sampler's --proc-match mode (--daemon pgbackrest), not a systemd MainPID. +# +# Credentials: on EC2, pgBackRest speaks IMDSv2 natively (repo1-s3-key-type=auto, +# since pgBackRest 2.39) and reads the instance-profile role directly. Off-AWS +# (dev / Debian, no IMDS) it falls back to static keys from the environment +# (repo1-s3-key-type=shared) — the same AWS_ACCESS_KEY_ID/SECRET path wal-g and +# walrus use via wal-g.env. Repo lives in the SAME bucket as wal-g under a +# SEPARATE prefix (repo1-path) so the two never collide. +# +# Usage: +# BUCKET=my-bucket [UPLOAD_CONCURRENCY=4] sudo ./05_install_pgbackrest.sh +# or: sudo ./05_install_pgbackrest.sh [UPLOAD_CONCURRENCY] +set -euo pipefail + +BUCKET="${BUCKET:-${1:-}}" +UPLOAD_CONCURRENCY="${UPLOAD_CONCURRENCY:-${2:-16}}" +AWS_REGION="${AWS_REGION:-us-east-1}" +STANZA="${PGBACKREST_STANZA:-walbench}" +REPO_PATH="${PGBACKREST_REPO_PATH:-/pgbackrest-bench}" +PGDATA="${PGDATA:-/dat/18/data}" +PGBIN="${PGBIN:-/usr/lib/postgresql/18/bin}" +CONF_DIR="/etc/pgbackrest" +CONF="${CONF_DIR}/pgbackrest.conf" +# Spool + logs on the data NVMe (not tmpfs, not the small root volume); async +# only writes tiny ack files to the spool, so disk pressure is negligible. +SPOOL_PATH="${PGBACKREST_SPOOL_PATH:-/dat/pgbackrest/spool}" +LOG_PATH="${PGBACKREST_LOG_PATH:-/dat/pgbackrest/log}" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo) to apt-install + write ${CONF}." >&2 + exit 1 +fi +if [[ -z "${BUCKET}" ]]; then + echo "ERROR: BUCKET is required (env BUCKET=... or first positional arg)." >&2 + exit 1 +fi + +echo "=== Installing pgbackrest (PGDG) ===" +# PGDG repo is already configured by 01_install_pg18.sh. +if ! command -v pgbackrest >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y pgbackrest +fi +pgbackrest version + +echo "=== Creating spool/log/config dirs (owned by postgres) ===" +install -d -o postgres -g postgres -m 0750 "${CONF_DIR}" +install -d -o postgres -g postgres -m 0750 "${SPOOL_PATH}" +install -d -o postgres -g postgres -m 0750 "${LOG_PATH}" + +# S3 auth: static keys from env when present (off-AWS / Debian), else IMDS role. +if [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" ]]; then + echo "=== pgbackrest S3 auth: shared (static keys from environment) ===" + S3_AUTH="repo1-s3-key-type=shared +repo1-s3-key=${AWS_ACCESS_KEY_ID} +repo1-s3-key-secret=${AWS_SECRET_ACCESS_KEY}" + [[ -n "${AWS_SESSION_TOKEN:-}" ]] && S3_AUTH="${S3_AUTH} +repo1-s3-token=${AWS_SESSION_TOKEN}" +else + echo "=== pgbackrest S3 auth: auto (EC2 instance-profile via IMDS) ===" + S3_AUTH="repo1-s3-key-type=auto" +fi + +echo "=== Writing ${CONF} (process-max=${UPLOAD_CONCURRENCY}, bucket=${BUCKET}) ===" +# process-max matches WALG_UPLOAD_CONCURRENCY so pgbackrest's async parallelism is +# the same knob as wal-g's background uploader — the throughput<->memory tradeoff +# is compared at equal fan-out. compress-type=lz4 matches WALG_COMPRESSION_METHOD. +umask 077 +tmp="$(mktemp)" +cat > "${tmp}" </' "${CONF}" +echo "pgbackrest $(pgbackrest version) ready for stanza '${STANZA}' -> s3://${BUCKET}${REPO_PATH}" diff --git a/bench/scripts/sut/10_init_pg.sh b/bench/scripts/sut/10_init_pg.sh new file mode 100755 index 0000000..1f09801 --- /dev/null +++ b/bench/scripts/sut/10_init_pg.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Initialize the PG18 cluster at /dat/18/data, write prod-mirrored +# postgresql.conf + a pg_hba.conf entry for the driver box, and (re)start the +# cluster. Idempotent: safe to re-run. +set -euo pipefail + +PGDATA="${PGDATA:-/dat/18/data}" +PGBIN="${PGBIN:-/usr/lib/postgresql/18/bin}" +# CIDR allowed to connect over the network (the driver SG / subnet). +DRIVER_CIDR="${DRIVER_CIDR:-}" +PG_LOG="${PG_LOG:-/dat/18/pg.log}" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo); it drops to 'postgres' internally." >&2 + exit 1 +fi + +if [[ -z "${DRIVER_CIDR}" ]]; then + echo "ERROR: DRIVER_CIDR is required (e.g. DRIVER_CIDR=10.0.0.0/24)." >&2 + exit 1 +fi + +if [[ ! -x "${PGBIN}/initdb" ]]; then + echo "ERROR: initdb not found at ${PGBIN}/initdb." >&2 + exit 1 +fi + +# PGDG auto-creates AND starts a default cluster '18/main' on port 5432 at +# install time; it holds the socket and would collide with our cluster +# (same port + socket dir). Drop it so 5432 is free. Idempotent. +if command -v pg_lsclusters >/dev/null 2>&1 \ + && pg_lsclusters -h 2>/dev/null | awk '{print $1"/"$2}' | grep -qx '18/main'; then + echo "=== Removing PGDG default cluster 18/main (frees port 5432) ===" + pg_dropcluster --stop 18 main || true +fi + +install -d -o postgres -g postgres -m 0750 "$(dirname "${PGDATA}")" + +echo "=== initdb (if needed) at ${PGDATA} ===" +if [[ -f "${PGDATA}/PG_VERSION" ]]; then + echo "Cluster already initialized; skipping initdb." +else + sudo -u postgres "${PGBIN}/initdb" -D "${PGDATA}" --data-checksums +fi + +echo "=== Writing postgresql.conf ===" +sudo -u postgres tee "${PGDATA}/postgresql.conf" >/dev/null </dev/null +fi + +echo "=== (Re)starting cluster ===" +if sudo -u postgres "${PGBIN}/pg_ctl" -D "${PGDATA}" status >/dev/null 2>&1; then + sudo -u postgres "${PGBIN}/pg_ctl" -D "${PGDATA}" -l "${PG_LOG}" restart -w -m fast +else + sudo -u postgres "${PGBIN}/pg_ctl" -D "${PGDATA}" -l "${PG_LOG}" start -w +fi + +echo "=== Cluster status ===" +sudo -u postgres "${PGBIN}/pg_ctl" -D "${PGDATA}" status +sudo -u postgres "${PGBIN}/psql" -p 5432 -c "SHOW server_version;" -c "SHOW archive_library;" diff --git a/bench/scripts/sut/11_write_walg_env.sh b/bench/scripts/sut/11_write_walg_env.sh new file mode 100755 index 0000000..ea18c35 --- /dev/null +++ b/bench/scripts/sut/11_write_walg_env.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Write the shared environment file consumed by BOTH daemons (wal-g and walrus). +# wal-rs has no IMDS support, so credentials must live in this file as plain env +# vars. +# +# Credential source, in order: +# 1. AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY already in env (off-AWS / dev / +# static keys; AWS_SESSION_TOKEN optional) — written verbatim. +# 2. otherwise IMDSv2 (EC2 instance role) — fetched here. +# +# Usage: +# BUCKET=my-bucket [UPLOAD_CONCURRENCY=4] sudo -E ./11_write_walg_env.sh +# or: sudo ./11_write_walg_env.sh [UPLOAD_CONCURRENCY] +set -euo pipefail + +BUCKET="${BUCKET:-${1:-}}" +UPLOAD_CONCURRENCY="${UPLOAD_CONCURRENCY:-${2:-16}}" +ENV_FILE="${ENV_FILE:-/etc/postgresql/wal-g.env}" +AWS_REGION="${AWS_REGION:-us-east-1}" +COMPRESSION_METHOD="${WALG_COMPRESSION_METHOD:-lz4}" +IMDS="http://169.254.169.254/latest" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo) to write ${ENV_FILE}." >&2 + exit 1 +fi + +if [[ -z "${BUCKET}" ]]; then + echo "ERROR: BUCKET is required (env BUCKET=... or first positional arg)." >&2 + exit 1 +fi + +# Storage prefix both daemons archive into. run.sh / run_op.sh scope it per +# tool+run for isolation; default keeps the shared bench prefix for setup/smoke. +WALG_S3_PREFIX="${WALG_S3_PREFIX:-s3://${BUCKET}/walg-bench}" + +ACCESS_KEY="${AWS_ACCESS_KEY_ID:-}" +SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-}" +SESSION_TOKEN="${AWS_SESSION_TOKEN:-}" + +if [[ -n "${ACCESS_KEY}" && -n "${SECRET_KEY}" ]]; then + echo "=== Using AWS credentials from environment ===" +else + echo "=== Fetching temporary credentials via IMDSv2 ===" + TOKEN="$(curl -sf -X PUT "${IMDS}/api/token" \ + -H 'X-aws-ec2-metadata-token-ttl-seconds: 21600')" + if [[ -z "${TOKEN}" ]]; then + echo "ERROR: no env credentials and failed to obtain IMDSv2 token." >&2 + exit 1 + fi + + ROLE="$(curl -sf -H "X-aws-ec2-metadata-token: ${TOKEN}" \ + "${IMDS}/meta-data/iam/security-credentials/")" + if [[ -z "${ROLE}" ]]; then + echo "ERROR: no IAM role attached to this instance." >&2 + exit 1 + fi + echo "IAM role: ${ROLE}" + + CREDS_JSON="$(curl -sf -H "X-aws-ec2-metadata-token: ${TOKEN}" \ + "${IMDS}/meta-data/iam/security-credentials/${ROLE}")" + + read_field() { + local key="$1" + if command -v jq >/dev/null 2>&1; then + printf '%s' "${CREDS_JSON}" | jq -r ".${key}" + else + printf '%s' "${CREDS_JSON}" \ + | python3 -c "import sys,json;print(json.load(sys.stdin)['${key}'])" + fi + } + + ACCESS_KEY="$(read_field AccessKeyId)" + SECRET_KEY="$(read_field SecretAccessKey)" + SESSION_TOKEN="$(read_field Token)" + echo "Credentials expire at: $(read_field Expiration)" +fi + +if [[ -z "${ACCESS_KEY}" || -z "${SECRET_KEY}" ]]; then + echo "ERROR: incomplete credentials (need AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY)." >&2 + exit 1 +fi + +echo "=== Writing ${ENV_FILE} (UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY}) ===" +install -d -o postgres -g postgres -m 0755 "$(dirname "${ENV_FILE}")" +umask 077 +tmp="$(mktemp)" +cat > "${tmp}" <> "${tmp}" +fi +install -o postgres -g postgres -m 0600 "${tmp}" "${ENV_FILE}" +rm -f "${tmp}" + +echo "Done. ${ENV_FILE}:" +# Show keys only, never secret values. +sed -E 's/^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN)=.*/\1=/' "${ENV_FILE}" diff --git a/bench/scripts/sut/20_walg.service b/bench/scripts/sut/20_walg.service new file mode 100644 index 0000000..1077ac7 --- /dev/null +++ b/bench/scripts/sut/20_walg.service @@ -0,0 +1,16 @@ +[Unit] +Description=wal-g WAL archive daemon (benchmark) +After=network-online.target postgresql.service +Wants=network-online.target + +[Service] +Type=simple +User=postgres +Group=postgres +EnvironmentFile=/etc/postgresql/wal-g.env +ExecStart=/usr/bin/wal-g daemon /tmp/wal-g +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/bench/scripts/sut/21_walrus.service b/bench/scripts/sut/21_walrus.service new file mode 100644 index 0000000..3310ad7 --- /dev/null +++ b/bench/scripts/sut/21_walrus.service @@ -0,0 +1,16 @@ +[Unit] +Description=walrus (wal-rs) WAL archive daemon (benchmark) +After=network-online.target postgresql.service +Wants=network-online.target + +[Service] +Type=simple +User=postgres +Group=postgres +EnvironmentFile=/etc/postgresql/wal-g.env +ExecStart=/usr/local/bin/walrus daemon --socket /tmp/wal-g +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/bench/scripts/sut/30_select_daemon.sh b/bench/scripts/sut/30_select_daemon.sh new file mode 100755 index 0000000..de2352f --- /dev/null +++ b/bench/scripts/sut/30_select_daemon.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Select exactly one archive daemon (wal-g or walrus). Stops both units, +# removes the shared socket, installs (if needed) and starts the chosen unit, +# waits for the socket to appear, and prints the active PID/cgroup. +# +# Usage: sudo ./30_select_daemon.sh walg|walrus +set -euo pipefail + +CHOICE="${1:-}" +SOCKET="${SOCKET:-/tmp/wal-g}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WALG_UNIT="wal-g.service" +WALRUS_UNIT="walrus.service" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo) to control systemd." >&2 + exit 1 +fi + +case "${CHOICE}" in + walg) chosen_unit="${WALG_UNIT}"; src_unit="20_walg.service" ;; + walrus) chosen_unit="${WALRUS_UNIT}"; src_unit="21_walrus.service" ;; + *) + echo "Usage: $0 walg|walrus" >&2 + exit 1 + ;; +esac + +echo "=== Installing systemd unit files ===" +install -m 0644 "${SCRIPT_DIR}/20_walg.service" "/etc/systemd/system/${WALG_UNIT}" +install -m 0644 "${SCRIPT_DIR}/21_walrus.service" "/etc/systemd/system/${WALRUS_UNIT}" +systemctl daemon-reload + +echo "=== Stopping both daemons ===" +systemctl stop "${WALG_UNIT}" 2>/dev/null || true +systemctl stop "${WALRUS_UNIT}" 2>/dev/null || true + +echo "=== Removing stale socket ${SOCKET} ===" +rm -f "${SOCKET}" + +echo "=== Starting ${chosen_unit} (from ${src_unit}) ===" +systemctl start "${chosen_unit}" + +echo "=== Waiting for socket ${SOCKET} ===" +for _ in $(seq 1 30); do + if [[ -S "${SOCKET}" ]]; then + break + fi + sleep 0.5 +done +if [[ ! -S "${SOCKET}" ]]; then + echo "ERROR: socket ${SOCKET} did not appear; recent logs:" >&2 + systemctl status "${chosen_unit}" --no-pager || true + journalctl -u "${chosen_unit}" -n 30 --no-pager || true + exit 1 +fi + +# Point archive_command at the chosen daemon's OWN client (walg_archive's +# extension does not interoperate with the walrus daemon; each tool's own +# daemon-client does). walrus needs the absolute WAL path; wal-g uses %f. +# Best-effort here (PG is up by this point in setup); run_one.sh sets it per cell. +PGDATA_DIR="/dat/18/data" +if [[ "${CHOICE}" == "walg" ]]; then + archive_cmd="/usr/bin/walg-daemon-client ${SOCKET} wal-push %f" +else + archive_cmd="/usr/local/bin/walrus daemon-client --socket ${SOCKET} wal-push ${PGDATA_DIR}/%p" +fi +echo "=== Setting archive_command for ${CHOICE} (and clearing archive_library) ===" +# Each ALTER SYSTEM in its own -c (cannot run inside a transaction block). +sudo -u postgres /usr/lib/postgresql/18/bin/psql -p 5432 -tA \ + -c "ALTER SYSTEM SET archive_library = '';" \ + -c "ALTER SYSTEM SET archive_command = '${archive_cmd}';" \ + -c "SELECT pg_reload_conf();" >/dev/null 2>&1 || true + +main_pid="$(systemctl show -p MainPID --value "${chosen_unit}")" +echo "=== Active daemon: ${chosen_unit} ===" +echo "MainPID: ${main_pid}" +echo "cgroup: $(systemctl show -p ControlGroup --value "${chosen_unit}")" +ls -l "${SOCKET}" +systemctl is-active "${chosen_unit}" diff --git a/bench/scripts/sut/40_smoke_test.sh b/bench/scripts/sut/40_smoke_test.sh new file mode 100755 index 0000000..12417a6 --- /dev/null +++ b/bench/scripts/sut/40_smoke_test.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Smoke test the currently-active archive daemon: force a few WAL switches, +# insert a tiny table, wait for archiving, then confirm WAL objects landed under +# s3:///walg-bench/wal_005/. FAIL loudly if nothing appears. +# +# Usage: BUCKET=my-bucket sudo ./40_smoke_test.sh (or pass BUCKET as $1) +set -euo pipefail + +BUCKET="${BUCKET:-${1:-}}" +PGBIN="${PGBIN:-/usr/lib/postgresql/18/bin}" +PSQL="${PGBIN}/psql" +SOCKET="${SOCKET:-/tmp/wal-g}" +AWS_REGION="${AWS_REGION:-us-east-1}" +WAL_SWITCHES="${WAL_SWITCHES:-5}" +WAIT_SECONDS="${WAIT_SECONDS:-90}" + +if [[ -z "${BUCKET}" ]]; then + echo "ERROR: BUCKET is required (env BUCKET=... or first positional arg)." >&2 + exit 1 +fi + +S3_WAL_PREFIX="s3://${BUCKET}/walg-bench/wal_005/" + +run_psql() { sudo -u postgres "${PSQL}" -X -v ON_ERROR_STOP=1 -p 5432 "$@"; } + +if [[ ! -S "${SOCKET}" ]]; then + echo "ERROR: daemon socket ${SOCKET} not present; start a daemon first." >&2 + exit 1 +fi + +echo "=== Baseline object count under ${S3_WAL_PREFIX} (before this daemon archives) ===" +# The prefix is SHARED by both daemons. Counting >0 would false-pass walrus on +# objects wal-g left earlier (and vice versa). Capture a baseline and require an +# INCREASE so the test proves THIS daemon archived. +before="$(aws s3 ls "${S3_WAL_PREFIX}" --region "${AWS_REGION}" 2>/dev/null | grep -c . || true)" +echo "baseline=${before}" + +echo "=== Generating WAL: tiny table + ${WAL_SWITCHES} forced switches ===" +run_psql -d postgres </dev/null + run_psql -d postgres -tAc \ + "INSERT INTO walg_smoke (payload) SELECT repeat('y',256) FROM generate_series(1,500);" +done +# Final switch so the last populated segment becomes archivable. +run_psql -d postgres -tAc "SELECT pg_switch_wal();" >/dev/null + +echo "=== Waiting up to ${WAIT_SECONDS}s for NEW archived WAL under ${S3_WAL_PREFIX} ===" +deadline=$(( SECONDS + WAIT_SECONDS )) +found=0 +count="${before}" +while (( SECONDS < deadline )); do + count="$(aws s3 ls "${S3_WAL_PREFIX}" --region "${AWS_REGION}" 2>/dev/null | grep -c . || true)" + if (( count > before )); then + found=1 + break + fi + sleep 3 +done + +echo "=== Archiver status ===" +run_psql -d postgres -c \ + "SELECT archived_count, failed_count, last_archived_wal FROM pg_stat_archiver;" || true + +if (( found == 0 )); then + echo "FAIL: object count under ${S3_WAL_PREFIX} did not rise past baseline ${before} (still ${count}) after ${WAIT_SECONDS}s — this daemon did not archive." >&2 + echo "Listing parent for diagnostics:" >&2 + aws s3 ls "s3://${BUCKET}/walg-bench/" --region "${AWS_REGION}" >&2 || true + exit 1 +fi + +echo "PASS: object count rose ${before} -> ${count} under ${S3_WAL_PREFIX} (this daemon archived $(( count - before )) new):" +aws s3 ls "${S3_WAL_PREFIX}" --region "${AWS_REGION}" | tail -n 10 diff --git a/bench/setup.sh b/bench/setup.sh new file mode 100755 index 0000000..213da38 --- /dev/null +++ b/bench/setup.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# setup.sh — bootstrap THIS host as a single-box benchmark System-Under-Test. +# +# Runs the numbered scripts/sut steps in order: mount NVMe (optional) -> install +# PG18 + toolchains -> build wal-g, walrus (from this repo), walg_archive -> init +# the PG18 cluster -> install + stanza pgbackrest -> create the bench role -> +# deploy the sampler -> write wal-g.env -> install both systemd units. +# +# Single-host counterpart of the external AWS fleet's setup_sut.sh: no SSH, no +# source upload (walrus builds straight from this repo), driver == this box. +# +# Run as root: sudo ./setup.sh (config from ./config.env) +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +SUT="${SCRIPT_DIR}/scripts/sut" + +# Export everything sourced so the sub-scripts inherit it (BUCKET, creds, etc). +set -a +# shellcheck source=config.env.example +. "${ENV_FILE:-${SCRIPT_DIR}/config.env}" +set +a + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: must run as root (use sudo)." >&2 + exit 1 +fi + +: "${BUCKET:?set BUCKET in config.env}" +: "${PGUSER:?set PGUSER in config.env}" +: "${PGPASSWORD:?set PGPASSWORD in config.env}" +: "${UPLOAD_CONCURRENCY:?set UPLOAD_CONCURRENCY in config.env}" + +# Toolchain owner + pg_hba CIDR. Single-host driver is loopback. +export BUILD_USER="${BUILD_USER:-${SUDO_USER:-ubuntu}}" +export DRIVER_CIDR="${DRIVER_CIDR:-127.0.0.1/32}" + +log() { printf '[setup %s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } + +cd "${SUT}" +chmod +x ./*.sh + +# 00 is AWS instance-store specific: it formats+mounts a spare NVMe at /dat. +# Skip on a box that already has /dat (or set SKIP_MOUNT=1 to point PGDATA at an +# existing fast disk yourself). +if [[ -n "${SKIP_MOUNT:-}" ]]; then + log "SKIP_MOUNT set — skipping 00_mount_nvme.sh (ensure /dat exists)" +else + log "00 mount NVMe"; bash ./00_mount_nvme.sh +fi +log "01 install PG18 + toolchains"; bash ./01_install_pg18.sh +log "02 build wal-g"; bash ./02_build_walg.sh +log "03 build walrus (this repo)"; bash ./03_build_walrus.sh +log "04 build walg_archive"; bash ./04_build_walg_archive.sh +log "10 init PG cluster"; bash ./10_init_pg.sh +log "05 install pgbackrest"; bash ./05_install_pgbackrest.sh + +log "create bench role '${PGUSER}'" +PSQL="sudo -u postgres /usr/lib/postgresql/18/bin/psql -p 5432" +if [[ "$(${PSQL} -tAc "SELECT 1 FROM pg_roles WHERE rolname='${PGUSER}'")" == "1" ]]; then + log "role ${PGUSER} exists; updating password" + ${PSQL} -c "ALTER ROLE \"${PGUSER}\" LOGIN PASSWORD '${PGPASSWORD}' CREATEDB;" +else + ${PSQL} -c "CREATE ROLE \"${PGUSER}\" LOGIN PASSWORD '${PGPASSWORD}' CREATEDB;" +fi + +log "build bench-tools (bench-sampler + bench-analyze) from ${SCRIPT_DIR}/tools" +build_home="$(getent passwd "${BUILD_USER}" | cut -d: -f6)" +cargo_bin="${build_home}/.cargo/bin/cargo" +sudo -u "${BUILD_USER}" -H bash -c "cd '${SCRIPT_DIR}/tools' && '${cargo_bin}' build --release" + +log "deploy bench-sampler + bench-analyze to /usr/local/bin" +install -m 0755 "${SCRIPT_DIR}/tools/target/release/bench-sampler" /usr/local/bin/bench-sampler +install -m 0755 "${SCRIPT_DIR}/tools/target/release/bench-analyze" /usr/local/bin/bench-analyze + +log "11 write wal-g.env"; bash ./11_write_walg_env.sh +log "install both systemd units (via 30, starts walg)"; bash ./30_select_daemon.sh walg + +log "bootstrap complete" +/usr/bin/wal-g --version || true +/usr/local/bin/walrus --version || true +echo +echo "Next: bash ${SUT}/40_smoke_test.sh then ${SCRIPT_DIR}/matrix.sh" diff --git a/bench/terraform/.terraform.lock.hcl b/bench/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..64e1bfc --- /dev/null +++ b/bench/terraform/.terraform.lock.hcl @@ -0,0 +1,88 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.60" + hashes = [ + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.9.0" + constraints = "~> 2.5" + hashes = [ + "h1:9rBZCMNpxKwMlRbWH2QpwD3kqUCAejdOZQ/aiiDObXQ=", + "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", + "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", + "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", + "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", + "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", + "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", + "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", + "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", + "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", + "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", + "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.6" + hashes = [ + "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", + "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", + "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", + "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", + "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", + "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", + "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", + "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", + "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", + "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", + "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", + "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.3.0" + constraints = "~> 4.0" + hashes = [ + "h1:j/BqLS2N2AScZyotd9nZpHdieJ7e5S8y+A+ZfIu8kL8=", + "zh:0ab58d6f8991d436c7d2dbd89ed814709b949b07ac5a54ee53b0aec1fa772a8b", + "zh:60b347abcb56f45d97c56f14d895069cd15a83993f199777f571b79fea3642ee", + "zh:6889be32640349230de3f23856e6f04e0e9ced4a84a27d3f552fa54684448218", + "zh:73f8e1ecf7135033165fb14b7e8bf4d656f3ce13065ec35762ea0481975328c7", + "zh:94ce25ee253eca0b42cae9c856b36bca8103b6453012d1b279c3623c805f2d42", + "zh:96bc6de9fd67bc446fd11257872e1ffb1029a996ed1d65a3f6b43f6d408ad9ab", + "zh:97c609a310a51bfd504d704e036d72064a84bf0bdb36cc08cd4cc66098212b41", + "zh:a12c16e94533c5bd123f75032576b9dc91dd5d5ccd5f7cf331d0f2e1adc55cf8", + "zh:c4f014f876adf7af57188795050bda5b0029d8c7d7773031102b6c36dcf1fc21", + "zh:d9b0a21583aaa3df3a95394fb949a3c515ff71c2ff5a1fc4a73d364aa90bfca5", + "zh:da510d22f0c6d71ad19a76406f106b782448f512375787ecfabb338ed1e311a7", + "zh:f0e9447a9ce3a24cdaa113089e65663c836d8b9bfdb915a1c0284e0112cab5c0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/bench/terraform/iam.tf b/bench/terraform/iam.tf new file mode 100644 index 0000000..01f5896 --- /dev/null +++ b/bench/terraform/iam.tf @@ -0,0 +1,50 @@ +# Instance role scoped to the bench bucket. 11_write_walg_env.sh bridges these +# IMDSv2 creds into wal-g.env (walrus has no IMDS credential chain); the aws CLI +# and pgbackrest read the instance role directly. + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "bench" { + name = "walrus-bench-${random_id.suffix.hex}" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +data "aws_iam_policy_document" "bench_s3" { + statement { + effect = "Allow" + + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + ] + + resources = [ + aws_s3_bucket.bench.arn, + "${aws_s3_bucket.bench.arn}/*", + ] + } +} + +resource "aws_iam_role_policy" "bench_s3" { + name = "walrus-bench-s3" + role = aws_iam_role.bench.id + policy = data.aws_iam_policy_document.bench_s3.json +} + +resource "aws_iam_instance_profile" "bench" { + name = "walrus-bench-${random_id.suffix.hex}" + role = aws_iam_role.bench.name +} diff --git a/bench/terraform/instances.tf b/bench/terraform/instances.tf new file mode 100644 index 0000000..e5a3f55 --- /dev/null +++ b/bench/terraform/instances.tf @@ -0,0 +1,70 @@ +data "aws_ami" "ubuntu_noble" { + most_recent = true + owners = ["099720109477"] # Canonical + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } +} + +resource "tls_private_key" "bench" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "aws_key_pair" "bench" { + key_name = "walrus-bench-${random_id.suffix.hex}" + public_key = tls_private_key.bench.public_key_openssh +} + +resource "local_sensitive_file" "ssh_key" { + content = tls_private_key.bench.private_key_pem + filename = "${path.module}/walrus_bench_key.pem" + file_permission = "0600" +} + +# All-in-one bench box: PG18 + wal-g/walrus daemons + local pgbench driver. +# The 'd' instance family ships a local NVMe instance-store; 00_mount_nvme.sh +# detects the non-root NVMe, mkfs.ext4, and mounts it at /dat (PGDATA + WAL + +# restore). Root holds the Go/Rust toolchains + build trees. +resource "aws_instance" "bench" { + ami = data.aws_ami.ubuntu_noble.id + instance_type = var.instance_type + availability_zone = local.az + subnet_id = aws_subnet.bench.id + vpc_security_group_ids = [aws_security_group.bench.id] + key_name = aws_key_pair.bench.key_name + iam_instance_profile = aws_iam_instance_profile.bench.name + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + + root_block_device { + volume_type = "gp3" + volume_size = 60 + } + + tags = { + Name = "walrus-bench" + Role = "sut" + } +} diff --git a/bench/terraform/main.tf b/bench/terraform/main.tf new file mode 100644 index 0000000..f52ceb1 --- /dev/null +++ b/bench/terraform/main.tf @@ -0,0 +1,33 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.60" + } + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + local = { + source = "hashicorp/local" + version = "~> 2.5" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile + + default_tags { + tags = { + Project = "walrus-bench" + } + } +} diff --git a/bench/terraform/network.tf b/bench/terraform/network.tf new file mode 100644 index 0000000..4ad2fbd --- /dev/null +++ b/bench/terraform/network.tf @@ -0,0 +1,75 @@ +# Dedicated VPC + public subnet so the box is isolated from any existing VPCs. +# Single host, so no intra-SG PG rule is needed (pgbench talks to PG over loopback). + +data "aws_availability_zones" "available" { + state = "available" + + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +locals { + az = data.aws_availability_zones.available.names[0] +} + +resource "aws_vpc" "bench" { + cidr_block = "10.78.0.0/16" + enable_dns_support = true + enable_dns_hostnames = true + + tags = { Name = "walrus-bench" } +} + +resource "aws_internet_gateway" "bench" { + vpc_id = aws_vpc.bench.id + tags = { Name = "walrus-bench" } +} + +resource "aws_subnet" "bench" { + vpc_id = aws_vpc.bench.id + cidr_block = "10.78.1.0/24" + availability_zone = local.az + map_public_ip_on_launch = true + + tags = { Name = "walrus-bench-public" } +} + +resource "aws_route_table" "bench" { + vpc_id = aws_vpc.bench.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.bench.id + } + + tags = { Name = "walrus-bench" } +} + +resource "aws_route_table_association" "bench" { + subnet_id = aws_subnet.bench.id + route_table_id = aws_route_table.bench.id +} + +resource "aws_security_group" "bench" { + name = "walrus-bench-${random_id.suffix.hex}" + description = "walrus-bench single box: SSH from my_ip, all egress" + vpc_id = aws_vpc.bench.id +} + +resource "aws_vpc_security_group_ingress_rule" "ssh" { + security_group_id = aws_security_group.bench.id + description = "SSH from operator IP" + cidr_ipv4 = var.my_ip + from_port = 22 + to_port = 22 + ip_protocol = "tcp" +} + +resource "aws_vpc_security_group_egress_rule" "all_out" { + security_group_id = aws_security_group.bench.id + description = "Allow all egress (S3, apt/PGDG, IMDS, etc.)" + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" +} diff --git a/bench/terraform/outputs.tf b/bench/terraform/outputs.tf new file mode 100644 index 0000000..83325be --- /dev/null +++ b/bench/terraform/outputs.tf @@ -0,0 +1,24 @@ +output "public_ip" { + description = "Public IP of the bench box (SSH + scp target)." + value = aws_instance.bench.public_ip +} + +output "bucket_name" { + description = "S3 bucket name (BUCKET in config.env; setup default WALG_S3_PREFIX = s3:///walg-bench, runs scope it per tool+run)." + value = aws_s3_bucket.bench.id +} + +output "ssh_key_path" { + description = "Local path to the generated SSH private key (mode 0600)." + value = local_sensitive_file.ssh_key.filename +} + +output "ssh_user" { + description = "SSH login user for the Ubuntu 24.04 AMI." + value = "ubuntu" +} + +output "region" { + description = "AWS region of the bench resources." + value = var.region +} diff --git a/bench/terraform/s3.tf b/bench/terraform/s3.tf new file mode 100644 index 0000000..d7510d5 --- /dev/null +++ b/bench/terraform/s3.tf @@ -0,0 +1,47 @@ +resource "random_id" "suffix" { + byte_length = 4 +} + +# Private bench bucket; force_destroy + a 7-day lifecycle so a forgotten teardown +# does not leak storage. setup/smoke default to s3:///walg-bench; runs +# scope walrus/wal-g and pgbackrest below tool-specific prefixes by run. +resource "aws_s3_bucket" "bench" { + bucket = "walrus-bench-${random_id.suffix.hex}" + force_destroy = true +} + +resource "aws_s3_bucket_public_access_block" "bench" { + bucket = aws_s3_bucket.bench.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_ownership_controls" "bench" { + bucket = aws_s3_bucket.bench.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "bench" { + bucket = aws_s3_bucket.bench.id + + rule { + id = "expire-bench-objects" + status = "Enabled" + + filter {} + + expiration { + days = 7 + } + + abort_incomplete_multipart_upload { + days_after_initiation = 1 + } + } +} diff --git a/bench/terraform/variables.tf b/bench/terraform/variables.tf new file mode 100644 index 0000000..0f62dfe --- /dev/null +++ b/bench/terraform/variables.tf @@ -0,0 +1,31 @@ +# Single-box IaC for the in-repo bench (driver == SUT == one host). The external +# multi-replica fleet lives elsewhere; this provisions exactly one all-in-one box +# that runs PG18 + the archivers + pgbench locally, per the bench/ README. + +variable "region" { + description = "AWS region for all bench resources." + type = string + default = "us-east-1" +} + +variable "profile" { + description = "AWS CLI named profile used for provisioning." + type = string + default = "pg-dev-postgresqladmindev" +} + +variable "instance_type" { + description = "All-in-one bench box: PG18 + wal-g/walrus daemons + local pgbench driver. Needs a local NVMe instance-store (the 'd' family) for /dat (PGDATA, WAL, restore)." + type = string + default = "m5d.2xlarge" +} + +variable "my_ip" { + description = "Your public IP in CIDR form (e.g. 203.0.113.4/32) allowed to SSH on port 22." + type = string + + validation { + condition = can(cidrhost(var.my_ip, 0)) + error_message = "my_ip must be a valid CIDR, e.g. 203.0.113.4/32." + } +} diff --git a/bench/tools/Cargo.lock b/bench/tools/Cargo.lock new file mode 100644 index 0000000..bc7782c --- /dev/null +++ b/bench/tools/Cargo.lock @@ -0,0 +1,744 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ab_glyph" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bench-tools" +version = "0.1.0" +dependencies = [ + "ab_glyph", + "anyhow", + "chrono", + "clap", + "csv", + "dejavu", + "libc", + "serde", + "serde_json", + "signal-hook", + "tiny-skia", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dejavu" +version = "2.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "017b75b5fbc5806d490da36acc5c4d0e4ae1b69890d02f24c268da971813fbc1" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "owned_ttf_parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b" +dependencies = [ + "ttf-parser", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "strict-num" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-skia" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47ffee5eaaf5527f630fb0e356b90ebdec84d5d18d937c5e440350f88c5a91ea" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "png", + "tiny-skia-path", +] + +[[package]] +name = "tiny-skia-path" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca365c3faccca67d06593c5980fa6c57687de727a03131735bb85f01fdeeb9" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", +] + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/bench/tools/Cargo.toml b/bench/tools/Cargo.toml new file mode 100644 index 0000000..1c7cd81 --- /dev/null +++ b/bench/tools/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "bench-tools" +version = "0.1.0" +edition = "2024" +rust-version = "1.89" +publish = false +description = "walrus bench harness: 1 Hz resource sampler + CSV/plot analyzer" + +# Standalone: NOT a member of the wal-rus package above this dir. Empty table +# makes this its own workspace root so cargo does not try to attach it to the +# parent package (which has no [workspace.members]). +[workspace] + +[[bin]] +name = "bench-sampler" +path = "src/bin/sampler.rs" + +[[bin]] +name = "bench-analyze" +path = "src/bin/analyze.rs" + +[dependencies] +ab_glyph = "0.2.32" +anyhow = "1.0.102" +chrono = { version = "0.4.45", default-features = false, features = ["clock"] } +clap = { version = "4.6.1", features = ["derive", "env"] } +csv = "1.4.0" +dejavu = "2.37.0" +libc = "0.2.186" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" +signal-hook = "0.4.4" +tiny-skia = "0.12.0" diff --git a/bench/tools/src/bin/analyze.rs b/bench/tools/src/bin/analyze.rs new file mode 100644 index 0000000..9b0b006 --- /dev/null +++ b/bench/tools/src/bin/analyze.rs @@ -0,0 +1,1233 @@ +//! bench-analyze — plot + summarize walrus vs wal-g vs pgbackrest runs +//! (Rust port of plot.py; tiny-skia rasterizes the canvas, ab_glyph draws text, +//! tiny-skia's png-format writes the PNG). +//! +//! Reads the 1 Hz sampler CSVs for one or more run dirs and emits, into --out: +//! mem_over_time.png two panels: VmRSS (top), VmPeak (bottom) +//! backlog.png *.ready backlog over time (archive keep-up) +//! upload_rate.png tx_bytes upload rate (MB/s) +//! cpu.png daemon CPU % over time +//! Replicas of a variant are aggregated: bold = median, band = min..max. +//! +//! Plus self-describing raw exports (every row carries run metadata): +//! samples_.csv long table, one row per sample per run +//! summary_.csv one row per run: metadata + aggregates +//! summary.json same per-run aggregates as JSON + +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; + +use ab_glyph::{Font, FontRef, PxScale, ScaleFont, point}; +use anyhow::{Context, Result}; +use clap::Parser; +use tiny_skia::{ + Color, FillRule, LineCap, LineJoin, Paint, Path as SkPath, PathBuilder, Pixmap, Rect, Stroke, + StrokeDash, Transform, +}; + +const KB: f64 = 1024.0; +const MB: f64 = 1024.0 * 1024.0; +const WAL_SEG_MB: f64 = 16.0; // archived_count -> MB approximation + +const META_KEYS: [&str; 7] = [ + "daemon", + "gomemlimit", + "scale", + "churn_rows", + "burst_seconds", + "upload_concurrency", + "captured_at", +]; + +const SAMPLE_COLS: [&str; 17] = [ + "run_label", + "variant", + "box", + "ts", + "t", + "t_int", + "vmrss_mb", + "vmpeak_mb", + "vmsize_mb", + "rssanon_mb", + "cpu_pct", + "archived_count", + "failed_count", + "ready_backlog", + "wal_gen_mb", + "tx_mb_s", + "archived_mb", +]; + +#[derive(Clone, Copy)] +struct Rgb(u8, u8, u8); + +const BG: Rgb = Rgb(0x29, 0x25, 0x22); +const FLOAT: Rgb = Rgb(0x34, 0x30, 0x2C); +const SEL: Rgb = Rgb(0x40, 0x3A, 0x36); +const MUTED: Rgb = Rgb(0xC1, 0xA7, 0x8E); +const FG: Rgb = Rgb(0xEC, 0xE1, 0xD7); + +fn variant_color(variant: &str) -> Option { + match variant { + "walrus" | "walrus-serial" => Some(Rgb(0xFA, 0xFF, 0x69)), + "walg" => Some(Rgb(0xFC, 0x3F, 0x1D)), + "pgbackrest" => Some(Rgb(0x27, 0x68, 0x9D)), + _ => None, + } +} + +const FALLBACK: [Rgb; 6] = [ + Rgb(0xA3, 0xA9, 0xCE), + Rgb(0x85, 0xB6, 0x95), + Rgb(0xCF, 0x9B, 0xC2), + Rgb(0x89, 0xB3, 0xB6), + Rgb(0xE4, 0x9B, 0x5D), + Rgb(0xB3, 0x80, 0xB0), +]; + +#[derive(Parser)] +#[command(about = "plot + summarize bench runs")] +struct Args { + /// run directory (repeatable; pair with --label in order) + #[arg(long = "run", required = true)] + runs: Vec, + /// label for the matching --run (repeatable) + #[arg(long = "label", required = true)] + labels: Vec, + /// output directory for plots + exports + #[arg(long)] + out: String, + /// timestamp tag for output filenames (default: now, UTC) + #[arg(long)] + stamp: Option, +} + +// -------------------------------------------------------------------------- +// Parsing helpers +// -------------------------------------------------------------------------- +fn pf(s: Option<&String>) -> Option { + let t = s?.trim(); + if t.is_empty() { None } else { t.parse().ok() } +} + +fn variant_of(label: &str) -> String { + // strip a trailing -b + if let Some(idx) = label.rfind("-b") { + let suffix = &label[idx + 2..]; + if !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()) { + return label[..idx].to_string(); + } + } + label.to_string() +} + +fn box_of(label: &str) -> String { + if let Some(idx) = label.rfind("-b") { + let suffix = &label[idx + 2..]; + if !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()) { + return suffix.to_string(); + } + } + "0".to_string() +} + +fn read_provenance(dir: &Path) -> HashMap { + let mut meta = HashMap::new(); + if let Ok(text) = std::fs::read_to_string(dir.join("provenance.txt")) { + for line in text.lines() { + if line.trim_start().starts_with('#') { + continue; + } + if let Some((k, v)) = line.split_once('=') { + meta.insert(k.trim().to_string(), v.trim().to_string()); + } + } + } + meta +} + +fn read_csv(dir: &Path, name: &str) -> Option>> { + let mut rdr = csv::ReaderBuilder::new() + .flexible(true) + .from_path(dir.join(name)) + .ok()?; + let headers = rdr.headers().ok()?.clone(); + let mut rows: Vec> = Vec::new(); + for rec in rdr.records().flatten() { + let m: HashMap = headers + .iter() + .zip(rec.iter()) + .map(|(h, v)| (h.to_string(), v.to_string())) + .collect(); + if m.get("ts").is_some_and(|s| !s.is_empty()) { + rows.push(m); + } + } + if rows.is_empty() { + return None; + } + rows.sort_by(|a, b| { + pf(a.get("ts")) + .unwrap_or(0.0) + .total_cmp(&pf(b.get("ts")).unwrap_or(0.0)) + }); + Some(rows) +} + +// -------------------------------------------------------------------------- +// Data model +// -------------------------------------------------------------------------- +struct Sample { + run_label: String, + variant: String, + boxid: String, + ts: f64, + t: f64, + t_int: i64, + vmrss_mb: Option, + vmpeak_mb: Option, + vmsize_mb: Option, + rssanon_mb: Option, + cpu_pct: Option, + archived_count: Option, + failed_count: Option, + ready_backlog: Option, + wal_gen_mb: Option, + tx_mb_s: Option, + archived_mb: Option, + meta: HashMap, +} + +impl Sample { + fn cell(&self, col: &str) -> String { + let num = |v: Option| v.map_or(String::new(), fmt_num); + match col { + "run_label" => self.run_label.clone(), + "variant" => self.variant.clone(), + "box" => self.boxid.clone(), + "ts" => format!("{:.3}", self.ts), + "t" => fmt_num(self.t), + "t_int" => self.t_int.to_string(), + "vmrss_mb" => num(self.vmrss_mb), + "vmpeak_mb" => num(self.vmpeak_mb), + "vmsize_mb" => num(self.vmsize_mb), + "rssanon_mb" => num(self.rssanon_mb), + "cpu_pct" => num(self.cpu_pct), + "archived_count" => num(self.archived_count), + "failed_count" => num(self.failed_count), + "ready_backlog" => num(self.ready_backlog), + "wal_gen_mb" => num(self.wal_gen_mb), + "tx_mb_s" => num(self.tx_mb_s), + "archived_mb" => num(self.archived_mb), + _ => self.meta.get(col).cloned().unwrap_or_default(), + } + } +} + +/// Extracts a plotted metric from a sample (None = absent at that tick). +type Getter = Box Option>; + +struct Run { + label: String, + variant: String, + boxid: String, + dir: String, + meta: HashMap, + samples: Vec, +} + +fn load_run(dir: &str, label: &str) -> Option { + let d = Path::new(dir); + // Degraded cells (failed burst workers, receiver shipped nothing) are stamped + // INVALID by the drivers; exclude them so a weaker workload is not averaged in. + if d.join("INVALID").exists() { + eprintln!("warning: {dir} marked INVALID, skipping"); + return None; + } + let Some(mem) = read_csv(d, "mem.csv") else { + eprintln!("warning: {dir} has no mem.csv, skipping"); + return None; + }; + let meta = read_provenance(d); + + let cpu: HashMap> = read_csv(d, "cpu.csv") + .unwrap_or_default() + .into_iter() + .map(|r| (r["ts"].clone(), pf(r.get("pct_cpu")))) + .collect(); + let arc: HashMap> = read_csv(d, "archive.csv") + .unwrap_or_default() + .into_iter() + .map(|r| (r["ts"].clone(), r)) + .collect(); + + let wal_rows = read_csv(d, "wal.csv").unwrap_or_default(); + let wal0 = wal_rows.iter().find_map(|r| pf(r.get("wal_bytes"))); + let wal: HashMap> = wal_rows + .iter() + .map(|r| { + let v = match (pf(r.get("wal_bytes")), wal0) { + (Some(wb), Some(w0)) => Some((wb - w0) / MB), + _ => None, + }; + (r["ts"].clone(), v) + }) + .collect(); + + // tx upload rate: derivative over consecutive net samples, keyed by later ts. + let mut net: HashMap> = HashMap::new(); + let (mut prev_ts, mut prev_tx): (Option, Option) = (None, None); + for r in read_csv(d, "net.csv").unwrap_or_default() { + let ts = pf(r.get("ts")); + let tx = pf(r.get("tx_bytes")); + let rate = match (prev_ts, prev_tx, ts, tx) { + (Some(pt), Some(px), Some(t), Some(x)) if t - pt > 0.0 && x - px >= 0.0 => { + Some((x - px) / (t - pt) / MB) + } + _ => None, + }; + net.insert(r["ts"].clone(), rate); + prev_ts = ts; + prev_tx = tx; + } + + let variant = variant_of(label); + let boxid = box_of(label); + let ts0 = pf(mem[0].get("ts")).unwrap_or(0.0); + let div = |r: &HashMap, k: &str, denom: f64| pf(r.get(k)).map(|v| v / denom); + + let samples = mem + .iter() + .map(|r| { + let tskey = r["ts"].clone(); + let ts = pf(Some(&tskey)).unwrap_or(0.0); + let t = ts - ts0; + let a = arc.get(&tskey); + let archived = a.and_then(|m| pf(m.get("archived_count"))); + Sample { + run_label: label.to_string(), + variant: variant.clone(), + boxid: boxid.clone(), + ts, + t, + t_int: t.round() as i64, + vmrss_mb: div(r, "vmrss_kb", KB), + vmpeak_mb: div(r, "vmpeak_kb", KB), + vmsize_mb: div(r, "vmsize_kb", KB), + rssanon_mb: div(r, "rssanon_kb", KB), + cpu_pct: cpu.get(&tskey).copied().flatten(), + archived_count: archived, + failed_count: a.and_then(|m| pf(m.get("failed_count"))), + ready_backlog: a.and_then(|m| pf(m.get("ready_backlog"))), + wal_gen_mb: wal.get(&tskey).copied().flatten(), + tx_mb_s: net.get(&tskey).copied().flatten(), + archived_mb: archived.map(|v| v * WAL_SEG_MB), + meta: meta.clone(), + } + }) + .collect(); + + Some(Run { + label: label.to_string(), + variant, + boxid, + dir: dir.to_string(), + meta, + samples, + }) +} + +// -------------------------------------------------------------------------- +// Aggregation +// -------------------------------------------------------------------------- +#[derive(Clone)] +struct Style { + color: Rgb, + dashed: bool, + z: i32, +} + +fn style_for(variant: &str, idx: usize) -> Style { + Style { + color: variant_color(variant).unwrap_or(FALLBACK[idx % FALLBACK.len()]), + dashed: variant.ends_with("-serial"), + z: if variant.starts_with("walrus") { 10 } else { 4 }, + } +} + +fn variants_ordered(labels: &[String]) -> Vec { + let mut vs: Vec = labels.iter().map(|l| variant_of(l)).collect(); + vs.sort(); + vs.dedup(); + vs.sort_by_key(|v| (!v.starts_with("walrus"), v.clone())); + vs +} + +fn median(xs: &mut [f64]) -> f64 { + xs.sort_by(f64::total_cmp); + let n = xs.len(); + if n == 0 { + 0.0 + } else if n % 2 == 1 { + xs[n / 2] + } else { + (xs[n / 2 - 1] + xs[n / 2]) / 2.0 + } +} + +struct Series { + label: String, + xs: Vec, + med: Vec, + lo: Vec, + hi: Vec, + style: Style, +} + +fn panel_series( + samples: &[&Sample], + variants: &[String], + style_map: &HashMap, + get: impl Fn(&Sample) -> Option, +) -> Vec { + // variant -> elapsed-second -> values + let mut buckets: HashMap<&str, BTreeMap>> = HashMap::new(); + for s in samples { + if let Some(v) = get(s) { + buckets + .entry(&s.variant) + .or_default() + .entry(s.t_int) + .or_default() + .push(v); + } + } + let mut out = Vec::new(); + for variant in variants { + let Some(tb) = buckets.get(variant.as_str()) else { + continue; + }; + let (mut xs, mut med, mut lo, mut hi) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for (t, vals) in tb { + let mut v = vals.clone(); + xs.push(*t as f64); + med.push(median(&mut v)); + lo.push(v.iter().copied().fold(f64::INFINITY, f64::min)); + hi.push(v.iter().copied().fold(f64::NEG_INFINITY, f64::max)); + } + out.push(Series { + label: variant.clone(), + xs, + med, + lo, + hi, + style: style_map[variant].clone(), + }); + } + out +} + +// -------------------------------------------------------------------------- +// Rendering (tiny-skia raster + ab_glyph text) +// +// We own a small canvas: tiny-skia gives AA strokes/fills and the PNG encoder +// (png-format feature); ab_glyph rasterizes glyph coverage that we blend in. +// No plotters, no `image`. Dashing is tiny-skia's native StrokeDash (pixel +// space), so the old data-space dash_segments hack is gone. +// -------------------------------------------------------------------------- +struct Panel { + title: String, + ylabel: String, + series: Vec, +} + +const W: u32 = 1180; +const HEADER_H: u32 = 48; +const PANEL_H: u32 = 340; + +struct Fonts { + regular: FontRef<'static>, + bold: FontRef<'static>, +} + +fn load_fonts() -> Result { + let load = |b: &'static [u8]| { + FontRef::try_from_slice(b).map_err(|_| anyhow::anyhow!("embedded font parse failed")) + }; + Ok(Fonts { + regular: load(dejavu::sans::regular())?, + bold: load(dejavu::sans::bold())?, + }) +} + +fn paint_rgb(c: Rgb, a: u8) -> Paint<'static> { + let mut p = Paint::default(); + p.set_color_rgba8(c.0, c.1, c.2, a); + p.anti_alias = true; + p +} + +fn polyline(pts: &[(f32, f32)]) -> Option { + let mut pb = PathBuilder::new(); + let (x0, y0) = *pts.first()?; + pb.move_to(x0, y0); + for &(x, y) in &pts[1..] { + pb.line_to(x, y); + } + pb.finish() +} + +fn stroke_poly( + pm: &mut Pixmap, + pts: &[(f32, f32)], + c: Rgb, + a: u8, + width: f32, + dash: Option<[f32; 2]>, +) { + let Some(path) = polyline(pts) else { return }; + let mut stroke = Stroke { + width, + line_cap: LineCap::Round, + line_join: LineJoin::Round, + ..Default::default() + }; + if let Some([on, off]) = dash { + stroke.dash = StrokeDash::new(vec![on, off], 0.0); + } + pm.stroke_path( + &path, + &paint_rgb(c, a), + &stroke, + Transform::identity(), + None, + ); +} + +fn fill_poly(pm: &mut Pixmap, pts: &[(f32, f32)], c: Rgb, a: u8) { + let mut pb = PathBuilder::new(); + let Some(&(x0, y0)) = pts.first() else { return }; + pb.move_to(x0, y0); + for &(x, y) in &pts[1..] { + pb.line_to(x, y); + } + pb.close(); + let Some(path) = pb.finish() else { return }; + pm.fill_path( + &path, + &paint_rgb(c, a), + FillRule::Winding, + Transform::identity(), + None, + ); +} + +fn fill_rect(pm: &mut Pixmap, x: f32, y: f32, w: f32, h: f32, c: Rgb, a: u8) { + if let Some(r) = Rect::from_xywh(x, y, w, h) { + pm.fill_rect(r, &paint_rgb(c, a), Transform::identity(), None); + } +} + +fn stroke_rect(pm: &mut Pixmap, x: f32, y: f32, w: f32, h: f32, c: Rgb, width: f32) { + stroke_poly( + pm, + &[(x, y), (x + w, y), (x + w, y + h), (x, y + h), (x, y)], + c, + 255, + width, + None, + ); +} + +/// "Nice" axis ticks: a round step (1/2/5 x 10^k) spanning [lo, hi]. +fn nice_ticks(lo: f64, hi: f64, target: usize) -> Vec { + if hi <= lo || target == 0 { + return vec![lo]; + } + let raw = (hi - lo) / target as f64; + let mag = 10f64.powf(raw.log10().floor()); + let norm = raw / mag; + let step = mag + * if norm < 1.5 { + 1.0 + } else if norm < 3.0 { + 2.0 + } else if norm < 7.0 { + 5.0 + } else { + 10.0 + }; + let mut t = (lo / step).ceil() * step; + let mut out = Vec::new(); + while t <= hi + step * 1e-9 { + out.push(t); + t += step; + } + out +} + +// --- text: rasterize a string into a premultiplied pixmap, then blit ---------- +fn text_width(font: &FontRef<'static>, px: f32, text: &str) -> f32 { + let sf = font.as_scaled(PxScale::from(px)); + let mut w = 0.0; + let mut prev = None; + for ch in text.chars() { + let g = font.glyph_id(ch); + if let Some(p) = prev { + w += sf.kern(p, g); + } + w += sf.h_advance(g); + prev = Some(g); + } + w +} + +fn text_pixmap(font: &FontRef<'static>, px: f32, text: &str, c: Rgb) -> Option { + let sf = font.as_scaled(PxScale::from(px)); + let w = (text_width(font, px, text).ceil() as u32 + 2).max(1); + let h = ((sf.ascent() - sf.descent()).ceil() as u32 + 2).max(1); + let mut pm = Pixmap::new(w, h)?; + let baseline = sf.ascent() + 1.0; + let data = pm.data_mut(); + let mut caret = 1.0f32; + let mut prev = None; + for ch in text.chars() { + let gid = font.glyph_id(ch); + if let Some(p) = prev { + caret += sf.kern(p, gid); + } + let glyph = gid.with_scale_and_position(PxScale::from(px), point(caret, baseline)); + caret += sf.h_advance(gid); + prev = Some(gid); + let Some(og) = font.outline_glyph(glyph) else { + continue; + }; + let bb = og.px_bounds(); + og.draw(|gx, gy, cov| { + let x = bb.min.x as i32 + gx as i32; + let y = bb.min.y as i32 + gy as i32; + if x < 0 || y < 0 || x as u32 >= w || y as u32 >= h { + return; + } + let a = (cov.clamp(0.0, 1.0) * 255.0).round() as u8; + if a == 0 { + return; + } + let i = ((y as u32 * w + x as u32) * 4) as usize; + // glyph boxes can overlap; keep the stronger coverage. premultiplied. + if a >= data[i + 3] { + let pre = |v: u8| ((v as u16 * a as u16) / 255) as u8; + data[i] = pre(c.0); + data[i + 1] = pre(c.1); + data[i + 2] = pre(c.2); + data[i + 3] = a; + } + }); + } + Some(pm) +} + +/// Blit a premultiplied src onto an opaque dst (src-over keeps dst opaque). +/// rotate_ccw=true places src rotated 90° counter-clockwise (vertical y-labels). +fn blit(dst: &mut Pixmap, src: &Pixmap, dx: i32, dy: i32, rotate_ccw: bool) { + let (dw, dh) = (dst.width(), dst.height()); + let (sw, sh) = (src.width(), src.height()); + let s = src.data(); + let d = dst.data_mut(); + for sy in 0..sh { + for sx in 0..sw { + let si = ((sy * sw + sx) * 4) as usize; + let a = s[si + 3]; + if a == 0 { + continue; + } + let (rx, ry) = if rotate_ccw { + (sy as i32, (sw - 1 - sx) as i32) + } else { + (sx as i32, sy as i32) + }; + let (px_, py_) = (dx + rx, dy + ry); + if px_ < 0 || py_ < 0 || px_ as u32 >= dw || py_ as u32 >= dh { + continue; + } + let di = ((py_ as u32 * dw + px_ as u32) * 4) as usize; + let inv = (255 - a) as u16; + for k in 0..3 { + d[di + k] = (s[si + k] as u16 + d[di + k] as u16 * inv / 255) as u8; + } + d[di + 3] = 255; + } + } +} + +fn text_at( + pm: &mut Pixmap, + font: &FontRef<'static>, + x: f32, + top: f32, + px: f32, + text: &str, + c: Rgb, +) { + if let Some(t) = text_pixmap(font, px, text, c) { + blit(pm, &t, x.round() as i32, top.round() as i32, false); + } +} + +fn text_center( + pm: &mut Pixmap, + font: &FontRef<'static>, + cx: f32, + top: f32, + px: f32, + text: &str, + c: Rgb, +) { + if let Some(t) = text_pixmap(font, px, text, c) { + blit( + pm, + &t, + (cx - t.width() as f32 / 2.0).round() as i32, + top.round() as i32, + false, + ); + } +} + +fn text_right( + pm: &mut Pixmap, + font: &FontRef<'static>, + right: f32, + cy: f32, + px: f32, + text: &str, + c: Rgb, +) { + if let Some(t) = text_pixmap(font, px, text, c) { + let x = (right - t.width() as f32).round() as i32; + let y = (cy - t.height() as f32 / 2.0).round() as i32; + blit(pm, &t, x, y, false); + } +} + +fn text_left_mid( + pm: &mut Pixmap, + font: &FontRef<'static>, + x: f32, + cy: f32, + px: f32, + text: &str, + c: Rgb, +) { + if let Some(t) = text_pixmap(font, px, text, c) { + blit( + pm, + &t, + x.round() as i32, + (cy - t.height() as f32 / 2.0).round() as i32, + false, + ); + } +} + +fn text_vert( + pm: &mut Pixmap, + font: &FontRef<'static>, + left: f32, + cy: f32, + px: f32, + text: &str, + c: Rgb, +) { + if let Some(t) = text_pixmap(font, px, text, c) { + // rotated box is t.height() wide x t.width() tall; center on cy. + let top = (cy - t.width() as f32 / 2.0).round() as i32; + blit(pm, &t, left.round() as i32, top, true); + } +} + +fn render( + panels: &[Panel], + out_path: &Path, + header: &str, + suffix: &str, + xlabel: &str, + fonts: &Fonts, +) -> Result<()> { + let n = panels.len() as u32; + let height = HEADER_H + n * PANEL_H; + let xmax = panels + .iter() + .flat_map(|p| p.series.iter()) + .filter_map(|s| s.xs.last().copied()) + .fold(0.0_f64, f64::max) + .max(1.0); + + let mut pm = Pixmap::new(W, height).context("alloc pixmap")?; + pm.fill(Color::from_rgba8(BG.0, BG.1, BG.2, 255)); + text_at(&mut pm, &fonts.bold, 24.0, 8.0, 20.0, header, FG); + if !suffix.is_empty() { + text_at(&mut pm, &fonts.regular, 24.0, 30.0, 14.0, suffix, MUTED); + } + for (i, panel) in panels.iter().enumerate() { + let py0 = HEADER_H + i as u32 * PANEL_H; + draw_panel(&mut pm, panel, py0, xmax, xlabel, i == 0, fonts); + } + pm.save_png(out_path) + .map_err(|e| anyhow::anyhow!("save {out_path:?}: {e}"))?; + Ok(()) +} + +fn draw_panel( + pm: &mut Pixmap, + panel: &Panel, + py0: u32, + xmax: f64, + xlabel: &str, + legend: bool, + fonts: &Fonts, +) { + let ymax = panel + .series + .iter() + .flat_map(|s| s.hi.iter().copied()) + .fold(0.0_f64, f64::max); + let ymax = if ymax > 0.0 { ymax * 1.05 } else { 1.0 }; + + let caption_h = if panel.title.is_empty() { 0.0 } else { 24.0 }; + let left = 76.0_f32; + let right = W as f32 - 12.0; + let top = py0 as f32 + 12.0 + caption_h; + let bottom = py0 as f32 + PANEL_H as f32 - 42.0; + + if !panel.title.is_empty() { + text_at( + pm, + &fonts.bold, + left, + py0 as f32 + 12.0, + 17.0, + &panel.title, + FG, + ); + } + + let sx = |x: f64| left + (x / xmax) as f32 * (right - left); + let sy = |y: f64| bottom - (y / ymax) as f32 * (bottom - top); + + for t in nice_ticks(0.0, ymax, 6) { + let yy = sy(t); + stroke_poly(pm, &[(left, yy), (right, yy)], SEL, 255, 1.0, None); + text_right(pm, &fonts.regular, left - 8.0, yy, 14.0, &fmt_num(t), MUTED); + } + for t in nice_ticks(0.0, xmax, 8) { + let xx = sx(t); + stroke_poly(pm, &[(xx, top), (xx, bottom)], SEL, 255, 1.0, None); + text_center( + pm, + &fonts.regular, + xx, + bottom + 6.0, + 14.0, + &fmt_num(t), + MUTED, + ); + } + // axis spines (left + bottom), slightly heavier + stroke_poly( + pm, + &[(left, top), (left, bottom), (right, bottom)], + SEL, + 255, + 1.5, + None, + ); + + text_center( + pm, + &fonts.regular, + (left + right) / 2.0, + bottom + 22.0, + 14.0, + xlabel, + MUTED, + ); + text_vert( + pm, + &fonts.regular, + 14.0, + (top + bottom) / 2.0, + 14.0, + &panel.ylabel, + MUTED, + ); + + // bands first (under every line) + for s in &panel.series { + if s.xs.len() < 2 { + continue; + } + let mut poly: Vec<(f32, f32)> = + s.xs.iter() + .zip(&s.lo) + .map(|(&x, &y)| (sx(x), sy(y))) + .collect(); + poly.extend(s.xs.iter().zip(&s.hi).rev().map(|(&x, &y)| (sx(x), sy(y)))); + fill_poly(pm, &poly, s.style.color, 36); + } + + // median lines by ascending z (walrus rides on top) + let mut ordered: Vec<&Series> = panel.series.iter().collect(); + ordered.sort_by_key(|s| s.style.z); + for s in ordered { + let pts: Vec<(f32, f32)> = + s.xs.iter() + .zip(&s.med) + .map(|(&x, &y)| (sx(x), sy(y))) + .collect(); + let dash = s.style.dashed.then_some([11.0, 7.0]); + stroke_poly(pm, &pts, s.style.color, 200, 1.25, dash); + } + + if legend && !panel.series.is_empty() { + draw_legend(pm, &panel.series, left + 12.0, top + 10.0, fonts); + } +} + +fn draw_legend(pm: &mut Pixmap, series: &[Series], x: f32, y: f32, fonts: &Fonts) { + let (pad, swatch, gap, row, fs) = (8.0_f32, 24.0_f32, 8.0_f32, 20.0_f32, 14.0_f32); + let tw = series + .iter() + .map(|s| text_width(&fonts.regular, fs, &s.label)) + .fold(0.0_f32, f32::max); + let bw = pad + swatch + gap + tw + pad; + let bh = pad * 2.0 + row * series.len() as f32; + fill_rect(pm, x, y, bw, bh, FLOAT, 255); + stroke_rect(pm, x, y, bw, bh, SEL, 1.0); + for (i, s) in series.iter().enumerate() { + let cy = y + pad + row * i as f32 + row / 2.0; + let lx = x + pad; + let dash = s.style.dashed.then_some([8.0, 5.0]); + stroke_poly( + pm, + &[(lx, cy), (lx + swatch, cy)], + s.style.color, + 255, + 1.25, + dash, + ); + text_left_mid(pm, &fonts.regular, lx + swatch + gap, cy, fs, &s.label, FG); + } +} + +// -------------------------------------------------------------------------- +// Summary + formatting +// -------------------------------------------------------------------------- +fn fmt_num(v: f64) -> String { + if v.is_finite() && v.fract() == 0.0 && v.abs() < 1e15 { + format!("{}", v as i64) + } else { + // %g-ish: trim trailing zeros from a 6-sig-figure rendering. + let s = format!("{v:.6}"); + let s = s.trim_end_matches('0').trim_end_matches('.'); + s.to_string() + } +} + +fn summarize(run: &Run) -> Vec<(String, serde_json::Value)> { + use serde_json::json; + let collect = |get: &dyn Fn(&Sample) -> Option| -> Vec { + run.samples.iter().filter_map(get).collect() + }; + + let mut row: Vec<(String, serde_json::Value)> = Vec::new(); + let mut push = |k: &str, v: serde_json::Value| row.push((k.to_string(), v)); + + push("run_label", json!(run.label)); + push("variant", json!(run.variant)); + push("box", json!(run.boxid)); + for k in META_KEYS { + push(k, json!(run.meta.get(k).cloned().unwrap_or_default())); + } + push("dir", json!(run.dir)); + + let vmrss = collect(&|s| s.vmrss_mb); + let mut vmrss_m = vmrss.clone(); + if !vmrss.is_empty() { + push( + "peak_vmrss_mb", + json!(vmrss.iter().copied().fold(f64::MIN, f64::max)), + ); + push("median_vmrss_mb", json!(median(&mut vmrss_m))); + } + let vmpeak = collect(&|s| s.vmpeak_mb); + if !vmpeak.is_empty() { + push( + "peak_vmpeak_mb", + json!(vmpeak.iter().copied().fold(f64::MIN, f64::max)), + ); + } + let vmsize = collect(&|s| s.vmsize_mb); + let mut vmsize_m = vmsize.clone(); + if !vmsize.is_empty() { + push( + "peak_vmsize_mb", + json!(vmsize.iter().copied().fold(f64::MIN, f64::max)), + ); + push("median_vmsize_mb", json!(median(&mut vmsize_m))); + } + let cpu = collect(&|s| s.cpu_pct); + if !cpu.is_empty() { + push( + "mean_cpu_pct", + json!(cpu.iter().sum::() / cpu.len() as f64), + ); + push( + "peak_cpu_pct", + json!(cpu.iter().copied().fold(f64::MIN, f64::max)), + ); + } + let backlog = collect(&|s| s.ready_backlog); + if !backlog.is_empty() { + push( + "max_backlog", + json!(backlog.iter().copied().fold(f64::MIN, f64::max) as i64), + ); + let last = run + .samples + .iter() + .rev() + .find_map(|s| s.ready_backlog) + .unwrap_or(0.0); + push("final_backlog", json!(last as i64)); + } + let archived = collect(&|s| s.archived_count); + if !archived.is_empty() { + let total = archived[archived.len() - 1] - archived[0]; + push("total_archived", json!(total as i64)); + let elapsed = run.samples[run.samples.len() - 1].t - run.samples[0].t; + if elapsed > 0.0 { + push("mean_drain_mb_s", json!(total * WAL_SEG_MB / elapsed)); + } + } + let failed = collect(&|s| s.failed_count); + if !failed.is_empty() { + push( + "total_failed", + json!((failed[failed.len() - 1] - failed[0]) as i64), + ); + } + row +} + +fn json_cell(v: &serde_json::Value) -> String { + match v { + serde_json::Value::Null => String::new(), + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + i.to_string() + } else if let Some(f) = n.as_f64() { + fmt_num(f) + } else { + n.to_string() + } + } + other => other.to_string(), + } +} + +fn main() -> Result<()> { + let args = Args::parse(); + let fonts = load_fonts()?; + if args.runs.len() != args.labels.len() { + anyhow::bail!("number of --run and --label must match"); + } + let stamp = args + .stamp + .clone() + .unwrap_or_else(|| chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string()); + let out = Path::new(&args.out); + std::fs::create_dir_all(out).with_context(|| format!("mkdir {out:?}"))?; + + let runs: Vec = args + .runs + .iter() + .zip(&args.labels) + .filter_map(|(d, l)| load_run(d, l)) + .collect(); + if runs.is_empty() { + anyhow::bail!("no loadable runs"); + } + + let all: Vec<&Sample> = runs.iter().flat_map(|r| r.samples.iter()).collect(); + let labels: Vec = runs.iter().map(|r| r.label.clone()).collect(); + let variants = variants_ordered(&labels); + let style_map: HashMap = variants + .iter() + .enumerate() + .map(|(i, v)| (v.clone(), style_for(v, i))) + .collect(); + let suffix = { + let m = &runs[0].meta; + let g = |k: &str| m.get(k).map(String::as_str).unwrap_or("?"); + format!( + "scale={} churn_rows={} burst={}s | {stamp}", + g("scale"), + g("churn_rows"), + g("burst_seconds") + ) + }; + + let panel = |ycol: &str, title: &str, ylabel: &str| -> Panel { + let get: Getter = match ycol { + "vmrss_mb" => Box::new(|s: &Sample| s.vmrss_mb), + "vmpeak_mb" => Box::new(|s: &Sample| s.vmpeak_mb), + "ready_backlog" => Box::new(|s: &Sample| s.ready_backlog), + "tx_mb_s" => Box::new(|s: &Sample| s.tx_mb_s), + "cpu_pct" => Box::new(|s: &Sample| s.cpu_pct), + _ => Box::new(|_: &Sample| None), + }; + Panel { + title: title.to_string(), + ylabel: ylabel.to_string(), + series: panel_series(&all, &variants, &style_map, get), + } + }; + + render( + &[ + panel( + "vmrss_mb", + "resident memory (vmrss) - median across replicas, band = min..max", + "vmrss (mb)", + ), + panel( + "vmpeak_mb", + "peak virtual memory (vmpeak) - the no-overcommit metric", + "vmpeak (mb)", + ), + ], + &out.join("mem_over_time.png"), + "memory over time: walrus vs wal-g vs pgbackrest", + &suffix, + "elapsed seconds", + &fonts, + )?; + render( + &[panel("ready_backlog", "", "*.ready backlog (segments)")], + &out.join("backlog.png"), + "archive backlog over time (lower = keeping up)", + &suffix, + "elapsed seconds", + &fonts, + )?; + render( + &[panel("tx_mb_s", "", "tx rate (mb/s)")], + &out.join("upload_rate.png"), + "network upload rate to s3 (tx_bytes derivative)", + &suffix, + "elapsed seconds", + &fonts, + )?; + render( + &[panel("cpu_pct", "", "cpu (%)")], + &out.join("cpu.png"), + "daemon cpu utilization over time", + &suffix, + "elapsed seconds", + &fonts, + )?; + + // --- raw exports ------------------------------------------------------- + let samples_path = out.join(format!("samples_{stamp}.csv")); + let mut w = csv::Writer::from_path(&samples_path)?; + w.write_record(SAMPLE_COLS.iter().chain(META_KEYS.iter()))?; + for s in &all { + let row: Vec = SAMPLE_COLS + .iter() + .chain(META_KEYS.iter()) + .map(|c| s.cell(c)) + .collect(); + w.write_record(&row)?; + } + w.flush()?; + + let rows: Vec> = runs.iter().map(summarize).collect(); + // CSV field order = first-seen union across rows. + let mut fields: Vec = Vec::new(); + for row in &rows { + for (k, _) in row { + if !fields.contains(k) { + fields.push(k.clone()); + } + } + } + let summary_csv = out.join(format!("summary_{stamp}.csv")); + let mut sw = csv::Writer::from_path(&summary_csv)?; + sw.write_record(&fields)?; + for row in &rows { + let map: HashMap<&str, &serde_json::Value> = + row.iter().map(|(k, v)| (k.as_str(), v)).collect(); + let rec: Vec = fields + .iter() + .map(|f| map.get(f.as_str()).map_or(String::new(), |v| json_cell(v))) + .collect(); + sw.write_record(&rec)?; + } + sw.flush()?; + + // summary.json: BTreeMap-backed objects => keys sorted (matches sort_keys). + let runs_json: Vec = rows + .iter() + .map(|row| { + serde_json::Value::Object(row.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + }) + .collect(); + let doc = serde_json::json!({ "stamp": stamp, "runs": runs_json }); + std::fs::write( + out.join("summary.json"), + serde_json::to_string_pretty(&doc)?, + )?; + + println!( + "wrote plots + {} + {} + summary.json", + samples_path.display(), + summary_csv.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn variant_and_box() { + assert_eq!(variant_of("walrus-serial-b0"), "walrus-serial"); + assert_eq!(box_of("walrus-serial-b0"), "0"); + assert_eq!(variant_of("walrus-r1"), "walrus-r1"); // -r1 is not -b + assert_eq!(box_of("walrus"), "0"); + assert_eq!(variant_of("pgbackrest-b12"), "pgbackrest"); + assert_eq!(box_of("pgbackrest-b12"), "12"); + } + + #[test] + fn median_odd_even() { + assert_eq!(median(&mut [3.0, 1.0, 2.0]), 2.0); + assert_eq!(median(&mut [4.0, 1.0, 3.0, 2.0]), 2.5); + } + + #[test] + fn fmt_num_int_vs_float() { + assert_eq!(fmt_num(42.0), "42"); + assert_eq!(fmt_num(2.5), "2.5"); + } + + #[test] + fn variant_order_walrus_first() { + let v = variants_ordered(&["walg-b0".into(), "walrus-b0".into(), "pgbackrest-b0".into()]); + assert_eq!(v[0], "walrus"); + } +} diff --git a/bench/tools/src/bin/sampler.rs b/bench/tools/src/bin/sampler.rs new file mode 100644 index 0000000..0b74ae9 --- /dev/null +++ b/bench/tools/src/bin/sampler.rs @@ -0,0 +1,828 @@ +//! bench-sampler — 1 Hz on-SUT resource sampler +//! +//! Writes one CSV per metric family, sharing a float-epoch `ts`. Schemas are +//! fixed by the bench contract (consumed by bench-analyze and plot.py alike): +//! mem.csv: ts,vmpeak_kb,vmsize_kb,vmhwm_kb,vmrss_kb,rssanon_kb,cg_current_bytes,cg_peak_bytes +//! cpu.csv: ts,pct_usr,pct_sys,pct_cpu +//! wal.csv: ts,wal_bytes +//! archive.csv: ts,archived_count,failed_count,ready_backlog,last_archived_age_s +//! net.csv: ts,tx_bytes,rx_bytes +//! +//! Memory/CPU come from /proc, network from /sys, PG metrics from a long-lived +//! `psql` reused across ticks (result framing via a sentinel row, respawned if +//! it dies). Daemonless archivers (pgbackrest) are followed by --proc-match, +//! which rescans /proc each tick and aggregates over the matching process tree. +//! +//! Self-test (no PostgreSQL): +//! sleep 30 & bench-sampler --pid $! --iface lo --no-pg \ +//! --outdir /tmp/samp --duration 3 --interval 1.0 + +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use clap::Parser; + +const SENTINEL: &str = "__SAMPLER_EOT__"; + +// Per-tick PG queries; each yields exactly one tuple line, framed by SENTINEL. +const PG_QUERIES: [&str; 2] = [ + "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(),'0/0');", + "SELECT archived_count, failed_count, \ + COALESCE(EXTRACT(EPOCH FROM (now() - last_archived_time)), -1) \ + FROM pg_stat_archiver;", +]; + +#[derive(Parser)] +#[command(about = "1 Hz on-SUT resource sampler")] +struct Args { + /// PID to sample + #[arg(long)] + pid: Option, + /// systemd unit; MainPID resolved via systemctl show (auto cgroup too) + #[arg(long)] + unit: Option, + /// do not exit if no PID can be resolved (mem/cpu columns left blank) + #[arg(long = "no-pid-required")] + no_pid_required: bool, + /// aggregate mem/cpu over ALL processes whose comm equals this name, + /// rescanned every tick (daemonless archivers). Excludes --pid/--unit. + #[arg(long = "proc-match")] + proc_match: Option, + /// archiver shorthand: walg|walrus -> that unit's MainPID, pgbackrest -> + /// proc-match (daemonless). Fills --unit/--proc-match when neither is given. + #[arg(long, value_parser = ["walg", "walrus", "pgbackrest"])] + daemon: Option, + /// cgroup v2 dir (default: auto from --unit) + #[arg(long)] + cgroup: Option, + /// network iface (default: auto default-route iface) + #[arg(long)] + iface: Option, + /// PGDATA path + #[arg(long, default_value = "/dat/18/data")] + pgdata: String, + /// directory for CSV outputs + #[arg(long)] + outdir: String, + /// sample interval seconds + #[arg(long, default_value_t = 1.0)] + interval: f64, + /// number of ticks to take (default: until SIGTERM) + #[arg(long)] + duration: Option, + /// psql conninfo (default: local socket, db walbench) + #[arg( + long, + default_value = "host=/var/run/postgresql user=postgres dbname=walbench" + )] + pg: String, + /// skip wal/archive queries (CSVs get headers only); PG-free self-test + #[arg(long = "no-pg")] + no_pg: bool, +} + +fn clk_tck() -> f64 { + let v = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + if v > 0 { v as f64 } else { 100.0 } +} + +// Diagnostic events go to stderr (bare). The orchestrator redirects the +// sampler's stderr to sampler.log in the result dir (run.sh/run_op.sh), so +// events land there co-located with the CSVs; standalone runs print to the +// terminal. Pipe through awk for timestamps if wanted; no own log file. + +// -------------------------------------------------------------------------- +// CSV sink: header on open, one comma-joined row per tick, flushed each write. +// -------------------------------------------------------------------------- +struct CsvSink { + fh: BufWriter, +} + +impl CsvSink { + fn create(dir: &Path, name: &str, header: &[&str]) -> Result { + let path = dir.join(name); + let mut fh = + BufWriter::new(File::create(&path).with_context(|| format!("create {path:?}"))?); + writeln!(fh, "{}", header.join(","))?; + fh.flush()?; + Ok(Self { fh }) + } + fn row(&mut self, cells: &[String]) { + let _ = writeln!(self.fh, "{}", cells.join(",")); + let _ = self.fh.flush(); + } +} + +fn ts_cell(ts: f64) -> String { + format!("{ts:.3}") +} +fn opt_u64(v: Option) -> String { + v.map_or(String::new(), |x| x.to_string()) +} + +// -------------------------------------------------------------------------- +// Resolution helpers +// -------------------------------------------------------------------------- +fn systemctl_value(unit: &str, prop: &str) -> Option { + let out = Command::new("systemctl") + .args(["show", "-p", prop, "--value", unit]) + .output() + .ok()?; + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if s.is_empty() { None } else { Some(s) } +} + +fn resolve_main_pid(unit: &str) -> Option { + let s = systemctl_value(unit, "MainPID")?; + s.parse::().ok().filter(|&p| p > 0) +} + +fn resolve_cgroup_path(unit: &str) -> Option { + let rel = systemctl_value(unit, "ControlGroup")?; + let path = format!("/sys/fs/cgroup/{}", rel.trim_start_matches('/')); + Path::new(&path).is_dir().then_some(path) +} + +/// Map an archiver token to a sampling target (unit, proc_match): walg/walrus +/// follow their systemd unit's MainPID; pgbackrest is daemonless (PG forks +/// archive-push per segment) so it is followed by proc-match. +fn daemon_target(daemon: &str) -> (Option, Option) { + match daemon { + "walg" => (Some("wal-g.service".into()), None), + "walrus" => (Some("walrus.service".into()), None), + "pgbackrest" => (None, Some("pgbackrest".into())), + _ => (None, None), + } +} + +fn list_pids_by_comm(name: &str) -> Vec { + let mut pids = Vec::new(); + let Ok(entries) = fs::read_dir("/proc") else { + return pids; + }; + for e in entries.flatten() { + let fname = e.file_name(); + let Some(s) = fname.to_str() else { continue }; + let Ok(pid) = s.parse::() else { continue }; + if let Ok(comm) = fs::read_to_string(format!("/proc/{pid}/comm")) + && comm.trim() == name + { + pids.push(pid); + } + } + pids +} + +fn detect_default_iface() -> Option { + // /proc/net/route: Iface col 0, Destination col 1 (00000000 = default). + let data = fs::read_to_string("/proc/net/route").ok()?; + for line in data.lines().skip(1) { + let f: Vec<&str> = line.split_whitespace().collect(); + if f.len() >= 2 && f[1] == "00000000" { + return Some(f[0].to_string()); + } + } + None +} + +// -------------------------------------------------------------------------- +// /proc readers +// -------------------------------------------------------------------------- +/// From /proc//status (kB) +#[derive(Default, Clone, Copy)] +struct MemStats { + vmpeak: Option, + vmsize: Option, + vmhwm: Option, + vmrss: Option, + rssanon: Option, +} + +impl MemStats { + const ZERO: MemStats = MemStats { + vmpeak: Some(0), + vmsize: Some(0), + vmhwm: Some(0), + vmrss: Some(0), + rssanon: Some(0), + }; + + fn slot(&mut self, key: &str) -> Option<&mut Option> { + Some(match key { + "VmPeak" => &mut self.vmpeak, + "VmSize" => &mut self.vmsize, + "VmHWM" => &mut self.vmhwm, + "VmRSS" => &mut self.vmrss, + "RssAnon" => &mut self.rssanon, + _ => return None, + }) + } + + /// Field-wise sum, missing treated as 0; result always present. + fn add(&mut self, o: MemStats) { + self.vmpeak = Some(self.vmpeak.unwrap_or(0) + o.vmpeak.unwrap_or(0)); + self.vmsize = Some(self.vmsize.unwrap_or(0) + o.vmsize.unwrap_or(0)); + self.vmhwm = Some(self.vmhwm.unwrap_or(0) + o.vmhwm.unwrap_or(0)); + self.vmrss = Some(self.vmrss.unwrap_or(0) + o.vmrss.unwrap_or(0)); + self.rssanon = Some(self.rssanon.unwrap_or(0) + o.rssanon.unwrap_or(0)); + } +} + +fn read_proc_status(pid: i32) -> MemStats { + let mut out = MemStats::default(); + let Ok(data) = fs::read_to_string(format!("/proc/{pid}/status")) else { + return out; + }; + for line in data.lines() { + let Some((key, rest)) = line.split_once(':') else { + continue; + }; + if let Some(slot) = out.slot(key) + && let Some(tok) = rest.split_whitespace().next() + && let Ok(v) = tok.parse::() + { + *slot = Some(v); + } + } + out +} + +/// (utime, stime) in clock ticks from /proc//stat. comm (field 2) may hold +/// spaces/parens, so split after the final ')'. None if the process is gone. +fn read_proc_cpu_jiffies(pid: i32) -> Option<(u64, u64)> { + let data = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let rparen = data.rfind(')')?; + let rest: Vec<&str> = data[rparen + 1..].split_whitespace().collect(); + // rest[0]=state; utime is overall field 14 -> index 11, stime 15 -> 12. + if rest.len() < 13 { + return None; + } + Some((rest[11].parse().ok()?, rest[12].parse().ok()?)) +} + +fn read_int_file(path: &str) -> Option { + let text = fs::read_to_string(path).ok()?; + let text = text.trim(); + if text == "max" { + None + } else { + text.parse().ok() + } +} + +// -------------------------------------------------------------------------- +// Persistent psql connection +// -------------------------------------------------------------------------- +#[derive(Default)] +struct PgRow { + wal_bytes: Option, + archived_count: Option, + failed_count: Option, + last_archived_age_s: Option, +} + +struct PsqlConn { + conninfo: String, + child: Option, + stdin: Option, + stdout: Option>, +} + +impl PsqlConn { + fn new(conninfo: String) -> Self { + let mut c = Self { + conninfo, + child: None, + stdin: None, + stdout: None, + }; + c.spawn(); + c + } + + fn spawn(&mut self) { + // -A unaligned, -t tuples-only, -q quiet, -X no psqlrc, -F '|' field sep. + let mut child = match Command::new("psql") + .args([ + "-Atq", + "-X", + "-F", + "|", + "-v", + "ON_ERROR_STOP=0", + &self.conninfo, + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + { + Ok(c) => c, + Err(e) => { + eprintln!("psql spawn failed: {e}"); + return; + } + }; + self.stdin = child.stdin.take(); + self.stdout = child.stdout.take().map(BufReader::new); + eprintln!("psql spawned pid={}", child.id()); + self.child = Some(child); + } + + fn alive(&mut self) -> bool { + match self.child.as_mut() { + Some(c) => matches!(c.try_wait(), Ok(None)), + None => false, + } + } + + fn kill(&mut self) { + if let Some(mut c) = self.child.take() { + let _ = c.kill(); + let _ = c.wait(); + } + self.stdin = None; + self.stdout = None; + } + + fn reset_archiver(&mut self) { + let _ = self.query_tick(); // ensure live first + if !self.alive() { + return; + } + if let (Some(stdin), Some(stdout)) = (self.stdin.as_mut(), self.stdout.as_mut()) { + if stdin + .write_all(b"SELECT pg_stat_reset_shared('archiver');\n") + .is_err() + { + return; + } + let _ = stdin.flush(); + let mut line = String::new(); + let _ = stdout.read_line(&mut line); + eprintln!("archiver stats reset"); + } + } + + fn query_tick(&mut self) -> PgRow { + if !self.alive() { + eprintln!("psql not alive; respawning"); + self.spawn(); + if !self.alive() { + return PgRow::default(); + } + } + let batch = format!("{}{}SELECT '{SENTINEL}';\n", PG_QUERIES[0], PG_QUERIES[1]); + let Some(stdin) = self.stdin.as_mut() else { + return PgRow::default(); + }; + if stdin.write_all(batch.as_bytes()).is_err() || stdin.flush().is_err() { + eprintln!("psql write failed; respawning next tick"); + self.kill(); + return PgRow::default(); + } + let Some(stdout) = self.stdout.as_mut() else { + return PgRow::default(); + }; + let mut lines: Vec = Vec::new(); + loop { + let mut line = String::new(); + match stdout.read_line(&mut line) { + Ok(0) => { + eprintln!("psql EOF mid-tick; respawning next tick"); + self.kill(); + return PgRow::default(); + } + Ok(_) => {} + Err(_) => { + self.kill(); + return PgRow::default(); + } + } + let s = line.trim_end_matches('\n'); + if s == SENTINEL { + break; + } + if s.is_empty() { + continue; + } + lines.push(s.to_string()); + } + parse_pg(&lines) + } + + fn close(&mut self) { + if self.alive() + && let Some(stdin) = self.stdin.as_mut() + { + let _ = stdin.write_all(b"\\q\n"); + let _ = stdin.flush(); + } + if let Some(c) = self.child.as_mut() { + let _ = c.wait(); + } + } +} + +fn parse_pg(lines: &[String]) -> PgRow { + let mut out = PgRow::default(); + if let Some(l) = lines.first() { + out.wal_bytes = nonempty(l.trim()); + } + if let Some(l) = lines.get(1) { + let parts: Vec<&str> = l.split('|').collect(); + if parts.len() >= 3 { + out.archived_count = nonempty(parts[0].trim()); + out.failed_count = nonempty(parts[1].trim()); + let age = parts[2].trim(); + // -1 sentinel => never archived yet => blank. + out.last_archived_age_s = (age != "-1").then(|| nonempty(age)).flatten(); + } + } + out +} + +fn nonempty(s: &str) -> Option { + (!s.is_empty()).then(|| s.to_string()) +} + +// -------------------------------------------------------------------------- +// Sampler +// -------------------------------------------------------------------------- +struct Sampler { + proc_match: Option, + pid: Option, + unit: Option, + cgroup: Option, + iface: Option, + pgdata: PathBuf, + clk_tck: f64, + mem: CsvSink, + cpu: CsvSink, + wal: CsvSink, + archive: CsvSink, + net: CsvSink, + psql: Option, + prev_cpu: Option<(u64, u64)>, + prev_cpu_ts: Option, + prev_cpu_map: HashMap, +} + +impl Sampler { + fn new(args: &Args) -> Result { + let outdir = Path::new(&args.outdir); + fs::create_dir_all(outdir).with_context(|| format!("mkdir {outdir:?}"))?; + + // --daemon is a bench shorthand: fill unit/proc_match from the token + // unless an explicit flag already set one. + let (mut unit, mut proc_match) = (args.unit.clone(), args.proc_match.clone()); + if let Some(d) = &args.daemon { + let (u, pm) = daemon_target(d); + unit = unit.or(u); + proc_match = proc_match.or(pm); + } + let mut pid = args.pid; + if proc_match.is_none() + && pid.is_none() + && let Some(u) = &unit + { + pid = resolve_main_pid(u); + eprintln!("resolved MainPID={pid:?} for unit={u}"); + } + + let mut cgroup = args.cgroup.clone(); + if proc_match.is_none() + && cgroup.is_none() + && let Some(u) = &unit + { + cgroup = resolve_cgroup_path(u); + eprintln!("resolved cgroup={cgroup:?} for unit={u}"); + } + + let iface = args.iface.clone().or_else(detect_default_iface); + eprintln!("using iface={iface:?}"); + if let Some(pm) = &proc_match { + eprintln!("proc-match mode: comm={pm:?}"); + } + + if proc_match.is_none() && pid.is_none() && !args.no_pid_required { + eprintln!("FATAL: no PID resolved and --no-pid-required not set"); + std::process::exit(2); + } + + let mut s = Self { + proc_match, + pid, + unit, + cgroup, + iface, + pgdata: PathBuf::from(&args.pgdata), + clk_tck: clk_tck(), + mem: CsvSink::create( + outdir, + "mem.csv", + &[ + "ts", + "vmpeak_kb", + "vmsize_kb", + "vmhwm_kb", + "vmrss_kb", + "rssanon_kb", + "cg_current_bytes", + "cg_peak_bytes", + ], + )?, + cpu: CsvSink::create(outdir, "cpu.csv", &["ts", "pct_usr", "pct_sys", "pct_cpu"])?, + wal: CsvSink::create(outdir, "wal.csv", &["ts", "wal_bytes"])?, + archive: CsvSink::create( + outdir, + "archive.csv", + &[ + "ts", + "archived_count", + "failed_count", + "ready_backlog", + "last_archived_age_s", + ], + )?, + net: CsvSink::create(outdir, "net.csv", &["ts", "tx_bytes", "rx_bytes"])?, + psql: None, + prev_cpu: None, + prev_cpu_ts: None, + prev_cpu_map: HashMap::new(), + }; + if !args.no_pg { + let mut conn = PsqlConn::new(args.pg.clone()); + conn.reset_archiver(); + s.psql = Some(conn); + } + Ok(s) + } + + fn refresh_pid_if_needed(&mut self) { + if self.proc_match.is_some() { + return; + } + let Some(unit) = self.unit.clone() else { + return; + }; + let cur = resolve_main_pid(&unit); + if cur != self.pid { + eprintln!("PID change for unit={unit}: {:?} -> {cur:?}", self.pid); + self.pid = cur; + self.prev_cpu = None; + self.prev_cpu_ts = None; + if self.cgroup.is_none() { + self.cgroup = resolve_cgroup_path(&unit); + } + } + } + + fn sample_mem(&mut self, ts: f64) { + let vals = if let Some(pm) = &self.proc_match { + // Sum each metric over the live tree. Per-proc lifetime highs + // (VmPeak/VmHWM) summed over the live set are a lower bound on the + // tree peak; run-level peak is max-over-time downstream. + let mut agg = MemStats::default(); + let mut found = false; + for pid in list_pids_by_comm(pm) { + agg.add(read_proc_status(pid)); + found = true; + } + // found => summed tree; else genuine 0 (async drained + exited). + if found { agg } else { MemStats::ZERO } + } else if let Some(pid) = self.pid { + read_proc_status(pid) + } else { + MemStats::default() + }; + let cg_cur = self + .cgroup + .as_ref() + .and_then(|c| read_int_file(&format!("{c}/memory.current"))); + let cg_peak = self + .cgroup + .as_ref() + .and_then(|c| read_int_file(&format!("{c}/memory.peak"))); + self.mem.row(&[ + ts_cell(ts), + opt_u64(vals.vmpeak), + opt_u64(vals.vmsize), + opt_u64(vals.vmhwm), + opt_u64(vals.vmrss), + opt_u64(vals.rssanon), + opt_u64(cg_cur), + opt_u64(cg_peak), + ]); + } + + fn sample_cpu(&mut self, ts: f64) { + if self.proc_match.is_some() { + self.sample_cpu_proctree(ts); + return; + } + let (mut usr, mut sys, mut cpu) = (String::new(), String::new(), String::new()); + let cur = self.pid.and_then(read_proc_cpu_jiffies); + if let (Some(cur), Some(prev), Some(pts)) = (cur, self.prev_cpu, self.prev_cpu_ts) { + let elapsed = ts - pts; + if elapsed > 0.0 { + let du = cur.0.saturating_sub(prev.0) as f64; + let ds = cur.1.saturating_sub(prev.1) as f64; + // ticks -> CPU-seconds / wall-seconds * 100. May exceed 100 on + // multi-threaded daemons. + let u = 100.0 * (du / self.clk_tck) / elapsed; + let s = 100.0 * (ds / self.clk_tck) / elapsed; + usr = format!("{:.2}", u.max(0.0)); + sys = format!("{:.2}", s.max(0.0)); + cpu = format!("{:.2}", (u + s).max(0.0)); + } + } + if let Some(cur) = cur { + self.prev_cpu = Some(cur); + self.prev_cpu_ts = Some(ts); + } + self.cpu.row(&[ts_cell(ts), usr, sys, cpu]); + } + + fn sample_cpu_proctree(&mut self, ts: f64) { + // Per-PID cumulative ticks diffed tick-over-tick; a freshly forked PID + // contributes its whole counter (~one interval), a vanished one drops + // out. Only combined pct_cpu is meaningful across a churning set. + let pm = self.proc_match.clone().unwrap(); + let mut cur_map: HashMap = HashMap::new(); + for pid in list_pids_by_comm(&pm) { + if let Some((u, s)) = read_proc_cpu_jiffies(pid) { + cur_map.insert(pid, u + s); + } + } + let mut cpu = String::new(); + if let Some(pts) = self.prev_cpu_ts { + let elapsed = ts - pts; + if elapsed > 0.0 { + let d_ticks: u64 = cur_map + .iter() + .map(|(pid, &total)| { + total.saturating_sub(self.prev_cpu_map.get(pid).copied().unwrap_or(0)) + }) + .sum(); + cpu = format!( + "{:.2}", + (100.0 * (d_ticks as f64 / self.clk_tck) / elapsed).max(0.0) + ); + } + } + self.prev_cpu_map = cur_map; + self.prev_cpu_ts = Some(ts); + self.cpu + .row(&[ts_cell(ts), String::new(), String::new(), cpu]); + } + + fn sample_net(&mut self, ts: f64) { + let (mut tx, mut rx) = (String::new(), String::new()); + if let Some(iface) = &self.iface { + let base = format!("/sys/class/net/{iface}/statistics"); + tx = opt_u64(read_int_file(&format!("{base}/tx_bytes"))); + rx = opt_u64(read_int_file(&format!("{base}/rx_bytes"))); + } + self.net.row(&[ts_cell(ts), tx, rx]); + } + + fn sample_pg(&mut self, ts: f64) { + let Some(psql) = self.psql.as_mut() else { + self.wal.row(&[ts_cell(ts), String::new()]); + self.archive.row(&[ + ts_cell(ts), + String::new(), + String::new(), + String::new(), + String::new(), + ]); + return; + }; + let res = psql.query_tick(); + let ready = self.count_ready(); + self.wal + .row(&[ts_cell(ts), res.wal_bytes.unwrap_or_default()]); + self.archive.row(&[ + ts_cell(ts), + res.archived_count.unwrap_or_default(), + res.failed_count.unwrap_or_default(), + opt_u64(ready), + res.last_archived_age_s.unwrap_or_default(), + ]); + } + + fn count_ready(&self) -> Option { + let dir = self.pgdata.join("pg_wal").join("archive_status"); + let entries = fs::read_dir(dir).ok()?; + Some( + entries + .flatten() + .filter(|e| e.file_name().to_string_lossy().ends_with(".ready")) + .count() as u64, + ) + } + + fn tick(&mut self) { + self.refresh_pid_if_needed(); + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + self.sample_mem(ts); + self.sample_cpu(ts); + self.sample_net(ts); + self.sample_pg(ts); + } + + fn close(&mut self) { + if let Some(psql) = self.psql.as_mut() { + psql.close(); + } + eprintln!("sampler stopped"); + } +} + +fn run(args: &Args, sampler: &mut Sampler, stop: &Arc) { + let interval = Duration::from_secs_f64(args.interval.max(0.0)); + let mut ticks = 0u64; + let mut next_at = Instant::now(); + while !stop.load(Ordering::Relaxed) { + sampler.tick(); + ticks += 1; + if args.duration.is_some_and(|d| ticks >= d) { + eprintln!("duration reached ({ticks} ticks); stopping"); + break; + } + next_at += interval; + let now = Instant::now(); + if next_at <= now { + next_at = now; // fell behind; resync, no burst catch-up + } else { + // Interruptible sleep so SIGTERM is honored promptly. + while !stop.load(Ordering::Relaxed) && Instant::now() < next_at { + let remaining = next_at.saturating_duration_since(Instant::now()); + std::thread::sleep(remaining.min(Duration::from_millis(100))); + } + } + } + sampler.close(); +} + +fn main() -> Result<()> { + let args = Args::parse(); + let mut sampler = Sampler::new(&args)?; + let stop = Arc::new(AtomicBool::new(false)); + for sig in [signal_hook::consts::SIGTERM, signal_hook::consts::SIGINT] { + signal_hook::flag::register(sig, Arc::clone(&stop))?; + } + run(&args, &mut sampler, &stop); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_pg_full() { + let r = parse_pg(&["123456".into(), "42|0|3.5".into()]); + assert_eq!(r.wal_bytes.as_deref(), Some("123456")); + assert_eq!(r.archived_count.as_deref(), Some("42")); + assert_eq!(r.failed_count.as_deref(), Some("0")); + assert_eq!(r.last_archived_age_s.as_deref(), Some("3.5")); + } + + #[test] + fn parse_pg_never_archived() { + // -1 age sentinel collapses to blank. + let r = parse_pg(&["10".into(), "0|0|-1".into()]); + assert_eq!(r.last_archived_age_s, None); + } + + #[test] + fn parse_pg_empty() { + let r = parse_pg(&[]); + assert!(r.wal_bytes.is_none() && r.archived_count.is_none()); + } + + #[test] + fn daemon_target_maps() { + assert_eq!(daemon_target("walg"), (Some("wal-g.service".into()), None)); + assert_eq!( + daemon_target("walrus"), + (Some("walrus.service".into()), None) + ); + assert_eq!( + daemon_target("pgbackrest"), + (None, Some("pgbackrest".into())) + ); + assert_eq!(daemon_target("nope"), (None, None)); + } +} diff --git a/ci/backup_mark.sh b/ci/backup_mark.sh index dcb5291..042c84c 100755 --- a/ci/backup_mark.sh +++ b/ci/backup_mark.sh @@ -22,7 +22,7 @@ walrus backup-list permanent_of() { # is_permanent is exposed as snake_case in backup-list --json walrus backup-list --json \ - | python3 -c 'import sys,json; b=json.load(sys.stdin)[0]; print("true" if b["is_permanent"] else "false")' + | jq -r 'if .[0].is_permanent then "true" else "false" end' } initial=$(permanent_of) diff --git a/ci/backup_show.sh b/ci/backup_show.sh index e60e79f..1e1c4fd 100755 --- a/ci/backup_show.sh +++ b/ci/backup_show.sh @@ -16,14 +16,9 @@ walrus backup-push walrus backup-show LATEST walrus backup-show LATEST --json \ - | python3 -c ' -import sys, json -o = json.load(sys.stdin) -assert "name" in o, o -assert "sentinel" in o, o -s = o["sentinel"] -# PascalCase keys mirror wal-g on-disk format -for k in ("Version", "StartTime", "FinishTime", "Hostname", "IsPermanent", "PgVersion"): - assert k in s, (k, s) -print("backup_show OK") -' + | jq -er ' + (["name","sentinel"] - keys) as $top + | (.sentinel | ["Version","StartTime","FinishTime","Hostname","IsPermanent","PgVersion"] - keys) as $sub + | if ($top|length) > 0 then error("missing keys: \($top)") + elif ($sub|length) > 0 then error("missing sentinel keys: \($sub)") + else "backup_show OK" end' diff --git a/ci/daemon.sh b/ci/daemon.sh index 61beef8..7848759 100755 --- a/ci/daemon.sh +++ b/ci/daemon.sh @@ -19,7 +19,11 @@ pgbench -p "$PGPORT" -h "$PGHOST" -i -s 1 postgres psql -p "$PGPORT" -h "$PGHOST" -c "SELECT pg_switch_wal()" postgres sleep 2 -WAL=$(ls "$PGDATA/pg_wal" | grep -E '^[0-9A-F]{24}$' | head -n1) +WAL="" +for f in "$PGDATA"/pg_wal/*; do + name=$(basename "$f") + [[ $name =~ ^[0-9A-F]{24}$ ]] && { WAL=$name; break; } +done [ -n "$WAL" ] || { echo "no WAL segment found"; exit 1; } SOCKET="$WORKROOT/wal-daemon.sock" diff --git a/ci/lib.sh b/ci/lib.sh index 826727c..411d387 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -17,7 +17,8 @@ export PATH="$PG_BIN:$PATH" WORKROOT=$(mktemp -d -t walrus-ci-XXXXXX) export PGDATA="$WORKROOT/pgdata" export PGHOST="$WORKROOT/run" -export PGUSER="$(id -un)" +PGUSER="$(id -un)" +export PGUSER export PGDATABASE=postgres export PGPORT=55435 @@ -44,7 +45,8 @@ storage_init() { s3) : "${MINIO_ENDPOINT:?set MINIO_ENDPOINT, e.g. http://127.0.0.1:9000}" local bucket="${WALRUS_S3_BUCKET:-walrus}" - export WALG_S3_PREFIX="s3://$bucket/$(basename "$WORKROOT")" + WALG_S3_PREFIX="s3://$bucket/$(basename "$WORKROOT")" + export WALG_S3_PREFIX export AWS_ENDPOINT_URL="$MINIO_ENDPOINT" export WALG_S3_FORCE_PATH_STYLE=true export AWS_REGION="${AWS_REGION:-us-east-1}" @@ -55,7 +57,8 @@ storage_init() { gcs) : "${FAKE_GCS_ENDPOINT:?set FAKE_GCS_ENDPOINT, e.g. http://127.0.0.1:4443}" local bucket="${WALRUS_GS_BUCKET:-walrus}" - export WALG_GS_PREFIX="gs://$bucket/$(basename "$WORKROOT")" + WALG_GS_PREFIX="gs://$bucket/$(basename "$WORKROOT")" + export WALG_GS_PREFIX export WALG_GS_ENDPOINT="$FAKE_GCS_ENDPOINT" WALG_ARCHIVE_ENV="WALG_GS_PREFIX=$WALG_GS_PREFIX WALG_GS_ENDPOINT=$WALG_GS_ENDPOINT" ;; diff --git a/examples/bench_increment.rs b/examples/bench_increment.rs deleted file mode 100644 index f18d943..0000000 --- a/examples/bench_increment.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! Micro-benchmark: wal-g `wi1` vs PG17 native INCREMENTAL -//! -//! For a synthetic paged file of size `FILE_BLOCKS * BLCKSZ`, with -//! `DIRTY_BLOCKS` randomly-selected dirty pages, measure: -//! - encode (header + page bodies into a Vec) -//! - apply (apply_increment_in_place onto a same-size in-memory target) -//! - on-disk header overhead -//! -//! Run with: `cargo run --release --example bench_increment` - -use std::io::{Cursor, Write}; -use std::time::Instant; - -use walrus::pg::backup::delta::PG_PAGE_SIZE; -use walrus::pg::backup::increment::{ - Format, apply_increment_in_place, write_increment_header, write_native_increment_header, -}; - -const ITERS: usize = 50; - -/// Deterministic "pseudo-random" block picker: stride through the file -fn pick_blocks(file_blocks: u32, count: u32) -> Vec { - assert!(count <= file_blocks); - if count == 0 { - return Vec::new(); - } - let stride = (file_blocks / count).max(1); - let mut out: Vec = (0..count).map(|i| (i * stride) % file_blocks).collect(); - out.sort(); - out.dedup(); - while (out.len() as u32) < count { - // pad with extra blocks if stride collapsed dups - let next = (out.last().copied().unwrap_or(0) + 1) % file_blocks; - if out.contains(&next) { - break; - } - out.push(next); - out.sort(); - } - out -} - -fn encode_wi1(file_size: u64, blocks: &[u32], page_template: &[u8]) -> Vec { - let mut buf = Vec::with_capacity(16 + blocks.len() * 4 + blocks.len() * PG_PAGE_SIZE as usize); - write_increment_header(&mut buf, file_size, blocks).unwrap(); - for _ in blocks { - buf.write_all(page_template).unwrap(); - } - buf -} - -fn encode_native(file_blocks: u32, blocks: &[u32], page_template: &[u8]) -> Vec { - let mut buf = Vec::with_capacity(16384 + blocks.len() * PG_PAGE_SIZE as usize); - write_native_increment_header(&mut buf, file_blocks, blocks).unwrap(); - for _ in blocks { - buf.write_all(page_template).unwrap(); - } - buf -} - -fn bench(label: &str, n: usize, mut f: F) -> u128 { - // Warmup - for _ in 0..3 { - f(); - } - let t0 = Instant::now(); - for _ in 0..n { - f(); - } - let elapsed = t0.elapsed(); - let per = elapsed / n as u32; - println!( - " {label:<40} total={:>9.2?} per={:>9.2?} ({n} iters)", - elapsed, per - ); - elapsed.as_nanos() / n as u128 -} - -fn run_case(file_blocks: u32, dirty: u32) { - println!( - "\n─── file={} MiB ({} blocks), dirty={} pages ({:.1}% density) ───", - (file_blocks as u64 * PG_PAGE_SIZE) / (1 << 20), - file_blocks, - dirty, - 100.0 * dirty as f64 / file_blocks as f64, - ); - - let blocks = pick_blocks(file_blocks, dirty); - let page = vec![0xAB; PG_PAGE_SIZE as usize]; - let file_size = file_blocks as u64 * PG_PAGE_SIZE; - - let wi1 = encode_wi1(file_size, &blocks, &page); - let native = encode_native(file_blocks, &blocks, &page); - println!( - " wire-size: wi1={} bytes native={} bytes diff={}", - wi1.len(), - native.len(), - native.len() as i64 - wi1.len() as i64, - ); - let wi1_header = 16 + blocks.len() * 4; - let native_header_unpadded = 12 + blocks.len() * 4; - let native_header_padded = native.len() - blocks.len() * PG_PAGE_SIZE as usize; - println!( - " header: wi1={} bytes native_raw={} bytes native_padded={} bytes", - wi1_header, native_header_unpadded, native_header_padded, - ); - - // Encode benchmarks - let wi1_ns = bench("encode wi1", ITERS, || { - let _ = encode_wi1(file_size, &blocks, &page); - }); - let native_ns = bench("encode native", ITERS, || { - let _ = encode_native(file_blocks, &blocks, &page); - }); - - // Apply benchmarks (fresh target each iter, so apply is comparable) - let target_template = vec![0u8; file_size as usize]; - let apply_wi1_ns = bench("apply wi1", ITERS, || { - let mut target = Cursor::new(target_template.clone()); - let mut inc = Cursor::new(&wi1); - let (sz, n, fmt) = apply_increment_in_place(&mut inc, &mut target).unwrap(); - debug_assert_eq!(sz, file_size); - debug_assert_eq!(n, blocks.len()); - debug_assert_eq!(fmt, Format::Wi1); - }); - let apply_native_ns = bench("apply native", ITERS, || { - let mut target = Cursor::new(target_template.clone()); - let mut inc = Cursor::new(&native); - let (_, _, fmt) = apply_increment_in_place(&mut inc, &mut target).unwrap(); - debug_assert_eq!(fmt, Format::Native); - }); - - let total_payload_mib = (blocks.len() as u64 * PG_PAGE_SIZE) as f64 / (1 << 20) as f64; - let mib_per_s = |ns: u128| { - if ns == 0 { - f64::INFINITY - } else { - total_payload_mib * 1_000_000_000.0 / ns as f64 - } - }; - println!( - " encode throughput: wi1={:>7.1} MiB/s native={:>7.1} MiB/s", - mib_per_s(wi1_ns), - mib_per_s(native_ns), - ); - println!( - " apply throughput: wi1={:>7.1} MiB/s native={:>7.1} MiB/s", - mib_per_s(apply_wi1_ns), - mib_per_s(apply_native_ns), - ); -} - -fn main() { - println!("walrus increment format micro-benchmark"); - println!("BLCKSZ = {} bytes; ITERS = {}", PG_PAGE_SIZE, ITERS); - - // 4 MiB / sparse delta (typical OLTP) - run_case(512, 5); - // 64 MiB / 5% dirty (moderate write workload) - run_case(8192, 410); - // 1 GiB rel segment / 1% dirty - run_case(131_072, 1310); - // 1 GiB / 50% dirty (worst case before falling back to full) - run_case(131_072, 65_536); -} diff --git a/src/storage/s3.rs b/src/storage/s3.rs index 8e0a34c..aea7d45 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -11,7 +11,7 @@ use std::time::{Duration, SystemTime}; use async_trait::async_trait; use aws_lc_rs::{digest, hmac}; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; use chrono::{DateTime, Utc}; use futures::{StreamExt, TryStreamExt, stream}; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; @@ -205,24 +205,21 @@ impl S3Storage { let mut parts: Vec<(u32, String)> = Vec::new(); let mut part_no: u32 = 0; - let mut buf = vec![0u8; PART_SIZE]; loop { - // fill buf up to PART_SIZE or EOF - let mut filled = 0usize; - while filled < buf.len() { - let n = body.read(&mut buf[filled..]).await?; - if n == 0 { + let mut buf = BytesMut::with_capacity(PART_SIZE); + while buf.len() < PART_SIZE { + if body.read_buf(&mut buf).await? == 0 { break; } - filled += n; } + let filled = buf.len(); if filled == 0 && part_no > 0 { break; } part_no += 1; let part_no_str = part_no.to_string(); - let chunk = Bytes::copy_from_slice(&buf[..filled]); + let chunk = buf.freeze(); // Per-part retry: chunk is already buffered, so transient failures // (5xx, transport) replay the same body without re-reading source