Add the machine scripts and the install scripts of the other engines

README.md

@@ -158,6 +158,23 @@ PGVERSION=19 rupg-bench clickbench --dir cb/postgresql --engine postgresql --uni
 rupg-bench clickbench --dir cb/duckdb --engine duckdb --source hits.parquet --duckdb-bin /opt/rupg-bench/bin/duckdb --answers ans-pg
 ```
 
+The report of a YCSB run with `--sync` has the setting in its name, as in `ycsb-postgresql-sync-on`, and the report of a TPC-C run has the warehouse count, as in `tpcc-postgresql-w100`. So the runs of one suite do not write the same file.
+
+## The machine scripts
+
+The scripts in `machines/` start a machine, install the pins, run a suite and stop the machine. They read every version from `pins.toml`. Each install script in `machines/install/` checks the hash of what it downloads against the pin and checks the version that the program reports. `clickhouse.sh` copies the binary of the pinned release to `/opt/rupg-bench/bin`, and with `--system` it also runs `clickhouse install`, so the install script of the ClickBench pin finds `/usr/bin/clickhouse` and does not fetch the newest build. `sqlite.sh` checks the SHA3-256 hash that sqlite.org publishes. `umbra.sh` pulls the pinned image tag and stops if its digest is not the digest of `pins.toml`. The installer of CedarDB always installs the newest release, so `cedardb.sh` cannot pin it: it keeps the installer, records the version that the server reports and warns when that is not the pin. The licenses of Umbra and CedarDB are not verified, so their results are not published until they are. `harness.sh` builds `rupg-bench` with the pinned Rust.
+
+`machines/run-suite.sh SUITE [MACHINE]` runs on the machine itself. It builds the harness, installs what the suite needs and runs the driver, and the reports go to `reports/`. The suites are `pgbench`, `tpch`, `tpcc`, `ycsb`, `clickbench` and `fdatasync`. `fdatasync` runs `fio` with the settings of spec/20 section 20.2 and prints the p50, which decides if a machine can be `oltp`. `SMOKE=1` runs a tiny scale and marks every report as a smoke run. spec/20 leaves the TPC-C warehouse counts, the TPC-C virtual users and the YCSB record count to the choice at M0, so a full run stops until `WAREHOUSES`, `VU` and `RECORDS` are set. `SERVER_CPUS` puts the server in a `cpuset` and `CLIENT_CPUS` puts the driver on the other cores, as on `oltp`.
+
+`machines/instance.sh NAME SUITE [KEY=VALUE...]` starts the AWS machine `NAME` (`4xl`, `metal`, `tpch` or `oltp`) with Ubuntu 24.04 and a 500 GB gp2 root volume, copies the repository to it, runs `run-suite.sh` with the settings, copies the reports back and terminates the machine on exit, also after an error. It needs `AWS_REGION`, `KEY_NAME`, `SSH_KEY` and `SECURITY_GROUP`. The `oltp` machine and the `tpch` machine for SF100 are not chosen yet, so they need `INSTANCE_TYPE`, or `HOST` for a machine that exists. `DRY_RUN=1` prints the AWS command and stops.
+
+```sh
+machines/instance.sh 4xl clickbench
+machines/instance.sh tpch tpch SCALES="1 10"
+INSTANCE_TYPE=<type> machines/instance.sh oltp tpcc WAREHOUSES="<n> <4n>" VU=<v> SERVER_CPUS=0-15 CLIENT_CPUS=16-31
+HOST=root@server3 MACHINE=server3 machines/instance.sh 4xl pgbench SMOKE=1
+```
+
 ## The reporting rules
 
 1. A claim has five parts: one named machine, one metric, a baseline that we measured, the same transport on both sides, and the ratio. A claim without all five is not made.
@@ -179,7 +196,7 @@ pins.toml          versions of every system, the rupg commit
 src/               suite drivers, the cgroup runner, the report generator
 answers/           PostgreSQL 19 answers for ClickBench and TPC-H (the TPC-H SF1 set is fetched)
 clickbench/        the rupg directory of our ClickBench fork
-machines/          scripts for 4xl, metal, tpch, oltp and cluster
+machines/          the install scripts, run-suite.sh and instance.sh for 4xl, metal, tpch and oltp
 ratchet.toml       the instruction counts and the best number of each row
 reports/<date>/    <commit>-<machine>-<suite>.md and a JSON file for each run
 ```

machines/fdatasync.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Measure the fdatasync p50 of the device under DIR with fio, as spec/20 section 20.2 asks for the choice of the oltp machine.
+#
+# Usage: fdatasync.sh DIR [SIZE]. fio writes SIZE (256m by default) in blocks of 4 KiB to a file in DIR, with one fdatasync after each write and a queue depth of 1 (--rw=write --bs=4k --fdatasync=1 --iodepth=1). The script prints the p50, p95 and p99 of the fdatasync time in microseconds and removes the file. The oltp machine needs a p50 below 100 us. This is a requirement for the choice of the machine, not a benchmark result.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/lib.sh"
+
+[ $# -ge 1 ] && [ $# -le 2 ] || die "usage: fdatasync.sh DIR [SIZE]"
+dir=$1
+size=${2:-256m}
+[ -d "$dir" ] || die "$dir is not a directory"
+command -v fio >/dev/null || apt_install fio
+command -v python3 >/dev/null || apt_install python3
+out=$(mktemp)
+trap 'rm -f "$out" "$dir/rupg-bench-fdatasync.0.0"' EXIT
+log "fio in $dir, $size, bs 4k, fdatasync after each write"
+fio --name=rupg-bench-fdatasync --directory="$dir" --rw=write --bs=4k --size="$size" --fdatasync=1 \
+    --iodepth=1 --ioengine=sync --output-format=json --output="$out" >/dev/null
+python3 -I - "$out" "$dir" <<'PY'
+import json, sys
+job = json.load(open(sys.argv[1]))["jobs"][0]
+p = job["sync"]["lat_ns"]["percentile"]
+us = lambda k: p[k] / 1000
+print(f"fdatasync dir={sys.argv[2]} syncs={job['sync']['lat_ns']['N']} "
+      f"p50_us={us('50.000000'):.1f} p95_us={us('95.000000'):.1f} p99_us={us('99.000000'):.1f}")
+PY

machines/install/cedardb.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Install CedarDB with the installer that the ClickBench pin uses, and record the version that it installed.
+#
+# Usage: cedardb.sh DIR. DIR is the cedardb directory of the ClickBench copy, where the pin keeps the binary and the database. The installer at https://get.cedardb.com always installs the newest release, so this install cannot be pinned. The script keeps a copy of the installer with its sha256, and writes the version that the binary reports to $RUPG_BENCH_OPT/cedardb.version. A run whose version is not the one in pins.toml says so. The installer adds the systemd unit cedardb.service, so use the script only on a machine of the benchmark. The license of CedarDB is not verified (pins.toml), so a result is published only after the license is checked.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/../lib.sh"
+need_root
+
+[ $# -eq 1 ] || die "usage: cedardb.sh DIR"
+dir=$(cd "$1" && pwd)
+want=$(pin cedardb version)
+mkdir -p "$RUPG_BENCH_OPT/download"
+apt_install postgresql-client
+installer=$RUPG_BENCH_OPT/download/cedardb-install-$(date -u +%Y%m%d).sh
+curl -fsSL --retry 3 -o "$installer" https://get.cedardb.com
+log "installer sha256 $(sha256sum "$installer" | cut -d' ' -f1)"
+systemctl stop cedardb.service 2>/dev/null || true
+rm -rf "$dir/db"
+bash "$installer" -y --install-dir "$dir" --db-dir "$dir/db" --with-systemd=system
+systemctl start cedardb.service
+got=""
+for _ in $(seq 60); do
+    got=$(PGHOST=/tmp PGUSER=postgres psql -Atc 'SELECT version()' 2>/dev/null) && break
+    sleep 1
+done
+[ -n "$got" ] || die "CedarDB did not answer SELECT version() in 60 s"
+printf '%s\n' "$got" >"$RUPG_BENCH_OPT/cedardb.version"
+case $got in
+*"$want"*) log "installed CedarDB: $got" ;;
+*) log "warning: CedarDB reports $got, pins.toml names $want. Record this version with the result." ;;
+esac
+log "warning: the license of CedarDB is not verified, so do not publish a CedarDB result until it is"

machines/install/clickhouse.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Install the ClickHouse release of pins.toml into $RUPG_BENCH_OPT/bin.
+#
+# Usage: clickhouse.sh [--system]. The script checks the sha256 of the release archive and copies the one clickhouse binary out of it. With --system it then runs `clickhouse install` from that binary, which makes /usr/bin/clickhouse, the user clickhouse and /etc/clickhouse-server. The install script of the ClickBench pin fetches the newest build with `curl https://clickhouse.com/ | sh` only when /usr/bin/clickhouse does not exist, so after --system the pin runs the pinned version. Use --system only on a machine of the benchmark, because it changes the system.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/../lib.sh"
+
+system=no
+case ${1:-} in
+"") ;;
+--system) system=yes ;;
+*) die "usage: clickhouse.sh [--system]" ;;
+esac
+version=$(pin clickhouse version)
+url=$(pin clickhouse url)
+sha=$(pin clickhouse sha256)
+bin=$RUPG_BENCH_OPT/bin
+mkdir -p "$bin" "$RUPG_BENCH_OPT/download"
+
+tgz=$RUPG_BENCH_OPT/download/clickhouse-common-static-$version-amd64.tgz
+fetch "$url" "$sha" "$tgz"
+tmp=$(mktemp -d "$RUPG_BENCH_OPT/download/clickhouse.XXXXXX")
+trap 'rm -rf "$tmp"' EXIT
+tar -xzf "$tgz" -C "$tmp" "clickhouse-common-static-$version/usr/bin/clickhouse"
+install -m 755 "$tmp/clickhouse-common-static-$version/usr/bin/clickhouse" "$bin/clickhouse.part"
+mv "$bin/clickhouse.part" "$bin/clickhouse"
+got=$("$bin/clickhouse" local --version)
+case $got in
+*" $version "* | *" $version") log "installed $bin/clickhouse: $got" ;;
+*) die "$bin/clickhouse reports $got, pins.toml wants $version" ;;
+esac
+
+if [ "$system" = yes ]; then
+    need_root
+    if [ -x /usr/bin/clickhouse ]; then
+        have=$(/usr/bin/clickhouse local --version)
+        case $have in
+        *" $version "* | *" $version") log "/usr/bin/clickhouse is already $version" ;;
+        *) die "/usr/bin/clickhouse reports $have, pins.toml wants $version. Remove it first." ;;
+        esac
+    else
+        log "clickhouse install --noninteractive"
+        (cd "$tmp" && "$bin/clickhouse" install --noninteractive >"$RUPG_BENCH_OPT/download/clickhouse-install.log" 2>&1) ||
+            die "clickhouse install failed, see $RUPG_BENCH_OPT/download/clickhouse-install.log"
+        log "installed /usr/bin/clickhouse: $(/usr/bin/clickhouse local --version)"
+    fi
+fi

machines/install/harness.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Build rupg-bench with the Rust release of pins.toml and copy the binary to $RUPG_BENCH_OPT/bin/rupg-bench.
+#
+# The script installs rustup when the machine has none, then the toolchain of rust-toolchain.toml. rustup-init is checked against the sha256 file that static.rust-lang.org publishes next to it. That checks the download, not a pin, and the toolchain itself is pinned. The build uses $JOBS jobs and the lock file of the repository. The script prints the path of the binary.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/../lib.sh"
+
+version=$(pin rust version)
+bin=$RUPG_BENCH_OPT/bin
+mkdir -p "$bin" "$RUPG_BENCH_OPT/download"
+if [ -f "$HOME/.cargo/env" ]; then
+    # shellcheck source=/dev/null
+    . "$HOME/.cargo/env"
+fi
+if ! command -v rustup >/dev/null; then
+    apt_install build-essential curl
+    url=https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init
+    init=$RUPG_BENCH_OPT/download/rustup-init
+    rm -f "$init"
+    fetch "$url" "$(curl -fsSL --retry 3 "$url.sha256" | cut -d' ' -f1)" "$init"
+    chmod 755 "$init"
+    "$init" -y -q --profile minimal --default-toolchain "$version" --no-modify-path >/dev/null
+    # shellcheck source=/dev/null
+    . "$HOME/.cargo/env"
+fi
+rustup toolchain install "$version" --profile minimal >/dev/null 2>&1
+cd "$RUPG_BENCH_ROOT"
+log "build rupg-bench with Rust $version and $JOBS jobs"
+CARGO_BUILD_JOBS=$JOBS cargo "+$version" build --release --locked --quiet
+install -m 755 target/release/rupg-bench "$bin/rupg-bench.part"
+mv "$bin/rupg-bench.part" "$bin/rupg-bench"
+log "installed $bin/rupg-bench: $("$bin/rupg-bench" --version)"
+printf '%s\n' "$bin/rupg-bench"

machines/install/sqlite.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Install the sqlite3 program of pins.toml into $RUPG_BENCH_OPT/bin.
+#
+# sqlite.org publishes a SHA3-256 hash for each download, not a SHA-256 hash, so the script checks that hash. The install script of the ClickBench pin installs sqlite3 from apt only when no sqlite3 is on the PATH, so put $RUPG_BENCH_OPT/bin first on the PATH and the pin runs the pinned version.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/../lib.sh"
+
+version=$(pin sqlite version)
+url=$(pin sqlite url)
+sha=$(pin sqlite sha3_256)
+bin=$RUPG_BENCH_OPT/bin
+mkdir -p "$bin" "$RUPG_BENCH_OPT/download"
+
+command -v unzip >/dev/null || apt_install unzip
+zip=$RUPG_BENCH_OPT/download/$(basename "$url")
+fetch "$url" "$sha" "$zip" sha3
+tmp=$(mktemp -d "$RUPG_BENCH_OPT/download/sqlite.XXXXXX")
+trap 'rm -rf "$tmp"' EXIT
+unzip -q -o -j "$zip" sqlite3 -d "$tmp"
+install -m 755 "$tmp/sqlite3" "$bin/sqlite3.part"
+mv "$bin/sqlite3.part" "$bin/sqlite3"
+got=$("$bin/sqlite3" --version)
+case $got in
+"$version "*) log "installed $bin/sqlite3: $got" ;;
+*) die "$bin/sqlite3 reports $got, pins.toml wants $version" ;;
+esac

machines/install/umbra.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Pull the Umbra image of pins.toml and check its digest.
+#
+# The ClickBench pin pulls the image by its tag, and a tag can move. So the script pulls the tag and stops when the digest is not the digest of pins.toml. It also writes the digest to $RUPG_BENCH_OPT/umbra.digest for the result. The license of Umbra is not verified (pins.toml), so a result is published only after the license is checked.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/../lib.sh"
+need_root
+
+image=$(pin umbra image)
+want=$(pin umbra digest)
+command -v docker >/dev/null || apt_install docker.io
+log "docker pull $image"
+docker pull -q "$image" >/dev/null
+digest=$(docker image inspect --format '{{index .RepoDigests 0}}' "$image")
+[ "${digest#*@}" = "$want" ] || die "$image has the digest ${digest#*@}, pins.toml wants $want. The tag moved."
+mkdir -p "$RUPG_BENCH_OPT"
+printf '%s\n' "$digest" >"$RUPG_BENCH_OPT/umbra.digest"
+log "pulled $image as $digest"
+log "warning: the license of Umbra is not verified, so do not publish an Umbra result until it is"

machines/instance.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+# Start a machine of spec/20 section 20.2 on AWS, run one suite on it with run-suite.sh, copy the reports back and stop the machine.
+#
+# Usage: instance.sh NAME SUITE [KEY=VALUE...]. NAME is 4xl, metal, tpch or oltp. SUITE and the KEY=VALUE settings go to machines/run-suite.sh on the machine, for example SMOKE=1 or SCALES="1 10".
+#
+#   4xl    c6a.4xlarge, 16 vCPU, 32 GiB, 500 GB gp2, as the ClickBench pin
+#   metal  c6a.metal, 192 vCPU, 384 GiB, 500 GB gp2
+#   tpch   c6a.4xlarge for SF1 and SF10. SF100 needs at least 64 vCPU, 256 GiB and local NVMe, so set INSTANCE_TYPE.
+#   oltp   not chosen yet (spec/20 section 20.2), so set INSTANCE_TYPE, or HOST for a machine that is not on AWS.
+#
+# The AWS settings come from the environment: AWS_REGION, KEY_NAME (the key pair), SSH_KEY (its private key file), SECURITY_GROUP (it must let ssh in), and SUBNET when the account has no default subnet. AMI is Ubuntu 24.04 by default. VOLUME_SIZE and VOLUME_TYPE change the root volume.
+#
+# HOST=user@host runs on a machine that exists, with no AWS call, and does not stop it. MACHINE sets the machine name of the reports (NAME by default). REMOTE_DIR is the copy of the repository on the machine (rupg-bench in the home directory). SSH_OPTS adds ssh options. DRY_RUN=1 prints the AWS command and stops.
+#
+# The machine is terminated when the script exits for any reason, after the reports and the log are copied back. The log goes to runs/tmp/.
+set -euo pipefail
+# shellcheck source=machines/lib.sh
+. "$(dirname "$0")/lib.sh"
+
+[ $# -ge 2 ] || die "usage: instance.sh NAME SUITE [KEY=VALUE...]"
+name=$1
+suite=$2
+shift 2
+settings=()
+for kv in "$@"; do
+    [[ $kv =~ ^[A-Z_][A-Z0-9_]*= ]] || die "the setting $kv is not KEY=VALUE"
+    settings+=("$kv")
+done
+
+volume_type=${VOLUME_TYPE:-gp2}
+volume_size=${VOLUME_SIZE:-500}
+case $name in
+4xl) type=c6a.4xlarge ;;
+metal) type=c6a.metal ;;
+tpch) type=${INSTANCE_TYPE:-c6a.4xlarge} ;;
+oltp)
+    [ -n "${INSTANCE_TYPE:-}${HOST:-}" ] || die "the oltp machine is not chosen yet: set INSTANCE_TYPE or HOST"
+    type=${INSTANCE_TYPE:-}
+    ;;
+*) die "unknown machine $name: use 4xl, metal, tpch or oltp" ;;
+esac
+machine=${MACHINE:-$name}
+remote_dir=${REMOTE_DIR:-rupg-bench}
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+mkdir -p "$RUPG_BENCH_ROOT/runs/tmp"
+logfile=$RUPG_BENCH_ROOT/runs/tmp/$stamp-$machine-$suite.log
+# shellcheck disable=SC2206
+ssh_opts=(-o BatchMode=yes -o ConnectTimeout=10 ${SSH_OPTS:-})
+instance=""
+
+launch() {
+    local ami subnet=() out
+    for v in AWS_REGION KEY_NAME SSH_KEY SECURITY_GROUP; do
+        [ -n "${!v:-}" ] || die "set $v, or set HOST to use a machine that exists"
+    done
+    ami=${AMI:-resolve:ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id}
+    [ -n "${SUBNET:-}" ] && subnet=(--subnet-id "$SUBNET")
+    local cmd=(aws ec2 run-instances --region "$AWS_REGION" --image-id "$ami" --instance-type "$type"
+        --key-name "$KEY_NAME" --security-group-ids "$SECURITY_GROUP" ${subnet[@]+"${subnet[@]}"}
+        --block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":$volume_size,\"VolumeType\":\"$volume_type\",\"DeleteOnTermination\":true}}]"
+        --instance-initiated-shutdown-behavior terminate
+        --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=rupg-bench-$machine-$suite}]"
+        --query 'Instances[0].InstanceId' --output text)
+    if [ "${DRY_RUN:-0}" = 1 ]; then
+        printf '%q ' "${cmd[@]}"
+        printf '\n'
+        exit 0
+    fi
+    out=$("${cmd[@]}")
+    instance=$out
+    log "started $instance ($type) for $machine"
+    aws ec2 wait instance-running --region "$AWS_REGION" --instance-ids "$instance"
+    local ip
+    ip=$(aws ec2 describe-instances --region "$AWS_REGION" --instance-ids "$instance" \
+        --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
+    HOST=ubuntu@$ip
+    ssh_opts+=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new)
+}
+
+finish() {
+    local status=$?
+    trap - EXIT
+    if [ -n "${HOST:-}" ]; then
+        log "copy the reports back from $HOST"
+        rsync -az -e "ssh ${ssh_opts[*]}" "$HOST:$remote_dir/reports/" "$RUPG_BENCH_ROOT/reports/" ||
+            log "warning: the copy of the reports failed"
+    fi
+    if [ -n "$instance" ]; then
+        log "terminate $instance"
+        aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids "$instance" >/dev/null
+        aws ec2 wait instance-terminated --region "$AWS_REGION" --instance-ids "$instance" ||
+            log "warning: $instance did not reach terminated, check the console"
+    fi
+    log "the log is $logfile"
+    exit "$status"
+}
+
+on_host() {
+    # shellcheck disable=SC2029
+    ssh "${ssh_opts[@]}" "$HOST" "$@"
+}
+
+if [ -z "${HOST:-}" ]; then
+    launch
+fi
+trap finish EXIT
+log "wait for ssh on $HOST"
+for i in $(seq 60); do
+    on_host true 2>/dev/null && break
+    [ "$i" -lt 60 ] || die "no ssh on $HOST after 5 minutes"
+    sleep 5
+done
+{
+    printf 'machine %s, instance %s, type %s, suite %s, started %s\n' "$machine" "${instance:-none}" "${type:-none}" "$suite" "$stamp"
+    on_host 'uname -a; lscpu | grep -E "^(Model name|CPU\(s\)|Thread|NUMA node\(s\))"; free -g | head -2; lsblk -d -o NAME,SIZE,ROTA,MODEL'
+} 2>&1 | tee "$logfile"
+log "copy the repository to $HOST:$remote_dir"
+on_host "mkdir -p $(printf '%q' "$remote_dir")"
+rsync -az --delete --exclude /target --exclude /reports --exclude /runs -e "ssh ${ssh_opts[*]}" \
+    "$RUPG_BENCH_ROOT/" "$HOST:$remote_dir/"
+remote="cd $(printf '%q' "$remote_dir") && env"
+for kv in ${settings[@]+"${settings[@]}"}; do
+    remote+=" $(printf '%q' "$kv")"
+done
+remote+=" machines/run-suite.sh $(printf '%q' "$suite") $(printf '%q' "$machine")"
+log "run $suite on $HOST"
+on_host "$remote" 2>&1 | tee -a "$logfile"