From 13c225f10a1c3e9223c4362d603adf9e35e6ec7b Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:44:04 +0700 Subject: [PATCH 1/3] Add the instruction count runner --- README.md | 9 +- ratchet.toml | 11 ++ src/instructions.rs | 310 ++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 140 ++++++++++++++++++++ 4 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 ratchet.toml create mode 100644 src/instructions.rs diff --git a/README.md b/README.md index e1753c6..16dbea0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The procedure is [`spec/20-benchmarks.md`](https://github.com/tamnd/rupg/blob/ma ## Status -Early. The crate builds and CI is green, but no suite runs yet. `rupg-bench gates` prints the twelve gates of spec/02 section 2.10, `rupg-bench pins` prints the pinned versions, `rupg-bench measure` runs the cgroup v2 runner, `rupg-bench load` runs the load cases, `rupg-bench answers` checks answer sets, `rupg-bench report` writes the report files, and `rupg-bench pgbench` runs the pgbench smoke run with its check. The first milestone, M0, builds the harness and measures every baseline that a later gate uses. Every command below is the planned interface from the spec. +Early. The crate builds and CI is green, but no suite runs yet. `rupg-bench gates` prints the twelve gates of spec/02 section 2.10, `rupg-bench pins` prints the pinned versions, `rupg-bench measure` runs the cgroup v2 runner, `rupg-bench load` runs the load cases, `rupg-bench answers` checks answer sets, `rupg-bench report` writes the report files, `rupg-bench pgbench` runs the pgbench smoke run with its check, and `rupg-bench instructions` counts the instructions of each query. The first milestone, M0, builds the harness and measures every baseline that a later gate uses. Every command below is the planned interface from the spec. ## The gates @@ -123,6 +123,13 @@ rupg-bench report --result idle.json --suite idle-base --machine server3 --smoke rupg-bench pgbench --unit postgresql@19-main --scale 1 --clients 4 --time 30 --smoke --report reports --machine server3 ``` +`rupg-bench instructions` is the instruction count runner of spec/21 section 21.14. It runs each query of a set under `perf stat -e instructions`. For DuckDB, perf counts the `duckdb` process with one thread. For a server, perf counts the cgroup of the server on all CPUs while `psql` sends the query, so the count is the work of the server and not the work of the client. Each query runs three times and the minimum counts. The count of `SELECT 1` in the same way is subtracted, so the net count is the work of the query. A set is a file with one query on each line, named `q0` to `q42` for ClickBench, or a directory of `.sql` files. `--ratchet ratchet.toml` fails when one query rises by more than 3 percent or the total by more than 1 percent. `--save` writes the table for the ratchet. The ratchet is filled only on the dedicated runner, so [`ratchet.toml`](ratchet.toml) has no table yet. + +```sh +rupg-bench instructions --set clickbench --queries queries.sql --duckdb hits.db --duckdb-bin duckdb +rupg-bench instructions --set clickbench --queries queries.sql --unit postgresql@19-main --ratchet ratchet.toml +``` + ## The reporting rules 1. A claim has five parts: one named machine, one metric, a baseline that we measured, the same transport on both sides, and the ratio. A claim without all five is not made. diff --git a/ratchet.toml b/ratchet.toml new file mode 100644 index 0000000..90e5ae1 --- /dev/null +++ b/ratchet.toml @@ -0,0 +1,11 @@ +# The instruction count ratchet of spec/21 section 21.14. +# +# The dedicated Linux x86-64 runner writes one table for each set with +# `rupg-bench instructions --set NAME --save FILE`, for example +# [instructions.clickbench]. Each key is a query and its value is the net +# instruction count of that query. `total` is the sum of the net counts. +# `rupg-bench instructions --ratchet ratchet.toml` fails when one query rises by +# more than 3 percent or the total by more than 1 percent. +# +# The file has no table yet. Counts from a shared machine such as server3 are +# smoke runs and do not belong here. diff --git a/src/instructions.rs b/src/instructions.rs new file mode 100644 index 0000000..84cefe6 --- /dev/null +++ b/src/instructions.rs @@ -0,0 +1,310 @@ +//! The instruction count runner of spec/21 section 21.14. +//! +//! The runner counts the instructions retired for each query of a fixed set with `perf stat -e instructions`. For an engine in the process of its client, such as DuckDB or rupg in embedded mode, perf counts the client process. For a server, perf counts the server cgroup on all CPUs (`-a -G`) while the client sends the query, so the count is the work of the server and not of the client. Each query runs `repeat` times and the minimum counts. The count of `SELECT 1` in the same way is the base, and the net count of a query is its count minus the base. +//! +//! `ratchet.toml` holds the net count of each query from the dedicated runner. A change fails when one query rises by more than 3 percent or the total by more than 1 percent. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::json::Json; +use crate::toml::{Doc, Value}; + +/// A query rises by at most 3 percent. +pub(crate) const QUERY_BUDGET: f64 = 0.03; +/// The total rises by at most 1 percent. +pub(crate) const TOTAL_BUDGET: f64 = 0.01; + +/// One query of the set. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Query { + pub(crate) name: String, + pub(crate) sql: String, +} + +/// Reads a set. A file has one query on each line, named `q0`, `q1` and so on, as in `queries.sql` of ClickBench. A directory has one query in each `.sql` file, as `qgen` writes them for TPC-H. +pub(crate) fn read_set(path: &Path) -> Result, String> { + let err = |e: std::io::Error| format!("{}: {e}", path.display()); + if path.is_dir() { + let mut files: Vec = std::fs::read_dir(path) + .map_err(err)? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|x| x == "sql")) + .collect(); + files.sort_by_key(|p| natural_key(&p.file_stem().unwrap_or_default().to_string_lossy())); + files + .iter() + .map(|p| { + let sql = + std::fs::read_to_string(p).map_err(|e| format!("{}: {e}", p.display()))?; + Ok(Query { + name: p.file_stem().unwrap_or_default().to_string_lossy().into_owned(), + sql, + }) + }) + .collect() + } else { + let text = std::fs::read_to_string(path).map_err(err)?; + Ok(text + .lines() + .filter(|l| !l.trim().is_empty()) + .enumerate() + .map(|(i, l)| Query { name: format!("q{i}"), sql: l.trim().to_owned() }) + .collect()) + } +} + +/// `q2` sorts before `q10`. +fn natural_key(name: &str) -> (String, u64) { + let digits = name.trim_start_matches(|c: char| !c.is_ascii_digit()); + let prefix = &name[..name.len() - digits.len()]; + (prefix.to_owned(), digits.parse().unwrap_or(u64::MAX)) +} + +/// How a query reaches the engine. +#[derive(Clone, Debug)] +pub(crate) enum Engine { + /// The client program with the engine in its process: argv before the query. + InProcess(Vec), + /// A server in a cgroup (the path under `/sys/fs/cgroup`) and its client: argv before the query. + Server { cgroup: String, client: Vec }, +} + +impl Engine { + /// The `perf stat` command for one query. perf writes its counts to `out`. + fn command(&self, sql: &str, out: &Path) -> Command { + let mut c = Command::new("perf"); + c.args(["stat", "-e", "instructions", "-x", ","]).arg("-o").arg(out); + let argv = match self { + Engine::InProcess(argv) => argv, + Engine::Server { cgroup, client } => { + c.args(["-a", "-G", cgroup]); + client + } + }; + c.arg("--").args(argv).arg(sql); + c.stdout(std::process::Stdio::null()); + c + } + + pub(crate) fn to_json(&self) -> Json { + match self { + Engine::InProcess(argv) => { + Json::obj().with("mode", "in process").with("client", argv.clone()) + } + Engine::Server { cgroup, client } => Json::obj() + .with("mode", "server cgroup") + .with("cgroup", cgroup.as_str()) + .with("client", client.clone()), + } + } +} + +/// The instruction count in the CSV output of `perf stat -x ,`. With `-G` perf prints one line for each cgroup, and the counts are added. +pub(crate) fn parse_perf(text: &str) -> Result { + let mut total = None; + for line in text.lines() { + let fields: Vec<&str> = line.split(',').collect(); + if fields.len() < 3 || !fields[2].starts_with("instructions") { + continue; + } + let n: u64 = + fields[0].parse().map_err(|_| format!("perf did not count instructions: {line:?}"))?; + *total.get_or_insert(0) += n; + } + total.ok_or_else(|| format!("perf printed no instruction count: {:?}", text.trim())) +} + +/// Runs one query once and returns its count. +fn count_once(engine: &Engine, sql: &str) -> Result { + let out = std::env::temp_dir().join(format!("rupg-bench-perf-{}.csv", std::process::id())); + let mut cmd = engine.command(sql, &out); + let result = cmd.output().map_err(|e| format!("perf: {e}")); + let text = std::fs::read_to_string(&out).unwrap_or_default(); + let _ = std::fs::remove_file(&out); + let result = result?; + if !result.status.success() { + return Err(format!( + "the query failed: {}\n{}", + result.status, + String::from_utf8_lossy(&result.stderr).trim() + )); + } + parse_perf(&text) +} + +/// The counts of one query. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Count { + pub(crate) name: String, + pub(crate) runs: Vec, + /// The minimum of the runs minus the base, at least 0. + pub(crate) net: u64, +} + +/// The counts of a set. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Counts { + pub(crate) base: Vec, + pub(crate) queries: Vec, +} + +impl Counts { + pub(crate) fn total(&self) -> u64 { + self.queries.iter().map(|q| q.net).sum() + } + + pub(crate) fn to_json(&self) -> Json { + let queries: Vec = self + .queries + .iter() + .map(|q| { + Json::obj() + .with("query", q.name.as_str()) + .with("net", q.net) + .with("runs", q.runs.clone()) + }) + .collect(); + Json::obj() + .with("base_runs", self.base.clone()) + .with("base", self.base.iter().min().copied().unwrap_or(0)) + .with("total_net", self.total()) + .with("queries", queries) + } + + /// A table for `ratchet.toml`. + pub(crate) fn ratchet_table(&self, set: &str, machine: &str, date: &str) -> String { + let mut out = format!( + "[instructions.{set}]\nmachine = \"{machine}\"\ndate = \"{date}\"\ntotal = {}\n", + self.total() + ); + for q in &self.queries { + out.push_str(&format!("{} = {}\n", q.name, q.net)); + } + out + } +} + +/// Counts each query of the set. `progress` gets one line for each query. +pub(crate) fn count( + engine: &Engine, + set: &[Query], + repeat: usize, + mut progress: impl FnMut(&str), +) -> Result { + let runs = |sql: &str| -> Result, String> { + (0..repeat).map(|_| count_once(engine, sql)).collect() + }; + let base = runs("SELECT 1")?; + let floor = base.iter().min().copied().unwrap_or(0); + let mut queries = Vec::new(); + for q in set { + let r = runs(&q.sql).map_err(|e| format!("{}: {e}", q.name))?; + let net = r.iter().min().copied().unwrap_or(0).saturating_sub(floor); + progress(&format!("{:<8} {net:>16} net instructions, runs {r:?}", q.name)); + queries.push(Count { name: q.name.clone(), runs: r, net }); + } + Ok(Counts { base, queries }) +} + +/// Compares the counts with the table `[instructions.]` of a ratchet file. It returns one line for each query and for the total that is over its budget. A query that the ratchet does not have is not compared. +pub(crate) fn compare(counts: &Counts, ratchet: &Doc, set: &str) -> Result, String> { + let name = format!("instructions.{set}"); + let table = ratchet.table(&name).ok_or(format!("the ratchet has no table [{name}]"))?; + let int = |key: &str| match table.get(key) { + Some(Value::Int(n)) => u64::try_from(*n).ok(), + _ => None, + }; + let rise = + |new: u64, old: u64| if old == 0 { 0.0 } else { (new as f64 - old as f64) / old as f64 }; + let mut over = Vec::new(); + for q in &counts.queries { + if let Some(old) = int(&q.name) { + let r = rise(q.net, old); + if r > QUERY_BUDGET { + over.push(format!( + "{}: {} against {old}, up {:.2} percent, budget 3", + q.name, + q.net, + r * 100.0 + )); + } + } + } + if let Some(old) = int("total") { + let r = rise(counts.total(), old); + if r > TOTAL_BUDGET { + over.push(format!( + "total: {} against {old}, up {:.2} percent, budget 1", + counts.total(), + r * 100.0 + )); + } + } + Ok(over) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn perf_csv() { + assert_eq!(parse_perf("52133065,,instructions:u,210013041,100.00,,\n"), Ok(52133065)); + let g = "# started on Wed Oct 7\n\n15155433,,instructions,system.slice/a.service,22704897,100.00,,\n"; + assert_eq!(parse_perf(g), Ok(15155433)); + assert!( + parse_perf(",,instructions,,0,100.00,,\n") + .unwrap_err() + .contains("did not count") + ); + assert!(parse_perf("").is_err()); + } + + #[test] + fn sets() { + let dir = std::env::temp_dir().join(format!("rupg-bench-set-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for (name, sql) in [("q10", "SELECT 10;"), ("q2", "SELECT 2;"), ("notes", "x")] { + let ext = if name == "notes" { "txt" } else { "sql" }; + std::fs::write(dir.join(format!("{name}.{ext}")), sql).unwrap(); + } + let set = read_set(&dir).unwrap(); + assert_eq!(set.iter().map(|q| q.name.as_str()).collect::>(), ["q2", "q10"]); + let file = dir.join("queries.sql"); + std::fs::write(&file, "SELECT 1;\n\nSELECT 2;\n").unwrap(); + let set = read_set(&file).unwrap(); + assert_eq!(set[1], Query { name: "q1".to_owned(), sql: "SELECT 2;".to_owned() }); + std::fs::remove_dir_all(&dir).unwrap(); + } + + fn counts(nets: &[u64]) -> Counts { + Counts { + base: vec![10], + queries: nets + .iter() + .enumerate() + .map(|(i, n)| Count { name: format!("q{i}"), runs: vec![n + 10], net: *n }) + .collect(), + } + } + + #[test] + fn budgets() { + let old = counts(&[1000, 2000, 3000]); + let text = old.ratchet_table("cb", "runner", "2026-10-07"); + assert!(text.starts_with("[instructions.cb]\nmachine = \"runner\"\ndate = \"2026-10-07\"\ntotal = 6000\nq0 = 1000\n")); + let doc = Doc::parse(&text).unwrap(); + // 2.9 percent on one query and 0.5 percent in total pass. + assert!(compare(&counts(&[1029, 2000, 3000]), &doc, "cb").unwrap().is_empty()); + // 3.1 percent on one query fails. + let over = compare(&counts(&[1031, 2000, 3000]), &doc, "cb").unwrap(); + assert_eq!(over.len(), 1); + assert!(over[0].starts_with("q0: 1031 against 1000, up 3.10 percent")); + // 2 percent on each query fails the total only. + let over = compare(&counts(&[1020, 2040, 3060]), &doc, "cb").unwrap(); + assert_eq!(over.len(), 1); + assert!(over[0].starts_with("total: 6120 against 6000")); + assert!(compare(&old, &doc, "tpch").is_err()); + } +} diff --git a/src/main.rs b/src/main.rs index 7f8eae7..bf5437f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod answers; mod args; mod cgroup; mod gates; +mod instructions; mod json; mod load; mod pg; @@ -53,6 +54,15 @@ commands: --conn takes libpq words: host, port, user, dbname, password (user=bench dbname=bench by default). --report writes the report files under DIR (see report). --smoke marks the result as a smoke run, which is not a baseline. + instructions --queries PATH --set NAME (--duckdb DB [--duckdb-bin B] [--threads N] | --unit UNIT | --attach PATH) + [--bin DIR] [--conn WORDS] [--repeat N] [--ratchet FILE] [--save FILE] [--smoke] [--json] [--report DIR [--machine M]] + count the instructions retired for each query with perf stat (spec/21 section 21.14). + PATH is a file with one query on each line or a directory of .sql files. + --duckdb counts the duckdb process. --unit and --attach count the server cgroup on all + CPUs while psql from --bin sends the query. Each query runs --repeat times (3) and the + minimum counts, minus the count of SELECT 1. --ratchet compares with [instructions.NAME] + of a ratchet file and fails over the budgets (3 percent a query, 1 percent in total). + --save writes that table for this run. --version print the version"; fn main() -> ExitCode { @@ -96,6 +106,7 @@ fn run(argv: &[String]) -> Result<(), String> { "load" => load_command(a)?, "report" => report_command(a)?, "answers" => answers_command(a)?, + "instructions" => instructions_command(a)?, "pgbench" => pgbench_command(a)?, "--version" | "-V" => println!("rupg-bench {}", env!("CARGO_PKG_VERSION")), "--help" | "-h" | "help" => println!("{USAGE}"), @@ -473,3 +484,132 @@ fn pgbench_run( Err(format!("wrong answer, the run is not a number: {}", failures.join("; "))) } } + +/// The `instructions` command. +fn instructions_command(mut a: args::Args) -> Result<(), String> { + let queries = a.value("queries").ok_or("usage: instructions needs --queries PATH")?; + let set_name = a.value("set").ok_or("usage: instructions needs --set NAME")?; + if !set_name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') { + return Err(format!( + "usage: the set name {set_name:?} may have only letters, digits, _ and -" + )); + } + let repeat: usize = number(&mut a, "repeat", 3)?; + if repeat == 0 { + return Err("usage: --repeat must be at least 1".to_owned()); + } + let ratchet = a.value("ratchet"); + let save = a.value("save"); + let smoke = a.flag("smoke"); + let json = a.flag("json"); + let report_dir = a.value("report"); + let machine = a.value("machine"); + let bin = a.value("bin").unwrap_or_else(|| "/usr/lib/postgresql/19/bin".to_owned()); + let conn = pg::Config::parse( + &a.value("conn").unwrap_or_else(|| "user=bench dbname=bench".to_owned()), + )?; + let engine = if let Some(db) = a.value("duckdb") { + let duckdb = a.value("duckdb-bin").unwrap_or_else(|| "duckdb".to_owned()); + let threads: u32 = number(&mut a, "threads", 1)?; + let argv = [ + duckdb, + "-readonly".to_owned(), + db, + "-cmd".to_owned(), + format!("SET threads = {threads}"), + "-c".to_owned(), + ]; + instructions::Engine::InProcess(argv.to_vec()) + } else { + if a.value_is_set("name") { + return Err("usage: instructions counts a running server, so give --unit or --attach" + .to_owned()); + } + let cg = target(&mut a)?; + let cgroup = cg + .path + .strip_prefix(cgroup::CGROUP_ROOT) + .map_err(|_| format!("{} is not under {}", cg.path.display(), cgroup::CGROUP_ROOT))? + .display() + .to_string(); + let psql = Path::new(&bin).join("psql").display().to_string(); + let mut client = vec![ + psql, + "-X".to_owned(), + "-q".to_owned(), + "-v".to_owned(), + "ON_ERROR_STOP=1".to_owned(), + ]; + client.extend(["-h".to_owned(), conn.host.clone(), "-p".to_owned(), conn.port.to_string()]); + client.extend(["-U".to_owned(), conn.user.clone(), "-d".to_owned(), conn.dbname.clone()]); + client.extend(["-o".to_owned(), "/dev/null".to_owned(), "-c".to_owned()]); + instructions::Engine::Server { cgroup, client } + }; + a.finish()?; + let set = instructions::read_set(Path::new(&queries))?; + if set.is_empty() { + return Err(format!("{queries} has no query")); + } + let meta = report::Meta::now(&format!("instructions-{set_name}"), machine, None, smoke); + let mut result = meta.to_json(); + if smoke { + result = + result.with("note", "smoke run: it shows that the runner works, it is not the ratchet"); + } + result = result + .with("set", set_name.as_str()) + .with("queries_from", queries.as_str()) + .with("repeat", repeat) + .with( + "kernel", + std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap_or_default().trim(), + ) + .with("engine", engine.to_json()); + let counts = instructions::count(&engine, &set, repeat, |line| { + if !json { + println!("{line}"); + } + })?; + result = result.with("counts", counts.to_json()); + let mut over = Vec::new(); + if let Some(file) = &ratchet { + let text = std::fs::read_to_string(file).map_err(|e| format!("{file}: {e}"))?; + let doc = toml::Doc::parse(&text).map_err(|e| format!("{file}: {e}"))?; + over = instructions::compare(&counts, &doc, &set_name)?; + result = result.with( + "ratchet", + Json::obj().with("file", file.as_str()).with("over_budget", over.clone()), + ); + } + if let Some(file) = &save { + let text = format!( + "# Instruction counts of spec/21 section 21.14, written by rupg-bench instructions --save.\n{}", + counts.ratchet_table(&set_name, &meta.machine, &meta.date) + ); + std::fs::write(file, text).map_err(|e| format!("{file}: {e}"))?; + } + if let Some(dir) = &report_dir { + let (md, js) = report::write(Path::new(dir), &result)?; + if !json { + println!("report {}\n {}", md.display(), js.display()); + } + } + if json { + print!("{}", result.pretty()); + } else { + println!("base {:?} for SELECT 1", counts.base); + println!( + "total {} net instructions in {} queries", + counts.total(), + counts.queries.len() + ); + for line in &over { + println!("OVER {line}"); + } + } + if over.is_empty() { + Ok(()) + } else { + Err(format!("{} counts are over their budget", over.len())) + } +} From 6005c2b16e5fcb2d4678663e4cdc4bb3d686ecd3 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:57:29 +0700 Subject: [PATCH 2/3] Name the engine in the report --- README.md | 4 ++-- src/instructions.rs | 17 +++++++++++------ src/main.rs | 21 ++++++++++++++++++--- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 16dbea0..767a261 100644 --- a/README.md +++ b/README.md @@ -123,11 +123,11 @@ rupg-bench report --result idle.json --suite idle-base --machine server3 --smoke rupg-bench pgbench --unit postgresql@19-main --scale 1 --clients 4 --time 30 --smoke --report reports --machine server3 ``` -`rupg-bench instructions` is the instruction count runner of spec/21 section 21.14. It runs each query of a set under `perf stat -e instructions`. For DuckDB, perf counts the `duckdb` process with one thread. For a server, perf counts the cgroup of the server on all CPUs while `psql` sends the query, so the count is the work of the server and not the work of the client. Each query runs three times and the minimum counts. The count of `SELECT 1` in the same way is subtracted, so the net count is the work of the query. A set is a file with one query on each line, named `q0` to `q42` for ClickBench, or a directory of `.sql` files. `--ratchet ratchet.toml` fails when one query rises by more than 3 percent or the total by more than 1 percent. `--save` writes the table for the ratchet. The ratchet is filled only on the dedicated runner, so [`ratchet.toml`](ratchet.toml) has no table yet. +`rupg-bench instructions` is the instruction count runner of spec/21 section 21.14. It runs each query of a set under `perf stat -e instructions`. For DuckDB, perf counts the `duckdb` process with one thread. For a server, perf counts the cgroup of the server on all CPUs while `psql` sends the query, so the count is the work of the server and not the work of the client. Each query runs three times and the minimum counts. The count of `SELECT 1` in the same way is subtracted, so the net count is the work of the query. A set is a file with one query on each line, named `q0` to `q42` for ClickBench, or a directory of `.sql` files. A server needs `--engine NAME`, and the report name has the set and the engine. `--ratchet ratchet.toml` fails when one query rises by more than 3 percent or the total by more than 1 percent. `--save` writes the table for the ratchet. The ratchet is filled only on the dedicated runner, so [`ratchet.toml`](ratchet.toml) has no table yet. ```sh rupg-bench instructions --set clickbench --queries queries.sql --duckdb hits.db --duckdb-bin duckdb -rupg-bench instructions --set clickbench --queries queries.sql --unit postgresql@19-main --ratchet ratchet.toml +rupg-bench instructions --set clickbench --queries queries.sql --engine postgresql --unit postgresql@19-main --ratchet ratchet.toml ``` ## The reporting rules diff --git a/src/instructions.rs b/src/instructions.rs index 84cefe6..8db55e9 100644 --- a/src/instructions.rs +++ b/src/instructions.rs @@ -154,8 +154,17 @@ impl Counts { self.queries.iter().map(|q| q.net).sum() } + /// The base and the total. The rows of the queries are in [`Counts::rows`]. pub(crate) fn to_json(&self) -> Json { - let queries: Vec = self + Json::obj() + .with("base_runs", self.base.clone()) + .with("base", self.base.iter().min().copied().unwrap_or(0)) + .with("total_net", self.total()) + } + + /// One row for each query. + pub(crate) fn rows(&self) -> Json { + let rows: Vec = self .queries .iter() .map(|q| { @@ -165,11 +174,7 @@ impl Counts { .with("runs", q.runs.clone()) }) .collect(); - Json::obj() - .with("base_runs", self.base.clone()) - .with("base", self.base.iter().min().copied().unwrap_or(0)) - .with("total_net", self.total()) - .with("queries", queries) + Json::from(rows) } /// A table for `ratchet.toml`. diff --git a/src/main.rs b/src/main.rs index bf5437f..f0120c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,7 +54,7 @@ commands: --conn takes libpq words: host, port, user, dbname, password (user=bench dbname=bench by default). --report writes the report files under DIR (see report). --smoke marks the result as a smoke run, which is not a baseline. - instructions --queries PATH --set NAME (--duckdb DB [--duckdb-bin B] [--threads N] | --unit UNIT | --attach PATH) + instructions --queries PATH --set NAME (--duckdb DB [--duckdb-bin B] [--threads N] | --engine NAME (--unit UNIT | --attach PATH)) [--bin DIR] [--conn WORDS] [--repeat N] [--ratchet FILE] [--save FILE] [--smoke] [--json] [--report DIR [--machine M]] count the instructions retired for each query with perf stat (spec/21 section 21.14). PATH is a file with one query on each line or a directory of .sql files. @@ -508,6 +508,7 @@ fn instructions_command(mut a: args::Args) -> Result<(), String> { let conn = pg::Config::parse( &a.value("conn").unwrap_or_else(|| "user=bench dbname=bench".to_owned()), )?; + let engine_name = a.value("engine"); let engine = if let Some(db) = a.value("duckdb") { let duckdb = a.value("duckdb-bin").unwrap_or_else(|| "duckdb".to_owned()); let threads: u32 = number(&mut a, "threads", 1)?; @@ -546,11 +547,24 @@ fn instructions_command(mut a: args::Args) -> Result<(), String> { instructions::Engine::Server { cgroup, client } }; a.finish()?; + let engine_name = match (&engine, engine_name) { + (_, Some(n)) => n, + (instructions::Engine::InProcess(_), None) => "duckdb".to_owned(), + (instructions::Engine::Server { .. }, None) => { + return Err("usage: give --engine NAME for a server, for example postgresql".to_owned()); + } + }; + if !engine_name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') { + return Err(format!( + "usage: the engine name {engine_name:?} may have only letters, digits, _ and -" + )); + } let set = instructions::read_set(Path::new(&queries))?; if set.is_empty() { return Err(format!("{queries} has no query")); } - let meta = report::Meta::now(&format!("instructions-{set_name}"), machine, None, smoke); + let meta = + report::Meta::now(&format!("instructions-{set_name}-{engine_name}"), machine, None, smoke); let mut result = meta.to_json(); if smoke { result = @@ -558,6 +572,7 @@ fn instructions_command(mut a: args::Args) -> Result<(), String> { } result = result .with("set", set_name.as_str()) + .with("engine_name", engine_name.as_str()) .with("queries_from", queries.as_str()) .with("repeat", repeat) .with( @@ -570,7 +585,7 @@ fn instructions_command(mut a: args::Args) -> Result<(), String> { println!("{line}"); } })?; - result = result.with("counts", counts.to_json()); + result = result.with("counts", counts.to_json()).with("queries", counts.rows()); let mut over = Vec::new(); if let Some(file) = &ratchet { let text = std::fs::read_to_string(file).map_err(|e| format!("{file}: {e}"))?; From d3389d3640552494ee5a24cb7daf8e90f72f44b7 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:47:46 +0700 Subject: [PATCH 3/3] Add the instruction count smoke reports for ClickBench --- ...-instructions-clickbench-duckdb-smoke.json | 246 +++++++++++++++++ ...r3-instructions-clickbench-duckdb-smoke.md | 81 ++++++ ...tructions-clickbench-postgresql-smoke.json | 257 ++++++++++++++++++ ...nstructions-clickbench-postgresql-smoke.md | 82 ++++++ 4 files changed, 666 insertions(+) create mode 100644 reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.json create mode 100644 reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.md create mode 100644 reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.json create mode 100644 reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.md diff --git a/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.json b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.json new file mode 100644 index 0000000..363a6d4 --- /dev/null +++ b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.json @@ -0,0 +1,246 @@ +{ + "suite": "instructions-clickbench-duckdb", + "machine": "server3", + "date": "2026-10-07", + "commit": "6005c2b1", + "smoke": true, + "note": "smoke run: it shows that the runner works, it is not the ratchet", + "set": "clickbench", + "engine_name": "duckdb", + "queries_from": "duckdb-queries.sql", + "repeat": 3, + "kernel": "6.8.0-106-generic", + "engine": { + "mode": "in process", + "client": [ + "/root/work/rupg-bench-opt/bin/duckdb", + "-readonly", + "hits1m.db", + "-cmd", + "SET threads = 1", + "-c" + ] + }, + "counts": { + "base_runs": [80916556, 78040265, 78382784], + "base": 78040265, + "total_net": 18064983237 + }, + "queries": [ + { + "query": "q0", + "net": 2658966, + "runs": [81782230, 81926639, 80699231] + }, + { + "query": "q1", + "net": 9794033, + "runs": [87834298, 88071044, 98749864] + }, + { + "query": "q2", + "net": 21379579, + "runs": [116705717, 100057642, 99419844] + }, + { + "query": "q3", + "net": 30056892, + "runs": [109395308, 108097157, 110024982] + }, + { + "query": "q4", + "net": 96061769, + "runs": [174102034, 176684910, 174479860] + }, + { + "query": "q5", + "net": 142365074, + "runs": [229474235, 220405339, 221811745] + }, + { + "query": "q6", + "net": 6391527, + "runs": [84431792, 91324585, 138004129] + }, + { + "query": "q7", + "net": 12274205, + "runs": [90314470, 98506520, 90877270] + }, + { + "query": "q8", + "net": 141955649, + "runs": [229414983, 220329937, 219995914] + }, + { + "query": "q9", + "net": 260171263, + "runs": [338211528, 374255484, 338243517] + }, + { + "query": "q10", + "net": 79267130, + "runs": [158599180, 157307395, 158798206] + }, + { + "query": "q11", + "net": 85726959, + "runs": [165710997, 163767224, 164860287] + }, + { + "query": "q12", + "net": 104967374, + "runs": [184804364, 184240163, 183007639] + }, + { + "query": "q13", + "net": 126302995, + "runs": [206476167, 205229570, 204343260] + }, + { + "query": "q14", + "net": 116660750, + "runs": [208069401, 194701015, 195197372] + }, + { + "query": "q15", + "net": 114154184, + "runs": [193169086, 192194449, 193594877] + }, + { + "query": "q16", + "net": 238392237, + "runs": [316985871, 380977329, 316432502] + }, + { + "query": "q17", + "net": 232148198, + "runs": [324993525, 310188463, 314307544] + }, + { + "query": "q18", + "net": 490274337, + "runs": [577124723, 568314602, 588599687] + }, + { + "query": "q19", + "net": 7260510, + "runs": [108010818, 86214489, 85300775] + }, + { + "query": "q20", + "net": 474816252, + "runs": [602058081, 557769957, 552856517] + }, + { + "query": "q21", + "net": 443819841, + "runs": [524793576, 521860106, 535583770] + }, + { + "query": "q22", + "net": 311919457, + "runs": [433448618, 389959722, 411982077] + }, + { + "query": "q23", + "net": 780471101, + "runs": [910784254, 858511366, 874619848] + }, + { + "query": "q24", + "net": 124617635, + "runs": [202796815, 222973585, 202657900] + }, + { + "query": "q25", + "net": 97594025, + "runs": [175634290, 192633348, 176087162] + }, + { + "query": "q26", + "net": 131454751, + "runs": [209495016, 218807746, 210925889] + }, + { + "query": "q27", + "net": 429523741, + "runs": [507564006, 517167810, 508441273] + }, + { + "query": "q28", + "net": 8580483958, + "runs": [8675722436, 8668755425, 8658524223] + }, + { + "query": "q29", + "net": 69401430, + "runs": [148583472, 150163834, 147441695] + }, + { + "query": "q30", + "net": 136512966, + "runs": [216352389, 229079759, 214553231] + }, + { + "query": "q31", + "net": 169098914, + "runs": [256408322, 258144163, 247139179] + }, + { + "query": "q32", + "net": 552858320, + "runs": [636146474, 630898585, 635693870] + }, + { + "query": "q33", + "net": 635693402, + "runs": [768820987, 714799322, 713733667] + }, + { + "query": "q34", + "net": 697613263, + "runs": [779471590, 775653528, 782402582] + }, + { + "query": "q35", + "net": 215031035, + "runs": [307080719, 293071300, 337445189] + }, + { + "query": "q36", + "net": 429593955, + "runs": [514337619, 509244140, 507634220] + }, + { + "query": "q37", + "net": 88197302, + "runs": [180336420, 166237567, 173174879] + }, + { + "query": "q38", + "net": 187898092, + "runs": [267594687, 265938357, 269324026] + }, + { + "query": "q39", + "net": 908767319, + "runs": [1023851467, 986807584, 1001875938] + }, + { + "query": "q40", + "net": 68201989, + "runs": [154972951, 146242254, 147902136] + }, + { + "query": "q41", + "net": 62543633, + "runs": [145026872, 140583898, 148454573] + }, + { + "query": "q42", + "net": 150607225, + "runs": [231513245, 232694981, 228647490] + } + ] +} diff --git a/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.md b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.md new file mode 100644 index 0000000..9dde39a --- /dev/null +++ b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-duckdb-smoke.md @@ -0,0 +1,81 @@ +# instructions-clickbench-duckdb on server3, 2026-10-07 + +**This is a smoke run.** It shows that the driver works. It ran on a small scale or on a shared machine, so its numbers are not baselines and no claim may use them. + +| Field | Value | +|---|---| +| suite | instructions-clickbench-duckdb | +| machine | server3 | +| date | 2026-10-07 | +| commit | 6005c2b1 | +| smoke | true | +| note | smoke run: it shows that the runner works, it is not the ratchet | +| set | clickbench | +| engine_name | duckdb | +| queries_from | duckdb-queries.sql | +| repeat | 3 | +| kernel | 6.8.0-106-generic | + +## engine + +| Field | Value | +|---|---| +| mode | in process | +| client | /root/work/rupg-bench-opt/bin/duckdb, -readonly, hits1m.db, -cmd, SET threads = 1, -c | + +## counts + +| Field | Value | +|---|---| +| base_runs | 80916556, 78040265, 78382784 | +| base | 78040265 | +| total_net | 18064983237 | + +## queries + +| query | net | runs | +|---|---|---| +| q0 | 2658966 | 81782230, 81926639, 80699231 | +| q1 | 9794033 | 87834298, 88071044, 98749864 | +| q2 | 21379579 | 116705717, 100057642, 99419844 | +| q3 | 30056892 | 109395308, 108097157, 110024982 | +| q4 | 96061769 | 174102034, 176684910, 174479860 | +| q5 | 142365074 | 229474235, 220405339, 221811745 | +| q6 | 6391527 | 84431792, 91324585, 138004129 | +| q7 | 12274205 | 90314470, 98506520, 90877270 | +| q8 | 141955649 | 229414983, 220329937, 219995914 | +| q9 | 260171263 | 338211528, 374255484, 338243517 | +| q10 | 79267130 | 158599180, 157307395, 158798206 | +| q11 | 85726959 | 165710997, 163767224, 164860287 | +| q12 | 104967374 | 184804364, 184240163, 183007639 | +| q13 | 126302995 | 206476167, 205229570, 204343260 | +| q14 | 116660750 | 208069401, 194701015, 195197372 | +| q15 | 114154184 | 193169086, 192194449, 193594877 | +| q16 | 238392237 | 316985871, 380977329, 316432502 | +| q17 | 232148198 | 324993525, 310188463, 314307544 | +| q18 | 490274337 | 577124723, 568314602, 588599687 | +| q19 | 7260510 | 108010818, 86214489, 85300775 | +| q20 | 474816252 | 602058081, 557769957, 552856517 | +| q21 | 443819841 | 524793576, 521860106, 535583770 | +| q22 | 311919457 | 433448618, 389959722, 411982077 | +| q23 | 780471101 | 910784254, 858511366, 874619848 | +| q24 | 124617635 | 202796815, 222973585, 202657900 | +| q25 | 97594025 | 175634290, 192633348, 176087162 | +| q26 | 131454751 | 209495016, 218807746, 210925889 | +| q27 | 429523741 | 507564006, 517167810, 508441273 | +| q28 | 8580483958 | 8675722436, 8668755425, 8658524223 | +| q29 | 69401430 | 148583472, 150163834, 147441695 | +| q30 | 136512966 | 216352389, 229079759, 214553231 | +| q31 | 169098914 | 256408322, 258144163, 247139179 | +| q32 | 552858320 | 636146474, 630898585, 635693870 | +| q33 | 635693402 | 768820987, 714799322, 713733667 | +| q34 | 697613263 | 779471590, 775653528, 782402582 | +| q35 | 215031035 | 307080719, 293071300, 337445189 | +| q36 | 429593955 | 514337619, 509244140, 507634220 | +| q37 | 88197302 | 180336420, 166237567, 173174879 | +| q38 | 187898092 | 267594687, 265938357, 269324026 | +| q39 | 908767319 | 1023851467, 986807584, 1001875938 | +| q40 | 68201989 | 154972951, 146242254, 147902136 | +| q41 | 62543633 | 145026872, 140583898, 148454573 | +| q42 | 150607225 | 231513245, 232694981, 228647490 | + diff --git a/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.json b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.json new file mode 100644 index 0000000..e977eb4 --- /dev/null +++ b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.json @@ -0,0 +1,257 @@ +{ + "suite": "instructions-clickbench-postgresql", + "machine": "server3", + "date": "2026-10-07", + "commit": "6005c2b1", + "smoke": true, + "note": "smoke run: it shows that the runner works, it is not the ratchet", + "set": "clickbench", + "engine_name": "postgresql", + "queries_from": "pg-queries.sql", + "repeat": 3, + "kernel": "6.8.0-106-generic", + "engine": { + "mode": "server cgroup", + "cgroup": "system.slice/system-postgresql.slice/postgresql@19-main.service", + "client": [ + "/usr/lib/postgresql/19/bin/psql", + "-X", + "-q", + "-v", + "ON_ERROR_STOP=1", + "-h", + "/var/run/postgresql", + "-p", + "5432", + "-U", + "bench", + "-d", + "bench", + "-o", + "/dev/null", + "-c" + ] + }, + "counts": { + "base_runs": [14559773, 12103536, 12137156], + "base": 12103536, + "total_net": 228920014998 + }, + "queries": [ + { + "query": "q0", + "net": 1127187385, + "runs": [2020638587, 1505921835, 1139290921] + }, + { + "query": "q1", + "net": 1987287841, + "runs": [2009303463, 2024866875, 1999391377] + }, + { + "query": "q2", + "net": 2204310222, + "runs": [2228008427, 2228011214, 2216413758] + }, + { + "query": "q3", + "net": 1313044674, + "runs": [1325148210, 1330447333, 1330371293] + }, + { + "query": "q4", + "net": 2437982822, + "runs": [2451496954, 2450086358, 2459061963] + }, + { + "query": "q5", + "net": 3852533451, + "runs": [3939211593, 3917598182, 3864636987] + }, + { + "query": "q6", + "net": 1260878026, + "runs": [1280892920, 1272981562, 1274918827] + }, + { + "query": "q7", + "net": 1988369831, + "runs": [2003153081, 2000473367, 2006959180] + }, + { + "query": "q8", + "net": 4497603372, + "runs": [4509706908, 4550722692, 4548290743] + }, + { + "query": "q9", + "net": 5707036356, + "runs": [5724988974, 5729721572, 5719139892] + }, + { + "query": "q10", + "net": 2102666011, + "runs": [2124708600, 2114769547, 2117246260] + }, + { + "query": "q11", + "net": 2131622355, + "runs": [2146134602, 2143725891, 2151532132] + }, + { + "query": "q12", + "net": 2360099940, + "runs": [2374816832, 2372203476, 2379399057] + }, + { + "query": "q13", + "net": 2610020619, + "runs": [2630146022, 2622124155, 2634749520] + }, + { + "query": "q14", + "net": 2193445094, + "runs": [2223107564, 2205548630, 2222259911] + }, + { + "query": "q15", + "net": 2261442555, + "runs": [2281940529, 2273546091, 2279454556] + }, + { + "query": "q16", + "net": 3321520270, + "runs": [3336860485, 3337065642, 3333623806] + }, + { + "query": "q17", + "net": 3134772104, + "runs": [3146875640, 3152073012, 3164559020] + }, + { + "query": "q18", + "net": 10601091652, + "runs": [10616244850, 10613195188, 10644463629] + }, + { + "query": "q19", + "net": 1205473419, + "runs": [1217576955, 1236916715, 1232003051] + }, + { + "query": "q20", + "net": 2649191434, + "runs": [2689553944, 2661294970, 2663482454] + }, + { + "query": "q21", + "net": 3269908544, + "runs": [3282012080, 3294954560, 3291572997] + }, + { + "query": "q22", + "net": 3580660698, + "runs": [3592764234, 3593460446, 3600493381] + }, + { + "query": "q23", + "net": 2656694172, + "runs": [2681145694, 2668797708, 2690219976] + }, + { + "query": "q24", + "net": 2134625471, + "runs": [2474653543, 2533533371, 2146729007] + }, + { + "query": "q25", + "net": 2100330156, + "runs": [2115861490, 2112433692, 2116506987] + }, + { + "query": "q26", + "net": 2117571838, + "runs": [2137303744, 2129675374, 2129750133] + }, + { + "query": "q27", + "net": 4299536956, + "runs": [4311640492, 4314649357, 4315867195] + }, + { + "query": "q28", + "net": 54758339550, + "runs": [54770443086, 54863077749, 54946788676] + }, + { + "query": "q29", + "net": 8937867227, + "runs": [8949970763, 9525629282, 9858776382] + }, + { + "query": "q30", + "net": 2496044779, + "runs": [2521723530, 2508148315, 2528970813] + }, + { + "query": "q31", + "net": 2532928859, + "runs": [2545032395, 2577892090, 2548224645] + }, + { + "query": "q32", + "net": 17178959741, + "runs": [17337943808, 17191063277, 17418697561] + }, + { + "query": "q33", + "net": 12237746444, + "runs": [12289383027, 12383684663, 12249849980] + }, + { + "query": "q34", + "net": 11487328821, + "runs": [11499432357, 12328037797, 12003709416] + }, + { + "query": "q35", + "net": 3473181175, + "runs": [3485284711, 3527900863, 3507832298] + }, + { + "query": "q36", + "net": 5624864572, + "runs": [5636968108, 5677111336, 5640079997] + }, + { + "query": "q37", + "net": 3394360398, + "runs": [3518054275, 3427695698, 3406463934] + }, + { + "query": "q38", + "net": 2497532965, + "runs": [2509636501, 2514159061, 2550746675] + }, + { + "query": "q39", + "net": 9855515082, + "runs": [10507321035, 9916183476, 9867618618] + }, + { + "query": "q40", + "net": 4758535255, + "runs": [4775388725, 4770638791, 4798933403] + }, + { + "query": "q41", + "net": 4625550391, + "runs": [4637653927, 4645591266, 4660138587] + }, + { + "query": "q42", + "net": 3954352471, + "runs": [3999171810, 3994153682, 3966456007] + } + ] +} diff --git a/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.md b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.md new file mode 100644 index 0000000..7b7b4c9 --- /dev/null +++ b/reports/2026-10-07/6005c2b1-server3-instructions-clickbench-postgresql-smoke.md @@ -0,0 +1,82 @@ +# instructions-clickbench-postgresql on server3, 2026-10-07 + +**This is a smoke run.** It shows that the driver works. It ran on a small scale or on a shared machine, so its numbers are not baselines and no claim may use them. + +| Field | Value | +|---|---| +| suite | instructions-clickbench-postgresql | +| machine | server3 | +| date | 2026-10-07 | +| commit | 6005c2b1 | +| smoke | true | +| note | smoke run: it shows that the runner works, it is not the ratchet | +| set | clickbench | +| engine_name | postgresql | +| queries_from | pg-queries.sql | +| repeat | 3 | +| kernel | 6.8.0-106-generic | + +## engine + +| Field | Value | +|---|---| +| mode | server cgroup | +| cgroup | system.slice/system-postgresql.slice/postgresql@19-main.service | +| client | /usr/lib/postgresql/19/bin/psql, -X, -q, -v, ON_ERROR_STOP=1, -h, /var/run/postgresql, -p, 5432, -U, bench, -d, bench, -o, /dev/null, -c | + +## counts + +| Field | Value | +|---|---| +| base_runs | 14559773, 12103536, 12137156 | +| base | 12103536 | +| total_net | 228920014998 | + +## queries + +| query | net | runs | +|---|---|---| +| q0 | 1127187385 | 2020638587, 1505921835, 1139290921 | +| q1 | 1987287841 | 2009303463, 2024866875, 1999391377 | +| q2 | 2204310222 | 2228008427, 2228011214, 2216413758 | +| q3 | 1313044674 | 1325148210, 1330447333, 1330371293 | +| q4 | 2437982822 | 2451496954, 2450086358, 2459061963 | +| q5 | 3852533451 | 3939211593, 3917598182, 3864636987 | +| q6 | 1260878026 | 1280892920, 1272981562, 1274918827 | +| q7 | 1988369831 | 2003153081, 2000473367, 2006959180 | +| q8 | 4497603372 | 4509706908, 4550722692, 4548290743 | +| q9 | 5707036356 | 5724988974, 5729721572, 5719139892 | +| q10 | 2102666011 | 2124708600, 2114769547, 2117246260 | +| q11 | 2131622355 | 2146134602, 2143725891, 2151532132 | +| q12 | 2360099940 | 2374816832, 2372203476, 2379399057 | +| q13 | 2610020619 | 2630146022, 2622124155, 2634749520 | +| q14 | 2193445094 | 2223107564, 2205548630, 2222259911 | +| q15 | 2261442555 | 2281940529, 2273546091, 2279454556 | +| q16 | 3321520270 | 3336860485, 3337065642, 3333623806 | +| q17 | 3134772104 | 3146875640, 3152073012, 3164559020 | +| q18 | 10601091652 | 10616244850, 10613195188, 10644463629 | +| q19 | 1205473419 | 1217576955, 1236916715, 1232003051 | +| q20 | 2649191434 | 2689553944, 2661294970, 2663482454 | +| q21 | 3269908544 | 3282012080, 3294954560, 3291572997 | +| q22 | 3580660698 | 3592764234, 3593460446, 3600493381 | +| q23 | 2656694172 | 2681145694, 2668797708, 2690219976 | +| q24 | 2134625471 | 2474653543, 2533533371, 2146729007 | +| q25 | 2100330156 | 2115861490, 2112433692, 2116506987 | +| q26 | 2117571838 | 2137303744, 2129675374, 2129750133 | +| q27 | 4299536956 | 4311640492, 4314649357, 4315867195 | +| q28 | 54758339550 | 54770443086, 54863077749, 54946788676 | +| q29 | 8937867227 | 8949970763, 9525629282, 9858776382 | +| q30 | 2496044779 | 2521723530, 2508148315, 2528970813 | +| q31 | 2532928859 | 2545032395, 2577892090, 2548224645 | +| q32 | 17178959741 | 17337943808, 17191063277, 17418697561 | +| q33 | 12237746444 | 12289383027, 12383684663, 12249849980 | +| q34 | 11487328821 | 11499432357, 12328037797, 12003709416 | +| q35 | 3473181175 | 3485284711, 3527900863, 3507832298 | +| q36 | 5624864572 | 5636968108, 5677111336, 5640079997 | +| q37 | 3394360398 | 3518054275, 3427695698, 3406463934 | +| q38 | 2497532965 | 2509636501, 2514159061, 2550746675 | +| q39 | 9855515082 | 10507321035, 9916183476, 9867618618 | +| q40 | 4758535255 | 4775388725, 4770638791, 4798933403 | +| q41 | 4625550391 | 4637653927, 4645591266, 4660138587 | +| q42 | 3954352471 | 3999171810, 3994153682, 3966456007 | +