From 18cf9a75904176604f4763869ed1c7c90faa8f94 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:50:00 +0700 Subject: [PATCH] Add the answer check and the report generator --- README.md | 17 ++- src/answers.rs | 349 +++++++++++++++++++++++++++++++++++++++++++++++++ src/json.rs | 251 ++++++++++++++++++++++++++++++++++- src/main.rs | 95 ++++++++++++++ src/report.rs | 336 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1042 insertions(+), 6 deletions(-) create mode 100644 src/answers.rs create mode 100644 src/report.rs diff --git a/README.md b/README.md index 62cb0a4..b878b35 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The procedure is [`spec/20-benchmarks.md`](https://github.com/tamnd/rupg/blob/ma ## Status -Early. The crate builds and CI is green, but no suite runs yet. `rupg-bench gates` prints the twelve gates of spec/02 section 2.10, `rupg-bench pins` prints the pinned versions, `rupg-bench measure` runs the cgroup v2 runner, and `rupg-bench load` runs the load cases. The first milestone, M0, builds the harness and measures every baseline that a later gate uses. Every command below is the planned interface from the spec. +Early. The crate builds and CI is green, but no suite runs yet. `rupg-bench gates` prints the twelve gates of spec/02 section 2.10, `rupg-bench pins` prints the pinned versions, `rupg-bench measure` runs the cgroup v2 runner, `rupg-bench load` runs the load cases, `rupg-bench answers` checks answer sets, and `rupg-bench report` writes the report files. The first milestone, M0, builds the harness and measures every baseline that a later gate uses. Every command below is the planned interface from the spec. ## The gates @@ -104,6 +104,19 @@ rupg-bench load --case a --source hits.parquet --name duckdb-load --du hits.db - duckdb hits.db -c "CREATE TABLE hits AS SELECT * FROM read_parquet('hits.parquet'); CHECKPOINT;" ``` +`rupg-bench answers` compares two answer sets as multisets of rows (spec/20 section 20.5). An answer set is a directory with one file `.tsv` for each query: a header line with the column names, then the rows in the text format of PostgreSQL `COPY`, with `\N` for NULL. So `COPY () TO STDOUT WITH (HEADER)` writes one. Floats match within a relative 1e-9. An expected file `.out` is a file of the TPC-H answer set at SF1, and its values match with the rules of TPC-H clause 2.1.3.5: equal column values and counts, sums within 100, and averages and ratios within 1 percent. The TPC-H tools are under the TPC EULA, so the harness fetches the answer set at the pin of `pins.toml` and this repository does not hold a copy. + +```sh +rupg-bench answers --expected tpch_tools_3.0.1/dbgen/answers --actual out/ +``` + +`rupg-bench report` writes `reports//--.json` and a Markdown file with the same name from the `--json` output of a command. The Markdown file comes from the JSON file only. A smoke run has `-smoke` at the end of its name and a line at the top that says that its numbers are not baselines. + +```sh +rupg-bench measure --unit cron.service --idle --json > idle.json +rupg-bench report --result idle.json --suite idle-base --machine server3 --smoke +``` + ## The reporting rules 1. A claim has five parts: one named machine, one metric, a baseline that we measured, the same transport on both sides, and the ratio. A claim without all five is not made. @@ -123,7 +136,7 @@ rupg-bench load --case a --source hits.parquet --name duckdb-load --du hits.db - ``` pins.toml versions of every system, the rupg commit src/ suite drivers, the cgroup runner, the report generator -answers/ PostgreSQL 19 answers for ClickBench and TPC-H, TPC-H SF1 set +answers/ PostgreSQL 19 answers for ClickBench and TPC-H (the TPC-H SF1 set is fetched) clickbench/ the rupg directory of our ClickBench fork machines/ scripts for 4xl, metal, tpch, oltp and cluster ratchet.toml the instruction counts and the best number of each row diff --git a/src/answers.rs b/src/answers.rs new file mode 100644 index 0000000..c425af8 --- /dev/null +++ b/src/answers.rs @@ -0,0 +1,349 @@ +//! Answer sets and the answer check of spec/20 sections 20.5 and 20.6. +//! +//! An answer set is a directory with one file for each query, `.tsv`. The first line has the column names. Each other line is one row in the text format of PostgreSQL `COPY`: a tab between fields, `\N` for NULL, and `\\`, `\t`, `\n` and `\r` for those bytes in a value. So `COPY () TO STDOUT WITH (HEADER)` writes an answer file with no other step, and PostgreSQL 15 and later print each float with the shortest exact digits. +//! +//! The check compares two answers as multisets of rows. It sorts both by all columns and then compares row by row. Two numbers match when they differ by at most the tolerance. Two texts match when they are equal after the spaces at the end are removed, because `char(n)` pads with spaces and the TPC-H answer files pad each column. + +use std::cmp::Ordering; +use std::fmt::Write as _; + +/// The answer of one query. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Answer { + pub(crate) columns: Vec, + pub(crate) rows: Vec>>, +} + +/// How two numbers are compared. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) enum Tolerance { + /// `|a - b| <= r * max(|a|, |b|)`. spec/20 section 20.5 sets 1e-9 for ClickBench. + Relative(f64), + /// The rules of TPC-H clause 2.1.3.5 for the answer set at SF1. The column name of the answer file gives the kind of value (see `tpch_kind`). A column value or a count must be equal after the actual value is rounded to the decimals of the expected value. A sum may differ by 100. An average, rounded to 2 decimals, must be within 1 percent. A ratio of sums must pass both rules. + Tpch, +} + +/// The tolerance of spec/20 section 20.5. +pub(crate) const RELATIVE: Tolerance = Tolerance::Relative(1e-9); + +/// Reads an answer in the `COPY` text format with a header line. +pub(crate) fn parse_tsv(text: &str) -> Result { + let mut lines = text.split('\n'); + let header = lines.next().ok_or("an answer file with no header")?; + let columns: Vec = header + .split('\t') + .map(unescape) + .collect::, _>>()? + .into_iter() + .map(Option::unwrap_or_default) + .collect(); + let mut rows = Vec::new(); + for (i, line) in lines.enumerate() { + if line.is_empty() { + continue; + } + let row: Vec> = line.split('\t').map(unescape).collect::>()?; + if row.len() != columns.len() { + return Err(format!( + "line {} has {} fields and the header has {}", + i + 2, + row.len(), + columns.len() + )); + } + rows.push(row); + } + Ok(Answer { columns, rows }) +} + +fn unescape(field: &str) -> Result, String> { + if field == "\\N" { + return Ok(None); + } + let mut out = String::with_capacity(field.len()); + let mut chars = field.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('\\') => out.push('\\'), + Some('t') => out.push('\t'), + Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + other => { + return Err(format!( + "a bad escape \\{} in {field:?}", + other.map(String::from).unwrap_or_default() + )); + } + } + } + Ok(Some(out)) +} + +/// Reads a file of the TPC-H answer set, for example `dbgen/answers/q1.out`: a header line, then one line for each row, with `|` between the fields and spaces that pad them. +pub(crate) fn parse_tpch(text: &str) -> Result { + let mut lines = text.lines().filter(|l| !l.trim().is_empty()); + let header = lines.next().ok_or("an answer file with no header")?; + let columns: Vec = header.split('|').map(|c| c.trim().to_owned()).collect(); + let mut rows = Vec::new(); + for line in lines { + let row: Vec> = line.split('|').map(|f| Some(f.trim().to_owned())).collect(); + if row.len() != columns.len() { + return Err(format!( + "{line:?} has {} fields and the header has {}", + row.len(), + columns.len() + )); + } + rows.push(row); + } + Ok(Answer { columns, rows }) +} + +/// The value as a number, when it has the form of one. `NaN` and `Infinity` stay text and must be equal. +fn number(v: &str) -> Option { + let v = v.trim(); + let ok = v.bytes().any(|b| b.is_ascii_digit()) + && v.bytes().all(|b| matches!(b, b'0'..=b'9' | b'+' | b'-' | b'.' | b'e' | b'E')); + if ok { v.parse().ok() } else { None } +} + +fn decimals(v: &str) -> usize { + let v = v.trim(); + let mantissa = v.split(['e', 'E']).next().unwrap_or(v); + mantissa.split_once('.').map_or(0, |(_, d)| d.len()) +} + +/// The kinds of value of TPC-H clause 2.1.3.5. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TpchKind { + /// A column value, a count, or a `sum(l_quantity)` (comment 4 of the clause). + Exact, + /// The result of a `sum`: within 100. + Sum, + /// The result of an `avg`: within 1 percent after rounding to 2 decimals. + Avg, + /// A ratio of sums, as in Q8, Q14 and Q17: both rules (comment 1 of the clause). + SumRatio, +} + +/// The kind of a column of the TPC-H answer set at SF1, from its name in the answer file. +pub(crate) fn tpch_kind(column: &str) -> TpchKind { + match column { + // Q1 sum_qty and Q18 col6 are sum(l_quantity). + "sum_qty" | "col6" => TpchKind::Exact, + "mkt_share" | "promo_revenue" | "avg_yearly" => TpchKind::SumRatio, + "revenue" | "total_revenue" | "value" | "totacctbal" => TpchKind::Sum, + c if c.starts_with("sum_") => TpchKind::Sum, + c if c.starts_with("avg_") => TpchKind::Avg, + _ => TpchKind::Exact, + } +} + +fn round(x: f64, decimals: usize) -> String { + let s = format!("{x:.decimals$}"); + if s.trim_start_matches(['-', '0', '.']).is_empty() { + s.trim_start_matches('-').to_owned() + } else { + s + } +} + +/// `0.99 * v <= round(r, 2) <= 1.01 * v`, for a negative `v` too. +fn within_one_percent(v: f64, r: f64) -> bool { + let r: f64 = round(r, 2).parse().unwrap_or(f64::NAN); + let (lo, hi) = if v < 0.0 { (1.01 * v, 0.99 * v) } else { (0.99 * v, 1.01 * v) }; + lo <= r && r <= hi +} + +fn value_matches( + expected: Option<&str>, + actual: Option<&str>, + tol: Tolerance, + column: &str, +) -> bool { + let (Some(e), Some(a)) = (expected, actual) else { + return expected.is_none() && actual.is_none(); + }; + if let (Some(x), Some(y)) = (number(e), number(a)) { + #[allow(clippy::float_cmp)] + if x == y { + return true; + } + return match tol { + Tolerance::Relative(r) => (x - y).abs() <= r * x.abs().max(y.abs()), + Tolerance::Tpch => match tpch_kind(column) { + TpchKind::Exact => round(y, decimals(e)) == round(x, decimals(e)), + TpchKind::Sum => (x - y).abs() <= 100.0, + TpchKind::Avg => within_one_percent(x, y), + TpchKind::SumRatio => (x - y).abs() <= 100.0 && within_one_percent(x, y), + }, + }; + } + e.trim_end_matches(' ') == a.trim_end_matches(' ') +} + +/// The order of the sort: NULL first, then numbers by value, then text. +fn cmp_value(a: Option<&str>, b: Option<&str>) -> Ordering { + match (a, b) { + (None, None) => Ordering::Equal, + (None, Some(_)) => Ordering::Less, + (Some(_), None) => Ordering::Greater, + (Some(x), Some(y)) => match (number(x), number(y)) { + (Some(p), Some(q)) => p.total_cmp(&q), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => x.trim_end_matches(' ').cmp(y.trim_end_matches(' ')), + }, + } +} + +fn sorted(rows: &[Vec>]) -> Vec<&Vec>> { + let mut out: Vec<&Vec>> = rows.iter().collect(); + out.sort_by(|a, b| { + a.iter() + .zip(b.iter()) + .map(|(x, y)| cmp_value(x.as_deref(), y.as_deref())) + .find(|o| o.is_ne()) + .unwrap_or(Ordering::Equal) + }); + out +} + +/// Compares two answers as multisets of rows. The names of the columns are not compared, because each system names an expression in its own way. The error names the first difference. +pub(crate) fn compare(expected: &Answer, actual: &Answer, tol: Tolerance) -> Result<(), String> { + if expected.columns.len() != actual.columns.len() { + return Err(format!( + "{} columns, expected {}", + actual.columns.len(), + expected.columns.len() + )); + } + if expected.rows.len() != actual.rows.len() { + return Err(format!("{} rows, expected {}", actual.rows.len(), expected.rows.len())); + } + for (i, (e, a)) in sorted(&expected.rows).into_iter().zip(sorted(&actual.rows)).enumerate() { + for (c, (x, y)) in e.iter().zip(a.iter()).enumerate() { + if !value_matches(x.as_deref(), y.as_deref(), tol, &expected.columns[c]) { + let mut msg = + format!("sorted row {} column {} ({}): ", i + 1, c + 1, expected.columns[c]); + let _ = write!(msg, "got {}, expected {}", show(y.as_deref()), show(x.as_deref())); + return Err(msg); + } + } + } + Ok(()) +} + +fn show(v: Option<&str>) -> String { + v.map_or_else(|| "NULL".to_owned(), |s| format!("{:?}", s.trim_end_matches(' '))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn answer(columns: &[&str], rows: &[&[Option<&str>]]) -> Answer { + Answer { + columns: columns.iter().map(|c| (*c).to_owned()).collect(), + rows: rows.iter().map(|r| r.iter().map(|v| v.map(str::to_owned)).collect()).collect(), + } + } + + #[test] + fn tsv_files() { + let a = answer(&["k", "v"], &[&[Some("a\tb\\c\nd"), None], &[Some(""), Some("\\N")]]); + assert_eq!(parse_tsv("k\tv\na\\tb\\\\c\\nd\t\\N\n\t\\\\N\n").unwrap(), a); + assert!(parse_tsv("a\tb\n1\n").unwrap_err().contains("line 2 has 1 fields")); + assert!(parse_tsv("a\n\\q\n").is_err()); + } + + #[test] + fn tpch_answer_files() { + let text = "c_name |c_custkey |col6 \nCustomer#000128120 | 128120|323.00\n"; + let a = parse_tpch(text).unwrap(); + assert_eq!(a.columns, ["c_name", "c_custkey", "col6"]); + assert_eq!( + a.rows, + [[ + Some("Customer#000128120".to_owned()), + Some("128120".to_owned()), + Some("323.00".to_owned()) + ]] + ); + let ours = answer( + &["c_name", "c_custkey", "sum"], + &[&[Some("Customer#000128120 "), Some("128120"), Some("323")]], + ); + assert_eq!(compare(&a, &ours, Tolerance::Tpch), Ok(())); + } + + #[test] + fn numbers_and_tolerances() { + let rel = RELATIVE; + let m = |e, a, tol, c| value_matches(e, a, tol, c); + assert!(m(Some("1e10"), Some("10000000000.000001"), rel, "x")); + assert!(!m(Some("1"), Some("1.00001"), rel, "x")); + assert!(m(Some("NaN"), Some("NaN"), rel, "x")); + assert!(!m(Some("NaN"), Some("nan"), rel, "x")); + assert!(!m(Some("1994-04-07"), Some("1994-04-08"), rel, "x")); + assert!(m(None, None, rel, "x")); + assert!(!m(None, Some(""), rel, "x")); + let t = Tolerance::Tpch; + // Column values and counts. + assert!(m(Some("9938.53"), Some("9938.53"), t, "s_acctbal")); + assert!(m(Some("323.00"), Some("323"), t, "o_totalprice")); + assert!(!m(Some("9938.53"), Some("9938.54"), t, "s_acctbal")); + assert!(!m(Some("50005"), Some("50004"), t, "custdist")); + assert!(m(Some("0"), Some("-0.0001"), t, "x")); + // Sums within 100. + assert!(m(Some("123141078.23"), Some("123141078.2283"), t, "revenue")); + assert!(m(Some("53758257134.87"), Some("53758257100.00"), t, "sum_disc_price")); + assert!(!m(Some("53758257134.87"), Some("53758257034.00"), t, "sum_disc_price")); + // Averages and ratios within 1 percent after rounding to 2 decimals. + assert!(m(Some("348406.02"), Some("348406.0542857143"), t, "avg_yearly")); + assert!(m(Some("0.05"), Some("0.049985295838397614"), t, "avg_disc")); + assert!(m(Some("16.38"), Some("16.3807"), t, "promo_revenue")); + assert!(!m(Some("16.38"), Some("16.60"), t, "promo_revenue")); + assert!(!m(Some("0.05"), Some("0.0449"), t, "avg_disc")); + assert!(!m(Some("37734107.00"), Some("37734107.5"), t, "sum_qty")); + assert!(!m(Some("348406.02"), Some("348606.02"), t, "avg_yearly")); + } + + #[test] + fn tpch_kinds() { + assert_eq!(tpch_kind("sum_charge"), TpchKind::Sum); + assert_eq!(tpch_kind("col6"), TpchKind::Exact); + assert_eq!(tpch_kind("sum_qty"), TpchKind::Exact); + assert_eq!(tpch_kind("avg_price"), TpchKind::Avg); + assert_eq!(tpch_kind("mkt_share"), TpchKind::SumRatio); + assert_eq!(tpch_kind("count_order"), TpchKind::Exact); + assert_eq!(tpch_kind("l_returnflag"), TpchKind::Exact); + } + + #[test] + fn multisets() { + let e = answer( + &["a", "b"], + &[&[Some("x"), Some("2")], &[Some("x"), Some("10")], &[None, Some("1")]], + ); + let a = answer( + &["a", "count"], + &[&[Some("x"), Some("10")], &[None, Some("1")], &[Some("x"), Some("2")]], + ); + assert_eq!(compare(&e, &a, RELATIVE), Ok(())); + let b = answer( + &["a", "b"], + &[&[Some("x"), Some("10")], &[None, Some("1")], &[Some("x"), Some("3")]], + ); + assert_eq!( + compare(&e, &b, RELATIVE).unwrap_err(), + "sorted row 2 column 2 (b): got \"3\", expected \"2\"" + ); + let c = answer(&["a", "b"], &[&[None, Some("1")]]); + assert_eq!(compare(&e, &c, RELATIVE).unwrap_err(), "1 rows, expected 3"); + } +} diff --git a/src/json.rs b/src/json.rs index 2f0f12d..0d9ccac 100644 --- a/src/json.rs +++ b/src/json.rs @@ -1,4 +1,6 @@ -//! A JSON writer for the result files. The harness has no dependencies, so this replaces `serde_json`. +//! A JSON writer and parser for the result files. The harness has no dependencies, so this replaces `serde_json`. +//! +//! The writer escapes each character that is not ASCII, so a result file is ASCII. use std::fmt::Write as _; @@ -113,10 +115,13 @@ fn quote(out: &mut String, s: &str) { '\n' => out.push_str("\\n"), '\r' => out.push_str("\\r"), '\t' => out.push_str("\\t"), - c if (c as u32) < 0x20 => { - let _ = write!(out, "\\u{:04x}", c as u32); + c if c.is_ascii() && (c as u32) >= 0x20 => out.push(c), + c => { + let mut units = [0u16; 2]; + for u in c.encode_utf16(&mut units) { + let _ = write!(out, "\\u{u:04x}"); + } } - c => out.push(c), } } out.push('"'); @@ -182,6 +187,210 @@ impl> From> for Json { } } +impl Json { + /// The value of `key` in an object. + pub(crate) fn get(&self, key: &str) -> Option<&Json> { + match self { + Json::Obj(entries) => entries.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + pub(crate) fn as_str(&self) -> Option<&str> { + match self { + Json::Str(s) => Some(s), + _ => None, + } + } + + pub(crate) fn as_bool(&self) -> Option { + match self { + Json::Bool(b) => Some(*b), + _ => None, + } + } + + /// Parses a JSON text. An error names the byte offset. + pub(crate) fn parse(text: &str) -> Result { + let mut p = Parser { b: text.as_bytes(), at: 0 }; + let v = p.value(0)?; + p.space(); + if p.at != p.b.len() { + return Err(p.err("text after the value")); + } + Ok(v) + } +} + +/// The parser. Nesting is limited, so a bad file cannot overflow the stack. +struct Parser<'a> { + b: &'a [u8], + at: usize, +} + +const MAX_DEPTH: usize = 128; + +impl Parser<'_> { + fn err(&self, what: &str) -> String { + format!("json at byte {}: {what}", self.at) + } + + fn space(&mut self) { + while self.at < self.b.len() && matches!(self.b[self.at], b' ' | b'\t' | b'\n' | b'\r') { + self.at += 1; + } + } + + fn eat(&mut self, word: &str) -> bool { + if self.b[self.at..].starts_with(word.as_bytes()) { + self.at += word.len(); + true + } else { + false + } + } + + fn value(&mut self, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(self.err("nested too deep")); + } + self.space(); + match self.b.get(self.at) { + None => Err(self.err("no value")), + Some(b'n') if self.eat("null") => Ok(Json::Null), + Some(b't') if self.eat("true") => Ok(Json::Bool(true)), + Some(b'f') if self.eat("false") => Ok(Json::Bool(false)), + Some(b'"') => Ok(Json::Str(self.string()?)), + Some(b'[') => { + self.at += 1; + let mut items = Vec::new(); + self.space(); + if self.eat("]") { + return Ok(Json::Arr(items)); + } + loop { + items.push(self.value(depth + 1)?); + self.space(); + if self.eat(",") { + continue; + } + if self.eat("]") { + return Ok(Json::Arr(items)); + } + return Err(self.err("expected , or ]")); + } + } + Some(b'{') => { + self.at += 1; + let mut entries = Vec::new(); + self.space(); + if self.eat("}") { + return Ok(Json::Obj(entries)); + } + loop { + self.space(); + if self.b.get(self.at) != Some(&b'"') { + return Err(self.err("expected a key")); + } + let key = self.string()?; + self.space(); + if !self.eat(":") { + return Err(self.err("expected :")); + } + let v = self.value(depth + 1)?; + entries.push((key, v)); + self.space(); + if self.eat(",") { + continue; + } + if self.eat("}") { + return Ok(Json::Obj(entries)); + } + return Err(self.err("expected , or }")); + } + } + Some(c) if *c == b'-' || c.is_ascii_digit() => self.number(), + Some(_) => Err(self.err("not a value")), + } + } + + fn number(&mut self) -> Result { + let start = self.at; + while self.at < self.b.len() + && matches!(self.b[self.at], b'-' | b'+' | b'.' | b'e' | b'E' | b'0'..=b'9') + { + self.at += 1; + } + let s = std::str::from_utf8(&self.b[start..self.at]).map_err(|_| self.err("bad number"))?; + if let Ok(n) = s.parse::() { + return Ok(Json::UInt(n)); + } + if let Ok(n) = s.parse::() { + return Ok(Json::Int(n)); + } + s.parse::().map(Json::Num).map_err(|_| self.err("bad number")) + } + + fn string(&mut self) -> Result { + self.at += 1; + let mut out = Vec::new(); + loop { + let Some(&c) = self.b.get(self.at) else { + return Err(self.err("a string with no end")); + }; + self.at += 1; + match c { + b'"' => { + return String::from_utf8(out) + .map_err(|_| self.err("a string that is not UTF-8")); + } + b'\\' => { + let Some(&e) = self.b.get(self.at) else { + return Err(self.err("a string with no end")); + }; + self.at += 1; + match e { + b'"' => out.push(b'"'), + b'\\' => out.push(b'\\'), + b'/' => out.push(b'/'), + b'b' => out.push(8), + b'f' => out.push(12), + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'u' => { + let mut code = self.hex4()?; + if (0xd800..0xdc00).contains(&code) { + if !self.eat("\\u") { + return Err(self.err("a lone surrogate")); + } + let low = self.hex4()?; + if !(0xdc00..0xe000).contains(&low) { + return Err(self.err("a lone surrogate")); + } + code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00); + } + let ch = char::from_u32(code).ok_or(self.err("a bad escape"))?; + let mut buf = [0u8; 4]; + out.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); + } + _ => return Err(self.err("a bad escape")), + } + } + c => out.push(c), + } + } + } + + fn hex4(&mut self) -> Result { + let s = self.b.get(self.at..self.at + 4).ok_or(self.err("a short escape"))?; + let s = std::str::from_utf8(s).map_err(|_| self.err("a bad escape"))?; + let n = u32::from_str_radix(s, 16).map_err(|_| self.err("a bad escape"))?; + self.at += 4; + Ok(n) + } +} + #[cfg(test)] mod tests { use super::*; @@ -198,4 +407,38 @@ mod tests { let want = "{\n \"name\": \"a \\\"b\\\"\\n\",\n \"times\": [0.5, 1.25],\n \"none\": null,\n \"nan\": null,\n \"rows\": [\n {\n \"n\": 1\n }\n ],\n \"empty\": {}\n}\n"; assert_eq!(j.pretty(), want); } + + #[test] + fn parse_what_the_writer_writes() { + let j = Json::obj() + .with("name", "a \"b\"\n\u{e9}\u{2014}\u{1f600}") + .with("times", vec![0.5, 1.25]) + .with("big", u64::MAX) + .with("neg", -3i64) + .with("none", Option::::None) + .with("yes", true) + .with("rows", vec![Json::obj().with("n", 1u64)]) + .with("empty", Json::obj()); + let text = j.pretty(); + assert!(text.is_ascii()); + assert!(text.contains("\\u00e9\\u2014\\ud83d\\ude00")); + assert_eq!(Json::parse(&text).unwrap(), j); + assert_eq!(j.get("neg"), Some(&Json::Int(-3))); + assert_eq!(j.get("yes").and_then(Json::as_bool), Some(true)); + } + + #[test] + fn parse_errors() { + assert_eq!( + Json::parse(" [1, 2.5e1, \"x\"] ").unwrap(), + Json::Arr(vec![Json::UInt(1), Json::Num(25.0), Json::Str("x".into())]) + ); + for bad in + ["", "[1,", "{\"a\" 1}", "[1] x", "\"abc", "\"\\q\"", "\"\\ud800\"", "nul", "[-]"] + { + assert!(Json::parse(bad).is_err(), "{bad:?}"); + } + let deep = "[".repeat(200) + &"]".repeat(200); + assert!(Json::parse(&deep).unwrap_err().contains("nested too deep")); + } } diff --git a/src/main.rs b/src/main.rs index aa8b91a..fa35f69 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,12 +11,14 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::cgroup::Cgroup; use crate::json::Json; +mod answers; mod args; mod cgroup; mod gates; mod json; mod load; mod pins; +mod report; mod toml; const USAGE: &str = "usage: rupg-bench [options] @@ -34,6 +36,14 @@ commands: run the steps of case A or case B of spec/20 section 20.3.1 on FILE (drop_caches, fincore, the timed cat), then time COMMAND, the load, in the cgroup runner. Case A records L, R and L / R. + report --result FILE [--out DIR] [--suite S] [--machine M] [--commit C] [--smoke] + write DIR//--.json and .md from a result file + (the --json output of a command). The options fill the fields that the file lacks. + --smoke marks the run as a smoke run, not a baseline. DIR is reports by default. + answers --expected PATH --actual PATH + compare two answer sets as multisets of rows (spec/20 section 20.5). + PATH is a file or a directory of .tsv files. An expected file + .out is a TPC-H answer file and is compared with the rules of TPC-H clause 2.1.3.5. --version print the version"; fn main() -> ExitCode { @@ -75,6 +85,8 @@ fn run(argv: &[String]) -> Result<(), String> { } "measure" => measure(a)?, "load" => load_command(a)?, + "report" => report_command(a)?, + "answers" => answers_command(a)?, "--version" | "-V" => println!("rupg-bench {}", env!("CARGO_PKG_VERSION")), "--help" | "-h" | "help" => println!("{USAGE}"), other => return Err(format!("usage: unknown command {other:?}\n{USAGE}")), @@ -236,3 +248,86 @@ fn load_command(mut a: args::Args) -> Result<(), String> { out.du(du) }) } + +/// The `report` command. +fn report_command(mut a: args::Args) -> Result<(), String> { + let file = a.value("result").ok_or("usage: report needs --result FILE")?; + let out = a.value("out").unwrap_or_else(|| "reports".to_owned()); + let suite = a.value("suite"); + let machine = a.value("machine"); + let commit = a.value("commit"); + let smoke = a.flag("smoke"); + a.finish()?; + let text = std::fs::read_to_string(&file).map_err(|e| format!("{file}: {e}"))?; + let mut result = Json::parse(&text).map_err(|e| format!("{file}: {e}"))?; + if report::Meta::from_json(&result).is_err() { + let suite = suite.ok_or(format!("usage: {file} has no suite, so give --suite"))?; + let meta = report::Meta::now(&suite, machine, commit, smoke); + let Json::Obj(entries) = result else { return Err(format!("{file} is not a JSON object")) }; + let Json::Obj(mut head) = meta.to_json() else { + unreachable!("Meta::to_json is an object") + }; + head.extend(entries.into_iter().filter(|(k, _)| { + !matches!(k.as_str(), "suite" | "machine" | "date" | "commit" | "smoke") + })); + result = Json::Obj(head); + } else if suite.is_some() || machine.is_some() || commit.is_some() || smoke { + return Err(format!( + "usage: {file} names its run already, so --suite, --machine, --commit and --smoke do not apply" + )); + } + let (md, json) = report::write(Path::new(&out), &result)?; + println!("{}\n{}", md.display(), json.display()); + Ok(()) +} + +/// The `answers` command. +fn answers_command(mut a: args::Args) -> Result<(), String> { + let expected = a.value("expected").ok_or("usage: answers needs --expected PATH")?; + let actual = a.value("actual").ok_or("usage: answers needs --actual PATH")?; + a.finish()?; + let (expected, actual) = (Path::new(&expected), Path::new(&actual)); + let pairs: Vec<(std::path::PathBuf, std::path::PathBuf)> = if expected.is_dir() { + let mut files: Vec<_> = std::fs::read_dir(expected) + .map_err(|e| format!("{}: {e}", expected.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| matches!(p.extension().and_then(|x| x.to_str()), Some("tsv" | "out"))) + .collect(); + files.sort(); + files + .into_iter() + .map(|e| { + let stem = + e.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(); + (e, actual.join(format!("{stem}.tsv"))) + }) + .collect() + } else { + vec![(expected.to_path_buf(), actual.to_path_buf())] + }; + if pairs.is_empty() { + return Err(format!("{} has no .tsv or .out file", expected.display())); + } + let mut wrong = 0; + for (e, act) in &pairs { + let read = + |p: &Path| std::fs::read_to_string(p).map_err(|err| format!("{}: {err}", p.display())); + let tpch = e.extension().is_some_and(|x| x == "out"); + let verdict = (|| { + let want = + if tpch { answers::parse_tpch(&read(e)?)? } else { answers::parse_tsv(&read(e)?)? }; + let got = answers::parse_tsv(&read(act)?)?; + let tol = if tpch { answers::Tolerance::Tpch } else { answers::RELATIVE }; + answers::compare(&want, &got, tol).map(|()| want.rows.len()) + })(); + let name = e.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(); + match verdict { + Ok(rows) => println!("{name:<8} ok, {rows} rows"), + Err(err) => { + wrong += 1; + println!("{name:<8} WRONG: {err}"); + } + } + } + if wrong > 0 { Err(format!("{wrong} of {} answers are wrong", pairs.len())) } else { Ok(()) } +} diff --git a/src/report.rs b/src/report.rs new file mode 100644 index 0000000..7c0d7cf --- /dev/null +++ b/src/report.rs @@ -0,0 +1,336 @@ +//! The report generator of spec/20 section 20.12. +//! +//! Each run writes `reports//--.json` with every number of the run, and `.md` next to it for a reader. The Markdown file is made from the JSON file only, so `rupg-bench report` can make it again from a JSON file. A smoke run has `-smoke` at the end of its name and a line at the top that says it is not a baseline. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::json::Json; + +/// The fields that name a result. Each driver sets them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Meta { + pub(crate) suite: String, + pub(crate) machine: String, + /// `YYYY-MM-DD` in UTC. + pub(crate) date: String, + /// The commit of the system that the run measures, or of the harness for a baseline run. + pub(crate) commit: String, + pub(crate) smoke: bool, +} + +impl Meta { + /// The meta of a run that starts now on this machine. The machine is the host name unless `machine` is given. + pub(crate) fn now( + suite: &str, + machine: Option, + commit: Option, + smoke: bool, + ) -> Meta { + let machine = machine.unwrap_or_else(|| { + fs::read_to_string("/proc/sys/kernel/hostname") + .map(|s| s.trim().to_owned()) + .unwrap_or_else(|_| "unknown".to_owned()) + }); + let commit = commit.unwrap_or_else(harness_commit); + Meta { suite: suite.to_owned(), machine, date: today(), commit, smoke } + } + + pub(crate) fn to_json(&self) -> Json { + Json::obj() + .with("suite", self.suite.as_str()) + .with("machine", self.machine.as_str()) + .with("date", self.date.as_str()) + .with("commit", self.commit.as_str()) + .with("smoke", self.smoke) + } + + pub(crate) fn from_json(j: &Json) -> Result { + let s = |k: &str| { + j.get(k) + .and_then(Json::as_str) + .map(str::to_owned) + .ok_or(format!("the result has no {k}")) + }; + Ok(Meta { + suite: s("suite")?, + machine: s("machine")?, + date: s("date")?, + commit: s("commit")?, + smoke: j.get("smoke").and_then(Json::as_bool).ok_or("the result has no smoke field")?, + }) + } + + /// `/--`, with `-smoke` for a smoke run. + pub(crate) fn stem(&self) -> PathBuf { + let mut name = format!("{}-{}-{}", self.commit, self.machine, self.suite); + if self.smoke { + name.push_str("-smoke"); + } + Path::new(&self.date).join(name) + } +} + +/// The short commit of the harness, or `unknown` outside a git checkout. +fn harness_commit() -> String { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")); + Command::new("git") + .arg("-C") + .arg(dir) + .args(["rev-parse", "--short=8", "HEAD"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_owned()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +/// Today in UTC as `YYYY-MM-DD`. +pub(crate) fn today() -> String { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + civil_date(secs / 86400) +} + +/// The date of a day number since 1970-01-01 (the algorithm of Howard Hinnant's `civil_from_days`). +fn civil_date(days: u64) -> String { + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}") +} + +/// Writes the JSON file and the Markdown file under `root` and returns their paths. +pub(crate) fn write(root: &Path, result: &Json) -> Result<(PathBuf, PathBuf), String> { + let meta = Meta::from_json(result)?; + let stem = root.join(meta.stem()); + let dir = stem.parent().ok_or("a report path with no directory")?; + fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let json = stem.with_extension("json"); + let md = stem.with_extension("md"); + fs::write(&json, result.pretty()).map_err(|e| format!("{}: {e}", json.display()))?; + fs::write(&md, markdown(result)?).map_err(|e| format!("{}: {e}", md.display()))?; + Ok((md, json)) +} + +/// The Markdown form of a result. Scalars go in a table of fields. An object gets a section with a table of its fields, and nested objects get dotted names. An array of objects gets a table with one row for each object. Text with more than one line goes in a code block. +pub(crate) fn markdown(result: &Json) -> Result { + let meta = Meta::from_json(result)?; + let mut out = format!("# {} on {}, {}\n\n", meta.suite, meta.machine, meta.date); + if meta.smoke { + out.push_str("**This is a smoke run.** It shows that the driver works. It ran on a small scale or on a shared machine, so its numbers are not baselines and no claim may use them.\n\n"); + } else { + out.push_str("This is a measured run with the rules of spec/20.\n\n"); + } + let Json::Obj(entries) = result else { return Err("a result must be an object".to_owned()) }; + let mut fields = Vec::new(); + let mut blocks = Vec::new(); + let mut sections = Vec::new(); + for (k, v) in entries { + match v { + Json::Obj(_) => sections.push((k.as_str(), v)), + Json::Arr(items) if items.iter().any(|i| matches!(i, Json::Obj(_))) => { + sections.push((k.as_str(), v)) + } + Json::Str(s) if needs_block(s) => blocks.push((k.clone(), s.clone())), + _ => fields.push((k.clone(), scalar(v))), + } + } + table(&mut out, &fields); + code_blocks(&mut out, &blocks); + for (k, v) in sections { + out.push_str(&format!("## {k}\n\n")); + match v { + Json::Arr(items) => rows(&mut out, items), + _ => { + let mut fields = Vec::new(); + let mut blocks = Vec::new(); + flatten("", v, &mut fields, &mut blocks); + table(&mut out, &fields); + code_blocks(&mut out, &blocks); + } + } + } + Ok(out) +} + +fn flatten( + prefix: &str, + v: &Json, + fields: &mut Vec<(String, String)>, + blocks: &mut Vec<(String, String)>, +) { + match v { + Json::Obj(entries) => { + for (k, v) in entries { + let name = if prefix.is_empty() { k.clone() } else { format!("{prefix}.{k}") }; + flatten(&name, v, fields, blocks); + } + } + Json::Arr(items) if items.iter().any(|i| matches!(i, Json::Obj(_))) => { + for (i, item) in items.iter().enumerate() { + flatten(&format!("{prefix}.{i}"), item, fields, blocks); + } + } + Json::Str(s) if needs_block(s) => blocks.push((prefix.to_owned(), s.clone())), + _ => fields.push((prefix.to_owned(), scalar(v))), + } +} + +/// Text with more than one line goes in a code block. So does text with a hyphen between two spaces, which the style check of the reports does not allow in running text. +fn needs_block(s: &str) -> bool { + s.contains('\n') || s.contains(" - ") +} + +/// The style check of the reports does not allow the en dash and the em dash, so the Markdown file shows them as JSON escapes. The JSON file escapes them too. +fn ascii_dashes(s: &str) -> String { + s.replace('\u{2013}', "\\u2013").replace('\u{2014}', "\\u2014") +} + +/// One cell. A `|` in text would end the cell, so it is escaped. +fn scalar(v: &Json) -> String { + let s = match v { + Json::Null => "null".to_owned(), + Json::Bool(b) => b.to_string(), + Json::Int(n) => n.to_string(), + Json::UInt(n) => n.to_string(), + Json::Num(x) => format_num(*x), + Json::Str(s) => s.clone(), + Json::Arr(items) => items.iter().map(scalar).collect::>().join(", "), + Json::Obj(_) => "(object)".to_owned(), + }; + ascii_dashes(&s).replace('|', "\\|").replace('\n', " ") +} + +/// A number with at most 6 decimals and no trailing zeros. +fn format_num(x: f64) -> String { + if !x.is_finite() { + return "null".to_owned(); + } + let s = format!("{x:.6}"); + let s = s.trim_end_matches('0').trim_end_matches('.'); + if s == "-0" { "0".to_owned() } else { s.to_owned() } +} + +fn table(out: &mut String, fields: &[(String, String)]) { + if fields.is_empty() { + return; + } + out.push_str("| Field | Value |\n|---|---|\n"); + for (k, v) in fields { + out.push_str(&format!("| {k} | {v} |\n")); + } + out.push('\n'); +} + +fn code_blocks(out: &mut String, blocks: &[(String, String)]) { + for (k, text) in blocks { + out.push_str(&format!("{k}:\n\n```\n{}\n```\n\n", ascii_dashes(text.trim_end()))); + } +} + +/// A table with one row for each object. The columns are the scalar fields of the objects, in the order of first use. +fn rows(out: &mut String, items: &[Json]) { + let mut columns: Vec = Vec::new(); + let mut flat: Vec> = Vec::new(); + for item in items { + let mut fields = Vec::new(); + let mut blocks = Vec::new(); + flatten("", item, &mut fields, &mut blocks); + for (k, _) in &fields { + if !columns.contains(k) { + columns.push(k.clone()); + } + } + flat.push(fields); + } + if columns.is_empty() { + return; + } + out.push_str(&format!("| {} |\n|{}\n", columns.join(" | "), "---|".repeat(columns.len()))); + for fields in flat { + let cells: Vec<&str> = columns + .iter() + .map(|c| fields.iter().find(|(k, _)| k == c).map_or("", |(_, v)| v.as_str())) + .collect(); + out.push_str(&format!("| {} |\n", cells.join(" | "))); + } + out.push('\n'); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Json { + Meta { + suite: "pgbench".to_owned(), + machine: "server3".to_owned(), + date: "2026-10-07".to_owned(), + commit: "abc12345".to_owned(), + smoke: true, + } + .to_json() + .with("tps", 411.4976171) + .with("output", "line one\nline | two") + .with("command", "a - b") + .with("dash", "x \u{2014} y") + .with( + "run", + Json::obj().with("cpu_usec", 10u64).with("io", Json::obj().with("rbytes", 2u64)), + ) + .with( + "queries", + vec![ + Json::obj().with("q", "Q1").with("times", vec![0.5, 0.25]), + Json::obj().with("q", "Q2").with("cold", 1.0), + ], + ) + } + + #[test] + fn the_name_of_a_report() { + let m = Meta::from_json(&sample()).unwrap(); + assert_eq!(m.stem(), Path::new("2026-10-07/abc12345-server3-pgbench-smoke")); + } + + #[test] + fn markdown_of_a_result() { + let md = markdown(&sample()).unwrap(); + assert!(md.starts_with("# pgbench on server3, 2026-10-07\n\n**This is a smoke run.**")); + assert!(md.contains("| tps | 411.497617 |")); + assert!(md.contains("output:\n\n```\nline one\nline | two\n```")); + assert!(md.contains("command:\n\n```\na - b\n```")); + assert!(md.contains("| dash | x \\u2014 y |")); + assert!(md.contains( + "## run\n\n| Field | Value |\n|---|---|\n| cpu_usec | 10 |\n| io.rbytes | 2 |" + )); + assert!( + md.contains( + "| q | times | cold |\n|---|---|---|\n| Q1 | 0.5, 0.25 | |\n| Q2 | | 1 |" + ) + ); + } + + #[test] + fn dates() { + assert_eq!(civil_date(0), "1970-01-01"); + assert_eq!(civil_date(20_733), "2026-10-07"); + assert_eq!(civil_date(11_016), "2000-02-29"); + } + + #[test] + fn numbers() { + assert_eq!(format_num(1.0), "1"); + assert_eq!(format_num(0.1234567), "0.123457"); + assert_eq!(format_num(-0.0000001), "0"); + } +}